From 685e59885718c11e371136c37bc4aa0d208dc9b2 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Mon, 25 May 2020 16:41:42 -0400 Subject: [PATCH 01/19] Add num2date_exact function to decode times using timedelta addition --- cftime/__init__.py | 2 +- cftime/_cftime.pyx | 143 ++++++++++++++++++++++++++ test/test_cftime.py | 239 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 381 insertions(+), 3 deletions(-) diff --git a/cftime/__init__.py b/cftime/__init__.py index 835aee08..82a4a228 100644 --- a/cftime/__init__.py +++ b/cftime/__init__.py @@ -5,5 +5,5 @@ DatetimeGregorian, DatetimeProlepticGregorian from ._cftime import microsec_units, millisec_units, \ sec_units, hr_units, day_units, min_units -from ._cftime import num2date, date2num, date2index, num2pydate +from ._cftime import num2date, date2num, date2index, num2pydate, num2date_exact from ._cftime import __version__ diff --git a/cftime/_cftime.pyx b/cftime/_cftime.pyx index cb45e0a6..3e97449e 100644 --- a/cftime/_cftime.pyx +++ b/cftime/_cftime.pyx @@ -7,12 +7,15 @@ from cpython.object cimport (PyObject_RichCompare, Py_LT, Py_LE, Py_EQ, from numpy cimport int64_t, int32_t import cython import numpy as np +cimport numpy as np +cimport cpython.datetime import re import sys import time from datetime import datetime as datetime_python from datetime import timedelta, MINYEAR import time # strftime +import warnings try: from itertools import izip as zip except ImportError: # python 3.x @@ -396,6 +399,146 @@ OverflowError in python datetime, probably because year < datetime.MINYEAR""" return cdftime.num2date(times) +UNIT_CONVERSION_FACTORS = { + "microseconds": 1, + "microsecond": 1, + "microsec": 1, + "microsecs": 1, + "milliseconds": 1000, + "millisecond": 1000, + "millisec": 1000, + "millisecs": 1000, + "msec": 1000, + "msecs": 1000, + "ms": 1000, + "seconds": 1000000, + "second": 1000000, + "sec": 1000000, + "secs": 1000000, + "s": 1000000, + "minutes": 60 * 1000000, + "minute": 60 * 1000000, + "min": 60 * 1000000, + "mins": 60 * 1000000, + "hours": 3600 * 1000000, + "hour": 3600 * 1000000, + "hr": 3600 * 1000000, + "hrs": 3600 * 1000000, + "h": 3600 * 1000000, + "day": 86400 * 1000000, + "days": 86400 * 1000000, + "d": 86400 * 1000000, + "month": 30 * 86400 * 1000000, # Only allowed for 360_day calendar + "months": 30 * 86400 * 1000000 +} + + +DATE_TYPES = { + "proleptic_gregorian": DatetimeProlepticGregorian, + "standard": DatetimeGregorian, + "noleap": DatetimeNoLeap, + "365_day": DatetimeNoLeap, + "all_leap": DatetimeAllLeap, + "366_day": DatetimeAllLeap, + "julian": DatetimeJulian, + "360_day": Datetime360Day, + "gregorian": DatetimeGregorian +} + + +def to_calendar_specific_datetime(datetime, calendar): + return DATE_TYPES[calendar]( + datetime.year, + datetime.month, + datetime.day, + datetime.hour, + datetime.minute, + datetime.second, + datetime.microsecond + ) + + +def cast_to_int_if_safe(num): + """Copied from xarray.coding.times.py""" + if isinstance(num, np.ma.core.MaskedArray): + int_num = np.ma.masked_array(num, dtype=np.int64) + else: + int_num = np.array(num, dtype=np.int64) + if (num == int_num).all(): + num = int_num + return num + + +def num2date_exact( + times, + units, + calendar='standard', + only_use_cftime_datetimes=True, + only_use_python_datetimes=False +): + """Decode times exactly with timedelta arithmetic.""" + calendar = calendar.lower() + basedate = _dateparse(units) + + can_use_python_datetime=\ + ((calendar == 'proleptic_gregorian' and basedate.year >= MINYEAR) or \ + (calendar in ['gregorian','standard'] and basedate > gregorian)) + if not only_use_cftime_datetimes and only_use_python_datetimes: + if not can_use_python_datetime: + msg='illegal calendar or reference date for python datetime' + raise ValueError(msg) + + unit, ignore = _datesplit(units) + + # real-world calendars limited to positive reference years. + if calendar in ['julian', 'standard', 'gregorian', 'proleptic_gregorian']: + if basedate.year == 0: + msg='zero not allowed as a reference year, does not exist in Julian or Gregorian calendars' + raise ValueError(msg) + + use_python_datetime = False + if only_use_python_datetimes and not only_use_cftime_datetimes: + # only_use_cftime_datetimes takes precendence + use_python_datetime = True + if not only_use_python_datetimes and not only_use_cftime_datetimes and can_use_python_datetime: + # if only_use_cftimes_datetimes and only_use_python_datetimes are False + # return python datetime if possible. + use_python_datetime = True + + if use_python_datetime: + basedate = datetime_python( + basedate.year, + basedate.month, + basedate.day, + basedate.hour, + basedate.minute, + basedate.second, + basedate.microsecond + ) + else: + basedate = to_calendar_specific_datetime(basedate, calendar) + + if unit not in UNIT_CONVERSION_FACTORS: + raise ValueError("Invalid units provided.") + + if unit in ["months", "month"] and calendar != "360_day": + raise ValueError("Units of months only valid for 360_day calendar.") + + factor = UNIT_CONVERSION_FACTORS[unit] + scaled_times = factor * times + + scaled_times = cast_to_int_if_safe(scaled_times) + if not np.issubdtype(scaled_times.dtype, np.integer): + warnings.warn( + "times must have integer dtype or be able to be safely cast to an " + "integer dtype to be decoded exactly. Falling back to the " + "inexact version of the num2date function.") + return num2date(times, units, calendar, only_use_cftime_datetimes, only_use_python_datetimes) + + shape = scaled_times.shape + deltas = scaled_times.astype("timedelta64[us]").astype(timedelta) + return basedate + deltas + def date2index(dates, nctime, calendar=None, select='exact'): """date2index(dates, nctime, calendar=None, select='exact') diff --git a/test/test_cftime.py b/test/test_cftime.py index bcd69ce5..67b5d81b 100644 --- a/test/test_cftime.py +++ b/test/test_cftime.py @@ -5,7 +5,7 @@ import sys import unittest from collections import namedtuple -from datetime import datetime, timedelta +from datetime import datetime, timedelta, MINYEAR import numpy as np import pytest @@ -17,7 +17,7 @@ from cftime import (DateFromJulianDay, Datetime360Day, DatetimeAllLeap, DatetimeGregorian, DatetimeJulian, DatetimeNoLeap, DatetimeProlepticGregorian, JulianDayFromDate, _parse_date, - date2index, date2num, num2date, utime) + date2index, date2num, num2date, utime, num2date_exact) try: from datetime import timezone @@ -1555,5 +1555,240 @@ def test_dayofyr_after_timedelta_addition(date_type): assert date_after_timedelta_addition.dayofyr == 3 +_SHAPES = [(4, ), (2, 2)] +_MICROSECOND_UNITS = ["microseconds", "microsecond", "microsec", "microsecs"] +_MILLISECOND_UNITS = ["milliseconds", "millisecond", "millisec", "millisecs", "ms"] +_SECOND_UNITS = ["seconds", "second", "sec", "secs", "s"] +_MINUTE_UNITS = ["minutes", "minute", "min", "mins"] +_HOUR_UNITS = ["hours", "hour", "hr", "hrs", "h"] +_DAY_UNITS = ["day", "days", "d"] +_MONTH_UNITS = ["month", "months"] +_DTYPES = [np.dtype("int64"), np.dtype("float64")] +_STANDARD_CALENDARS = [ + "standard", + "gregorian", + "proleptic_gregorian" +] +_REAL_WORLD_CALENDARS = [ + "julian", + "standard", + "gregorian", + "proleptic_gregorian" +] +_ARTIFICIAL_CALENDARS = ["noleap", "all_leap", "360_day"] + + +@pytest.fixture(params=_SHAPES) +def shape(request): + return request.param + + +@pytest.fixture(params=_DTYPES) +def dtype(request): + return request.param + + +@pytest.fixture(params=list(_EXPECTED_DATE_TYPES.keys())) +def calendar(request): + return request.param + + +@pytest.mark.parametrize("unit", _MICROSECOND_UNITS) +def test_num2date_exact_microsecond_units(calendar, unit, shape, dtype): + date_type = _EXPECTED_DATE_TYPES[calendar] + expected = np.array([date_type(2000, 1, 1, 0, 0, 0, 1), + date_type(2000, 1, 1, 0, 0, 0, 2), + date_type(2000, 1, 1, 0, 0, 0, 3), + date_type(2000, 1, 1, 0, 0, 0, 4)]).reshape(shape) + numeric_times = np.array([1, 2, 3, 4]).reshape(shape).astype(dtype) + units = f"{unit} since 2000-01-01" + result = num2date_exact(numeric_times, units=units, calendar=calendar) + np.testing.assert_equal(result, expected) + + +@pytest.mark.parametrize("unit", _MILLISECOND_UNITS) +def test_num2date_exact_millisecond_units(calendar, unit, shape, dtype): + date_type = _EXPECTED_DATE_TYPES[calendar] + expected = np.array([date_type(2000, 1, 1, 0, 0, 0, 1000), + date_type(2000, 1, 1, 0, 0, 0, 2000), + date_type(2000, 1, 1, 0, 0, 0, 3000), + date_type(2000, 1, 1, 0, 0, 0, 4000)]).reshape(shape) + numeric_times = np.array([1, 2, 3, 4]).reshape(shape).astype(dtype) + units = f"{unit} since 2000-01-01" + result = num2date_exact(numeric_times, units=units, calendar=calendar) + np.testing.assert_equal(result, expected) + + +@pytest.mark.parametrize("unit", _SECOND_UNITS) +def test_num2date_exact_second_units(calendar, unit, shape, dtype): + date_type = _EXPECTED_DATE_TYPES[calendar] + expected = np.array([date_type(2000, 1, 1, 0, 0, 1, 0), + date_type(2000, 1, 1, 0, 0, 2, 0), + date_type(2000, 1, 1, 0, 0, 3, 0), + date_type(2000, 1, 1, 0, 0, 4, 0)]).reshape(shape) + numeric_times = np.array([1, 2, 3, 4]).reshape(shape).astype(dtype) + units = f"{unit} since 2000-01-01" + result = num2date_exact(numeric_times, units=units, calendar=calendar) + np.testing.assert_equal(result, expected) + + +@pytest.mark.parametrize("unit", _MINUTE_UNITS) +def test_num2date_exact_minute_units(calendar, unit, shape, dtype): + date_type = _EXPECTED_DATE_TYPES[calendar] + expected = np.array([date_type(2000, 1, 1, 0, 1, 0, 0), + date_type(2000, 1, 1, 0, 2, 0, 0), + date_type(2000, 1, 1, 0, 3, 0, 0), + date_type(2000, 1, 1, 0, 4, 0, 0)]).reshape(shape) + numeric_times = np.array([1, 2, 3, 4]).reshape(shape).astype(dtype) + units = f"{unit} since 2000-01-01" + result = num2date_exact(numeric_times, units=units, calendar=calendar) + np.testing.assert_equal(result, expected) + + +@pytest.mark.parametrize("unit", _HOUR_UNITS) +def test_num2date_exact_hour_units(calendar, unit, shape, dtype): + date_type = _EXPECTED_DATE_TYPES[calendar] + expected = np.array([date_type(2000, 1, 1, 1, 0, 0, 0), + date_type(2000, 1, 1, 2, 0, 0, 0), + date_type(2000, 1, 1, 3, 0, 0, 0), + date_type(2000, 1, 1, 4, 0, 0, 0)]).reshape(shape) + numeric_times = np.array([1, 2, 3, 4]).reshape(shape).astype(dtype) + units = f"{unit} since 2000-01-01" + result = num2date_exact(numeric_times, units=units, calendar=calendar) + np.testing.assert_equal(result, expected) + + +@pytest.mark.parametrize("unit", _DAY_UNITS) +def test_num2date_exact_day_units(calendar, unit, shape, dtype): + date_type = _EXPECTED_DATE_TYPES[calendar] + expected = np.array([date_type(2000, 1, 2, 0, 0, 0, 0), + date_type(2000, 1, 3, 0, 0, 0, 0), + date_type(2000, 1, 4, 0, 0, 0, 0), + date_type(2000, 1, 5, 0, 0, 0, 0)]).reshape(shape) + numeric_times = np.array([1, 2, 3, 4]).reshape(shape).astype(dtype) + units = f"{unit} since 2000-01-01" + result = num2date_exact(numeric_times, units=units, calendar=calendar) + np.testing.assert_equal(result, expected) + + +@pytest.mark.parametrize("unit", _MONTH_UNITS) +def test_num2date_exact_month_units(calendar, unit, shape, dtype): + date_type = _EXPECTED_DATE_TYPES[calendar] + expected = np.array([date_type(2000, 2, 1, 0, 0, 0, 0), + date_type(2000, 3, 1, 0, 0, 0, 0), + date_type(2000, 4, 1, 0, 0, 0, 0), + date_type(2000, 5, 1, 0, 0, 0, 0)]).reshape(shape) + numeric_times = np.array([1, 2, 3, 4]).reshape(shape).astype(dtype) + units = f"{unit} since 2000-01-01" + + if calendar != "360_day": + with pytest.raises(ValueError): + num2date_exact(numeric_times, units=units, calendar=calendar) + else: + result = num2date_exact(numeric_times, units=units, calendar=calendar) + np.testing.assert_equal(result, expected) + + +def test_num2date_exact_only_use_python_datetimes(calendar, shape, dtype): + date_type = datetime + expected = np.array([date_type(2000, 1, 2, 0, 0, 0, 0), + date_type(2000, 1, 3, 0, 0, 0, 0), + date_type(2000, 1, 4, 0, 0, 0, 0), + date_type(2000, 1, 5, 0, 0, 0, 0)]).reshape(shape) + numeric_times = np.array([1, 2, 3, 4]).reshape(shape).astype(dtype) + units = f"days since 2000-01-01" + if calendar not in _STANDARD_CALENDARS: + with pytest.raises(ValueError): + num2date_exact(numeric_times, units=units, calendar=calendar, + only_use_python_datetimes=True, + only_use_cftime_datetimes=False) + else: + result = num2date_exact(numeric_times, units=units, calendar=calendar, + only_use_python_datetimes=True, + only_use_cftime_datetimes=False) + np.testing.assert_equal(result, expected) + + +def test_num2date_exact_use_pydatetime_if_possible(calendar, shape, dtype): + date_type = datetime + if calendar not in _STANDARD_CALENDARS: + date_type = _EXPECTED_DATE_TYPES[calendar] + else: + date_type = datetime + + expected = np.array([date_type(2000, 1, 2, 0, 0, 0, 0), + date_type(2000, 1, 3, 0, 0, 0, 0), + date_type(2000, 1, 4, 0, 0, 0, 0), + date_type(2000, 1, 5, 0, 0, 0, 0)]).reshape(shape) + numeric_times = np.array([1, 2, 3, 4]).reshape(shape).astype(dtype) + units = f"days since 2000-01-01" + result = num2date_exact(numeric_times, units=units, calendar=calendar, + only_use_python_datetimes=False, + only_use_cftime_datetimes=False) + np.testing.assert_equal(result, expected) + + +@pytest.mark.parametrize( + ["standard_calendar", "breakpoint"], + [("proleptic_gregorian", f"{MINYEAR - 1}-12-31T23:59:59.999999"), + ("gregorian", "1582-10-15"), + ("standard", "1582-10-15")] +) +def test_num2date_exact_only_use_python_datetimes_invalid_basedate( + standard_calendar, + breakpoint +): + units = f"days since {breakpoint}" + numeric_times = np.array([1, 2, 3, 4]) + with pytest.raises(ValueError, match="illegal calendar or reference date"): + num2date_exact( + numeric_times, + units=units, + calendar=standard_calendar, + only_use_python_datetimes=True, + only_use_cftime_datetimes=False + ) + + +@pytest.mark.parametrize("real_world_calendar", _REAL_WORLD_CALENDARS) +def test_num2date_exact_invalid_zero_reference_year(real_world_calendar): + units = "days since 0000-01-01" + numeric_times = np.array([1, 2, 3, 4]) + with pytest.raises(ValueError, match="zero not allowed as a reference"): + num2date_exact( + numeric_times, + units=units, + calendar=real_world_calendar + ) + + +@pytest.mark.parametrize("artificial_calendar", _ARTIFICIAL_CALENDARS) +def test_num2date_exact_valid_zero_reference_year(artificial_calendar): + units = "days since 0000-01-01" + numeric_times = np.array([1, 2, 3, 4]) + num2date_exact(numeric_times, units=units, calendar=artificial_calendar) + + +def test_num2date_exact_uncastable_values(calendar): + units = "days since 2000-01-01" + numeric_times = np.array([1.0, np.pi]) + with pytest.warns(UserWarning, match="Falling back to the inexact"): + num2date_exact(numeric_times, units=units, calendar=calendar) + + +def test_num2date_exact_masked_array(calendar): + date_type = _EXPECTED_DATE_TYPES[calendar] + expected = np.array([date_type(2000, 1, 1, 1, 0, 0, 0), + date_type(2000, 1, 1, 2, 0, 0, 0), + date_type(2000, 1, 1, 3, 0, 0, 0), + date_type(2000, 1, 1, 4, 0, 0, 0)]) + mask = [False, False, True, False] + expected = np.ma.masked_array(expected, mask=mask) + numeric_times = np.ma.masked_array([1, 2, 3, 4], mask=mask) + units = f"hours since 2000-01-01" + result = num2date_exact(numeric_times, units=units, calendar=calendar) + np.testing.assert_equal(result, expected) + + if __name__ == '__main__': unittest.main() From 28d206c460e58b7a4c76940484360ebfdcd21739 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Mon, 25 May 2020 16:43:51 -0400 Subject: [PATCH 02/19] Remove unused cython imports --- cftime/_cftime.pyx | 2 -- 1 file changed, 2 deletions(-) diff --git a/cftime/_cftime.pyx b/cftime/_cftime.pyx index 3e97449e..1ff6dbcd 100644 --- a/cftime/_cftime.pyx +++ b/cftime/_cftime.pyx @@ -7,8 +7,6 @@ from cpython.object cimport (PyObject_RichCompare, Py_LT, Py_LE, Py_EQ, from numpy cimport int64_t, int32_t import cython import numpy as np -cimport numpy as np -cimport cpython.datetime import re import sys import time From 03e8f92e2e3e7fe90aa4b7c691c10db85d9feb4c Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Mon, 25 May 2020 16:50:59 -0400 Subject: [PATCH 03/19] Provide nicer error message for invalid time units --- cftime/_cftime.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cftime/_cftime.pyx b/cftime/_cftime.pyx index 1ff6dbcd..d3848a79 100644 --- a/cftime/_cftime.pyx +++ b/cftime/_cftime.pyx @@ -517,7 +517,7 @@ def num2date_exact( basedate = to_calendar_specific_datetime(basedate, calendar) if unit not in UNIT_CONVERSION_FACTORS: - raise ValueError("Invalid units provided.") + raise ValueError(f"Unsupported time units provided, '{unit}'.") if unit in ["months", "month"] and calendar != "360_day": raise ValueError("Units of months only valid for 360_day calendar.") From 0692a9ff574e1a2417760a249d3182e538a66800 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Mon, 25 May 2020 16:52:42 -0400 Subject: [PATCH 04/19] Update comment regarding xarray.coding.times --- cftime/_cftime.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cftime/_cftime.pyx b/cftime/_cftime.pyx index d3848a79..d57a8f5a 100644 --- a/cftime/_cftime.pyx +++ b/cftime/_cftime.pyx @@ -457,7 +457,7 @@ def to_calendar_specific_datetime(datetime, calendar): def cast_to_int_if_safe(num): - """Copied from xarray.coding.times.py""" + """Adapted from xarray.coding.times.py""" if isinstance(num, np.ma.core.MaskedArray): int_num = np.ma.masked_array(num, dtype=np.int64) else: From 90f9876b2ac432129763abe85ee891558fe1de91 Mon Sep 17 00:00:00 2001 From: Spencer Clark Date: Mon, 25 May 2020 18:00:12 -0400 Subject: [PATCH 05/19] Use real_datetime instead of datetime; add docstring --- cftime/_cftime.pyx | 48 +++++++++++++++++++++++++++++++++++++++++++-- test/test_cftime.py | 5 ++--- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/cftime/_cftime.pyx b/cftime/_cftime.pyx index d57a8f5a..8f62786c 100644 --- a/cftime/_cftime.pyx +++ b/cftime/_cftime.pyx @@ -474,7 +474,51 @@ def num2date_exact( only_use_cftime_datetimes=True, only_use_python_datetimes=False ): - """Decode times exactly with timedelta arithmetic.""" + """Decode times exactly with timedelta arithmetic. + + Return datetime objects given numeric time values. The units + of the numeric time values are described by the `units` argument + and the `calendar` keyword. The returned datetime objects represent + UTC with no time-zone offset, even if the specified + `units` contain a time-zone offset. + + **`times`**: numeric time values. + + **`units`**: a string of the form `