From fba0773832e98a2af519b80491fe5a538e9b8e1e Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Thu, 16 Jul 2020 18:50:45 -0600 Subject: [PATCH 1/8] potential solution for roundtrip issue with microseconds (issue #187) --- cftime/_cftime.pyx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cftime/_cftime.pyx b/cftime/_cftime.pyx index cc0a8e09..bccdcaf7 100644 --- a/cftime/_cftime.pyx +++ b/cftime/_cftime.pyx @@ -363,9 +363,15 @@ def cast_to_int(num): if np.any(num < _MIN_INT64) or np.any(num > _MAX_INT64): raise OverflowError('time values outside range of 64 bit signed integers') if isinstance(num, np.ma.core.MaskedArray): - int_num = np.ma.masked_array(np.rint(num), dtype=np.int64) + #int_num = np.ma.masked_array(np.rint(num), dtype=np.int64) + # use ceil instead of rint to preserve microseconds on roundtrip + # (issue #187) + int_num = np.ma.masked_array(np.ceil(num), dtype=np.int64) else: - int_num = np.array(np.rint(num), dtype=np.int64) + #int_num = np.array(np.rint(num), dtype=np.int64) + # use ceil instead of rint to preserve microseconds on roundtrip + # (issue #187) + int_num = np.array(np.ceil(num), dtype=np.int64) return int_num From 96a9bb50479c052665242ecd5cecb39b9fbd07a4 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Thu, 16 Jul 2020 20:54:24 -0600 Subject: [PATCH 2/8] update --- cftime/_cftime.pyx | 20 ++++++++++++-------- test/test_cftime.py | 5 +++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/cftime/_cftime.pyx b/cftime/_cftime.pyx index bccdcaf7..092c0d13 100644 --- a/cftime/_cftime.pyx +++ b/cftime/_cftime.pyx @@ -363,15 +363,19 @@ def cast_to_int(num): if np.any(num < _MIN_INT64) or np.any(num > _MAX_INT64): raise OverflowError('time values outside range of 64 bit signed integers') if isinstance(num, np.ma.core.MaskedArray): - #int_num = np.ma.masked_array(np.rint(num), dtype=np.int64) - # use ceil instead of rint to preserve microseconds on roundtrip - # (issue #187) - int_num = np.ma.masked_array(np.ceil(num), dtype=np.int64) + int_num = np.ma.masked_array(np.rint(num), dtype=np.int64) + # use ceil instead of rint if 1 microsec less than a second + # or floor if 1 microsec greater than a second (issue #187) + int_num = np.ma.where(int_num%10000000 == 1, \ + np.ma.masked_array(np.floor(num),dtype=np.int64), int_num) + int_num = np.ma.where(int_num%10000000 == 9999999, \ + np.ma.masked_array(np.ceil(num),dtype=np.int64), int_num) else: - #int_num = np.array(np.rint(num), dtype=np.int64) - # use ceil instead of rint to preserve microseconds on roundtrip - # (issue #187) - int_num = np.array(np.ceil(num), dtype=np.int64) + int_num = np.array(np.rint(num), dtype=np.int64) + int_num = np.where(int_num%10000000 == 1, \ + np.array(np.floor(num),dtype=np.int64), int_num) + int_num = np.where(int_num%10000000 == 9999999, \ + np.array(np.ceil(num),dtype=np.int64), int_num) return int_num diff --git a/test/test_cftime.py b/test/test_cftime.py index 75f5c5dd..32bcedfd 100644 --- a/test/test_cftime.py +++ b/test/test_cftime.py @@ -805,6 +805,11 @@ def test_tz_naive(self): time2 = date2num(date,units,calendar=calendar) date2 = num2date(time2,units,calendar=calendar) assert(date2 == refdate) +# issue #187 - roundtrip near second boundary + dt1 = datetime(1810, 4, 24, 16, 15, 10) + units = 'days since -4713-01-01 12:00' + dt2 = num2date(date2num(dt1, units), units) + assert(dt1 == dt2) class TestDate2index(unittest.TestCase): From 9624d36e0eb44bfa1da9d8ed55739108274e4cb1 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Thu, 16 Jul 2020 21:09:28 -0600 Subject: [PATCH 3/8] update --- Changelog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Changelog b/Changelog index c831bb13..df50c48d 100644 --- a/Changelog +++ b/Changelog @@ -1,7 +1,10 @@ -version 1.2.1 (not yeat released) +version 1.2.1 (not yet released) ================================= * num2date uses 'proleptic_gregorian' scheme when basedate is post-Gregorian but date is pre-Gregorian (issue #182). + * Fix for issue #187 (have date2num round to the nearest second when within 1 + microsecond). + version 1.2.0 (release tag v1.2.0rel) ===================================== From b7b501df8ed2b8f2a3b4d4dc1ddd2d88af47365a Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Fri, 17 Jul 2020 11:12:06 -0600 Subject: [PATCH 4/8] fix rounding to nearest second, only do this if units not millisecs/ms --- cftime/_cftime.pyx | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/cftime/_cftime.pyx b/cftime/_cftime.pyx index 092c0d13..1a7c3c97 100644 --- a/cftime/_cftime.pyx +++ b/cftime/_cftime.pyx @@ -356,7 +356,7 @@ _MAX_INT64 = np.iinfo("int64").max _MIN_INT64 = np.iinfo("int64").min -def cast_to_int(num): +def cast_to_int(num, units=None): if num.dtype.kind in "iu": return num else: @@ -366,16 +366,18 @@ def cast_to_int(num): int_num = np.ma.masked_array(np.rint(num), dtype=np.int64) # use ceil instead of rint if 1 microsec less than a second # or floor if 1 microsec greater than a second (issue #187) - int_num = np.ma.where(int_num%10000000 == 1, \ - np.ma.masked_array(np.floor(num),dtype=np.int64), int_num) - int_num = np.ma.where(int_num%10000000 == 9999999, \ - np.ma.masked_array(np.ceil(num),dtype=np.int64), int_num) + if units not in microsec_units and units not in millisec_units: + int_num = np.ma.where(int_num%1000000 == 1, \ + np.ma.masked_array(np.floor(num),dtype=np.int64), int_num) + int_num = np.ma.where(int_num%1000000 == 999999, \ + np.ma.masked_array(np.ceil(num),dtype=np.int64), int_num) else: int_num = np.array(np.rint(num), dtype=np.int64) - int_num = np.where(int_num%10000000 == 1, \ - np.array(np.floor(num),dtype=np.int64), int_num) - int_num = np.where(int_num%10000000 == 9999999, \ - np.array(np.ceil(num),dtype=np.int64), int_num) + if units not in microsec_units and units not in millisec_units: + int_num = np.where(int_num%1000000 == 1, \ + np.array(np.floor(num),dtype=np.int64), int_num) + int_num = np.where(int_num%1000000 == 999999, \ + np.array(np.ceil(num),dtype=np.int64), int_num) return int_num @@ -496,7 +498,7 @@ def num2date( times = np.asanyarray(times) # Allow list as input times = upcast_times(times) scaled_times = scale_times(times, factor) - scaled_times = cast_to_int(scaled_times) + scaled_times = cast_to_int(scaled_times,units=unit) # Through np.timedelta64, convert integers scaled to have units of # microseconds to datetime.timedelta objects, the timedelta type compatible From 17f0c472551b249e9956750a53cfecead8f69c19 Mon Sep 17 00:00:00 2001 From: jswhit Date: Fri, 17 Jul 2020 13:15:27 -0600 Subject: [PATCH 5/8] fix for issue #188 --- Changelog | 2 ++ cftime/_cftime.pyx | 4 ++-- test/test_cftime.py | 6 ++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Changelog b/Changelog index df50c48d..5b77138b 100644 --- a/Changelog +++ b/Changelog @@ -4,6 +4,8 @@ version 1.2.1 (not yet released) (issue #182). * Fix for issue #187 (have date2num round to the nearest second when within 1 microsecond). + * Fix for issue #188 (leap years calculated incorrectly for negative years in + proleptic_gregorian calendar). version 1.2.0 (release tag v1.2.0rel) diff --git a/cftime/_cftime.pyx b/cftime/_cftime.pyx index 1a7c3c97..b0b27a1a 100644 --- a/cftime/_cftime.pyx +++ b/cftime/_cftime.pyx @@ -1509,9 +1509,9 @@ cdef _is_leap(int year, calendar): if calendar == 'proleptic_gregorian' or (calendar == 'standard' and year > 1581): if tyear % 4: # not divisible by 4 leap = False - elif year % 100: # not divisible by 100 + elif tyear % 100: # not divisible by 100 leap = True - elif year % 400: # not divisible by 400 + elif tyear % 400: # not divisible by 400 leap = False else: leap = True diff --git a/test/test_cftime.py b/test/test_cftime.py index 32bcedfd..c324ed54 100644 --- a/test/test_cftime.py +++ b/test/test_cftime.py @@ -810,6 +810,12 @@ def test_tz_naive(self): units = 'days since -4713-01-01 12:00' dt2 = num2date(date2num(dt1, units), units) assert(dt1 == dt2) +# issue #188 - leap years calculated incorrectly for negative years in proleptic_gregorian calendar + dt1 = datetime(2020, 4, 24, 16, 15, 10) + units = 'days since -4713-01-01 12:00' + cal = 'proleptic_gregorian' + dt2 = num2date(date2num(dt1, units, cal), units, cal) + assert(dt1 == dt2) class TestDate2index(unittest.TestCase): From 60fc69fe0ab4e5519c78ce4932efebee532a1214 Mon Sep 17 00:00:00 2001 From: jswhit Date: Fri, 17 Jul 2020 19:04:05 -0600 Subject: [PATCH 6/8] update --- README.md | 2 ++ cftime/_cftime.pyx | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index da7c4605..df5dbf2c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Time-handling functionality from netcdf4-python ## News For details on the latest updates, see the [Changelog](https://github.com/Unidata/cftime/blob/master/Changelog). +??/??/2020: Version 1.2.1 released. Fixes a couple of regressions introduced in 1.2.0. See Changelog for details. + 7/06/2020: version 1.2.0 released. New microsecond accurate algorithm for date2num/num2date contributed by [spencerkclark](https://github.com/spencerkclark). Bugs fixed in masked array handling. 5/12/2020: version 1.1.3 released. Add isoformat method for compatibility with python datetime (issue #152). diff --git a/cftime/_cftime.pyx b/cftime/_cftime.pyx index ddd20380..03d66dcc 100644 --- a/cftime/_cftime.pyx +++ b/cftime/_cftime.pyx @@ -53,7 +53,7 @@ cdef int32_t* days_per_month_array = [ _rop_lookup = {Py_LT: '__gt__', Py_LE: '__ge__', Py_EQ: '__eq__', Py_GT: '__lt__', Py_GE: '__le__', Py_NE: '__ne__'} -__version__ = '1.2.0' +__version__ = '1.2.1' # Adapted from http://delete.me.uk/2005/03/iso8601.html # Note: This regex ensures that all ISO8601 timezone formats are accepted - but, due to legacy support for other timestrings, not all incorrect formats can be rejected. From 363a2c121dfa93ffd99ee437014a3f0c3e8fe343 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 18 Jul 2020 13:13:12 -0600 Subject: [PATCH 7/8] Update Changelog Co-authored-by: Spencer Clark --- Changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog b/Changelog index 024faeea..a3527710 100644 --- a/Changelog +++ b/Changelog @@ -5,7 +5,7 @@ version 1.2.1 (not yet released) * fix 1.2.0 regression (date2num no longer works with numpy scalar array inputs, issue #185). * Fix for issue #187 (have date2num round to the nearest second when within 1 microsecond). - * Fix for issue #188 (leap years calculated incorrectly for negative years in + * Fix for issue #189 (leap years calculated incorrectly for negative years in proleptic_gregorian calendar). version 1.2.0 (release tag v1.2.0rel) From 376f868ef39dd914276e640e8ba8e5c1123fe128 Mon Sep 17 00:00:00 2001 From: Jeff Whitaker Date: Sat, 18 Jul 2020 13:16:16 -0600 Subject: [PATCH 8/8] update --- test/test_cftime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_cftime.py b/test/test_cftime.py index f03b34dc..3e68b5ee 100644 --- a/test/test_cftime.py +++ b/test/test_cftime.py @@ -813,7 +813,7 @@ def test_tz_naive(self): units = 'days since -4713-01-01 12:00' dt2 = num2date(date2num(dt1, units), units) assert(dt1 == dt2) -# issue #188 - leap years calculated incorrectly for negative years in proleptic_gregorian calendar +# issue #189 - leap years calculated incorrectly for negative years in proleptic_gregorian calendar dt1 = datetime(2020, 4, 24, 16, 15, 10) units = 'days since -4713-01-01 12:00' cal = 'proleptic_gregorian'