From b49f837e6fed645275edf8602c0d687630be9a41 Mon Sep 17 00:00:00 2001 From: Victor Gavro Date: Wed, 17 Oct 2018 23:01:16 +0300 Subject: [PATCH 1/9] Timestamp field added --- marshmallow/fields.py | 35 +++++++++++++++++++++++++++++++++++ marshmallow/utils.py | 12 ++++++++++++ 2 files changed, 47 insertions(+) diff --git a/marshmallow/fields.py b/marshmallow/fields.py index 3a1fef299..d60de4f3e 100644 --- a/marshmallow/fields.py +++ b/marshmallow/fields.py @@ -1149,6 +1149,41 @@ def _deserialize(self, value, attr, data): self.fail('invalid') +class Timestamp(Field): + """Timestamp field, converts to datetime. + + :param timezone: Timezone of timestamp (defaults to UTC), should be tzinfo object. + :param bool ms: Milliseconds instead of seconds, defaults to `False`. For javascript + compatibility. + :param bool naive: Should deserialize to timezone-naive or timezone-aware datetime. + Defaults to `False`, so all datetimes will be timezone-aware with `timezone`. + On `True` timezone-naive datetimes will be converted to `timezone` on serialization. + :param bool as_int: If `True`, timestamp will be serialized to int instead of float, + so datetime microseconds precision can be lost. Note that this affects milliseconds also, + because 1 millisecond is 1000 microseconds. Defaults to `False`. + :param kwargs: The same keyword arguments that :class:`Field` receives. + """ + def __init__(self, timezone=utils.UTC, ms=False, naive=False, as_int=False, **kwargs): + self.timezone = timezone + self.ms = ms + self.naive = naive + self.as_int = as_int + super(Timestamp, self).__init__(**kwargs) + + def _serialize(self, value, attr, obj): + if value is None: + return None + value = utils.to_timestamp(value, self.timezone, self.ms) + return int(value) if self.as_int else value + + def _deserialize(self, value, attr, data): + try: + return utils.from_timestamp(value, None if self.naive else self.timezone, self.ms) + except (ValueError, OverflowError, OSError): + # Timestamp exceeds limits, ValueError needed for Python < 3.3 + self.fail('invalid') + + class Dict(Field): """A dict field. Supports dicts and dict-like objects. Optionally composed with another `Field` class or instance. diff --git a/marshmallow/utils.py b/marshmallow/utils.py index 168a2f169..2841b2014 100644 --- a/marshmallow/utils.py +++ b/marshmallow/utils.py @@ -317,6 +317,18 @@ def to_iso_date(date, *args, **kwargs): return datetime.date.isoformat(date) +def from_timestamp(value, tzinfo=UTC, ms=False): + return (datetime.utcfromtimestamp((float(value) * 1000) if ms else float(value)) + .replace(tzinfo=tzinfo)) + + +def to_timestamp(dt, tzinfo=UTC, ms=False): + if dt.tzinfo is None: + dt = dt.replace(tzinfo=tzinfo) + return (dt.astimezone(tzinfo).replace(tzinfo=UTC).timestamp() * + (1000 if ms else 1)) + + def ensure_text_type(val): if isinstance(val, binary_type): val = val.decode('utf-8') From ae99b0e8ce4f21b8bc09f1e86421bf207d4cd686 Mon Sep 17 00:00:00 2001 From: Victor Gavro Date: Wed, 17 Oct 2018 23:21:15 +0300 Subject: [PATCH 2/9] Timestamp: documentation fixes --- marshmallow/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/marshmallow/fields.py b/marshmallow/fields.py index d60de4f3e..06b16e864 100644 --- a/marshmallow/fields.py +++ b/marshmallow/fields.py @@ -1153,11 +1153,12 @@ class Timestamp(Field): """Timestamp field, converts to datetime. :param timezone: Timezone of timestamp (defaults to UTC), should be tzinfo object. + Timezone-aware datetimes will be converted to this before serialization, + timezone-naive datetimes will be serialized as is (in timestamp timezone). :param bool ms: Milliseconds instead of seconds, defaults to `False`. For javascript compatibility. :param bool naive: Should deserialize to timezone-naive or timezone-aware datetime. Defaults to `False`, so all datetimes will be timezone-aware with `timezone`. - On `True` timezone-naive datetimes will be converted to `timezone` on serialization. :param bool as_int: If `True`, timestamp will be serialized to int instead of float, so datetime microseconds precision can be lost. Note that this affects milliseconds also, because 1 millisecond is 1000 microseconds. Defaults to `False`. From 54222d0f21ba98971d2b0deba00d08df53fa9b24 Mon Sep 17 00:00:00 2001 From: Victor Gavro Date: Thu, 18 Oct 2018 00:23:00 +0300 Subject: [PATCH 3/9] added get_tzinfo (parsing timezone from string) --- marshmallow/fields.py | 4 ++-- marshmallow/utils.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/marshmallow/fields.py b/marshmallow/fields.py index 06b16e864..1929a984d 100644 --- a/marshmallow/fields.py +++ b/marshmallow/fields.py @@ -1152,7 +1152,7 @@ def _deserialize(self, value, attr, data): class Timestamp(Field): """Timestamp field, converts to datetime. - :param timezone: Timezone of timestamp (defaults to UTC), should be tzinfo object. + :param timezone: Timezone of timestamp (defaults to UTC). Timezone-aware datetimes will be converted to this before serialization, timezone-naive datetimes will be serialized as is (in timestamp timezone). :param bool ms: Milliseconds instead of seconds, defaults to `False`. For javascript @@ -1165,7 +1165,7 @@ class Timestamp(Field): :param kwargs: The same keyword arguments that :class:`Field` receives. """ def __init__(self, timezone=utils.UTC, ms=False, naive=False, as_int=False, **kwargs): - self.timezone = timezone + self.timezone = utils.get_tzinfo(timezone) self.ms = ms self.naive = naive self.as_int = as_int diff --git a/marshmallow/utils.py b/marshmallow/utils.py index 2841b2014..db752c9e7 100644 --- a/marshmallow/utils.py +++ b/marshmallow/utils.py @@ -24,7 +24,7 @@ dateutil_available = False try: - from dateutil import parser + from dateutil import parser, tz dateutil_available = True except ImportError: dateutil_available = False @@ -329,6 +329,16 @@ def to_timestamp(dt, tzinfo=UTC, ms=False): (1000 if ms else 1)) +def get_tzinfo(value): + if isinstance(value, datetime.tzinfo): + return value + elif value == 'UTC': + return UTC + elif dateutil_available: + return tz.gettz(value) + raise ValueError('Unknown timezone and dateutil not available') + + def ensure_text_type(val): if isinstance(val, binary_type): val = val.decode('utf-8') From 4485fb4bb9250bf439ada2ad92e6def0ef7df7c5 Mon Sep 17 00:00:00 2001 From: Victor Gavro Date: Thu, 18 Oct 2018 01:22:35 +0300 Subject: [PATCH 4/9] fields.Timestamp: tests added, related fixes --- marshmallow/utils.py | 2 +- tests/test_fields.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/marshmallow/utils.py b/marshmallow/utils.py index db752c9e7..1144a3409 100644 --- a/marshmallow/utils.py +++ b/marshmallow/utils.py @@ -318,7 +318,7 @@ def to_iso_date(date, *args, **kwargs): def from_timestamp(value, tzinfo=UTC, ms=False): - return (datetime.utcfromtimestamp((float(value) * 1000) if ms else float(value)) + return (datetime.datetime.utcfromtimestamp((float(value) / 1000) if ms else float(value)) .replace(tzinfo=tzinfo)) diff --git a/tests/test_fields.py b/tests/test_fields.py index de5ced4d4..c0046cc71 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- +import datetime as dt + import pytest from marshmallow import fields, Schema, ValidationError, EXCLUDE, INCLUDE, RAISE from marshmallow.marshalling import missing from marshmallow.exceptions import StringNotCollectionError +from marshmallow.utils import UTC from tests.base import ALL_FIELDS @@ -240,3 +243,22 @@ class MySchema(Schema): elif field_unknown == RAISE or (schema_unknown == RAISE and not field_unknown): with pytest.raises(ValidationError): MySchema().load({'nested': {'x': 1}}) + + +class TestTimestamp: + class UTC_plus_3(dt.tzinfo): + def utcoffset(self, dt): + return dt.timedelta(hours=3) + UTC_plus_3 = UTC_plus_3() + + @pytest.mark.parametrize('timestamp,datetime,kwargs', ( + (-1.5, dt.datetime(1969, 12, 31, 23, 59, 58, 500000, tzinfo=UTC), {}), + (-1500, dt.datetime(1969, 12, 31, 23, 59, 58, 500000, tzinfo=UTC), {'ms': True}), + (0, dt.datetime(1970, 1, 1, tzinfo=UTC), {'timezone': 'UTC', 'ms': True}), + (0, dt.datetime(1970, 1, 1, tzinfo=UTC_plus_3), {'timezone': UTC_plus_3}), + (0, dt.datetime(1970, 1, 1, tzinfo=None), {'naive': True}), + )) + def test_load_dump(self, timestamp, datetime, kwargs): + field = fields.Timestamp(**kwargs) + assert field.deserialize(timestamp) == datetime + assert field._serialize(datetime, '', object()) == timestamp From 7ba6897abaf2073a5028916efd5d6cdbfb8dcf12 Mon Sep 17 00:00:00 2001 From: Victor Gavro Date: Thu, 18 Oct 2018 01:37:57 +0300 Subject: [PATCH 5/9] fields.Timestamp: tests improved --- tests/test_fields.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index c0046cc71..cac7cf2c2 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -247,18 +247,21 @@ class MySchema(Schema): class TestTimestamp: class UTC_plus_3(dt.tzinfo): - def utcoffset(self, dt): + def utcoffset(self, dt_): return dt.timedelta(hours=3) + def dst(self, dt_): + return dt.timedelta(0) UTC_plus_3 = UTC_plus_3() @pytest.mark.parametrize('timestamp,datetime,kwargs', ( (-1.5, dt.datetime(1969, 12, 31, 23, 59, 58, 500000, tzinfo=UTC), {}), (-1500, dt.datetime(1969, 12, 31, 23, 59, 58, 500000, tzinfo=UTC), {'ms': True}), (0, dt.datetime(1970, 1, 1, tzinfo=UTC), {'timezone': 'UTC', 'ms': True}), - (0, dt.datetime(1970, 1, 1, tzinfo=UTC_plus_3), {'timezone': UTC_plus_3}), (0, dt.datetime(1970, 1, 1, tzinfo=None), {'naive': True}), + (0, dt.datetime(1969, 12, 31, 21, tzinfo=UTC), {'timezone': UTC_plus_3}), + (0, dt.datetime(1970, 1, 1, 3, tzinfo=UTC_plus_3), {'timezone': UTC}), )) def test_load_dump(self, timestamp, datetime, kwargs): field = fields.Timestamp(**kwargs) - assert field.deserialize(timestamp) == datetime + assert not (field.deserialize(timestamp) - datetime).total_seconds() assert field._serialize(datetime, '', object()) == timestamp From 0c676ce58b8a95d78a3292cfbefdca89af39b60d Mon Sep 17 00:00:00 2001 From: Victor Gavro Date: Thu, 18 Oct 2018 01:44:47 +0300 Subject: [PATCH 6/9] fix related to pre-commit "add-trailing-comma" hook failing on travis --- tests/test_fields.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index cac7cf2c2..6a6967105 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -253,14 +253,16 @@ def dst(self, dt_): return dt.timedelta(0) UTC_plus_3 = UTC_plus_3() - @pytest.mark.parametrize('timestamp,datetime,kwargs', ( - (-1.5, dt.datetime(1969, 12, 31, 23, 59, 58, 500000, tzinfo=UTC), {}), - (-1500, dt.datetime(1969, 12, 31, 23, 59, 58, 500000, tzinfo=UTC), {'ms': True}), - (0, dt.datetime(1970, 1, 1, tzinfo=UTC), {'timezone': 'UTC', 'ms': True}), - (0, dt.datetime(1970, 1, 1, tzinfo=None), {'naive': True}), - (0, dt.datetime(1969, 12, 31, 21, tzinfo=UTC), {'timezone': UTC_plus_3}), - (0, dt.datetime(1970, 1, 1, 3, tzinfo=UTC_plus_3), {'timezone': UTC}), - )) + @pytest.mark.parametrize( + 'timestamp,datetime,kwargs', ( + (-1.5, dt.datetime(1969, 12, 31, 23, 59, 58, 500000, tzinfo=UTC), {}), + (-1500, dt.datetime(1969, 12, 31, 23, 59, 58, 500000, tzinfo=UTC), {'ms': True}), + (0, dt.datetime(1970, 1, 1, tzinfo=UTC), {'timezone': 'UTC', 'ms': True}), + (0, dt.datetime(1970, 1, 1, tzinfo=None), {'naive': True}), + (0, dt.datetime(1969, 12, 31, 21, tzinfo=UTC), {'timezone': UTC_plus_3}), + (0, dt.datetime(1970, 1, 1, 3, tzinfo=UTC_plus_3), {'timezone': UTC}), + ), + ) def test_load_dump(self, timestamp, datetime, kwargs): field = fields.Timestamp(**kwargs) assert not (field.deserialize(timestamp) - datetime).total_seconds() From 1cd4d24d2981a43cb2b064009a113801f4e7cad8 Mon Sep 17 00:00:00 2001 From: Victor Gavro Date: Thu, 18 Oct 2018 02:04:06 +0300 Subject: [PATCH 7/9] to_timestamp python2 compatibility --- marshmallow/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/marshmallow/utils.py b/marshmallow/utils.py index 1144a3409..7a6f20ac9 100644 --- a/marshmallow/utils.py +++ b/marshmallow/utils.py @@ -317,6 +317,9 @@ def to_iso_date(date, *args, **kwargs): return datetime.date.isoformat(date) +_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) + + def from_timestamp(value, tzinfo=UTC, ms=False): return (datetime.datetime.utcfromtimestamp((float(value) / 1000) if ms else float(value)) .replace(tzinfo=tzinfo)) @@ -325,8 +328,8 @@ def from_timestamp(value, tzinfo=UTC, ms=False): def to_timestamp(dt, tzinfo=UTC, ms=False): if dt.tzinfo is None: dt = dt.replace(tzinfo=tzinfo) - return (dt.astimezone(tzinfo).replace(tzinfo=UTC).timestamp() * - (1000 if ms else 1)) + return ((dt.astimezone(tzinfo).replace(tzinfo=UTC) - _EPOCH) + .total_seconds() * (1000 if ms else 1)) def get_tzinfo(value): From 4ac26d7008eef8ebb0353dbcc21f4e985dadac0a Mon Sep 17 00:00:00 2001 From: Victor Gavro Date: Thu, 18 Oct 2018 02:26:36 +0300 Subject: [PATCH 8/9] utils to_timestamp/from_timestamp minor style improvements --- marshmallow/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/marshmallow/utils.py b/marshmallow/utils.py index 7a6f20ac9..bf7887001 100644 --- a/marshmallow/utils.py +++ b/marshmallow/utils.py @@ -317,19 +317,19 @@ def to_iso_date(date, *args, **kwargs): return datetime.date.isoformat(date) -_EPOCH = datetime.datetime(1970, 1, 1, tzinfo=UTC) +_EPOCH = datetime.datetime(1970, 1, 1) def from_timestamp(value, tzinfo=UTC, ms=False): - return (datetime.datetime.utcfromtimestamp((float(value) / 1000) if ms else float(value)) + value = float(value) + return (datetime.datetime.utcfromtimestamp((value / 1000) if ms else value) .replace(tzinfo=tzinfo)) def to_timestamp(dt, tzinfo=UTC, ms=False): - if dt.tzinfo is None: - dt = dt.replace(tzinfo=tzinfo) - return ((dt.astimezone(tzinfo).replace(tzinfo=UTC) - _EPOCH) - .total_seconds() * (1000 if ms else 1)) + if dt.tzinfo is not None: + dt = dt.astimezone(tzinfo).replace(tzinfo=None) + return (dt - _EPOCH).total_seconds() * (1000 if ms else 1) def get_tzinfo(value): From c57dd26f5171553c2065388a0eb4082dec8703f5 Mon Sep 17 00:00:00 2001 From: Victor Gavro Date: Thu, 18 Oct 2018 17:18:02 +0300 Subject: [PATCH 9/9] fixes for utils.get_tzinfo, tests added --- marshmallow/utils.py | 9 ++++++--- tests/test_utils.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/marshmallow/utils.py b/marshmallow/utils.py index bf7887001..fc43a7c9c 100644 --- a/marshmallow/utils.py +++ b/marshmallow/utils.py @@ -332,13 +332,16 @@ def to_timestamp(dt, tzinfo=UTC, ms=False): return (dt - _EPOCH).total_seconds() * (1000 if ms else 1) -def get_tzinfo(value): +def get_tzinfo(value, use_dateutil=True): if isinstance(value, datetime.tzinfo): return value elif value == 'UTC': return UTC - elif dateutil_available: - return tz.gettz(value) + elif dateutil_available and use_dateutil: + tzinfo = tz.gettz(value) + if tzinfo is None: + raise ValueError('Unknown timezone', value) + return tzinfo raise ValueError('Unknown timezone and dateutil not available') diff --git a/tests/test_utils.py b/tests/test_utils.py index bf10a1bb3..bf2eb6200 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -237,3 +237,14 @@ def __call__(self, foo, bar): for func in [f1, f2, f3]: assert utils.get_func_args(func) == ['foo', 'bar'] + +@pytest.mark.parametrize('use_dateutil', [True, False]) +def test_get_tzinfo(use_dateutil): + if use_dateutil: + assert utils.get_tzinfo('Europe/Kiev', use_dateutil) + with pytest.raises(ValueError): + utils.get_tzinfo('Europe/Maaaskva', use_dateutil) + else: + assert utils.get_tzinfo('UTC', use_dateutil) + with pytest.raises(ValueError): + utils.get_tzinfo('Europe/Kiev', use_dateutil)