From d6f3edd285cff80e0fdc9c3533a2665c9575c98d Mon Sep 17 00:00:00 2001 From: Victor Fernandez de Alba Date: Fri, 11 Aug 2017 17:10:47 +0200 Subject: [PATCH 01/11] Change the serialization of the datetime objects, and store them as offset-aware UTC-based objects --- CHANGES.rst | 5 +- docs/source/serialization.rst | 6 +- src/plone/restapi/deserializer/dxfields.py | 3 +- src/plone/restapi/tests/dxtypes.py | 5 +- src/plone/restapi/tests/test_content_patch.py | 53 ++++++++++++++++++ src/plone/restapi/tests/test_content_post.py | 55 +++++++++++++++++++ .../tests/test_dxfield_deserializer.py | 7 ++- 7 files changed, 126 insertions(+), 8 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index a93d7a7cbf..5cbc4a5e5c 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,13 +11,16 @@ New Features: - Translate titles in @workflow. [csenger] - + - Add skipped tests from @breadcrumbs and @navigation now that the expansion is in place [sneridagh] - Add endpoints for locking/unlocking. [buchi] +- The datetime objects are now stored as offset-aware UTC-based objects + [sneridagh] + 1.0a20 (2017-07-24) ------------------- diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst index 333cf7ee4c..077c9f2244 100644 --- a/docs/source/serialization.rst +++ b/docs/source/serialization.rst @@ -24,6 +24,10 @@ Python JSON ``DateTime('2015/11/23 19:45:55')`` ``'2015-11-23T19:45:55'`` ======================================= ====================================== +.. warning:: + All datetimes objects will be serialized adding the proper time zone information, storing an offset-aware object on it. + In case of using zope.schema date validators you should also use a datetime object that also contains offset-aware object as the validator value. + RichText fields --------------- @@ -155,4 +159,4 @@ UID ``'9b6a4eadb9074dde97d86171bb332ae9'`` IntId ``123456`` Path ``'/plone/doc1'`` URL ``'http://localhost:8080/plone/doc1'`` -======================================= ====================================== \ No newline at end of file +======================================= ====================================== diff --git a/src/plone/restapi/deserializer/dxfields.py b/src/plone/restapi/deserializer/dxfields.py index 7bcf61eed8..8b5058e349 100644 --- a/src/plone/restapi/deserializer/dxfields.py +++ b/src/plone/restapi/deserializer/dxfields.py @@ -46,8 +46,7 @@ def __call__(self, value): try: # Parse ISO 8601 string with Zope's DateTime module # and convert to a timezone naive datetime in local time - value = DateTime(value).toZone(DateTime().localZone()).asdatetime( - ).replace(tzinfo=None) + value = DateTime(value).toZone('UTC').asdatetime() except (SyntaxError, DateTimeError) as e: raise ValueError(e.message) diff --git a/src/plone/restapi/tests/dxtypes.py b/src/plone/restapi/tests/dxtypes.py index ae90b312ff..8a9419686d 100644 --- a/src/plone/restapi/tests/dxtypes.py +++ b/src/plone/restapi/tests/dxtypes.py @@ -21,6 +21,7 @@ from zope.schema.vocabulary import SimpleTerm from zope.schema.vocabulary import SimpleVocabulary +import pytz INDEXES = ( ("test_int_field", "FieldIndex"), @@ -92,8 +93,8 @@ class IDXTestDocumentSchema(model.Schema): test_maxlength_field = schema.TextLine(required=False, max_length=10) test_constraint_field = schema.TextLine(required=False, constraint=lambda x: u'00' in x) - test_datetime_min_field = schema.Datetime(required=False, - min=datetime(2000, 1, 1)) + test_datetime_min_field = schema.Datetime( + required=False, min=datetime(2000, 1, 1, 0, 0, 0, 0, pytz.UTC)) test_time_min_field = schema.Time(required=False, min=time(1)) test_timedelta_min_field = schema.Timedelta(required=False, min=timedelta(100)) diff --git a/src/plone/restapi/tests/test_content_patch.py b/src/plone/restapi/tests/test_content_patch.py index 513af250ee..ad1b484164 100644 --- a/src/plone/restapi/tests/test_content_patch.py +++ b/src/plone/restapi/tests/test_content_patch.py @@ -9,7 +9,10 @@ from plone.app.testing import login from plone.app.testing import setRoles from plone.restapi.testing import PLONE_RESTAPI_DX_FUNCTIONAL_TESTING +from plone.restapi.testing import RelativeSession +from DateTime import DateTime +import datetime import requests import transaction import unittest @@ -22,8 +25,14 @@ class TestContentPatch(unittest.TestCase): def setUp(self): self.app = self.layer['app'] self.portal = self.layer['portal'] + self.portal_url = self.portal.absolute_url() setRoles(self.portal, TEST_USER_ID, ['Member']) login(self.portal, SITE_OWNER_NAME) + + self.api_session = RelativeSession(self.portal_url) + self.api_session.headers.update({'Accept': 'application/json'}) + self.api_session.auth = (SITE_OWNER_NAME, SITE_OWNER_PASSWORD) + self.portal.invokeFactory( 'Document', id='doc1', @@ -78,3 +87,47 @@ def test_patch_document_returns_401_unauthorized(self): data='{"title": "Patched Document"}', ) self.assertEqual(401, response.status_code) + + def test_patch_feed_event_with_get_contents(self): + start_date = DateTime(datetime.datetime.today() + + datetime.timedelta(days=1)).ISO8601() + end_date = DateTime(datetime.datetime.today() + + datetime.timedelta(days=1, hours=1)).ISO8601() + response = self.api_session.post( + '/', + json={ + "title": "An Event", + "@type": "Event", + "start": start_date, + "end": end_date, + "timezone": "Europe/Vienna" + }, + ) + + self.assertEqual(201, response.status_code) + + response = response.json() + event_id = response['id'] + two_days_ahead = DateTime(datetime.datetime.today() + + datetime.timedelta(days=2)) + response = self.api_session.patch( + '/{}'.format(event_id), + json={ + "start": response['start'], + "end": two_days_ahead.ISO8601() + } + ) + + self.assertEqual(204, response.status_code) + + response = self.api_session.get('/{}'.format(event_id)) + response = response.json() + + self.assertEquals( + DateTime(response['end']).day(), + two_days_ahead.day() + ) + self.assertEquals( + DateTime(response['end']).hour(), + two_days_ahead.hour() + ) diff --git a/src/plone/restapi/tests/test_content_post.py b/src/plone/restapi/tests/test_content_post.py index 7edab292fe..db276190e1 100644 --- a/src/plone/restapi/tests/test_content_post.py +++ b/src/plone/restapi/tests/test_content_post.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from datetime import datetime from Products.CMFCore.utils import getToolByName from plone.app.testing import SITE_OWNER_NAME from plone.app.testing import SITE_OWNER_PASSWORD @@ -196,3 +197,57 @@ def test_id_from_filename(self): self.assertEqual(201, response.status_code) transaction.begin() self.assertIn('test.txt', self.portal.folder1) + + +class TestEventCT(unittest.TestCase): + + layer = PLONE_RESTAPI_DX_FUNCTIONAL_TESTING + + def setUp(self): + self.app = self.layer['app'] + self.portal = self.layer['portal'] + setRoles(self.portal, TEST_USER_ID, ['Member']) + login(self.portal, SITE_OWNER_NAME) + self.portal.invokeFactory( + 'Folder', + id='folder1', + title='My Folder' + ) + wftool = getToolByName(self.portal, 'portal_workflow') + wftool.doActionFor(self.portal.folder1, 'publish') + transaction.commit() + + def test_post_to_folder_creates_event_with_TZ(self): + response = requests.post( + self.portal.folder1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "@type": "Event", + "id": "myevent", + "title": "My Event", + "start": datetime(2013, 1, 1, 10, 0).isoformat(), + "end": datetime(2013, 1, 1, 12, 0).isoformat(), + "timezone": 'Europe/Madrid' + }, + ) + self.assertEqual(201, response.status_code) + self.assertEqual( + response.json()['start'], u'2013-01-01T10:00:00+01:00') + + response = requests.post( + self.portal.folder1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "@type": "Event", + "id": "myevent2", + "title": "My Event", + "start": datetime(2018, 1, 1, 10, 0).isoformat(), + "end": datetime(2018, 1, 1, 12, 0).isoformat(), + "timezone": 'Asia/Saigon' + }, + ) + self.assertEqual(201, response.status_code) + self.assertEqual( + response.json()['start'], u'2018-01-01T10:00:00+07:00') diff --git a/src/plone/restapi/tests/test_dxfield_deserializer.py b/src/plone/restapi/tests/test_dxfield_deserializer.py index 3e6f663238..6adf54ee43 100644 --- a/src/plone/restapi/tests/test_dxfield_deserializer.py +++ b/src/plone/restapi/tests/test_dxfield_deserializer.py @@ -12,6 +12,7 @@ from zope.component import getMultiAdapter from zope.schema.interfaces import ValidationError +import pytz import unittest @@ -79,12 +80,14 @@ def test_datetime_deserialization_returns_datetime(self): value = self.deserialize('test_datetime_field', u'2015-12-20T10:39:54.361Z') self.assertTrue(isinstance(value, datetime), 'Not a ') - self.assertEqual(datetime(2015, 12, 20, 10, 39, 54, 361000), value) + self.assertEqual( + datetime(2015, 12, 20, 10, 39, 54, 361000, pytz.UTC), value) def test_datetime_deserialization_handles_timezone(self): value = self.deserialize('test_datetime_field', u'2015-12-20T10:39:54.361+01') - self.assertEqual(datetime(2015, 12, 20, 9, 39, 54, 361000), value) + self.assertEqual( + datetime(2015, 12, 20, 9, 39, 54, 361000, pytz.UTC), value) def test_decimal_deserialization_returns_decimal(self): value = self.deserialize('test_decimal_field', u'1.1') From f7c4dec961c5bb3e8164c994eb5127ae4abaec1d Mon Sep 17 00:00:00 2001 From: Victor Fernandez de Alba Date: Fri, 11 Aug 2017 17:36:45 +0200 Subject: [PATCH 02/11] Fix typo --- docs/source/serialization.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst index 077c9f2244..4ecbaec351 100644 --- a/docs/source/serialization.rst +++ b/docs/source/serialization.rst @@ -25,7 +25,7 @@ Python JSON ======================================= ====================================== .. warning:: - All datetimes objects will be serialized adding the proper time zone information, storing an offset-aware object on it. + All datetimes objects will be deserialized adding the proper time zone information, storing an offset-aware object on it. In case of using zope.schema date validators you should also use a datetime object that also contains offset-aware object as the validator value. From 776ff6541b970890dc401055e69d9b0a1284cb41 Mon Sep 17 00:00:00 2001 From: Victor Fernandez de Alba Date: Fri, 11 Aug 2017 20:06:38 +0200 Subject: [PATCH 03/11] Improve tests and documentation --- docs/source/serialization.rst | 1 + src/plone/restapi/tests/test_content_post.py | 39 ++++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst index 4ecbaec351..0b19511969 100644 --- a/docs/source/serialization.rst +++ b/docs/source/serialization.rst @@ -26,6 +26,7 @@ Python JSON .. warning:: All datetimes objects will be deserialized adding the proper time zone information, storing an offset-aware object on it. + For Plone 5 (Dexterity plone.app.event powered) Event content type, you should send always naive (preferred and default if serializing using .toJSON Javascript API) or UTC-based offset datetime ISO8601 strings. In case of using zope.schema date validators you should also use a datetime object that also contains offset-aware object as the validator value. diff --git a/src/plone/restapi/tests/test_content_post.py b/src/plone/restapi/tests/test_content_post.py index db276190e1..30b8ea1766 100644 --- a/src/plone/restapi/tests/test_content_post.py +++ b/src/plone/restapi/tests/test_content_post.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from datetime import datetime +from DateTime import DateTime from Products.CMFCore.utils import getToolByName from plone.app.testing import SITE_OWNER_NAME from plone.app.testing import SITE_OWNER_PASSWORD @@ -11,6 +12,7 @@ from plone.restapi.testing import PLONE_RESTAPI_DX_FUNCTIONAL_TESTING from plone.restapi.testing import PLONE_RESTAPI_AT_FUNCTIONAL_TESTING +import pytz import requests import transaction import unittest @@ -217,7 +219,24 @@ def setUp(self): wftool.doActionFor(self.portal.folder1, 'publish') transaction.commit() - def test_post_to_folder_creates_event_with_TZ(self): + def test_post_to_folder_creates_event_with_correct_TZ(self): + response = requests.post( + self.portal.folder1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "@type": "Event", + "id": "myevent2", + "title": "My Event", + "start": DateTime(datetime(2018, 1, 1, 10, 0)).ISO8601(), + "end": DateTime(datetime(2018, 1, 1, 12, 0)).ISO8601(), + "timezone": 'Asia/Saigon' + }, + ) + self.assertEqual(201, response.status_code) + self.assertEqual( + response.json()['start'], u'2018-01-01T10:00:00+07:00') + response = requests.post( self.portal.folder1.absolute_url(), headers={'Accept': 'application/json'}, @@ -226,8 +245,8 @@ def test_post_to_folder_creates_event_with_TZ(self): "@type": "Event", "id": "myevent", "title": "My Event", - "start": datetime(2013, 1, 1, 10, 0).isoformat(), - "end": datetime(2013, 1, 1, 12, 0).isoformat(), + "start": DateTime(datetime(2013, 1, 1, 10, 0)).ISO8601(), + "end": DateTime(datetime(2013, 1, 1, 12, 0)).ISO8601(), "timezone": 'Europe/Madrid' }, ) @@ -235,19 +254,23 @@ def test_post_to_folder_creates_event_with_TZ(self): self.assertEqual( response.json()['start'], u'2013-01-01T10:00:00+01:00') + def test_post_creates_event_with_correct_TZ_using_UTC_offset(self): response = requests.post( self.portal.folder1.absolute_url(), headers={'Accept': 'application/json'}, auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), json={ "@type": "Event", - "id": "myevent2", + "id": "myevent", "title": "My Event", - "start": datetime(2018, 1, 1, 10, 0).isoformat(), - "end": datetime(2018, 1, 1, 12, 0).isoformat(), - "timezone": 'Asia/Saigon' + "start": datetime( + 2013, 1, 1, 10, 0, 0, 0, pytz.UTC).isoformat(), + "end": datetime( + 2013, 1, 1, 12, 0, 0, 0, pytz.UTC).isoformat(), + "timezone": 'Europe/Madrid' }, ) + self.assertEqual(201, response.status_code) self.assertEqual( - response.json()['start'], u'2018-01-01T10:00:00+07:00') + response.json()['start'], u'2013-01-01T10:00:00+01:00') From 8ee4b32cb408de178d0e2217dfee1b317b942a6f Mon Sep 17 00:00:00 2001 From: Victor Fernandez de Alba Date: Thu, 24 Aug 2017 16:53:27 +0200 Subject: [PATCH 04/11] Add conditional tests for Plone 5 --- plone-5.0.x.cfg | 8 ++++ src/plone/restapi/__init__.py | 8 ++++ src/plone/restapi/tests/test_content_post.py | 39 ++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/plone-5.0.x.cfg b/plone-5.0.x.cfg index 3997cd81e9..9d88485cc0 100644 --- a/plone-5.0.x.cfg +++ b/plone-5.0.x.cfg @@ -3,3 +3,11 @@ extends = base.cfg http://dist.plone.org/release/5.0.7/versions.cfg versions.cfg + +parts += omelette + +[omelette] +recipe = collective.recipe.omelette +eggs = + ${instance:eggs} + ${test:eggs} diff --git a/src/plone/restapi/__init__.py b/src/plone/restapi/__init__.py index 65cd614f77..1112c2ac76 100644 --- a/src/plone/restapi/__init__.py +++ b/src/plone/restapi/__init__.py @@ -17,6 +17,14 @@ try: pkg_resources.get_distribution('plone.app.contenttypes') HAS_PLONE_APP_CONTENTTYPES = True + + event_version = pkg_resources.get_distribution('plone.app.event').version + if pkg_resources.parse_version(event_version) > \ + pkg_resources.parse_version('1.99'): + HAS_PLONE_APP_EVENT_20 = True + else: + HAS_PLONE_APP_EVENT_20 = False + except pkg_resources.DistributionNotFound: # pragma: no cover HAS_PLONE_APP_CONTENTTYPES = False diff --git a/src/plone/restapi/tests/test_content_post.py b/src/plone/restapi/tests/test_content_post.py index 30b8ea1766..c26d51a4cd 100644 --- a/src/plone/restapi/tests/test_content_post.py +++ b/src/plone/restapi/tests/test_content_post.py @@ -11,6 +11,7 @@ from plone.app.testing import setRoles from plone.restapi.testing import PLONE_RESTAPI_DX_FUNCTIONAL_TESTING from plone.restapi.testing import PLONE_RESTAPI_AT_FUNCTIONAL_TESTING +from plone.restapi import HAS_PLONE_APP_EVENT_20 import pytz import requests @@ -219,6 +220,7 @@ def setUp(self): wftool.doActionFor(self.portal.folder1, 'publish') transaction.commit() + @unittest.skipIf(HAS_PLONE_APP_EVENT_20, 'Valid for p.a.event<2.0 only') def test_post_to_folder_creates_event_with_correct_TZ(self): response = requests.post( self.portal.folder1.absolute_url(), @@ -254,6 +256,7 @@ def test_post_to_folder_creates_event_with_correct_TZ(self): self.assertEqual( response.json()['start'], u'2013-01-01T10:00:00+01:00') + @unittest.skipIf(HAS_PLONE_APP_EVENT_20, 'Valid for p.a.event<2.0 only') def test_post_creates_event_with_correct_TZ_using_UTC_offset(self): response = requests.post( self.portal.folder1.absolute_url(), @@ -274,3 +277,39 @@ def test_post_creates_event_with_correct_TZ_using_UTC_offset(self): self.assertEqual(201, response.status_code) self.assertEqual( response.json()['start'], u'2013-01-01T10:00:00+01:00') + + @unittest.skipIf(not HAS_PLONE_APP_EVENT_20, 'Valid for p.a.event<2.0 only') # noqa + def test_pavent_20_post_to_folder_creates_event_with_correct_TZ(self): + response = requests.post( + self.portal.folder1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "@type": "Event", + "id": "myevent2", + "title": "My Event", + "start": pytz.timezone('Asia/Saigon').localize(datetime(2018, 1, 1, 10, 0)).isoformat(), # noqa + "end": pytz.timezone('Asia/Saigon').localize(datetime(2018, 1, 1, 12, 0)).isoformat(), # noqa + }, + ) + self.assertEqual(201, response.status_code) + self.assertEqual( + response.json()['start'], u'2018-01-01T03:00:00+00:00') + + response = requests.post( + self.portal.folder1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "@type": "Event", + "id": "myevent", + "title": "My Event", + "start": pytz.timezone('Europe/Berlin').localize(datetime( + 2013, 1, 1, 10, 0)).isoformat(), + "end": pytz.timezone('Europe/Berlin').localize(datetime( + 2013, 1, 1, 12, 0)).isoformat(), + }, + ) + self.assertEqual(201, response.status_code) + self.assertEqual( + response.json()['start'], u'2013-01-01T09:00:00+00:00') From d578263f3a086b882021e53f9b061e97891d7e14 Mon Sep 17 00:00:00 2001 From: Victor Fernandez de Alba Date: Thu, 24 Aug 2017 17:49:36 +0200 Subject: [PATCH 05/11] Add solution for dates comparision in PATCH operations and related test --- src/plone/restapi/deserializer/dxcontent.py | 9 +++- src/plone/restapi/tests/test_content_patch.py | 53 ++++++++++++++++--- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/src/plone/restapi/deserializer/dxcontent.py b/src/plone/restapi/deserializer/dxcontent.py index bed41a5927..8a28394aaf 100644 --- a/src/plone/restapi/deserializer/dxcontent.py +++ b/src/plone/restapi/deserializer/dxcontent.py @@ -78,7 +78,14 @@ def __call__(self, validate_all=False, data=None): # noqa: ignore=C901 'message': e.doc(), 'field': name, 'error': e}) else: field_data[name] = value - if value != dm.get(): + try: + if value != dm.get(): + dm.set(value) + modified = True + except TypeError: + # Most probably due to offset-naive and offset + # aware objects, set the value as they most likely + # are not the same dm.set(value) modified = True diff --git a/src/plone/restapi/tests/test_content_patch.py b/src/plone/restapi/tests/test_content_patch.py index ad1b484164..8b6fef733d 100644 --- a/src/plone/restapi/tests/test_content_patch.py +++ b/src/plone/restapi/tests/test_content_patch.py @@ -11,11 +11,13 @@ from plone.restapi.testing import PLONE_RESTAPI_DX_FUNCTIONAL_TESTING from plone.restapi.testing import RelativeSession from DateTime import DateTime +from datetime import datetime +from datetime import timedelta -import datetime import requests import transaction import unittest +import pytz class TestContentPatch(unittest.TestCase): @@ -89,10 +91,10 @@ def test_patch_document_returns_401_unauthorized(self): self.assertEqual(401, response.status_code) def test_patch_feed_event_with_get_contents(self): - start_date = DateTime(datetime.datetime.today() + - datetime.timedelta(days=1)).ISO8601() - end_date = DateTime(datetime.datetime.today() + - datetime.timedelta(days=1, hours=1)).ISO8601() + start_date = DateTime(datetime.today() + + timedelta(days=1)).ISO8601() + end_date = DateTime(datetime.today() + + timedelta(days=1, hours=1)).ISO8601() response = self.api_session.post( '/', json={ @@ -108,8 +110,8 @@ def test_patch_feed_event_with_get_contents(self): response = response.json() event_id = response['id'] - two_days_ahead = DateTime(datetime.datetime.today() + - datetime.timedelta(days=2)) + two_days_ahead = DateTime(datetime.today() + + timedelta(days=2)) response = self.api_session.patch( '/{}'.format(event_id), json={ @@ -131,3 +133,40 @@ def test_patch_feed_event_with_get_contents(self): DateTime(response['end']).hour(), two_days_ahead.hour() ) + + def test_patch_document_with_expires(self): + response = requests.patch( + self.portal.doc1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "expires": datetime( + 2013, 1, 1, 10, 0).isoformat() + } + ) + + self.assertEqual(204, response.status_code) + + response = requests.patch( + self.portal.doc1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "expires": datetime( + 2013, 1, 1, 10, 0).isoformat() + } + ) + + self.assertEqual(204, response.status_code) + + response = requests.patch( + self.portal.doc1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "expires": pytz.timezone('Europe/Berlin').localize(datetime( + 2013, 1, 1, 10, 0)).isoformat() + } + ) + + self.assertEqual(204, response.status_code) From 4dd3ff02e5679e419cac492e78d5bc68d5303061 Mon Sep 17 00:00:00 2001 From: Victor Fernandez de Alba Date: Fri, 25 Aug 2017 11:10:09 +0200 Subject: [PATCH 06/11] Add proper documentation explaining both behaviors of p.a.event --- docs/source/deserialization.rst | 87 +++++++++++++++++++++++++++++++++ docs/source/index.rst | 2 +- docs/source/serialization.rst | 6 --- 3 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 docs/source/deserialization.rst diff --git a/docs/source/deserialization.rst b/docs/source/deserialization.rst new file mode 100644 index 0000000000..b9a0246957 --- /dev/null +++ b/docs/source/deserialization.rst @@ -0,0 +1,87 @@ +Deserialization +=============== + +It's worth to note an special case when deserializing Datetimes objects, and how plone.restapi will handle them. + +Although not supported by Plone itself yet, plone.restapi will store all the Datetimes that will be handling along with its timezone converted to UTC. +This will provide a common ground for all the datetimes operations. + +There is an special case when using datetimes objects in p.a.event, and its behavior is different due to implementation differences for versions 1.x (Plone 4) and 2.x and above (Plone 5). + +.. warning:: + In case of using zope.schema date validators you should also use a datetime object that also contains offset-aware object as the validator value. + +.. note:: + This does not applies in case that you are using Plone 4 with no Dexterity support at all or not p.a.event installed. + +p.a.event 1.x in Plone 4 +------------------------ + +The implementation of p.a.event in 1.x requires to provide a `timezone` schema property along with the Event type information, otherwise the creation fails, because it's a required field, like this:: + +.. code-block:: json + + { + "@type": "Event", + "id": "myevent2", + "title": "My Event", + "start": "2018-01-01T10:00:00", + "end": "2018-01-01T12:00:00", + "timezone": "Asia/Saigon" + } + +The final stored datetime takes this field into account and adds the correct offset to the content type (abbreviated JSON response):: + +.. code-block:: json + + { + "@id": "http://localhost:55001/plone/folder1/myevent2", + "@type": "Event", + "UID": "bcfc3914ea174cc1aa8042147edfa33a", + "creators": ["admin"], + "description": "", + "end": "2018-01-01T12:00:00+07:00", + "id": "myevent2", + "start": "2018-01-01T10:00:00+07:00", + "timezone": "Asia/Saigon", + "title": "My Event", + } + +and builds the `start` and `end` fields with the proper timezone, depending on the `timezone` field. It also returns the datetime object with the proper timezone offset. + +If using Plone 4 and p.a.event 1.x you should construct the Event type using this approach, otherwise the Event object will be created with a wrong timezones. + +This approach was counterintuitive, and thus, it was changed it Plone 5 version of p.a.event. + +p.a.event 2.x in Plone 5 +------------------------ + +The implementation of p.a.event in 2.x no longer requires to provide a `timezone` schema property, because the timezone is computed taking the timezone already existent in dates supplied:: + +.. code-block:: json + + { + "@type": "Event", + "id": "myevent2", + "title": "My Event", + "start": "2018-01-01T10:00:00+07:00", + "end": "2018-01-01T12:00:00+07:00", + } + +You should pass the timezone information in the ISO8601 format, otherwise the system will fallback to UTC. The response given is also computed given the timezone information supplied, but this time it's UTC based: + +.. code-block:: json + + { + "@id": "http://localhost:55001/plone/folder1/myevent2", + "@type": "Event", + "UID": "4c031960718246db86c97685f83047ee", + "creators": ["admin"], + "description": "", + "end": "2018-01-01T05:00:00+00:00", + "id": "myevent2", + "start": "2018-01-01T03:00:00+00:00", + "title": "My Event", + } + +This approach is better because all Javascript serializers/deserializers works with UTC based dates (e.g. .toJSON Javascript API). diff --git a/docs/source/index.rst b/docs/source/index.rst index a0d8af7206..f48c87c063 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -39,6 +39,7 @@ Contents breadcrumbs navigation serialization + deserialization searching tusupload vocabularies @@ -57,4 +58,3 @@ Appendix, Indices and tables glossary * :ref:`genindex` - diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst index 0b19511969..b9ef9d74a2 100644 --- a/docs/source/serialization.rst +++ b/docs/source/serialization.rst @@ -24,12 +24,6 @@ Python JSON ``DateTime('2015/11/23 19:45:55')`` ``'2015-11-23T19:45:55'`` ======================================= ====================================== -.. warning:: - All datetimes objects will be deserialized adding the proper time zone information, storing an offset-aware object on it. - For Plone 5 (Dexterity plone.app.event powered) Event content type, you should send always naive (preferred and default if serializing using .toJSON Javascript API) or UTC-based offset datetime ISO8601 strings. - In case of using zope.schema date validators you should also use a datetime object that also contains offset-aware object as the validator value. - - RichText fields --------------- From 9112b291bbdad514de8f1991ab3115fe0dd1c32a Mon Sep 17 00:00:00 2001 From: Victor Fernandez de Alba Date: Wed, 30 Aug 2017 10:48:20 +0200 Subject: [PATCH 07/11] Fix typos --- docs/source/deserialization.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/deserialization.rst b/docs/source/deserialization.rst index b9a0246957..67ff7f023e 100644 --- a/docs/source/deserialization.rst +++ b/docs/source/deserialization.rst @@ -6,13 +6,13 @@ It's worth to note an special case when deserializing Datetimes objects, and how Although not supported by Plone itself yet, plone.restapi will store all the Datetimes that will be handling along with its timezone converted to UTC. This will provide a common ground for all the datetimes operations. -There is an special case when using datetimes objects in p.a.event, and its behavior is different due to implementation differences for versions 1.x (Plone 4) and 2.x and above (Plone 5). +There is a special case when using datetimes objects in p.a.event, and its behavior is different due to implementation differences for versions 1.x (Plone 4) and 2.x and above (Plone 5). .. warning:: In case of using zope.schema date validators you should also use a datetime object that also contains offset-aware object as the validator value. .. note:: - This does not applies in case that you are using Plone 4 with no Dexterity support at all or not p.a.event installed. + This does not apply in case that you are using Plone 4 with no Dexterity support at all or not p.a.event installed. p.a.event 1.x in Plone 4 ------------------------ From 7f5dd84ef6ca8a2124c4b669cb7a02d2ae9f5e13 Mon Sep 17 00:00:00 2001 From: Victor Fernandez de Alba Date: Wed, 30 Aug 2017 10:51:00 +0200 Subject: [PATCH 08/11] Correct old comment for a proper new one --- src/plone/restapi/deserializer/dxfields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plone/restapi/deserializer/dxfields.py b/src/plone/restapi/deserializer/dxfields.py index 8b5058e349..20a46b2899 100644 --- a/src/plone/restapi/deserializer/dxfields.py +++ b/src/plone/restapi/deserializer/dxfields.py @@ -45,7 +45,7 @@ class DatetimeFieldDeserializer(DefaultFieldDeserializer): def __call__(self, value): try: # Parse ISO 8601 string with Zope's DateTime module - # and convert to a timezone naive datetime in local time + # and convert to a UTC based timezone as datetime value = DateTime(value).toZone('UTC').asdatetime() except (SyntaxError, DateTimeError) as e: raise ValueError(e.message) From ba84c0f2b984df44d1a0d3df5933857f96e73e73 Mon Sep 17 00:00:00 2001 From: Victor Fernandez de Alba Date: Wed, 30 Aug 2017 11:29:49 +0200 Subject: [PATCH 09/11] Implement a better handling of the use case when two dates are compared on update --- src/plone/restapi/deserializer/dxcontent.py | 25 ++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/plone/restapi/deserializer/dxcontent.py b/src/plone/restapi/deserializer/dxcontent.py index 8a28394aaf..393ac4f844 100644 --- a/src/plone/restapi/deserializer/dxcontent.py +++ b/src/plone/restapi/deserializer/dxcontent.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from AccessControl import getSecurityManager +from datetime import datetime from plone.autoform.interfaces import WRITE_PERMISSIONS_KEY from plone.dexterity.interfaces import IDexterityContent from plone.dexterity.utils import iterSchemata @@ -23,6 +24,8 @@ from .mixins import OrderingMixin +import pytz + @implementer(IDeserializeFromJson) @adapter(IDexterityContent, Interface) @@ -78,14 +81,20 @@ def __call__(self, validate_all=False, data=None): # noqa: ignore=C901 'message': e.doc(), 'field': name, 'error': e}) else: field_data[name] = value - try: - if value != dm.get(): - dm.set(value) - modified = True - except TypeError: - # Most probably due to offset-naive and offset - # aware objects, set the value as they most likely - # are not the same + dm_value = dm.get() + + # This is required in case that we can compare + # offset-naive and offset aware objects. We convert all + # offset-naive datetimes to UTC timezone first and then + # compare them + if isinstance(value, datetime) and \ + isinstance(dm_value, datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=pytz.UTC) + if dm_value.tzinfo is None: + dm_value = dm_value.replace(tzinfo=pytz.UTC) + + if value != dm_value: dm.set(value) modified = True From 7169beeb58e8e505d893849046f26a6ba191f588 Mon Sep 17 00:00:00 2001 From: Victor Fernandez de Alba Date: Wed, 30 Aug 2017 12:15:22 +0200 Subject: [PATCH 10/11] Correct another typo --- docs/source/deserialization.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/deserialization.rst b/docs/source/deserialization.rst index 67ff7f023e..4cae7e450f 100644 --- a/docs/source/deserialization.rst +++ b/docs/source/deserialization.rst @@ -49,7 +49,7 @@ The final stored datetime takes this field into account and adds the correct off and builds the `start` and `end` fields with the proper timezone, depending on the `timezone` field. It also returns the datetime object with the proper timezone offset. -If using Plone 4 and p.a.event 1.x you should construct the Event type using this approach, otherwise the Event object will be created with a wrong timezones. +If using Plone 4 and p.a.event 1.x you should construct the Event type using this approach, otherwise the Event object will be created with a wrong timezone. This approach was counterintuitive, and thus, it was changed it Plone 5 version of p.a.event. From d7290c01e6411b7f756c6d80eb59c1fb2bec8147 Mon Sep 17 00:00:00 2001 From: Alexander Loechel Date: Tue, 3 Oct 2017 18:48:15 +0200 Subject: [PATCH 11/11] fix requested typos and error on except case --- docs/source/deserialization.rst | 8 ++++---- src/plone/restapi/__init__.py | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/source/deserialization.rst b/docs/source/deserialization.rst index 4cae7e450f..c889ddba7c 100644 --- a/docs/source/deserialization.rst +++ b/docs/source/deserialization.rst @@ -1,9 +1,9 @@ Deserialization =============== -It's worth to note an special case when deserializing Datetimes objects, and how plone.restapi will handle them. +It's worth to note a special case when deserializing datetime objects, and how plone.restapi will handle them. -Although not supported by Plone itself yet, plone.restapi will store all the Datetimes that will be handling along with its timezone converted to UTC. +Although not supported by Plone itself yet, plone.restapi will store datetime objects that will be handling along with its timezone converted to UTC. This will provide a common ground for all the datetimes operations. There is a special case when using datetimes objects in p.a.event, and its behavior is different due to implementation differences for versions 1.x (Plone 4) and 2.x and above (Plone 5). @@ -51,12 +51,12 @@ and builds the `start` and `end` fields with the proper timezone, depending on t If using Plone 4 and p.a.event 1.x you should construct the Event type using this approach, otherwise the Event object will be created with a wrong timezone. -This approach was counterintuitive, and thus, it was changed it Plone 5 version of p.a.event. +This approach was counterintuitive, and thus, it was changed for Plone 5 by p.a.event version 2. p.a.event 2.x in Plone 5 ------------------------ -The implementation of p.a.event in 2.x no longer requires to provide a `timezone` schema property, because the timezone is computed taking the timezone already existent in dates supplied:: +The implementation of p.a.event 2.x no longer requires to provide a `timezone` schema property, because the timezone is computed taking the timezone already existent in dates supplied:: .. code-block:: json diff --git a/src/plone/restapi/__init__.py b/src/plone/restapi/__init__.py index 1112c2ac76..cb518eee23 100644 --- a/src/plone/restapi/__init__.py +++ b/src/plone/restapi/__init__.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- + from AccessControl import allow_module from AccessControl.Permissions import add_user_folders -from Products.PluggableAuthService.PluggableAuthService import registerMultiPlugin # noqa +from Products.PluggableAuthService.PluggableAuthService import registerMultiPlugin # NOQA: E501 from plone.restapi.pas import plugin import pkg_resources @@ -11,7 +12,7 @@ try: pkg_resources.get_distribution('plone.app.testing') REGISTER_TEST_TYPES = True -except pkg_resources.DistributionNotFound: # pragma: no cover +except pkg_resources.DistributionNotFound: REGISTER_TEST_TYPES = False try: @@ -25,8 +26,9 @@ else: HAS_PLONE_APP_EVENT_20 = False -except pkg_resources.DistributionNotFound: # pragma: no cover +except pkg_resources.DistributionNotFound: HAS_PLONE_APP_CONTENTTYPES = False + HAS_PLONE_APP_EVENT_20 = False def initialize(context):