diff --git a/CHANGES.rst b/CHANGES.rst index 667fe1d535..6b526a7702 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,6 +9,12 @@ Changelog - Disallow None and u'' when TextLine is required. Refs #351. [jaroel] + +- The datetime objects are now stored as offset-aware UTC-based objects + [sneridagh] + +- Add skipped tests from @breadcrumbs and @navigation now that the expansion is in place + [sneridagh] 1.0a21 (2017-09-23) diff --git a/docs/source/deserialization.rst b/docs/source/deserialization.rst new file mode 100644 index 0000000000..c889ddba7c --- /dev/null +++ b/docs/source/deserialization.rst @@ -0,0 +1,87 @@ +Deserialization +=============== + +It's worth to note a special case when deserializing datetime objects, and how plone.restapi will handle them. + +Although not supported by Plone itself yet, plone.restapi will store datetime objects that will be handling along with its timezone converted to UTC. +This will provide a common ground for all the datetimes operations. + +There is a special case when using datetimes objects in p.a.event, and its behavior is different due to implementation differences for versions 1.x (Plone 4) and 2.x and above (Plone 5). + +.. warning:: + In case of using zope.schema date validators you should also use a datetime object that also contains offset-aware object as the validator value. + +.. note:: + This does not apply in case that you are using Plone 4 with no Dexterity support at all or not p.a.event installed. + +p.a.event 1.x in Plone 4 +------------------------ + +The implementation of p.a.event in 1.x requires to provide a `timezone` schema property along with the Event type information, otherwise the creation fails, because it's a required field, like this:: + +.. code-block:: json + + { + "@type": "Event", + "id": "myevent2", + "title": "My Event", + "start": "2018-01-01T10:00:00", + "end": "2018-01-01T12:00:00", + "timezone": "Asia/Saigon" + } + +The final stored datetime takes this field into account and adds the correct offset to the content type (abbreviated JSON response):: + +.. code-block:: json + + { + "@id": "http://localhost:55001/plone/folder1/myevent2", + "@type": "Event", + "UID": "bcfc3914ea174cc1aa8042147edfa33a", + "creators": ["admin"], + "description": "", + "end": "2018-01-01T12:00:00+07:00", + "id": "myevent2", + "start": "2018-01-01T10:00:00+07:00", + "timezone": "Asia/Saigon", + "title": "My Event", + } + +and builds the `start` and `end` fields with the proper timezone, depending on the `timezone` field. It also returns the datetime object with the proper timezone offset. + +If using Plone 4 and p.a.event 1.x you should construct the Event type using this approach, otherwise the Event object will be created with a wrong timezone. + +This approach was counterintuitive, and thus, it was changed for Plone 5 by p.a.event version 2. + +p.a.event 2.x in Plone 5 +------------------------ + +The implementation of p.a.event 2.x no longer requires to provide a `timezone` schema property, because the timezone is computed taking the timezone already existent in dates supplied:: + +.. code-block:: json + + { + "@type": "Event", + "id": "myevent2", + "title": "My Event", + "start": "2018-01-01T10:00:00+07:00", + "end": "2018-01-01T12:00:00+07:00", + } + +You should pass the timezone information in the ISO8601 format, otherwise the system will fallback to UTC. The response given is also computed given the timezone information supplied, but this time it's UTC based: + +.. code-block:: json + + { + "@id": "http://localhost:55001/plone/folder1/myevent2", + "@type": "Event", + "UID": "4c031960718246db86c97685f83047ee", + "creators": ["admin"], + "description": "", + "end": "2018-01-01T05:00:00+00:00", + "id": "myevent2", + "start": "2018-01-01T03:00:00+00:00", + "title": "My Event", + } + +This approach is better because all Javascript serializers/deserializers works with UTC based dates (e.g. .toJSON Javascript API). diff --git a/docs/source/index.rst b/docs/source/index.rst index 99a2ec55c5..d3a04b6090 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -39,6 +39,7 @@ Contents breadcrumbs navigation serialization + deserialization searching tusupload vocabularies @@ -58,4 +59,3 @@ Appendix, Indices and tables glossary * :ref:`genindex` - diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst index 333cf7ee4c..b9ef9d74a2 100644 --- a/docs/source/serialization.rst +++ b/docs/source/serialization.rst @@ -24,7 +24,6 @@ Python JSON ``DateTime('2015/11/23 19:45:55')`` ``'2015-11-23T19:45:55'`` ======================================= ====================================== - RichText fields --------------- @@ -155,4 +154,4 @@ UID ``'9b6a4eadb9074dde97d86171bb332ae9'`` IntId ``123456`` Path ``'/plone/doc1'`` URL ``'http://localhost:8080/plone/doc1'`` -======================================= ====================================== \ No newline at end of file +======================================= ====================================== diff --git a/plone-5.0.x.cfg b/plone-5.0.x.cfg index 3997cd81e9..9d88485cc0 100644 --- a/plone-5.0.x.cfg +++ b/plone-5.0.x.cfg @@ -3,3 +3,11 @@ extends = base.cfg http://dist.plone.org/release/5.0.7/versions.cfg versions.cfg + +parts += omelette + +[omelette] +recipe = collective.recipe.omelette +eggs = + ${instance:eggs} + ${test:eggs} diff --git a/src/plone/restapi/__init__.py b/src/plone/restapi/__init__.py index 65cd614f77..cb518eee23 100644 --- a/src/plone/restapi/__init__.py +++ b/src/plone/restapi/__init__.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- + from AccessControl import allow_module from AccessControl.Permissions import add_user_folders -from Products.PluggableAuthService.PluggableAuthService import registerMultiPlugin # noqa +from Products.PluggableAuthService.PluggableAuthService import registerMultiPlugin # NOQA: E501 from plone.restapi.pas import plugin import pkg_resources @@ -11,14 +12,23 @@ try: pkg_resources.get_distribution('plone.app.testing') REGISTER_TEST_TYPES = True -except pkg_resources.DistributionNotFound: # pragma: no cover +except pkg_resources.DistributionNotFound: REGISTER_TEST_TYPES = False try: pkg_resources.get_distribution('plone.app.contenttypes') HAS_PLONE_APP_CONTENTTYPES = True -except pkg_resources.DistributionNotFound: # pragma: no cover + + event_version = pkg_resources.get_distribution('plone.app.event').version + if pkg_resources.parse_version(event_version) > \ + pkg_resources.parse_version('1.99'): + HAS_PLONE_APP_EVENT_20 = True + else: + HAS_PLONE_APP_EVENT_20 = False + +except pkg_resources.DistributionNotFound: HAS_PLONE_APP_CONTENTTYPES = False + HAS_PLONE_APP_EVENT_20 = False def initialize(context): diff --git a/src/plone/restapi/deserializer/dxcontent.py b/src/plone/restapi/deserializer/dxcontent.py index bed41a5927..393ac4f844 100644 --- a/src/plone/restapi/deserializer/dxcontent.py +++ b/src/plone/restapi/deserializer/dxcontent.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from AccessControl import getSecurityManager +from datetime import datetime from plone.autoform.interfaces import WRITE_PERMISSIONS_KEY from plone.dexterity.interfaces import IDexterityContent from plone.dexterity.utils import iterSchemata @@ -23,6 +24,8 @@ from .mixins import OrderingMixin +import pytz + @implementer(IDeserializeFromJson) @adapter(IDexterityContent, Interface) @@ -78,7 +81,20 @@ def __call__(self, validate_all=False, data=None): # noqa: ignore=C901 'message': e.doc(), 'field': name, 'error': e}) else: field_data[name] = value - if value != dm.get(): + dm_value = dm.get() + + # This is required in case that we can compare + # offset-naive and offset aware objects. We convert all + # offset-naive datetimes to UTC timezone first and then + # compare them + if isinstance(value, datetime) and \ + isinstance(dm_value, datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=pytz.UTC) + if dm_value.tzinfo is None: + dm_value = dm_value.replace(tzinfo=pytz.UTC) + + if value != dm_value: dm.set(value) modified = True diff --git a/src/plone/restapi/deserializer/dxfields.py b/src/plone/restapi/deserializer/dxfields.py index 3af62dbac6..4861ed7449 100644 --- a/src/plone/restapi/deserializer/dxfields.py +++ b/src/plone/restapi/deserializer/dxfields.py @@ -64,9 +64,8 @@ class DatetimeFieldDeserializer(DefaultFieldDeserializer): def __call__(self, value): try: # Parse ISO 8601 string with Zope's DateTime module - # and convert to a timezone naive datetime in local time - value = DateTime(value).toZone(DateTime().localZone()).asdatetime( - ).replace(tzinfo=None) + # and convert to a UTC based timezone as datetime + value = DateTime(value).toZone('UTC').asdatetime() except (SyntaxError, DateTimeError) as e: raise ValueError(e.message) diff --git a/src/plone/restapi/tests/dxtypes.py b/src/plone/restapi/tests/dxtypes.py index ae90b312ff..8a9419686d 100644 --- a/src/plone/restapi/tests/dxtypes.py +++ b/src/plone/restapi/tests/dxtypes.py @@ -21,6 +21,7 @@ from zope.schema.vocabulary import SimpleTerm from zope.schema.vocabulary import SimpleVocabulary +import pytz INDEXES = ( ("test_int_field", "FieldIndex"), @@ -92,8 +93,8 @@ class IDXTestDocumentSchema(model.Schema): test_maxlength_field = schema.TextLine(required=False, max_length=10) test_constraint_field = schema.TextLine(required=False, constraint=lambda x: u'00' in x) - test_datetime_min_field = schema.Datetime(required=False, - min=datetime(2000, 1, 1)) + test_datetime_min_field = schema.Datetime( + required=False, min=datetime(2000, 1, 1, 0, 0, 0, 0, pytz.UTC)) test_time_min_field = schema.Time(required=False, min=time(1)) test_timedelta_min_field = schema.Timedelta(required=False, min=timedelta(100)) diff --git a/src/plone/restapi/tests/test_content_patch.py b/src/plone/restapi/tests/test_content_patch.py index 513af250ee..8b6fef733d 100644 --- a/src/plone/restapi/tests/test_content_patch.py +++ b/src/plone/restapi/tests/test_content_patch.py @@ -9,10 +9,15 @@ from plone.app.testing import login from plone.app.testing import setRoles from plone.restapi.testing import PLONE_RESTAPI_DX_FUNCTIONAL_TESTING +from plone.restapi.testing import RelativeSession +from DateTime import DateTime +from datetime import datetime +from datetime import timedelta import requests import transaction import unittest +import pytz class TestContentPatch(unittest.TestCase): @@ -22,8 +27,14 @@ class TestContentPatch(unittest.TestCase): def setUp(self): self.app = self.layer['app'] self.portal = self.layer['portal'] + self.portal_url = self.portal.absolute_url() setRoles(self.portal, TEST_USER_ID, ['Member']) login(self.portal, SITE_OWNER_NAME) + + self.api_session = RelativeSession(self.portal_url) + self.api_session.headers.update({'Accept': 'application/json'}) + self.api_session.auth = (SITE_OWNER_NAME, SITE_OWNER_PASSWORD) + self.portal.invokeFactory( 'Document', id='doc1', @@ -78,3 +89,84 @@ def test_patch_document_returns_401_unauthorized(self): data='{"title": "Patched Document"}', ) self.assertEqual(401, response.status_code) + + def test_patch_feed_event_with_get_contents(self): + start_date = DateTime(datetime.today() + + timedelta(days=1)).ISO8601() + end_date = DateTime(datetime.today() + + timedelta(days=1, hours=1)).ISO8601() + response = self.api_session.post( + '/', + json={ + "title": "An Event", + "@type": "Event", + "start": start_date, + "end": end_date, + "timezone": "Europe/Vienna" + }, + ) + + self.assertEqual(201, response.status_code) + + response = response.json() + event_id = response['id'] + two_days_ahead = DateTime(datetime.today() + + timedelta(days=2)) + response = self.api_session.patch( + '/{}'.format(event_id), + json={ + "start": response['start'], + "end": two_days_ahead.ISO8601() + } + ) + + self.assertEqual(204, response.status_code) + + response = self.api_session.get('/{}'.format(event_id)) + response = response.json() + + self.assertEquals( + DateTime(response['end']).day(), + two_days_ahead.day() + ) + self.assertEquals( + DateTime(response['end']).hour(), + two_days_ahead.hour() + ) + + def test_patch_document_with_expires(self): + response = requests.patch( + self.portal.doc1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "expires": datetime( + 2013, 1, 1, 10, 0).isoformat() + } + ) + + self.assertEqual(204, response.status_code) + + response = requests.patch( + self.portal.doc1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "expires": datetime( + 2013, 1, 1, 10, 0).isoformat() + } + ) + + self.assertEqual(204, response.status_code) + + response = requests.patch( + self.portal.doc1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "expires": pytz.timezone('Europe/Berlin').localize(datetime( + 2013, 1, 1, 10, 0)).isoformat() + } + ) + + self.assertEqual(204, response.status_code) diff --git a/src/plone/restapi/tests/test_content_post.py b/src/plone/restapi/tests/test_content_post.py index 7edab292fe..c26d51a4cd 100644 --- a/src/plone/restapi/tests/test_content_post.py +++ b/src/plone/restapi/tests/test_content_post.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from datetime import datetime +from DateTime import DateTime from Products.CMFCore.utils import getToolByName from plone.app.testing import SITE_OWNER_NAME from plone.app.testing import SITE_OWNER_PASSWORD @@ -9,7 +11,9 @@ from plone.app.testing import setRoles from plone.restapi.testing import PLONE_RESTAPI_DX_FUNCTIONAL_TESTING from plone.restapi.testing import PLONE_RESTAPI_AT_FUNCTIONAL_TESTING +from plone.restapi import HAS_PLONE_APP_EVENT_20 +import pytz import requests import transaction import unittest @@ -196,3 +200,116 @@ def test_id_from_filename(self): self.assertEqual(201, response.status_code) transaction.begin() self.assertIn('test.txt', self.portal.folder1) + + +class TestEventCT(unittest.TestCase): + + layer = PLONE_RESTAPI_DX_FUNCTIONAL_TESTING + + def setUp(self): + self.app = self.layer['app'] + self.portal = self.layer['portal'] + setRoles(self.portal, TEST_USER_ID, ['Member']) + login(self.portal, SITE_OWNER_NAME) + self.portal.invokeFactory( + 'Folder', + id='folder1', + title='My Folder' + ) + wftool = getToolByName(self.portal, 'portal_workflow') + wftool.doActionFor(self.portal.folder1, 'publish') + transaction.commit() + + @unittest.skipIf(HAS_PLONE_APP_EVENT_20, 'Valid for p.a.event<2.0 only') + def test_post_to_folder_creates_event_with_correct_TZ(self): + response = requests.post( + self.portal.folder1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "@type": "Event", + "id": "myevent2", + "title": "My Event", + "start": DateTime(datetime(2018, 1, 1, 10, 0)).ISO8601(), + "end": DateTime(datetime(2018, 1, 1, 12, 0)).ISO8601(), + "timezone": 'Asia/Saigon' + }, + ) + self.assertEqual(201, response.status_code) + self.assertEqual( + response.json()['start'], u'2018-01-01T10:00:00+07:00') + + response = requests.post( + self.portal.folder1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "@type": "Event", + "id": "myevent", + "title": "My Event", + "start": DateTime(datetime(2013, 1, 1, 10, 0)).ISO8601(), + "end": DateTime(datetime(2013, 1, 1, 12, 0)).ISO8601(), + "timezone": 'Europe/Madrid' + }, + ) + self.assertEqual(201, response.status_code) + self.assertEqual( + response.json()['start'], u'2013-01-01T10:00:00+01:00') + + @unittest.skipIf(HAS_PLONE_APP_EVENT_20, 'Valid for p.a.event<2.0 only') + def test_post_creates_event_with_correct_TZ_using_UTC_offset(self): + response = requests.post( + self.portal.folder1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "@type": "Event", + "id": "myevent", + "title": "My Event", + "start": datetime( + 2013, 1, 1, 10, 0, 0, 0, pytz.UTC).isoformat(), + "end": datetime( + 2013, 1, 1, 12, 0, 0, 0, pytz.UTC).isoformat(), + "timezone": 'Europe/Madrid' + }, + ) + + self.assertEqual(201, response.status_code) + self.assertEqual( + response.json()['start'], u'2013-01-01T10:00:00+01:00') + + @unittest.skipIf(not HAS_PLONE_APP_EVENT_20, 'Valid for p.a.event<2.0 only') # noqa + def test_pavent_20_post_to_folder_creates_event_with_correct_TZ(self): + response = requests.post( + self.portal.folder1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "@type": "Event", + "id": "myevent2", + "title": "My Event", + "start": pytz.timezone('Asia/Saigon').localize(datetime(2018, 1, 1, 10, 0)).isoformat(), # noqa + "end": pytz.timezone('Asia/Saigon').localize(datetime(2018, 1, 1, 12, 0)).isoformat(), # noqa + }, + ) + self.assertEqual(201, response.status_code) + self.assertEqual( + response.json()['start'], u'2018-01-01T03:00:00+00:00') + + response = requests.post( + self.portal.folder1.absolute_url(), + headers={'Accept': 'application/json'}, + auth=(SITE_OWNER_NAME, SITE_OWNER_PASSWORD), + json={ + "@type": "Event", + "id": "myevent", + "title": "My Event", + "start": pytz.timezone('Europe/Berlin').localize(datetime( + 2013, 1, 1, 10, 0)).isoformat(), + "end": pytz.timezone('Europe/Berlin').localize(datetime( + 2013, 1, 1, 12, 0)).isoformat(), + }, + ) + self.assertEqual(201, response.status_code) + self.assertEqual( + response.json()['start'], u'2013-01-01T09:00:00+00:00') diff --git a/src/plone/restapi/tests/test_dxfield_deserializer.py b/src/plone/restapi/tests/test_dxfield_deserializer.py index bf32186745..a7e2bdedd2 100644 --- a/src/plone/restapi/tests/test_dxfield_deserializer.py +++ b/src/plone/restapi/tests/test_dxfield_deserializer.py @@ -12,6 +12,7 @@ from zope.component import getMultiAdapter from zope.schema.interfaces import ValidationError +import pytz import unittest @@ -79,12 +80,14 @@ def test_datetime_deserialization_returns_datetime(self): value = self.deserialize('test_datetime_field', u'2015-12-20T10:39:54.361Z') self.assertTrue(isinstance(value, datetime), 'Not a ') - self.assertEqual(datetime(2015, 12, 20, 10, 39, 54, 361000), value) + self.assertEqual( + datetime(2015, 12, 20, 10, 39, 54, 361000, pytz.UTC), value) def test_datetime_deserialization_handles_timezone(self): value = self.deserialize('test_datetime_field', u'2015-12-20T10:39:54.361+01') - self.assertEqual(datetime(2015, 12, 20, 9, 39, 54, 361000), value) + self.assertEqual( + datetime(2015, 12, 20, 9, 39, 54, 361000, pytz.UTC), value) def test_decimal_deserialization_returns_decimal(self): value = self.deserialize('test_decimal_field', u'1.1')