diff --git a/kobocat-template/static/css/kobo-branding.css b/kobocat-template/static/css/kobo-branding.css index 45cfcc0fc..47306af30 100644 --- a/kobocat-template/static/css/kobo-branding.css +++ b/kobocat-template/static/css/kobo-branding.css @@ -330,7 +330,7 @@ section { .single-project__header .container, .projects-list__header .container, .data-page__header .container { - max-width: 984px; + max-width: 960px; } .single-project__header .fa-caret-right, @@ -681,4 +681,4 @@ i.fa-trash-o { td.footable-first-column { padding-right: 0px !important; } -} \ No newline at end of file +} diff --git a/kobocat-template/static/css/kobo-single-project.css b/kobocat-template/static/css/kobo-single-project.css index 69fd8bb66..c96ce6fd7 100644 --- a/kobocat-template/static/css/kobo-single-project.css +++ b/kobocat-template/static/css/kobo-single-project.css @@ -189,7 +189,7 @@ button.single-project__button-delete { } .dashboard__left { - width: 72%; + width: 60%; float: left; margin-left: 1%; } @@ -197,6 +197,7 @@ button.single-project__button-delete { .dashboard__right { width: 25%; float: left; + margin-left: -13%; } h2.dashboard__group-label { @@ -205,9 +206,8 @@ h2.dashboard__group-label { } .dashboard__submissions { - width: 55%; + width: 33%; float: left; - margin-right: 5%; position: relative; } @@ -268,7 +268,7 @@ a.dashboard__button-download-data:hover span { } .dashboard__submissions a.dashboard__button { - width: 30%; + width: 60%; } a.dashboard__button-twoline span { @@ -351,8 +351,8 @@ a.dashboard__formbutton-refresh { .dashboard__download__drop { position: absolute; - right: 22px; - bottom: -150px; + left: 0; + bottom: -128px; width: 124px; border: 1px solid #E7E8E9; background: white; @@ -629,4 +629,4 @@ span.poshytip { .dashboard__download__drop { bottom: -190px; } -} \ No newline at end of file +} diff --git a/kobocat-template/templates/base.html b/kobocat-template/templates/base.html index e67996755..8399f7caa 100644 --- a/kobocat-template/templates/base.html +++ b/kobocat-template/templates/base.html @@ -1,97 +1,102 @@ {% load static %} - + {% block title %}{{ SITE_NAME }}{% endblock %} - + {% if GOOGLE_SITE_VERIFICATION %} - + {% endif %} - {% if csrf_token %}{% endif %} + {% if csrf_token %} + {% endif %} {% block styles %} - - - - - - - - - - - - + + + + + + + + + + - + - - - - - + + + + + - + {% endblock %} {% block additional-headers %}{% endblock %} - - - {% block body %} - {% if not user.is_authenticated %} - - -
-
- We no longer recommend using the legacy interface. Please access all our new features in the new interface (new map, table, reports, labeled data exports, etc.). If you have deployed your project directly in the legacy interface, there is no way to use these new features. This legacy interface will be removed shortly. -
-
- -
-
- - Support -
-
- - {% else %} - - - {% include "topbar.html"%} - - {% endif %} - - {% block before-content %}{% endblock %} - -
- {% block message %} + + +{% block body %} + {% if not user.is_authenticated %} + + +
+
+ + We no longer recommend using the legacy interface. Please access all our new features in the new interface (new map, table, reports, labeled data exports, etc.). +
+
+ +
+
+ + Support +
+
+ + {% else %} + + + {% include "topbar.html" %} + + {% endif %} + +
+ {% block message %} {% if message or messages or message_list %} - {% include "message.html" %} + {% include "message.html" %} {% endif %} - {% endblock %} - {% block content %} + {% endblock %} + {% block content %} {% if template %}{% include template %}{% endif %} {{ content|safe }} - {% endblock %} -
+ {% endblock %} +
- {% include "footer.html" %} +{% include "footer.html" %} - {% block below-content %}{% endblock %} +{% block below-content %}{% endblock %} - {% block javascript %} +{% block javascript %} @@ -108,25 +113,32 @@ - {% endblock %} +{% endblock %} - {% block additional-javascript %}{% endblock %} +{% block additional-javascript %}{% endblock %} - {% block google-analytics %} +{% block google-analytics %} {% if GOOGLE_ANALYTICS_PROPERTY_ID %} - + {% endif %} - {% endblock %} +{% endblock %} - - {% endblock %} + +{% endblock %} diff --git a/kobocat-template/templates/dashboard.html b/kobocat-template/templates/dashboard.html index cfc087dca..2444f6959 100644 --- a/kobocat-template/templates/dashboard.html +++ b/kobocat-template/templates/dashboard.html @@ -64,12 +64,11 @@

{% blocktrans %}Publish a Form Upload XLSForm{% endblocktrans

- {{ content_user.username }} + {{ content_user.username }} {% if profile.name %} | {{ profile.name }} {% endif %}

- {% trans "View Profile Page" %}
diff --git a/kobocat-template/templates/data_view.html b/kobocat-template/templates/data_view.html deleted file mode 100644 index e3453f318..000000000 --- a/kobocat-template/templates/data_view.html +++ /dev/null @@ -1,97 +0,0 @@ -{% extends 'base.html' %} -{% load i18n %} -{% load static %} - -{% block additional-headers %} -{% load i18n %} - - - - -{% endblock %} - -{% block before-content %} - - -
-
-

{% trans "Data View" %}

-
-
- -{% endblock %} - -{% block content %} -
-
- -
-
-
-
-
-
-{% endblock %} - -{% block javascript %} - {{ block.super }} - - - - - - - - - - - -{% endblock %} diff --git a/kobocat-template/templates/instance.html b/kobocat-template/templates/instance.html deleted file mode 100644 index 7f2676861..000000000 --- a/kobocat-template/templates/instance.html +++ /dev/null @@ -1,190 +0,0 @@ -{% extends 'base.html' %} -{% load i18n %} -{% load static %} - -{% block before-content %} -{% load i18n %} - - - -
-
-

{% trans "Browse Form Data" %}

-
-
- -{% endblock %} - -{% block content %} -{% load i18n %} -
-

{% trans "Loading..." %}

- Loading... -
-{% if messages %} -
{{messages}}
-{% endif %} -
- - -{% endblock %} - -{% block javascript %} -{{ block.super }} - - - - - - - - - - -{% endblock %} diff --git a/kobocat-template/templates/map.html b/kobocat-template/templates/map.html index 5711925ac..3d95696fe 100644 --- a/kobocat-template/templates/map.html +++ b/kobocat-template/templates/map.html @@ -3,12 +3,12 @@ {% load static %} {% block additional-headers %} - + - - - - @@ -16,274 +16,275 @@ {% block body %} - - {% include "topbar.html" %} - - - {% endif %} - - diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index 81be884eb..1adcbd160 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -52,38 +52,42 @@ def setUp(self): self._add_permissions_to_user(AnonymousUser()) self.maxDiff = None - def publish_xls_form(self): - data = { - 'owner': self.user.username, - 'public': False, - 'public_data': False, - 'description': u'transportation_2011_07_25', - 'downloadable': True, - 'allows_sms': False, - 'encrypted': False, - 'sms_id_string': u'transportation_2011_07_25', - 'id_string': u'transportation_2011_07_25', - 'title': u'transportation_2011_07_25', - } - - path = os.path.join( - settings.ONADATA_DIR, "apps", "main", "tests", "fixtures", - "transportation", "transportation.xls") + def publish_xls_form(self, path=None, data=None, assert_=True): + if not data: + data = { + 'owner': self.user.username, + 'public': False, + 'public_data': False, + 'description': u'transportation_2011_07_25', + 'downloadable': True, + 'encrypted': False, + 'id_string': u'transportation_2011_07_25', + 'title': u'transportation_2011_07_25', + } + + if not path: + path = os.path.join( + settings.ONADATA_DIR, "apps", "main", "tests", "fixtures", + "transportation", "transportation.xls") xform_list_url = reverse('xform-list') with open(path) as xls_file: post_data = {'xls_file': xls_file} response = self.client.post(xform_list_url, data=post_data) - self.assertEqual(response.status_code, 201) - self.xform = XForm.objects.all().order_by('pk').reverse()[0] - data.update({ - 'url': - 'http://testserver/api/v1/forms/%s' % (self.xform.pk) - }) - - self.assertDictContainsSubset(data, response.data) - self.form_data = response.data + + if not assert_: + return response + + self.assertEqual(response.status_code, 201) + self.xform = XForm.objects.all().order_by('pk').reverse()[0] + data.update({ + 'url': + 'http://testserver/api/v1/forms/%s' % (self.xform.pk) + }) + + self.assertDictContainsSubset(data, response.data) + self.form_data = response.data def user_profile_data(self): return { diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 5fe3aaba6..88300d3f8 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -4,7 +4,6 @@ import os import re from datetime import datetime -from xml.dom import minidom, Node import pytz import requests @@ -13,6 +12,7 @@ from guardian.shortcuts import assign_perm from httmock import HTTMock, all_requests from rest_framework import status +from xml.dom import minidom, Node from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ TestAbstractViewSet @@ -29,7 +29,7 @@ def enketo_mock(url, request): response = requests.Response() response.status_code = 201 response._content = \ - '{\n "url": "https:\\/\\/dmfrm.enketo.org\\/webform",\n'\ + '{\n "url": "https:\\/\\/dmfrm.enketo.org\\/webform",\n' \ ' "code": "200"\n}' return response @@ -39,7 +39,7 @@ def enketo_error_mock(url, request): response = requests.Response() response.status_code = 400 response._content = \ - '{\n "message": "no account exists for this OpenRosa server",\n'\ + '{\n "message": "no account exists for this OpenRosa server",\n' \ ' "code": "200"\n}' return response @@ -79,7 +79,7 @@ def test_form_list_other_user_access(self): alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} self._login_user_and_profile(extra_post_data=alice_data) self.assertEqual(self.user.username, 'alice') - self.assertNotEqual(previous_user, self.user) + self.assertNotEqual(previous_user, self.user) request = self.factory.get('/', **self.extra) response = self.view(request) self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -94,7 +94,7 @@ def test_form_list_filter_by_user(self): alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} self._login_user_and_profile(extra_post_data=alice_data) self.assertEqual(self.user.username, 'alice') - self.assertNotEqual(previous_user, self.user) + self.assertNotEqual(previous_user, self.user) assign_perm(CAN_VIEW_XFORM, self.user, self.xform) view = XFormViewSet.as_view({ @@ -158,7 +158,7 @@ def test_form_format(self): } request = self.factory.get('/', **self.extra) # test for unsupported format - response = view(request, pk=formid, format='csvzip') + response = view(request, pk=formid, format='xlsx') self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) # test for supported formats @@ -181,13 +181,13 @@ def test_form_format(self): n for n in response_doc.getElementsByTagName("h:head")[0].childNodes if n.nodeType == Node.ELEMENT_NODE and - n.tagName == "model"][0] + n.tagName == "model"][0] # check for UUID and remove uuid_nodes = [ node for node in model_node.childNodes if node.nodeType == Node.ELEMENT_NODE - and node.getAttribute("nodeset") == "/transportation_2011_07_25/formhub/uuid"] + and node.getAttribute("nodeset") == "/transportation_2011_07_25/formhub/uuid"] self.assertEqual(len(uuid_nodes), 1) uuid_node = uuid_nodes[0] uuid_node.setAttribute("calculate", "''") @@ -238,34 +238,6 @@ def test_form_tags(self): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data, []) - def test_enketo_url_no_account(self): - self.publish_xls_form() - view = XFormViewSet.as_view({ - 'get': 'enketo' - }) - formid = self.xform.pk - # no tags - request = self.factory.get('/', **self.extra) - with HTTMock(enketo_error_mock): - response = view(request, pk=formid) - data = {'message': "Enketo not properly configured."} - - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertEqual(response.data, data) - - def test_enketo_url(self): - self.publish_xls_form() - view = XFormViewSet.as_view({ - 'get': 'enketo' - }) - formid = self.xform.pk - # no tags - request = self.factory.get('/', **self.extra) - with HTTMock(enketo_mock): - response = view(request, pk=formid) - data = {"enketo_url": "https://dmfrm.enketo.org/webform"} - self.assertEqual(response.data, data) - def test_publish_xlsform(self): view = XFormViewSet.as_view({ 'post': 'create' @@ -276,11 +248,9 @@ def test_publish_xlsform(self): 'public_data': False, 'description': 'transportation_2011_07_25', 'downloadable': True, - 'allows_sms': False, 'encrypted': False, - 'sms_id_string': 'transportation_2011_07_25', 'id_string': 'transportation_2011_07_25', - 'title': 'transportation_2011_07_25', + 'title': 'transportation_2011_07_25' } path = os.path.join( settings.ONADATA_DIR, "apps", "main", "tests", "fixtures", @@ -293,7 +263,7 @@ def test_publish_xlsform(self): xform = self.user.xforms.get(uuid=response.data.get('uuid')) data.update({ 'url': - 'http://testserver/api/v1/forms/%s' % xform.pk + 'http://testserver/api/v1/forms/%s' % xform.pk }) self.assertDictContainsSubset(data, response.data) self.assertTrue(xform.user.pk == self.user.pk) @@ -390,7 +360,7 @@ def test_set_form_bad_value(self): self.assertFalse(self.xform.__getattribute__(key)) self.assertEqual(response.data, {'shared': - ["'String' value must be either True or False."]}) + ["'String' value must be either True or False."]}) def test_set_form_bad_key(self): self.publish_xls_form() @@ -429,7 +399,6 @@ def test_xform_serializer_none(self): 'require_auth': False, 'description': '', 'downloadable': False, - 'allows_sms': False, 'uuid': '', 'instances_with_geopoints': False, 'num_of_submissions': 0, @@ -474,3 +443,24 @@ def test_csv_import_fail_invalid_field_post(self): response = view(request, pk=self.xform.id) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIsNotNone(response.data.get('error')) + + def test_cannot_publish_id_string_starting_with_number(self): + data = { + 'owner': self.user.username, + 'public': False, + 'public_data': False, + 'description': '2011_07_25_transportation', + 'downloadable': True, + 'encrypted': False, + 'id_string': '2011_07_25_transportation', + 'title': '2011_07_25_transportation', + } + + xls_path = os.path.join(settings.ONADATA_DIR, 'apps', 'main', 'tests', + 'fixtures', 'transportation', + 'transportation.id_starts_with_num.xls') + count = XForm.objects.count() + response = self.publish_xls_form(xls_path, data, assert_=False) + self.assertTrue('Names must begin with a letter' in response.content) + self.assertEqual(response.status_code, 400) + self.assertEqual(XForm.objects.count(), count) diff --git a/onadata/apps/api/tools.py b/onadata/apps/api/tools.py index 321d35490..34a6465ea 100644 --- a/onadata/apps/api/tools.py +++ b/onadata/apps/api/tools.py @@ -18,7 +18,7 @@ from rest_framework import exceptions from taggit.forms import TagField -from onadata.apps.main.forms import QuickConverter +from onadata.apps.main.forms import QuickConverterForm from onadata.apps.main.models import UserProfile from onadata.apps.viewer.models.parsed_instance import datetime_from_str from onadata.libs.utils.logger_tools import publish_form @@ -45,23 +45,6 @@ def _get_id_for_type(record, mongo_field): return {"$substr": [mongo_str, 0, 10]} if isinstance(date_field, datetime)\ else mongo_str -# TODO verify tests without this method, then delete -# def get_accessible_forms(owner=None, shared_form=False, shared_data=False): -# xforms = XForm.objects.filter() -# -# if shared_form and not shared_data: -# xforms = xforms.filter(shared=True) -# elif (shared_form and shared_data) or \ -# (owner == 'public' and not shared_form and not shared_data): -# xforms = xforms.filter(Q(shared=True) | Q(shared_data=True)) -# elif not shared_form and shared_data: -# xforms = xforms.filter(shared_data=True) -# -# if owner != 'public': -# xforms = xforms.filter(user__username=owner) -# -# return xforms.distinct() - def publish_xlsform(request, user, existing_xform=None): """ @@ -84,7 +67,7 @@ def publish_xlsform(request, user, existing_xform=None): ) def set_form(): - form = QuickConverter(request.POST, request.FILES) + form = QuickConverterForm(request.POST, request.FILES) if existing_xform: return form.publish(user, existing_xform.id_string) else: diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index 5dbdfd46d..bf1f32fee 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -379,8 +379,6 @@ class DataViewSet(AnonymousUserPublicFormsMixin, ModelViewSet): renderers.XLSRenderer, renderers.XLSXRenderer, renderers.CSVRenderer, - renderers.CSVZIPRenderer, - renderers.SAVZIPRenderer, renderers.RawXMLRenderer ] diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index a01c44a6b..a07373419 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -45,8 +45,6 @@ 'xls': Export.XLS_EXPORT, 'xlsx': Export.XLS_EXPORT, 'csv': Export.CSV_EXPORT, - 'csvzip': Export.CSV_ZIP_EXPORT, - 'savzip': Export.SAV_ZIP_EXPORT, } @@ -67,8 +65,6 @@ def _get_extension_from_export_type(export_type): if export_type == Export.XLS_EXPORT: extension = 'xlsx' - elif export_type in [Export.CSV_ZIP_EXPORT, Export.SAV_ZIP_EXPORT]: - extension = 'zip' return extension @@ -165,8 +161,7 @@ def response_for_format(form, format=None): def should_regenerate_export(xform, export_type, request): return should_create_new_export(xform, export_type) or\ 'start' in request.GET or 'end' in request.GET or\ - 'query' in request.GET or 'meta' in request.GET or\ - 'token' in request.GET + 'query' in request.GET def value_for_type(form, field, value): @@ -191,8 +186,7 @@ def log_export(request, xform, export_type): }, audit, request) -def custom_response_handler(request, xform, query, export_type, - token=None, meta=None): +def custom_response_handler(request, xform, query, export_type): export_type = _get_export_type(export_type) # check if we need to re-generate, @@ -237,7 +231,6 @@ class XFormViewSet(AnonymousUserPublicFormsMixin, LabelsMixin, ModelViewSet): account. - `xls_file`: the xlsform file. -- `xls_url`: the url to an xlsform - `owner`: username to the target account (Optional)
@@ -247,11 +240,6 @@ class XFormViewSet(AnonymousUserPublicFormsMixin, LabelsMixin, ModelViewSet):
 >       curl -X POST -F xls_file=@/path/to/form.xls \
 https://example.com/api/v1/forms
 >
-> OR post an xlsform url
->
->       curl -X POST -d \
-"xls_url=https://example.com/ukanga/forms/tutorial/form.xls" \
-https://example.com/api/v1/forms
 
 > Response
 >
@@ -260,9 +248,7 @@ class XFormViewSet(AnonymousUserPublicFormsMixin, LabelsMixin, ModelViewSet):
 >           "formid": 28058,
 >           "uuid": "853196d7d0a74bca9ecfadbf7e2f5c1f",
 >           "id_string": "Birds",
->           "sms_id_string": "Birds",
 >           "title": "Birds",
->           "allows_sms": false,
 >           "description": "",
 >           "downloadable": true,
 >           "encrypted": false,
@@ -317,9 +303,7 @@ class XFormViewSet(AnonymousUserPublicFormsMixin, LabelsMixin, ModelViewSet):
 >           "formid": 28058,
 >           "uuid": "853196d7d0a74bca9ecfadbf7e2f5c1f",
 >           "id_string": "Birds",
->           "sms_id_string": "Birds",
 >           "title": "Birds",
->           "allows_sms": false,
 >           "description": "",
 >           "downloadable": true,
 >           "encrypted": false,
@@ -353,9 +337,7 @@ class XFormViewSet(AnonymousUserPublicFormsMixin, LabelsMixin, ModelViewSet):
 >           "formid": 28058,
 >           "uuid": "853196d7d0a74bca9ecfadbf7e2f5c1f",
 >           "id_string": "Birds",
->           "sms_id_string": "Birds",
 >           "title": "Birds",
->           "allows_sms": false,
 >           "description": "Le description",
 >           "downloadable": true,
 >           "encrypted": false,
@@ -370,7 +352,7 @@ class XFormViewSet(AnonymousUserPublicFormsMixin, LabelsMixin, ModelViewSet):
 
 You may overwrite the form's contents while preserving its submitted data,
 `id_string` and all other attributes, by sending a `PATCH` that includes
-`xls_file` or `text_xls_form`. Use with caution, as this may compromise the
+`xls_file`. Use with caution, as this may compromise the
 methodology of your study!
 
 
@@ -539,22 +521,6 @@ class XFormViewSet(AnonymousUserPublicFormsMixin, LabelsMixin, ModelViewSet):
 >
 >        HTTP 200 OK
 
-## Get webform/enketo link
-
-
-GET /api/v1/forms/{pk}/enketo
- -> Request -> -> curl -X GET \ -https://example.com/api/v1/forms/28058/enketo -> -> Response -> -> {"enketo_url": "https://h6ic6.enketo.org/webform"} -> -> HTTP 200 OK - ## Get form data in xls, csv format. Get form data exported as xls, csv, csv zip, sav zip format. @@ -564,11 +530,6 @@ class XFormViewSet(AnonymousUserPublicFormsMixin, LabelsMixin, ModelViewSet): - `pk` - is the form unique identifier - `format` - is the data export format i.e csv, xls, csvzip, savzip -Params for the custom xls report - -- `meta` - the metadata id containing the template url -- `token` - the template url -
 GET /api/v1/forms/{pk}.{format}
 
@@ -583,46 +544,6 @@ class XFormViewSet(AnonymousUserPublicFormsMixin, LabelsMixin, ModelViewSet): > > HTTP 200 OK -> Example 2 Custom XLS reports (beta) -> -> curl -X GET https://example.com/api/v1/forms/28058.xls?meta=12121 -> or -> curl -X GET https://example.com/api/v1/forms/28058.xls?token={url} -> -> XLS file is downloaded -> -> Response -> -> HTTP 200 OK - -## Clone a form to a specific user account - -You can clone a form to a specific user account using `GET` with - -- `username` of the user you want to clone the form to - -
-GET /api/v1/forms/{pk}/clone
-
- -> Example -> -> curl -X GET https://example.com/api/v1/forms/123/clone \ --d username=alice - -> Response -> -> HTTP 201 CREATED -> { -> "url": "https://example.com/api/v1/forms/124", -> "formid": 124, -> "uuid": "853196d7d0a74bca9ecfadbf7e2f5c1e", -> "id_string": "Birds_cloned_1", -> "sms_id_string": "Birds_cloned_1", -> "title": "Birds_cloned_1", -> ... -> } - ## Import CSV data to existing form - `csv_file` a valid csv file with exported \ @@ -649,8 +570,6 @@ class XFormViewSet(AnonymousUserPublicFormsMixin, LabelsMixin, ModelViewSet): renderers.XLSRenderer, renderers.XLSXRenderer, renderers.CSVRenderer, - renderers.CSVZIPRenderer, - renderers.SAVZIPRenderer, renderers.RawXMLRenderer ] queryset = XForm.objects.all() @@ -673,9 +592,8 @@ def create(self, request, *args, **kwargs): if isinstance(survey, XForm): xform = XForm.objects.get(pk=survey.pk) # The XForm has been created, but `publish_xlsform` relies on - # `onadata.apps.main.forms.QuickConverter`, which uses standard - # Django forms and only recognizes the `xls_file`, `xls_url`, - # `dropbox_xls_url`, and `text_xls_form` fields. + # `onadata.apps.main.forms.QuickConverterForm`, which uses standard + # Django forms and only recognizes the `xls_file` fields. # Use the DRF serializer to update the XForm with values for other # fields. serializer = XFormSerializer( @@ -694,7 +612,7 @@ def create(self, request, *args, **kwargs): return Response(survey, status=status.HTTP_400_BAD_REQUEST) def update(self, request, pk, *args, **kwargs): - if 'xls_file' in request.FILES or 'text_xls_form' in request.data: + if 'xls_file' in request.FILES: # A new XLSForm has been uploaded and will replace the existing # form existing_xform = get_object_or_404(XForm, pk=pk) @@ -729,25 +647,6 @@ def form(self, request, format='json', **kwargs): return response - @detail_route(methods=['GET']) - def enketo(self, request, **kwargs): - self.object = self.get_object() - form_url = _get_form_url(self.object.user.username) - - data = {'message': _("Enketo not properly configured.")} - http_status = status.HTTP_400_BAD_REQUEST - - try: - url = enketo_url(form_url, self.object.id_string) - except EnketoError: - pass - else: - if url: - http_status = status.HTTP_200_OK - data = {"enketo_url": url} - - return Response(data, http_status) - def retrieve(self, request, *args, **kwargs): xform = self.get_object() export_type = kwargs.get('format') diff --git a/onadata/apps/export/__init__.py b/onadata/apps/export/__init__.py deleted file mode 100644 index 91f7c6c25..000000000 --- a/onadata/apps/export/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals, print_function, division, absolute_import -################################################# -# THIS APP IS DEAD CODE AND SHOULD BE EXCISED # -# EVERY SINGLE ENDPOINT 500s EXCEPT export_menu # -################################################# diff --git a/onadata/apps/export/templates/export/export_html.html b/onadata/apps/export/templates/export/export_html.html deleted file mode 100644 index e29f975af..000000000 --- a/onadata/apps/export/templates/export/export_html.html +++ /dev/null @@ -1,52 +0,0 @@ -{% extends 'base.html' %} -{% load i18n %} - -
-{% block content %} - - - -

{{ title }}

- -{{ table }} - - - -{% endblock %} -
diff --git a/onadata/apps/export/templates/export/export_menu.html b/onadata/apps/export/templates/export/export_menu.html deleted file mode 100644 index e22105856..000000000 --- a/onadata/apps/export/templates/export/export_menu.html +++ /dev/null @@ -1,163 +0,0 @@ -{% extends 'base.html' %} -{% load i18n %} - -
-{% block content %} - - - -

{% trans "Exports" %}:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -{% for lang in languages %} - - {% if lang == "_default" %} - - - - - - - - - - - - {% else %} - - - - - - - - - - - {% endif %} - -{% endfor %} - - -
{% trans "Groups included" %}{% trans "Groups excluded" %}
{% trans "HTML view" %}{% trans "CSV" %}{% trans "XLS" %}{% trans "HTML view" %}{% trans "CSV" %}{% trans "XLS" %}
- - {% trans "XML values and headers" %} - - - - {% trans "XML values and headers" %} - - - - {% trans "XML values and headers" %} - - - - {% trans "XML values and headers" %} - - - - {% trans "XML values and headers" %} - - - - {% trans "XML values and headers" %} - -
- - {% trans "Default labels" %} - - - - {% trans "Default labels" %} - - - - {% trans "Default labels" %} - - - - {% trans "Default labels" %} - - - - {% trans "Default labels" %} - - - - {% trans "Default labels" %} - -
- - {% trans lang %} - - - - {% trans lang %} - - - - {% trans lang %} - - - - {% trans lang %} - - - - {% trans lang %} - - - - {% trans lang %} - -
- -{% endblock %} -
diff --git a/onadata/apps/export/urls.py b/onadata/apps/export/urls.py deleted file mode 100644 index 5938b71bb..000000000 --- a/onadata/apps/export/urls.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals, print_function, division, absolute_import -################################################# -# THIS APP IS DEAD CODE AND SHOULD BE EXCISED # -# EVERY SINGLE ENDPOINT 500s EXCEPT export_menu # -################################################# - -from django.conf.urls import patterns, url - - -urlpatterns = patterns( - '', - url(r"(?P[^/]+)/$", - 'onadata.apps.export.views.export_menu', - name='formpack_export_menu'), - url(r"(?P[^/]+).csv$", - 'onadata.apps.export.views.csv_export', - name='formpack_csv_export'), - url(r"(?P[^/]+).xlsx$", - 'onadata.apps.export.views.xlsx_export', - name='formpack_xlsx_export'), - url(r"(?P[^/]+).html$", - 'onadata.apps.export.views.html_export', - name='formpack_html_export') -) diff --git a/onadata/apps/export/views.py b/onadata/apps/export/views.py deleted file mode 100644 index 148b278e5..000000000 --- a/onadata/apps/export/views.py +++ /dev/null @@ -1,164 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals, print_function, division, absolute_import - -import uuid -from datetime import datetime - -from django.conf import settings -from django.contrib.auth.models import User -from django.http import HttpResponse, HttpResponseForbidden -from django.shortcuts import render, get_object_or_404 -from django.utils.safestring import mark_safe -from django.utils.translation import ugettext as _ -from formpack import FormPack -from path import tempdir -from pure_pagination import Paginator, EmptyPage, PageNotAnInteger - -from onadata.libs.utils.user_auth import has_permission - - -################################################# -# THIS APP IS DEAD CODE AND SHOULD BE EXCISED # -# EVERY SINGLE ENDPOINT 500s EXCEPT export_menu # -################################################# - - -def readable_xform_required(func): - def _wrapper(request, username, id_string): - owner = get_object_or_404(User, username=username) - xform = get_object_or_404(owner.xforms, id_string=id_string) - if not has_permission(xform, owner, request): - return HttpResponseForbidden(_('Not shared.')) - return func(request, username, id_string) - return _wrapper - - -def get_instances_for_user_and_form(user, form_id): - userform_id = '{}_{}'.format(user, form_id) - query = {'_userform_id': userform_id} - return settings.MONGO_DB.instances.find(query) - - -def build_formpack(username, id_string): - user = User.objects.get(username=username) - xform = user.xforms.get(id_string=id_string) - schema = { - "id_string": id_string, - "version": 'v1', - "content": xform.to_kpi_content_schema(), - } - return FormPack([schema], id_string) - - -def build_export(request, username, id_string): - - hierarchy_in_labels = request.REQUEST.get( - 'hierarchy_in_labels', '' - ).lower() == 'true' - group_sep = request.REQUEST.get('groupsep', '/') - lang = request.REQUEST.get('lang', None) - - options = {'versions': 'v1', - 'header_lang': lang, - 'group_sep': group_sep, - 'translation': lang, - 'hierarchy_in_labels': hierarchy_in_labels, - 'copy_fields': ('_id', '_uuid', '_submission_time'), - 'force_index': True} - - formpack = build_formpack(username, id_string) - return formpack.export(**options) - - -def build_export_filename(export, extension): - form_type = 'labels' - if not export.translation: - form_type = "values" - elif export.translation != "_default": - form_type = export.translation - - return "{title} - {form_type} - {date:%Y-%m-%d-%H-%M}.{ext}".format( - form_type=form_type, - date=datetime.utcnow(), - title=export.title, - ext=extension - ) - - -@readable_xform_required -def export_menu(request, username, id_string): - - form_pack = build_formpack(username, id_string) - - context = { - 'languages': form_pack.available_translations, - 'username': username, - 'id_string': id_string - } - - return render(request, 'export/export_menu.html', context) - - -@readable_xform_required -def xlsx_export(request, username, id_string): - - export = build_export(request, username, id_string) - data = [("v1", get_instances_for_user_and_form(username, id_string))] - - with tempdir() as d: - tempfile = d / str(uuid.uuid4()) - export.to_xlsx(tempfile, data) - xlsx = tempfile.bytes() - - name = build_export_filename(export, 'xlsx') - ct = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' - response = HttpResponse(xlsx, content_type=ct) - response['Content-Disposition'] = 'attachment; filename="%s"' % name - return response - - -@readable_xform_required -def csv_export(request, username, id_string): - - export = build_export(request, username, id_string) - data = [("v1", get_instances_for_user_and_form(username, id_string))] - - name = build_export_filename(export, 'csv') - response = HttpResponse(content_type='text/csv') - response['Content-Disposition'] = 'attachment; filename="%s"' % name - - for line in export.to_csv(data): - response.write(line + "\n") - - return response - - -@readable_xform_required -def html_export(request, username, id_string): - - limit = request.REQUEST.get('limit', 100) - - cursor = get_instances_for_user_and_form(username, id_string) - paginator = Paginator(cursor, limit, request=request) - - try: - page = paginator.page(request.REQUEST.get('page', 1)) - except (EmptyPage, PageNotAnInteger): - try: - page = paginator.page(1) - except (EmptyPage, PageNotAnInteger): - page = None - - context = { - 'page': page, - 'table': [] - } - - if page: - data = [("v1", page.object_list)] - export = build_export(request, username, id_string) - context['table'] = mark_safe("\n".join(export.to_html(data))) - context['title'] = id_string - - return render(request, 'export/export_html.html', context) - diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 61a308969..d13a539ec 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -100,7 +100,9 @@ def update_xform_submission_count_delete(sender, instance, **kwargs): xform.num_of_submissions -= 1 if xform.num_of_submissions < 0: xform.num_of_submissions = 0 - xform.save(update_fields=['num_of_submissions']) + # Update `date_modified` to detect outdated exports + # with deleted instances + xform.save(update_fields=['num_of_submissions', 'date_modified']) profile_qs = User.profile.get_queryset() try: profile = profile_qs.select_for_update()\ diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 9c6a996c7..be8da2b91 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -1,16 +1,13 @@ # coding: utf-8 from __future__ import unicode_literals, print_function, division, absolute_import -import io import json import os import re from cStringIO import StringIO -from datetime import datetime from hashlib import md5 from xml.sax import saxutils -import pytz from django.conf import settings from django.contrib.auth.models import User from django.core.exceptions import ObjectDoesNotExist @@ -18,7 +15,6 @@ from django.core.urlresolvers import reverse from django.db import models from django.db.models.signals import post_save, post_delete -from django.utils import timezone from django.utils.encoding import smart_text from django.utils.translation import ugettext_lazy, ugettext as _ from guardian.shortcuts import ( @@ -39,11 +35,6 @@ ) from onadata.libs.models.base_model import BaseModel -try: - from formpack.utils.xls_to_ss_structure import xls_to_dicts -except ImportError: - xls_to_dicts = False - XFORM_TITLE_LENGTH = 255 title_pattern = re.compile(r"([^<]+)") @@ -70,16 +61,8 @@ class XForm(BaseModel): shared = models.BooleanField(default=False) shared_data = models.BooleanField(default=False) downloadable = models.BooleanField(default=True) - allows_sms = models.BooleanField(default=False) encrypted = models.BooleanField(default=False) - # the following fields are filled in automatically - sms_id_string = models.SlugField( - editable=False, - verbose_name=ugettext_lazy("SMS ID"), - max_length=MAX_ID_LENGTH, - default='' - ) id_string = models.SlugField( editable=False, verbose_name=ugettext_lazy("ID"), @@ -206,16 +189,6 @@ def save(self, *args, **kwargs): raise XLSFormError(_('In strict mode, the XForm ID must be a ' 'valid slug and contain no spaces.')) - if not self.sms_id_string: - try: - # try to guess the form's wanted sms_id_string - # from it's json rep (from XLSForm) - # otherwise, use id_string to ensure uniqueness - self.sms_id_string = json.loads(self.json).get('sms_keyword', - self.id_string) - except: - self.sms_id_string = self.id_string - super(XForm, self).save(*args, **kwargs) def __unicode__(self): @@ -248,7 +221,8 @@ def time_of_last_submission(self): def time_of_last_submission_update(self): try: - # we also consider deleted instances in this case + # We don't need to filter on `deleted_at` field anymore. + # Instances are really deleted and not flagged as deleted. return self.instances.latest("date_modified").date_modified except ObjectDoesNotExist: pass @@ -271,7 +245,7 @@ def public_forms(cls): def _xls_file_io(self): """ - pulls the xls file from remote storage + Pulls the xls file from remote storage this should be used sparingly """ @@ -285,35 +259,6 @@ def _xls_file_io(self): else: return StringIO(ff.read()) - def to_kpi_content_schema(self): - """ - Parses xlsform structure into json representation - of spreadsheet structure. - """ - if not xls_to_dicts: - raise ImportError('formpack module needed') - content = xls_to_dicts(self._xls_file_io()) - # a temporary fix to the problem of list_name alias - return json.loads(re.sub('list name', 'list_name', - json.dumps(content, indent=4))) - - def to_xlsform(self): - """ - Generate an XLS format XLSForm copy of this form. - """ - file_path = self.xls.name - default_storage = get_storage_class()() - - if file_path != '' and default_storage.exists(file_path): - with default_storage.open(file_path) as xlsform_file: - if file_path.endswith('.csv'): - xlsform_io = convert_csv_to_xls(xlsform_file.read()) - else: - xlsform_io= io.BytesIO(xlsform_file.read()) - return xlsform_io - else: - return None - @property def settings(self): """ diff --git a/onadata/apps/logger/templates/list_xforms.html b/onadata/apps/logger/templates/list_xforms.html deleted file mode 100644 index 237c45350..000000000 --- a/onadata/apps/logger/templates/list_xforms.html +++ /dev/null @@ -1,42 +0,0 @@ -{% load i18n %} - - - - - - - - - - - - - {% for xform in xforms %} - - - - - - - - {% endfor %} - -
{% trans "Downloadable by Phone" %}{% trans "Form ID" %}{% trans "Number of Submissions" %}{% trans "Time of Last Submission" %}{% trans "CSV" %}
- - {% if xform.downloadable %} - [ {% trans 'yes' %} ] - {% else %} - [ {% trans 'no' %} ] - {% endif %} - - {{ xform.id_string }}{{ xform.submission_count }} - {% if xform.time_of_last_submission %} - {{ xform.time_of_last_submission }} - {% endif %} - {% trans "Download CSV" %}
- - diff --git a/onadata/apps/logger/templates/submission.html b/onadata/apps/logger/templates/submission.html deleted file mode 100644 index e95388b3c..000000000 --- a/onadata/apps/logger/templates/submission.html +++ /dev/null @@ -1,14 +0,0 @@ -{% load i18n %} - -

{% trans "Your submission was successful!" %}

- - - diff --git a/onadata/apps/logger/tests/test_parsing.py b/onadata/apps/logger/tests/test_parsing.py index 96f1a0f2c..450089bed 100644 --- a/onadata/apps/logger/tests/test_parsing.py +++ b/onadata/apps/logger/tests/test_parsing.py @@ -29,7 +29,7 @@ def _publish_and_submit_new_repeats(self): "../fixtures/new_repeats/new_repeats.xls" ) self._publish_xls_file_and_set_xform(xls_file_path) - self.assertEqual(self.response.status_code, 200) + self.assertEqual(self.response.status_code, 201) # submit an instance xml_submission_file_path = os.path.join( @@ -161,7 +161,7 @@ def test_parse_xform_nested_repeats_multiple_nodes(self): "../fixtures/new_repeats/new_repeats.xls" ) self._publish_xls_file_and_set_xform(xls_file_path) - self.assertEqual(self.response.status_code, 200) + self.assertEqual(self.response.status_code, 201) # submit an instance xml_submission_file_path = os.path.join( diff --git a/onadata/apps/logger/tests/test_webforms.py b/onadata/apps/logger/tests/test_webforms.py index d3332b0d6..02f036388 100644 --- a/onadata/apps/logger/tests/test_webforms.py +++ b/onadata/apps/logger/tests/test_webforms.py @@ -3,13 +3,9 @@ import os import requests -import unittest - -from django.core.urlresolvers import reverse from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models.instance import Instance -from onadata.apps.logger.views import edit_data from onadata.apps.logger.xform_instance_parser import get_uuid_from_xml from onadata.libs.utils.logger_tools import inject_instanceid @@ -33,20 +29,6 @@ def __load_fixture(self, *path): with open(os.path.join(os.path.dirname(__file__), *path), 'r') as f: return f.read() - @unittest.skip('Fails under Django 1.6') - def test_edit_url(self): - instance = Instance.objects.order_by('id').reverse()[0] - edit_url = reverse(edit_data, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'data_id': instance.id - }) - with HTTMock(enketo_edit_mock): - response = self.client.get(edit_url) - self.assertEqual(response.status_code, 302) - self.assertEqual(response['location'], - 'https://hmh2a.enketo.formhub.org') - def test_inject_instanceid(self): """ Test that 1 and only 1 instance id exists or is injected diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index a5e7d7cb0..64688ee99 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -16,7 +16,6 @@ from django.contrib import messages from django.core.files.storage import get_storage_class from django.core.files import File -from django.core.urlresolvers import reverse from django.http import (HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, @@ -41,28 +40,22 @@ from onadata.apps.logger.models.instance import Instance from onadata.apps.logger.models.xform import XForm from onadata.libs.utils.log import audit_log, Actions -from onadata.libs.utils.viewer_tools import enketo_url -from onadata.libs.utils.viewer_tools import image_urls_dict from onadata.libs.utils.logger_tools import ( safe_create_instance, OpenRosaResponseBadRequest, OpenRosaResponse, BaseOpenRosaResponse, - inject_instanceid, - remove_xform, publish_xml_form, - publish_form) + publish_form, +) from onadata.libs.utils.logger_tools import response_with_mimetype_and_name -from onadata.libs.utils.decorators import is_owner from onadata.libs.utils.user_auth import (helper_auth_helper, has_permission, - has_edit_permission, HttpResponseNotAuthorized, add_cors_headers, ) -from onadata.libs.utils.viewer_tools import _get_form_url -from ...koboform.pyxform_utils import convert_csv_to_xls from .tasks import generate_stats_zip +from ...koboform.pyxform_utils import convert_csv_to_xls IO_ERROR_STRINGS = [ 'request data read error', @@ -90,15 +83,6 @@ def _parse_int(num): pass -def _html_submission_response(request, instance): - data = {} - data['username'] = instance.xform.user.username - data['id_string'] = instance.xform.id_string - data['domain'] = Site.objects.get(id=settings.SITE_ID).domain - - return render(request, "submission.html", data) - - def _submission_response(request, instance): data = {} data['message'] = _("Successful submission.") @@ -313,11 +297,7 @@ def submission(request, username=None): "id_string": instance.xform.id_string }, audit, request) - # response as html if posting with a UUID - if not username and uuid: - response = _html_submission_response(request, instance) - else: - response = _submission_response(request, instance) + response = _submission_response(request, instance) # ODK needs two things for a form to be considered successful # 1) the status code needs to be 201 (created) @@ -417,6 +397,7 @@ def download_xlsform(request, username, id_string): return HttpResponseRedirect("/%s" % username) + def download_jsonform(request, username, id_string): owner = get_object_or_404(User, username__iexact=username) xform = get_object_or_404(XForm, user__username__iexact=username, @@ -441,130 +422,6 @@ def download_jsonform(request, username, id_string): return response -@is_owner -@require_POST -def delete_xform(request, username, id_string): - xform = get_object_or_404(XForm, user__username__iexact=username, - id_string__exact=id_string) - - # delete xform and submissions - remove_xform(xform) - - audit = {} - audit_log( - Actions.FORM_DELETED, request.user, xform.user, - _("Deleted form '%(id_string)s'.") % - { - 'id_string': xform.id_string, - }, audit, request) - return HttpResponseRedirect('/') - - -@is_owner -def toggle_downloadable(request, username, id_string): - xform = XForm.objects.get(user__username__iexact=username, - id_string__exact=id_string) - xform.downloadable = not xform.downloadable - xform.save() - audit = {} - audit_log( - Actions.FORM_UPDATED, request.user, xform.user, - _("Made form '%(id_string)s' %(downloadable)s.") % - { - 'id_string': xform.id_string, - 'downloadable': - _("downloadable") if xform.downloadable else _("un-downloadable") - }, audit, request) - return HttpResponseRedirect("/%s" % username) - - -def enter_data(request, username, id_string): - owner = get_object_or_404(User, username__iexact=username) - xform = get_object_or_404(XForm, user__username__iexact=username, - id_string__exact=id_string) - if not has_edit_permission(xform, owner, request): - return HttpResponseForbidden(_('Not shared.')) - - form_url = _get_form_url(username) - - try: - url = enketo_url(form_url, xform.id_string) - if not url: - return HttpResponseRedirect(reverse('onadata.apps.main.views.show', - kwargs={'username': username, - 'id_string': id_string})) - return HttpResponseRedirect(url) - except Exception as e: - data = {} - owner = User.objects.get(username__iexact=username) - data['profile'], created = \ - UserProfile.objects.get_or_create(user=owner) - data['xform'] = xform - data['content_user'] = owner - data['form_view'] = True - data['message'] = { - 'type': 'alert-error', - 'text': "Enketo error, reason: %s" % e} - messages.add_message( - request, messages.WARNING, - _("Enketo error: enketo replied %s") % e, fail_silently=True) - return render(request, "profile.html", data) - - return HttpResponseRedirect(reverse('onadata.apps.main.views.show', - kwargs={'username': username, - 'id_string': id_string})) - - -def edit_data(request, username, id_string, data_id): - context = RequestContext(request) - owner = User.objects.get(username__iexact=username) - xform = get_object_or_404( - XForm, user__username__iexact=username, id_string__exact=id_string) - instance = get_object_or_404( - Instance, pk=data_id, xform=xform) - instance_attachments = image_urls_dict(instance) - if not has_edit_permission(xform, owner, request): - return HttpResponseForbidden(_('Not shared.')) - if not hasattr(settings, 'ENKETO_URL'): - return HttpResponseRedirect(reverse( - 'onadata.apps.main.views.show', - kwargs={'username': username, 'id_string': id_string})) - - url = '%sdata/edit_url' % settings.ENKETO_URL - # see commit 220f2dad0e for tmp file creation - injected_xml = inject_instanceid(instance.xml, instance.uuid) - return_url = request.build_absolute_uri( - reverse( - 'onadata.apps.viewer.views.instance', - kwargs={ - 'username': username, - 'id_string': id_string} - ) + "#/" + str(instance.id)) - form_url = _get_form_url(username) - - try: - url = enketo_url( - form_url, xform.id_string, instance_xml=injected_xml, - instance_id=instance.uuid, return_url=return_url, - instance_attachments=instance_attachments - ) - except Exception as e: - context.message = { - 'type': 'alert-error', - 'text': "Enketo error, reason: %s" % e} - messages.add_message( - request, messages.WARNING, - _("Enketo error: enketo replied %s") % e, fail_silently=True) - else: - if url: - context.enketo = url - return HttpResponseRedirect(url) - return HttpResponseRedirect( - reverse('onadata.apps.main.views.show', - kwargs={'username': username, - 'id_string': id_string})) - - def view_submission_list(request, username): form_user = get_object_or_404(User, username__iexact=username) profile, created = \ diff --git a/onadata/apps/main/forms.py b/onadata/apps/main/forms.py index de027a5d4..082b932ba 100644 --- a/onadata/apps/main/forms.py +++ b/onadata/apps/main/forms.py @@ -1,87 +1,13 @@ # coding: utf-8 from __future__ import unicode_literals, print_function, division, absolute_import -import re -import urllib2 -from urlparse import urlparse -from StringIO import StringIO - from django import forms -from django.contrib.auth.models import User -from django.contrib.sites.models import Site -from django.core.files.base import ContentFile -from django.core.files.storage import default_storage -from django.core.validators import URLValidator from django.forms import ModelForm -from django.utils.translation import ugettext as _, ugettext_lazy -from django.conf import settings -from recaptcha.client import captcha -from registration.forms import RegistrationFormUniqueEmail -from registration.models import RegistrationProfile - -from pyxform.xls2json_backends import csv_to_dict +from django.utils.translation import ugettext_lazy from onadata.apps.main.models import UserProfile -from onadata.apps.viewer.models.data_dictionary import upload_to -from onadata.libs.utils.country_field import COUNTRIES from onadata.libs.utils.logger_tools import publish_xls_form -FORM_LICENSES_CHOICES = ( - ('No License', ugettext_lazy('No License')), - ('https://creativecommons.org/licenses/by/3.0/', - ugettext_lazy('Attribution CC BY')), - ('https://creativecommons.org/licenses/by-sa/3.0/', - ugettext_lazy('Attribution-ShareAlike CC BY-SA')), -) - -DATA_LICENSES_CHOICES = ( - ('No License', ugettext_lazy('No License')), - ('http://opendatacommons.org/licenses/pddl/summary/', - ugettext_lazy('PDDL')), - ('http://opendatacommons.org/licenses/by/summary/', - ugettext_lazy('ODC-BY')), - ('http://opendatacommons.org/licenses/odbl/summary/', - ugettext_lazy('ODBL')), -) - -PERM_CHOICES = ( - ('view', ugettext_lazy('Can view')), - ('edit', ugettext_lazy('Can edit')), - ('report', ugettext_lazy('Can submit to')), - ('validate', ugettext_lazy('Can validate')), - ('remove', ugettext_lazy('Remove permissions')), -) - - -class DataLicenseForm(forms.Form): - value = forms.ChoiceField(choices=DATA_LICENSES_CHOICES, - widget=forms.Select( - attrs={'disabled': 'disabled', - 'id': 'data-license'})) - - -class FormLicenseForm(forms.Form): - value = forms.ChoiceField(choices=FORM_LICENSES_CHOICES, - widget=forms.Select( - attrs={'disabled': 'disabled', - 'id': 'form-license'})) - - -class PermissionForm(forms.Form): - for_user = forms.CharField( - widget=forms.TextInput( - attrs={ - 'id': 'autocomplete', - 'data-provide': 'typeahead', - 'autocomplete': 'off' - }) - ) - perm_type = forms.ChoiceField(choices=PERM_CHOICES, widget=forms.Select()) - - def __init__(self, username): - self.username = username - super(PermissionForm, self).__init__() - class UserProfileForm(ModelForm): class Meta: @@ -90,150 +16,6 @@ class Meta: fields = ('require_auth',) -class UserProfileFormRegister(forms.Form): - - REGISTRATION_REQUIRE_CAPTCHA = settings.REGISTRATION_REQUIRE_CAPTCHA - RECAPTCHA_PUBLIC_KEY = settings.RECAPTCHA_PUBLIC_KEY - RECAPTCHA_HTML = captcha.displayhtml(settings.RECAPTCHA_PUBLIC_KEY, - use_ssl=settings.RECAPTCHA_USE_SSL) - - name = forms.CharField(widget=forms.TextInput(), required=True, - max_length=255) - city = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - country = forms.ChoiceField(widget=forms.Select(), required=False, - choices=COUNTRIES, initial='ZZ') - organization = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - home_page = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - twitter = forms.CharField(widget=forms.TextInput(), required=False, - max_length=255) - - recaptcha_challenge_field = forms.CharField(required=False, max_length=512) - recaptcha_response_field = forms.CharField( - max_length=100, required=settings.REGISTRATION_REQUIRE_CAPTCHA) - - def save(self, new_user): - new_profile = \ - UserProfile(user=new_user, name=self.cleaned_data['name'], - city=self.cleaned_data['city'], - country=self.cleaned_data['country'], - organization=self.cleaned_data['organization'], - home_page=self.cleaned_data['home_page'], - twitter=self.cleaned_data['twitter']) - new_profile.save() - return new_profile - - -# order of inheritance control order of form display -class RegistrationFormUserProfile(RegistrationFormUniqueEmail, - UserProfileFormRegister): - - class Meta: - # JNM TEMPORARY - model = User - fields = ('username', 'email') - - # Those names conflicts with existing url patterns. Indeed, the root - # url pattern stating states / but you have /admin for - # as admin url, /support as support url, etc. So you want to avoid - # account created with those names. - _reserved_usernames = [ - 'accounts', - 'about', - 'admin', - 'clients', - 'data', - 'formhub', - 'forms', - 'maps', - 'odk', - 'ona', - 'people', - 'public', - 'submit', - 'submission', - 'support', - 'syntax', - 'xls2xform', - 'users', - 'worldbank', - 'unicef', - 'who', - 'wb', - 'wfp', - 'save', - 'ei', - 'modilabs', - 'mvp', - 'unido', - 'unesco', - 'savethechildren', - 'worldvision', - 'afsis' - ] - - username = forms.CharField(widget=forms.TextInput(), max_length=30) - email = forms.EmailField(widget=forms.TextInput()) - - legal_usernames_re = re.compile("^\w+$") - - def clean(self): - cleaned_data = super(UserProfileFormRegister, self).clean() - - # don't check captcha if it's disabled - if not self.REGISTRATION_REQUIRE_CAPTCHA: - if 'recaptcha_response_field' in self._errors: - del self._errors['recaptcha_response_field'] - return cleaned_data - - response = captcha.submit( - cleaned_data.get('recaptcha_challenge_field'), - cleaned_data.get('recaptcha_response_field'), - settings.RECAPTCHA_PRIVATE_KEY, - None) - - if not response.is_valid: - raise forms.ValidationError(_("The Captcha is invalid. " - "Please, try again.")) - return cleaned_data - - # This code use to be in clean_username. Now clean_username is just - # a convenience proxy to this method. This method is here to allow - # the UserProfileSerializer to validate the username without reinventing - # the wheel while still avoiding the need to instancate the form. A even - # cleaner way would be a shared custom validator. - @classmethod - def validate_username(cls, username): - username = username.lower() - if username in cls._reserved_usernames: - raise forms.ValidationError( - _('%s is a reserved name, please choose another') % username) - elif not cls.legal_usernames_re.search(username): - raise forms.ValidationError( - _('username may only contain alpha-numeric characters and ' - 'underscores')) - try: - User.objects.get(username=username) - except User.DoesNotExist: - return username - raise forms.ValidationError(_('%s already exists') % username) - - def clean_username(self): - return self.validate_username(self.cleaned_data['username']) - - -class SourceForm(forms.Form): - source = forms.FileField(label=ugettext_lazy("Source document"), - required=True) - - -class SupportDocForm(forms.Form): - doc = forms.FileField(label=ugettext_lazy("Supporting document"), - required=True) - - class MediaForm(forms.Form): media = forms.FileField(label=ugettext_lazy("Media upload"), required=True) @@ -245,97 +27,14 @@ def clean_media(self): allowed .png .jpg .mp3 .3gp .wav') -class QuickConverterFile(forms.Form): - xls_file = forms.FileField( - label=ugettext_lazy('XLS File'), required=False) - - -class QuickConverterURL(forms.Form): - xls_url = forms.URLField(label=ugettext_lazy('XLS URL'), - required=False) - +class QuickConverterForm(forms.Form): -class QuickConverterDropboxURL(forms.Form): - dropbox_xls_url = forms.URLField( - label=ugettext_lazy('XLS URL'), required=False) - - -class QuickConverterTextXlsForm(forms.Form): - text_xls_form = forms.CharField( - label=ugettext_lazy('XLSForm Representation'), required=False) - - -class QuickConverter(QuickConverterFile, QuickConverterURL, - QuickConverterDropboxURL, QuickConverterTextXlsForm): - validate = URLValidator() + xls_file = forms.FileField( + label=ugettext_lazy('XLS File'), required=True) def publish(self, user, id_string=None): if self.is_valid(): - # If a text (csv) representation of the xlsform is present, - # this will save the file and pass it instead of the 'xls_file' - # field. - if 'text_xls_form' in self.cleaned_data\ - and self.cleaned_data['text_xls_form'].strip(): - csv_data = self.cleaned_data['text_xls_form'] - # "Note that any text-based field - such as CharField or - # EmailField - always cleans the input into a Unicode string" - # (https://docs.djangoproject.com/en/1.8/ref/forms/api/#django.forms.Form.cleaned_data). - csv_data = csv_data.encode('utf-8') - # requires that csv forms have a settings with an id_string or - # form_id - _sheets = csv_to_dict(StringIO(csv_data)) - try: - _settings = _sheets['settings'][0] - if 'id_string' in _settings: - _name = '%s.csv' % _settings['id_string'] - else: - _name = '%s.csv' % _settings['form_id'] - except (KeyError, IndexError) as e: - raise ValueError('CSV XLSForms must have a settings sheet' - ' and id_string or form_id') - - cleaned_xls_file = \ - default_storage.save( - upload_to(None, _name, user.username), - ContentFile(csv_data)) - else: - cleaned_xls_file = self.cleaned_data['xls_file'] + cleaned_xls_file = self.cleaned_data['xls_file'] - if not cleaned_xls_file: - cleaned_url = self.cleaned_data['xls_url'] - if cleaned_url.strip() == '': - cleaned_url = self.cleaned_data['dropbox_xls_url'] - cleaned_xls_file = urlparse(cleaned_url) - cleaned_xls_file = \ - '_'.join(cleaned_xls_file.path.split('/')[-2:]) - if cleaned_xls_file[-4:] != '.xls': - cleaned_xls_file += '.xls' - cleaned_xls_file = \ - upload_to(None, cleaned_xls_file, user.username) - self.validate(cleaned_url) - xls_data = ContentFile(urllib2.urlopen(cleaned_url).read()) - cleaned_xls_file = \ - default_storage.save(cleaned_xls_file, xls_data) # publish the xls return publish_xls_form(cleaned_xls_file, user, id_string) - - -class ActivateSMSSupportFom(forms.Form): - - enable_sms_support = forms.TypedChoiceField(coerce=lambda x: x == 'True', - choices=((False, 'No'), - (True, 'Yes')), - widget=forms.Select, - label=ugettext_lazy( - "Enable SMS Support")) - sms_id_string = forms.CharField(max_length=50, required=True, - label=ugettext_lazy("SMS Keyword")) - - def clean_sms_id_string(self): - sms_id_string = self.cleaned_data.get('sms_id_string', '').strip() - - if not re.match(r'^[a-z0-9\_\-]+$', sms_id_string): - raise forms.ValidationError("id_string can only contain alphanum" - " characters") - - return sms_id_string diff --git a/onadata/apps/main/google_doc.py b/onadata/apps/main/google_doc.py deleted file mode 100644 index d0b7fb86d..000000000 --- a/onadata/apps/main/google_doc.py +++ /dev/null @@ -1,204 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals, print_function, division, absolute_import - -import re -import urllib2 - -from django.template.loader import render_to_string -from django.template.defaultfilters import slugify - - -class Section(dict): - """ - A class used to represent a section of a page. A section should - have certain fields. 'level' denotes how nested this section is in - the document, like h1, h2, etc. 'id' is a string used to link to - this section. 'title' will be printed at the top the - section. 'content' is the html that will be printed as the meat of - the section. Notice that we use the 'section.html' template to - render a section as html, and the url provides a link that will be - used in the page's table of contents. - """ - - FIELDS = ['level', 'id', 'title', 'content'] - - def to_html(self): - return render_to_string('section.html', self) - - def url(self): - return '%(title)s' % self - - -class TreeNode(list): - """ - This simple tree class will be used to construct the table of - contents for the page. - """ - - def __init__(self, value=None, parent=None): - self.value = value - self.parent = parent - list.__init__(self) - - def add_child(self, value): - child = TreeNode(value, self) - self.append(child) - return child - - -class GoogleDoc(object): - """ - This class provides a structure for dealing with a Google - Document. Most use cases will initialize a GoogleDoc by passing a - url to the init. This should be a public url that links to an html - version of the document. You can find this url by publishing your - Google Document to the web and copying the url. - - The primary method this class provides is 'to_html' which renders - this document as html in the Twitter Bootstrap format. - """ - - def __init__(self, url=None): - if url is not None: - self.set_html_from_url(url) - - def set_html_from_url(self, url): - f = urllib2.urlopen(url) - self.set_html(f.read()) - f.close() - - def set_html(self, html): - """ - When setting the html for this Google Document we do two - things: - - 1. We extract the content from the html. Using a regular - expression we pull the meat of the document out of the body - of the html, we also cut off the footer Google adds on - automatically. - - 2. We extract the various sections from the content of the - document. Again using a regular expression, we look for h1, - h2, ... tags to split the document up into sections. Note: - it is important when you are writing your Google Document - to use the heading text styles, so this code will split - things correctly. - """ - self._html = html - self._extract_content() - self._extract_sections() - - def _extract_content(self): - m = re.search(r'(.*)