Remote pack

pull.py

@@ -0,0 +1,96 @@
+from formpack.remote_pack import RemoteFormPack, FORMPACK_DATA_DIR
+
+import os
+import json
+import argparse
+import importlib
+
+
+parser = argparse.ArgumentParser(description='Initialize RemoteFormPack.')
+
+parser.add_argument('--refresh-data',
+                    dest='refresh_data',
+                    help='flush data cache',
+                    action='store_true')
+
+parser.add_argument('--print-stats',
+                    dest='print_stats',
+                    help='print stats from the dataset',
+                    action='store_true')
+
+parser.add_argument('--print-survey',
+                    dest='print_survey',
+                    help='print survey questions',
+                    action='store_true')
+
+parser.add_argument('--print-submissions',
+                    dest='print_submissions',
+                    help='print submissions from the dataset',
+                    action='store_true')
+
+parser.add_argument('--run-module',
+                    dest='run_module',
+                    help='import and run a module',
+                    action='store')
+
+parser.add_argument('--account',
+                    dest='account',
+                    help='server:account corresponding to local config',
+                    action='store')
+
+parser.add_argument('uid',
+                    nargs=1,
+                    help='formid',
+                    action='store')
+
+
+def load_pack(uid, account):
+    accounts_file = os.environ.get('KOBO_ACCOUNTS', False)
+    if not accounts_file:
+        accounts_file = os.path.join(FORMPACK_DATA_DIR, 'accounts.json')
+    if not os.path.exists(accounts_file):
+        raise ValueError('need an accounts json file in {}'.format(
+                         accounts_file))
+    with open(accounts_file, 'r') as ff:
+        accounts = json.loads(ff.read())
+    try:
+        _account = accounts[account]
+    except KeyError:
+        raise ValueError('accounts.json needs a configuration for {}'.format(
+                         account))
+    return RemoteFormPack(uid=uid,
+                          token=_account['token'],
+                          api_url=_account['api_url'],
+                          )
+
+
+def run(args):
+    rpack = load_pack(uid=args.uid[0],
+                      account=args.account)
+
+    if args.refresh_data:
+        print('clearing submissions')
+        rpack.clear_submissions()
+
+    rpack.pull()
+    formpk = rpack.create_pack()
+
+    if args.print_stats:
+        print(json.dumps(formpk._stats, indent=2))
+
+    if args.print_survey:
+        print(json.dumps(formpk.get_survey(), indent=2))
+
+    if args.print_submissions:
+        print(json.dumps(list(rpack.submissions), indent=2))
+
+    if args.run_module:
+        _mod = importlib.import_module(args.run_module)
+        if not hasattr(_mod, 'run') and hasattr(_mod.run, '__call__'):
+            _mod.run(formpk, submissions=rpack.submissions)
+        else:
+            raise ValueError('run-module parameter must be an importable '
+                             'module with a method "run"')
+
+if __name__ == '__main__':
+    run(parser.parse_args())

requirements.txt

@@ -6,4 +6,5 @@ path.py==8.1.2
 PyExcelerate==0.6.7
 pyquery==1.2.11
 pyxform==0.9.22
-statistics==1.0.3.5
+requests==2.13.0
+statistics==1.0.3.5

src/formpack/pack.py

@@ -23,8 +23,6 @@
 
 
 class FormPack(object):
-
-    # TODO: make a clear signature for __init__
     def __init__(self, versions=None, title='Submissions', id_string=None,
                  default_version_id_key='__version__',
                  strict_schema=False,
@@ -47,14 +45,14 @@ def __init__(self, versions=None, title='Submissions', id_string=None,
         self.root_node_name = root_node_name
 
         self.title = title
+
         self.strict_schema = strict_schema
 
         self.asset_type = asset_type
-
         self.load_all_versions(versions)
 
     def __repr__(self):
-        return '<FormPack %s>' % self._stats()
+        return '<FormPack %s>' % self._stats_str
 
     def version_id_keys(self, _versions=None):
         # if no parameter is passed, default to 'all'
@@ -67,6 +65,12 @@ def version_id_keys(self, _versions=None):
                 _id_keys.append(_id_key)
         return _id_keys
 
+    @property
+    def latest_version(self):
+        if len(self.versions) > 0:
+            return self.versions.values()[-1]
+        else:
+            raise ValueError('No versions available.')
 
     @property
     def available_translations(self):
@@ -93,15 +97,27 @@ def __getitem__(self, index):
         except IndexError:
             raise IndexError('version at index %d is not available' % index)
 
+    @property
     def _stats(self):
         _stats = OrderedDict()
+        _stats['title'] = self.title
         _stats['id_string'] = self.id_string
-        _stats['versions'] = len(self.versions)
-        # _stats['submissions'] = self.submissions_count()
-        _stats['row_count'] = len(self[-1].schema.get('content', {})
-                                                 .get('survey', []))
+        _vs = self.versions.values()
+        if len(_vs) > 0:
+            _content = _vs[-1].schema.get('content', {})
+            _survey = _content.get('survey', [])
+            _stats['row_count'] = len(_survey)
+
+        _versions = []
+        for (vid, version) in self.versions.items():
+            _versions.append(version._stats())
+        _stats['versions'] = _versions
+        return _stats
+
+    @property
+    def _stats_str(self):
         # returns stats in the format [ key="value" ]
-        return '\n\t'.join('%s="%s"' % item for item in _stats.items())
+        return '\n\t'.join('%s="%s"' % item for item in self._stats.items())
 
     def load_all_versions(self, versions):
         for schema in versions:
@@ -123,6 +139,8 @@ def load_version(self, schema):
             unique accross an entire FormPack. It can be None, but only for
             one version in the FormPack.
         """
+        if 'content' not in schema:
+            raise ValueError('''"content" is not an available key: {}'''.format(schema.keys()))
         replace_aliases(schema['content'], in_place=True)
         expand_content(schema['content'], in_place=True)
 
@@ -161,6 +179,14 @@ def load_version(self, schema):
 
         self.versions[form_version.id] = form_version
 
+    def _latest_change(self):
+        _lvs = len(self.versions)
+        _keys = self.versions.keys()
+        if _lvs > 1:
+            v1 = _keys[_lvs - 2]
+            v2 = _keys[_lvs - 1]
+        return self.version_diff(v1, v2)
+
     def version_diff(self, vn1, vn2):
         v1 = self.versions[vn1]
         v2 = self.versions[vn2]
@@ -266,6 +292,11 @@ def to_dict(self, **kwargs):
             out[u'asset_type'] = self.asset_type
         return out
 
+    def get_survey(self):
+        return [
+            row for row in self.latest_version.rows(include_groups=True)
+        ]
+
     def to_json(self, **kwargs):
         return json.dumps(self.to_dict(), **kwargs)
 

src/formpack/remote_pack.py

@@ -0,0 +1,166 @@
+# coding: utf-8
+
+from __future__ import (unicode_literals, print_function,
+                        absolute_import, division)
+
+from .pack import FormPack
+
+import json
+import errno
+import requests
+from urlparse import urlparse
+from argparse import Namespace as Ns
+from os import (path, makedirs, unlink)
+
+FORMPACK_DATA_DIR = path.join(path.expanduser('~'),
+                              '.formpack')
+
+
+def mkdir_p(_path):
+    try:
+        makedirs(_path)
+    except OSError as exc:  # Python >2.5
+        if exc.errno == errno.EEXIST and path.isdir(_path):
+            pass
+        else:
+            raise
+
+
+class RemoteFormPack:
+    def __init__(self, uid,
+                 token,
+                 api_url,
+                 data_dir=None):
+        self.uid = uid
+        self.api_token = token
+        self.api_url = api_url
+        self._data_dir = data_dir or FORMPACK_DATA_DIR
+
+        self.data_dir = path.join(self._data_dir, self.uid)
+        mkdir_p(self.data_dir)
+        self.paths = {
+            'versions/': self.path('versions'),
+            'data': self.path('data.json'),
+            'context': self.path('context.json'),
+            'asset': self.path('asset.json'),
+        }
+        self._versions_dir = self.path('versions')
+        self._data_path = self.path('data.json')
+        self._context_path = self.path('context.json')
+        mkdir_p(self.path('versions'))
+        self._asset_url = '{}{}'.format(self.api_url, self.uid)
+        self.asset = Ns(**self._query_asset())
+        self.context = Ns(**self._query_kcform())
+
+    def path(self, *args):
+        return path.join(self.data_dir, *args)
+
+    def _query_asset(self):
+        if not path.exists(self.path('asset.json')):
+            ad = requests.get('{}/?format=json'.format(self._asset_url),
+                              headers=self._headers(),
+                              ).json()
+            if 'detail' in ad and ad['detail'] == 'Invalid token.':
+                raise ValueError('Invalid token. Is it the correct server?')
+            elif 'detail' in ad:
+                raise ValueError("Error querying API: {}".format(
+                                 ad['detail']))
+            # content is ultimately pulled form the "version" file
+            del ad['content']
+            with open(self.path('asset.json'), 'w') as ff:
+                ff.write(json.dumps(ad, indent=2))
+            return ad
+        else:
+            with open(self.path('asset.json'), 'r') as ff:
+                return json.loads(ff.read())
+
+    def _query_kcform(self):
+        asset = self.asset
+        if not path.exists(self.path('context.json')):
+            _deployment_identifier = asset.deployment__identifier
+            _deployment = urlparse(_deployment_identifier)
+            ctx = {
+                'kc_api_url': '{}://{}/api/v1'.format(_deployment.scheme,
+                                                      _deployment.netloc),
+            }
+            _url = '{}/forms?id_string={}'.format(ctx['kc_api_url'],
+                                                  self.uid)
+            r2 = requests.get(_url, headers=self._headers()).json()
+            ctx['kc_formid'] = r2[0]['formid']
+            with open(self.path('context.json'), 'w') as ff:
+                ff.write(json.dumps(ctx, indent=2))
+            return ctx
+        else:
+            with open(self.path('context.json'), 'r') as ff:
+                return json.loads(ff.read())
+
+    def pull(self):
+        if not path.exists(self.path('data.json')):
+            _data_url = '{}/data/{}?{}'.format(self.context.kc_api_url,
+                                               self.context.kc_formid,
+                                               'format=json',
+                                               )
+            _data = requests.get(_data_url, headers=self._headers()).json()
+            with open(self.path('data.json'), 'w') as ff:
+                ff.write(json.dumps(_data, indent=2))
+            _version_ids = set([i['__version__'] for i in _data])
+            for version_id in _version_ids:
+                self.load_version(version_id)
+
+    def load_version(self, version_id):
+        _version_path = path.join(self.path('versions'),
+                                  '{}.json'.format(version_id)
+                                  )
+        if not path.exists(_version_path):
+            _version_url = '{}/{}/{}/?format=json'.format(
+                self._asset_url,
+                'versions',
+                version_id)
+            vd = requests.get(_version_url, headers=self._headers()).json()
+            if vd.get('detail') == 'Not found.':
+                raise Exception('Version not found')
+            with open(_version_path, 'w') as ff:
+                ff.write(json.dumps(vd, indent=2))
+            return vd
+        else:
+            with open(_version_path, 'r') as ff:
+                return json.loads(ff.read())
+
+    def _headers(self, upd={}):
+        return dict({'Content-Type': 'application/json',
+                    'Authorization': 'Token {}'.format(self.api_token),
+                     }, **upd)
+
+    def create_pack(self):
+        if not path.exists(self._data_path):
+            raise Exception('cannot generate formpack without running '
+                            'remote_pack.pull()')
+        _version_ids = set([s['__version__'] for s in self.submissions])
+        self.versions = []
+        for version_id in _version_ids:
+            _v = self.load_version(version_id)
+            _v['version'] = version_id
+            _v['date_deployed'] = _v.pop('date_deployed', None)
+            self.versions.append(_v)
+        return FormPack(versions=self.versions, id_string=self.uid,
+                        title=self.asset.name,
+                        )
+
+    def stats(self):
+        pck = self.create_pack()
+        _stats = pck._stats()
+        return _stats
+
+    def _submissions(self):
+        with open(self.path('data.json'), 'r') as ff:
+            return json.loads(ff.read())
+
+    def clear_submissions(self):
+        _data_path = self.path('data.json')
+        if path.exists(_data_path):
+            unlink(_data_path)
+
+    @property
+    def submissions(self):
+        for submission in self._submissions():
+            yield submission