feat: store import logs and add API to access them

cheminfo · Jan 8, 2020 · 89edc1d · 89edc1d
1 parent 5e417a4
commit 89edc1d
Show file tree

Hide file tree

Showing 14 changed files with 297 additions and 50 deletions.
diff --git a/API.md b/API.md
@@ -21,7 +21,7 @@
 
 | Method | Route                                   |               Action                |                Description                 |
 | :----: | :-------------------------------------- | :---------------------------------: | :----------------------------------------: |
-|  POST  | `/db/:dbname/entry`                     |      Insert / Update an entry       |     Based on \_id or $id of the entry      |
+|  POST  | `/db/:dbname/entry`                     |      Insert / Update an entry       |     Based on \_id or \$id of the entry     |
 |  GET   | `/db/:dbname/entry/_all`                |           Get all entries           |       Returns an array of documents        |
 |  HEAD  | `/db/:dbname/entry/:uuid`               | Get HTTP headers about the document | Similar to HEAD /:dbname/:docid in CouchDB |
 |  GET   | `/db/:dbname/entry/:uuid`               |        Get an entry by UUID         |                                            |
@@ -54,6 +54,7 @@
 
 | Method | Route                                  |                Action                 |                     Description                      |
 | :----: | :------------------------------------- | :-----------------------------------: | :--------------------------------------------------: |
+|  GET   | `/db/:dbname/groups`                   |            Get all groups             |                                                      |
 |  GET   | `/db/:dbname/group/:name`              |          Get a group by name          |                                                      |
 |  PUT   | `/db/:dbname/group/:name`              |            Create a group             |                                                      |
 | DELETE | `/db/:dbname/group/:name`              |            Remove a group             |                                                      |
@@ -72,6 +73,13 @@
 |  GET   | `/db/:dbname/token/:tokenid`     |     Get information about a token      |                                                                            |
 | DELETE | `/db/:dbname/token/:tokenid`     |             Delete a token             |                                                                            |
 
+### Importations
+
+| Method | Route                       |       Action        |     Description     |
+| :----: | :-------------------------- | :-----------------: | :-----------------: |
+|  GET   | `/db/:dbname/imports`       | Get list of imports | Params: limit, skip |
+|  GET   | `/db/:dbname/imports/:uuid` |  Get import by id   |                     |
+
 ### Zenodo
 
 | Method | Route                       |          Action           |                       Description                       |

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -0,0 +1,9 @@
+version: '3'
+services:
+  db:
+    image: couchdb:2.3
+    environment:
+      COUCHDB_USER: admin
+      COUCHDB_PASSWORD: admin
+    ports:
+      - 127.0.0.1:5984:5984
diff --git a/src/constants.js b/src/constants.js
@@ -9,6 +9,7 @@ const globalRightTypes = [
   'readGroup',
   'writeGroup',
   'createGroup',
+  'readImport',
   'owner',
 ];
 

diff --git a/src/couch/group.js b/src/couch/group.js
@@ -253,11 +253,14 @@ const methods = {
     await this.open();
     // Find all the ldap groups
     debug.trace('sync all ldap groups');
-    groups = await this._db.queryView('documentByType', {
-      key: 'group',
-      include_docs: true,
-    });
-    groups = groups.map((group) => group.doc);
+    groups = await this._db.queryView(
+      'documentByType',
+      {
+        key: 'group',
+        include_docs: true,
+      },
+      { onlyDoc: true },
+    );
 
     groups = groups.filter((group) => group.DN);
     for (let i = 0; i < groups.length; i++) {

diff --git a/src/couch/imports.js b/src/couch/imports.js
@@ -0,0 +1,77 @@
+'use strict';
+
+const CouchError = require('../util/CouchError');
+const debug = require('../util/debug')('main:imports');
+
+const validateMethods = require('./validate');
+
+const methods = {
+  async logImport(toLog) {
+    toLog.$type = 'import';
+    toLog.$creationDate = Date.now();
+    await this._db.insertDocument(toLog);
+  },
+
+  async getImports(user, query) {
+    await this.open();
+    debug('get imports (%s)', user);
+
+    const hasRight = await validateMethods.checkRightAnyGroup(
+      this,
+      user,
+      'readImport',
+    );
+    if (!hasRight) {
+      throw new CouchError(
+        'user is missing read right on imports',
+        'unauthorized',
+      );
+    }
+
+    const imports = await this._db.queryView(
+      'importsByDate',
+      {
+        descending: true,
+        include_docs: true,
+        limit: query.limit || 10,
+        skip: query.skip || 0,
+      },
+      { onlyDoc: true },
+    );
+
+    return imports;
+  },
+
+  async getImport(user, uuid) {
+    await this.open();
+    debug('get import (%s, %s)', user, uuid);
+
+    const hasRight = await validateMethods.checkRightAnyGroup(
+      this,
+      user,
+      'readImport',
+    );
+    if (!hasRight) {
+      throw new CouchError(
+        'user is missing read right on imports',
+        'unauthorized',
+      );
+    }
+
+    const doc = await this._db.getDocument(uuid);
+    if (!doc) {
+      throw new CouchError('document not found', 'not found');
+    }
+    if (doc.$type !== 'import') {
+      throw new CouchError(
+        `wrong document type: ${doc.$type}. Expected: import`,
+      );
+    }
+
+    return doc;
+  },
+};
+
+module.exports = {
+  methods,
+};
diff --git a/src/couch/index.js b/src/couch/index.js
@@ -102,6 +102,7 @@ extendCouch('query');
 extendCouch('right');
 extendCouch('token');
 extendCouch('user');
+extendCouch('imports');
 
 function extendCouch(name) {
   // eslint-disable-next-line import/no-dynamic-require

diff --git a/src/design/validateDocUpdate.js b/src/design/validateDocUpdate.js
@@ -13,7 +13,7 @@ module.exports = function(newDoc, oldDoc, userCtx) {
   if (newDoc._deleted) {
     return;
   }
-  var validTypes = ['entry', 'group', 'db', 'log', 'user', 'token'];
+  var validTypes = ['entry', 'group', 'db', 'log', 'user', 'token', 'import'];
   var validRights = ['create', 'read', 'write', 'createGroup'];
   // see http://emailregex.com/
   var validEmail = /^.+@.+$/;
@@ -160,7 +160,7 @@ module.exports = function(newDoc, oldDoc, userCtx) {
       throw { forbidden: 'Tokens are immutable' };
     }
     if (newDoc.$kind !== 'entry' && newDoc.$kind !== 'user') {
-      throw { forbidden: 'Only entry tokens are supported' };
+      throw { forbidden: 'Only entry and user tokens are supported' };
     }
     if (
       !newDoc.$id ||
@@ -173,5 +173,36 @@ module.exports = function(newDoc, oldDoc, userCtx) {
     if (newDoc.$kind === 'entry' && !newDoc.uuid) {
       throw { forbidden: 'token is missing fields' };
     }
+  } else if (newDoc.$type === 'import') {
+    if (oldDoc) {
+      throw { forbidden: 'import logs are immutable' };
+    }
+    if (
+      (!newDoc.$creationDate,
+      !newDoc.name || !newDoc.filename || !newDoc.status)
+    ) {
+      throw { forbidden: 'import is missing fields' };
+    }
+    if (newDoc.status === 'SUCCESS') {
+      if (
+        !newDoc.result ||
+        !newDoc.result.uuid ||
+        !newDoc.result.kind ||
+        !newDoc.result.id ||
+        !newDoc.result.owner
+      ) {
+        throw { forbidden: 'success import is missing fields' };
+      }
+    } else if (newDoc.status === 'ERROR') {
+      if (
+        !newDoc.error ||
+        typeof newDoc.error.message !== 'string' ||
+        typeof newDoc.error.stack !== 'string'
+      ) {
+        throw { forbidden: 'error import is missing fields' };
+      }
+    } else {
+      throw { forbidden: 'Bad import status: ' + newDoc.status };
+    }
   }
 };
diff --git a/src/design/views.js b/src/design/views.js
@@ -180,3 +180,11 @@ views.tokenByOwner = {
     emit(doc.$owner);
   },
 };
+
+views.importsByDate = {
+  map: function(doc) {
+    if (doc.$type !== 'import') return;
+    emit(doc.date);
+  },
+  reduce: '_count',
+};
diff --git a/src/import/import.js b/src/import/import.js
@@ -21,7 +21,32 @@ exports.import = async function importFile(
 
   const baseImport = new BaseImport(filePath, database);
   const result = new ImportResult();
-  await config(baseImport, result);
+
+  const { couch, filename } = baseImport;
+
+  try {
+    await config(baseImport, result);
+  } catch (e) {
+    await couch
+      .logImport({
+        name: importName,
+        filename,
+        status: 'ERROR',
+        error: {
+          message: e.message || '',
+          stack: e.stack || '',
+        },
+      })
+      .catch((error) => {
+        debug.error(
+          'error while logging import error for (%s)',
+          filename,
+          error,
+        );
+      });
+    throw e;
+  }
+
   if (result.isSkipped) {
     return { skip: 'skip' };
   }
@@ -30,6 +55,27 @@ exports.import = async function importFile(
   if (dryRun) {
     return { skip: 'dryRun', result };
   }
-  await saveResult(baseImport, result);
-  return { ok: true };
+  const uuid = await saveResult(baseImport, result);
+
+  await couch
+    .logImport({
+      name: importName,
+      filename,
+      status: 'SUCCESS',
+      result: {
+        uuid,
+        id: result.id,
+        kind: result.kind,
+        owner: result.owner,
+      },
+    })
+    .catch((error) => {
+      debug.error(
+        'error while logging import success for (%s)',
+        filename,
+        error,
+      );
+    });
+
+  return { ok: true, result };
 };
diff --git a/src/import/saveResult.js b/src/import/saveResult.js
@@ -102,4 +102,6 @@ module.exports = async function saveResult(importBase, result) {
       },
     );
   }
+
+  return document.id;
 };
diff --git a/src/server/middleware/couch.js b/src/server/middleware/couch.js
@@ -476,6 +476,17 @@ exports.deleteTokenById = composeWithError(async (ctx) => {
   respondOk(ctx);
 });
 
+exports.getImports = composeWithError(async (ctx) => {
+  ctx.body = await ctx.state.couch.getImports(ctx.state.userEmail, ctx.query);
+});
+
+exports.getImport = composeWithError(async (ctx) => {
+  ctx.body = await ctx.state.couch.getImport(
+    ctx.state.userEmail,
+    ctx.params.uuid,
+  );
+});
+
 function processCouchQuery(ctx) {
   for (let i = 0; i < couchNeedsParse.length; i++) {
     if (ctx.query[couchNeedsParse[i]]) {

diff --git a/src/server/routes/api.js b/src/server/routes/api.js
@@ -189,6 +189,10 @@ router.get('/:dbname/token', couch.getTokens);
 router.get('/:dbname/token/:tokenid', couch.getTokenById);
 router.delete('/:dbname/token/:tokenid', couch.deleteTokenById);
 
+// Importations
+router.get('/:dbname/imports', couch.getImports);
+router.get('/:dbname/import/:uuid', couch.getImport);
+
 // Zenodo
 if (config.zenodo === true) {
   const zenodo = require('../middleware/zenodo');

diff --git a/test/homeDirectories/main/test-new-import/error/import.js b/test/homeDirectories/main/test-new-import/error/import.js
@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = async function errorImport() {
+  throw new Error('this import is wrong');
+};