diff --git a/API.md b/API.md index 637bde94..031ea690 100644 --- a/API.md +++ b/API.md @@ -21,7 +21,7 @@ | Method | Route | Action | Description | | :----: | :-------------------------------------- | :---------------------------------: | :----------------------------------------: | -| POST | `/db/:dbname/entry` | Insert / Update an entry | Based on \_id or $id of the entry | +| POST | `/db/:dbname/entry` | Insert / Update an entry | Based on \_id or \$id of the entry | | GET | `/db/:dbname/entry/_all` | Get all entries | Returns an array of documents | | HEAD | `/db/:dbname/entry/:uuid` | Get HTTP headers about the document | Similar to HEAD /:dbname/:docid in CouchDB | | GET | `/db/:dbname/entry/:uuid` | Get an entry by UUID | | @@ -54,6 +54,7 @@ | Method | Route | Action | Description | | :----: | :------------------------------------- | :-----------------------------------: | :--------------------------------------------------: | +| GET | `/db/:dbname/groups` | Get all groups | | | GET | `/db/:dbname/group/:name` | Get a group by name | | | PUT | `/db/:dbname/group/:name` | Create a group | | | DELETE | `/db/:dbname/group/:name` | Remove a group | | @@ -72,6 +73,13 @@ | GET | `/db/:dbname/token/:tokenid` | Get information about a token | | | DELETE | `/db/:dbname/token/:tokenid` | Delete a token | | +### Importations + +| Method | Route | Action | Description | +| :----: | :-------------------------- | :-----------------: | :-----------------: | +| GET | `/db/:dbname/imports` | Get list of imports | Params: limit, skip | +| GET | `/db/:dbname/imports/:uuid` | Get import by id | | + ### Zenodo | Method | Route | Action | Description | diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..98f57705 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,9 @@ +version: '3' +services: + db: + image: couchdb:2.3 + environment: + COUCHDB_USER: admin + COUCHDB_PASSWORD: admin + ports: + - 127.0.0.1:5984:5984 diff --git a/src/constants.js b/src/constants.js index 826d99d5..d33b549f 100644 --- a/src/constants.js +++ b/src/constants.js @@ -9,6 +9,7 @@ const globalRightTypes = [ 'readGroup', 'writeGroup', 'createGroup', + 'readImport', 'owner', ]; diff --git a/src/couch/group.js b/src/couch/group.js index 4630dd47..98276eae 100644 --- a/src/couch/group.js +++ b/src/couch/group.js @@ -253,11 +253,14 @@ const methods = { await this.open(); // Find all the ldap groups debug.trace('sync all ldap groups'); - groups = await this._db.queryView('documentByType', { - key: 'group', - include_docs: true, - }); - groups = groups.map((group) => group.doc); + groups = await this._db.queryView( + 'documentByType', + { + key: 'group', + include_docs: true, + }, + { onlyDoc: true }, + ); groups = groups.filter((group) => group.DN); for (let i = 0; i < groups.length; i++) { diff --git a/src/couch/imports.js b/src/couch/imports.js new file mode 100644 index 00000000..34c54772 --- /dev/null +++ b/src/couch/imports.js @@ -0,0 +1,77 @@ +'use strict'; + +const CouchError = require('../util/CouchError'); +const debug = require('../util/debug')('main:imports'); + +const validateMethods = require('./validate'); + +const methods = { + async logImport(toLog) { + toLog.$type = 'import'; + toLog.$creationDate = Date.now(); + await this._db.insertDocument(toLog); + }, + + async getImports(user, query) { + await this.open(); + debug('get imports (%s)', user); + + const hasRight = await validateMethods.checkRightAnyGroup( + this, + user, + 'readImport', + ); + if (!hasRight) { + throw new CouchError( + 'user is missing read right on imports', + 'unauthorized', + ); + } + + const imports = await this._db.queryView( + 'importsByDate', + { + descending: true, + include_docs: true, + limit: query.limit || 10, + skip: query.skip || 0, + }, + { onlyDoc: true }, + ); + + return imports; + }, + + async getImport(user, uuid) { + await this.open(); + debug('get import (%s, %s)', user, uuid); + + const hasRight = await validateMethods.checkRightAnyGroup( + this, + user, + 'readImport', + ); + if (!hasRight) { + throw new CouchError( + 'user is missing read right on imports', + 'unauthorized', + ); + } + + const doc = await this._db.getDocument(uuid); + if (!doc) { + throw new CouchError('document not found', 'not found'); + } + if (doc.$type !== 'import') { + throw new CouchError( + `wrong document type: ${doc.$type}. Expected: import`, + ); + } + + return doc; + }, +}; + +module.exports = { + methods, +}; diff --git a/src/couch/index.js b/src/couch/index.js index 91a57e26..04aa0b4d 100644 --- a/src/couch/index.js +++ b/src/couch/index.js @@ -102,6 +102,7 @@ extendCouch('query'); extendCouch('right'); extendCouch('token'); extendCouch('user'); +extendCouch('imports'); function extendCouch(name) { // eslint-disable-next-line import/no-dynamic-require diff --git a/src/design/validateDocUpdate.js b/src/design/validateDocUpdate.js index c7e2b9bd..29b672fa 100644 --- a/src/design/validateDocUpdate.js +++ b/src/design/validateDocUpdate.js @@ -13,7 +13,7 @@ module.exports = function(newDoc, oldDoc, userCtx) { if (newDoc._deleted) { return; } - var validTypes = ['entry', 'group', 'db', 'log', 'user', 'token']; + var validTypes = ['entry', 'group', 'db', 'log', 'user', 'token', 'import']; var validRights = ['create', 'read', 'write', 'createGroup']; // see http://emailregex.com/ var validEmail = /^.+@.+$/; @@ -160,7 +160,7 @@ module.exports = function(newDoc, oldDoc, userCtx) { throw { forbidden: 'Tokens are immutable' }; } if (newDoc.$kind !== 'entry' && newDoc.$kind !== 'user') { - throw { forbidden: 'Only entry tokens are supported' }; + throw { forbidden: 'Only entry and user tokens are supported' }; } if ( !newDoc.$id || @@ -173,5 +173,36 @@ module.exports = function(newDoc, oldDoc, userCtx) { if (newDoc.$kind === 'entry' && !newDoc.uuid) { throw { forbidden: 'token is missing fields' }; } + } else if (newDoc.$type === 'import') { + if (oldDoc) { + throw { forbidden: 'import logs are immutable' }; + } + if ( + (!newDoc.$creationDate, + !newDoc.name || !newDoc.filename || !newDoc.status) + ) { + throw { forbidden: 'import is missing fields' }; + } + if (newDoc.status === 'SUCCESS') { + if ( + !newDoc.result || + !newDoc.result.uuid || + !newDoc.result.kind || + !newDoc.result.id || + !newDoc.result.owner + ) { + throw { forbidden: 'success import is missing fields' }; + } + } else if (newDoc.status === 'ERROR') { + if ( + !newDoc.error || + typeof newDoc.error.message !== 'string' || + typeof newDoc.error.stack !== 'string' + ) { + throw { forbidden: 'error import is missing fields' }; + } + } else { + throw { forbidden: 'Bad import status: ' + newDoc.status }; + } } }; diff --git a/src/design/views.js b/src/design/views.js index 982b9131..312d2db9 100644 --- a/src/design/views.js +++ b/src/design/views.js @@ -180,3 +180,11 @@ views.tokenByOwner = { emit(doc.$owner); }, }; + +views.importsByDate = { + map: function(doc) { + if (doc.$type !== 'import') return; + emit(doc.date); + }, + reduce: '_count', +}; diff --git a/src/import/import.js b/src/import/import.js index 73f23b4c..ac965d05 100644 --- a/src/import/import.js +++ b/src/import/import.js @@ -21,7 +21,32 @@ exports.import = async function importFile( const baseImport = new BaseImport(filePath, database); const result = new ImportResult(); - await config(baseImport, result); + + const { couch, filename } = baseImport; + + try { + await config(baseImport, result); + } catch (e) { + await couch + .logImport({ + name: importName, + filename, + status: 'ERROR', + error: { + message: e.message || '', + stack: e.stack || '', + }, + }) + .catch((error) => { + debug.error( + 'error while logging import error for (%s)', + filename, + error, + ); + }); + throw e; + } + if (result.isSkipped) { return { skip: 'skip' }; } @@ -30,6 +55,27 @@ exports.import = async function importFile( if (dryRun) { return { skip: 'dryRun', result }; } - await saveResult(baseImport, result); - return { ok: true }; + const uuid = await saveResult(baseImport, result); + + await couch + .logImport({ + name: importName, + filename, + status: 'SUCCESS', + result: { + uuid, + id: result.id, + kind: result.kind, + owner: result.owner, + }, + }) + .catch((error) => { + debug.error( + 'error while logging import success for (%s)', + filename, + error, + ); + }); + + return { ok: true, result }; }; diff --git a/src/import/saveResult.js b/src/import/saveResult.js index 3153b7ef..1a1482aa 100644 --- a/src/import/saveResult.js +++ b/src/import/saveResult.js @@ -102,4 +102,6 @@ module.exports = async function saveResult(importBase, result) { }, ); } + + return document.id; }; diff --git a/src/server/middleware/couch.js b/src/server/middleware/couch.js index cdd0e449..658ddfa7 100644 --- a/src/server/middleware/couch.js +++ b/src/server/middleware/couch.js @@ -476,6 +476,17 @@ exports.deleteTokenById = composeWithError(async (ctx) => { respondOk(ctx); }); +exports.getImports = composeWithError(async (ctx) => { + ctx.body = await ctx.state.couch.getImports(ctx.state.userEmail, ctx.query); +}); + +exports.getImport = composeWithError(async (ctx) => { + ctx.body = await ctx.state.couch.getImport( + ctx.state.userEmail, + ctx.params.uuid, + ); +}); + function processCouchQuery(ctx) { for (let i = 0; i < couchNeedsParse.length; i++) { if (ctx.query[couchNeedsParse[i]]) { diff --git a/src/server/routes/api.js b/src/server/routes/api.js index bfedcfb1..d295a9ce 100644 --- a/src/server/routes/api.js +++ b/src/server/routes/api.js @@ -189,6 +189,10 @@ router.get('/:dbname/token', couch.getTokens); router.get('/:dbname/token/:tokenid', couch.getTokenById); router.delete('/:dbname/token/:tokenid', couch.deleteTokenById); +// Importations +router.get('/:dbname/imports', couch.getImports); +router.get('/:dbname/import/:uuid', couch.getImport); + // Zenodo if (config.zenodo === true) { const zenodo = require('../middleware/zenodo'); diff --git a/test/homeDirectories/main/test-new-import/error/import.js b/test/homeDirectories/main/test-new-import/error/import.js new file mode 100644 index 00000000..87426f3d --- /dev/null +++ b/test/homeDirectories/main/test-new-import/error/import.js @@ -0,0 +1,5 @@ +'use strict'; + +module.exports = async function errorImport() { + throw new Error('this import is wrong'); +}; diff --git a/test/unit/import/import.js b/test/unit/import/import.js index 14af7f34..a46bf181 100644 --- a/test/unit/import/import.js +++ b/test/unit/import/import.js @@ -24,45 +24,62 @@ describe('import', () => { beforeEach(async function() { importCouch = await testUtils.resetDatabase(databaseName); }); - test('full import', () => { - return importFile(databaseName, 'simple', textFile1).then(() => { - return importCouch.getEntryById('test.txt', 'a@a.com').then((data) => { - expect(data).toBeDefined(); - expect(data.$content).toBeDefined(); - // Check that new content has been merged - expect(data.$content.sideEffect).toBe(true); - - // Check that the correct owners have been added - expect(data.$owners).toEqual(['a@a.com', 'group1', 'group2', 'group3']); - - // Main metadata - let metadata = data.$content.jpath.in.document[0]; - expect(metadata).toBeDefined(); - // metadata has been added - expect(metadata.hasMetadata).toBe(true); - // a reference to the attachment has been added - expect(metadata.field.filename).toBe('jpath/in/document/test.txt'); - // Reference has been added - expect(metadata.reference).toBe('test.txt'); - - // Additional metadata - metadata = data.$content.other.jpath[0]; - expect(metadata).toBeDefined(); - expect(metadata.hasMetadata).toBe(true); - expect(metadata.reference).toBe('testRef'); - expect(metadata.testField.filename).toBe( - 'other/jpath/testFilename.txt', - ); - - // check attachments - const mainAttachment = data._attachments['jpath/in/document/test.txt']; - const secondaryAttachment = - data._attachments['other/jpath/testFilename.txt']; - expect(mainAttachment).toBeDefined(); - expect(secondaryAttachment).toBeDefined(); - expect(mainAttachment.content_type).toBe('text/plain'); - expect(secondaryAttachment.content_type).toBe('text/plain'); - }); + test('full import', async () => { + await importFile(databaseName, 'simple', textFile1); + const data = await importCouch.getEntryById('test.txt', 'a@a.com'); + expect(data).toBeDefined(); + expect(data.$content).toBeDefined(); + // Check that new content has been merged + expect(data.$content.sideEffect).toBe(true); + + // Check that the correct owners have been added + expect(data.$owners).toEqual(['a@a.com', 'group1', 'group2', 'group3']); + + // Main metadata + let metadata = data.$content.jpath.in.document[0]; + expect(metadata).toBeDefined(); + // metadata has been added + expect(metadata.hasMetadata).toBe(true); + // a reference to the attachment has been added + expect(metadata.field.filename).toBe('jpath/in/document/test.txt'); + // Reference has been added + expect(metadata.reference).toBe('test.txt'); + + // Additional metadata + metadata = data.$content.other.jpath[0]; + expect(metadata).toBeDefined(); + expect(metadata.hasMetadata).toBe(true); + expect(metadata.reference).toBe('testRef'); + expect(metadata.testField.filename).toBe('other/jpath/testFilename.txt'); + + // check attachments + const mainAttachment = data._attachments['jpath/in/document/test.txt']; + const secondaryAttachment = + data._attachments['other/jpath/testFilename.txt']; + expect(mainAttachment).toBeDefined(); + expect(secondaryAttachment).toBeDefined(); + expect(mainAttachment.content_type).toBe('text/plain'); + expect(secondaryAttachment.content_type).toBe('text/plain'); + + // check log + const importLogs = await importCouch._db.queryView( + 'importsByDate', + { + descending: true, + include_docs: true, + }, + { onlyDoc: true }, + ); + expect(importLogs).toHaveLength(1); + expect(importLogs[0]).toMatchObject({ + name: 'simple', + filename: 'test.txt', + status: 'SUCCESS', + }); + expect(importLogs[0].result).toMatchObject({ + id: 'test.txt', + owner: 'a@a.com', + kind: 'sample', }); }); @@ -132,6 +149,30 @@ describe('import', () => { ); expect(attachmentData.toString('utf8')).toBe('test2'); }); + + test('error when the import function throws', async () => { + await expect(importFile(databaseName, 'error', textFile1)).rejects.toThrow( + /this import is wrong/, + ); + + // check log + const importLogs = await importCouch._db.queryView( + 'importsByDate', + { + descending: true, + include_docs: true, + }, + { onlyDoc: true }, + ); + expect(importLogs).toHaveLength(1); + expect(importLogs[0]).toMatchObject({ + name: 'error', + filename: 'test.txt', + status: 'ERROR', + }); + expect(importLogs[0].error.message).toBe('this import is wrong'); + expect(typeof importLogs[0].error.stack).toBe('string'); + }); }); function readStream(stream) {