From b49431ea2a453d2f2cfe2ccb0971202cc0550f87 Mon Sep 17 00:00:00 2001 From: Bertrand Marron Date: Mon, 10 Dec 2018 18:45:20 +0100 Subject: [PATCH 1/4] Add name to all enqueue calls --- plugins/publish-to-udata/models/Dataset.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/publish-to-udata/models/Dataset.js b/plugins/publish-to-udata/models/Dataset.js index 51757ff..34ad0ad 100644 --- a/plugins/publish-to-udata/models/Dataset.js +++ b/plugins/publish-to-udata/models/Dataset.js @@ -151,7 +151,7 @@ schema.method('asyncUpdate', function (data = {}) { throw new Error('Dataset not published') } - return enqueue('udata-sync-one', { + return enqueue('udata-sync-one', `update: ${this._id}`, { ...data, recordId: this._id, action: 'update' @@ -198,7 +198,7 @@ schema.method('asyncPublish', async function ({organizationId}) { throw new Error('Dataset already published') } - await enqueue('udata-sync-one', { + await enqueue('udata-sync-one', `publish: ${this._id}`, { recordId: this._id, action: 'publish', organizationId @@ -235,7 +235,7 @@ schema.method('asyncUnpublish', function () { throw new Error('Dataset not published') } - return enqueue('udata-sync-one', { + return enqueue('udata-sync-one', `unpublish: ${this._id}`, { recordId: this._id, action: 'unpublish' }) @@ -252,7 +252,7 @@ schema.method('transferTo', async function (targetOrganization, force = false) { }) schema.static('asyncSynchronizeAll', data => { - return enqueue('udata-sync-all', data) + return enqueue('udata-sync-all', 'udata-sync-all', data) }) mongoose.model('Dataset', schema) From 6210181c4770d261de6e58d3382cb165c48bc90d Mon Sep 17 00:00:00 2001 From: Bertrand Marron Date: Mon, 10 Dec 2018 18:45:31 +0100 Subject: [PATCH 2/4] Reduce concurrency on udata-sync-one --- plugins/publish-to-udata/jobs/definition.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/publish-to-udata/jobs/definition.js b/plugins/publish-to-udata/jobs/definition.js index d7eb4b3..5f36ef4 100644 --- a/plugins/publish-to-udata/jobs/definition.js +++ b/plugins/publish-to-udata/jobs/definition.js @@ -12,7 +12,7 @@ module.exports = [ { name: 'udata-sync-one', - concurrency: 10, + concurrency: 7, options: { jobIdKey: 'recordId' } From 98ee324539dfa72c07c986c06d0c98826e7b722d Mon Sep 17 00:00:00 2001 From: Bertrand Marron Date: Mon, 10 Dec 2018 18:45:53 +0100 Subject: [PATCH 3/4] Increase timeout on udata-sync-all --- plugins/publish-to-udata/jobs/definition.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/publish-to-udata/jobs/definition.js b/plugins/publish-to-udata/jobs/definition.js index 5f36ef4..caa95bf 100644 --- a/plugins/publish-to-udata/jobs/definition.js +++ b/plugins/publish-to-udata/jobs/definition.js @@ -6,7 +6,7 @@ module.exports = [ concurrency: 1, options: { jobId: 'unique', // Only one at a time - timeout: ms('20m') + timeout: ms('1h') } }, From 7a72f1de1bc2e0fc9da8e284e39c739db3d6fe30 Mon Sep 17 00:00:00 2001 From: Bertrand Marron Date: Mon, 10 Dec 2018 18:57:56 +0100 Subject: [PATCH 4/4] Fix publication error messages matching --- plugins/publish-to-udata/geogw.js | 18 +++++++++++++----- .../jobs/udata-sync-one/index.js | 2 +- plugins/publish-to-udata/models/Dataset.js | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/plugins/publish-to-udata/geogw.js b/plugins/publish-to-udata/geogw.js index 41a1580..e02f077 100644 --- a/plugins/publish-to-udata/geogw.js +++ b/plugins/publish-to-udata/geogw.js @@ -7,11 +7,19 @@ const client = got.extend({ }) async function getRecord(recordId) { - const {body} = await client.get(`/records/${recordId}`, { - json: true - }) - - return body + try { + const {body} = await client.get(`/records/${recordId}`, { + json: true + }) + + return body + } catch (error) { + if (error.statusCode === 404) { + throw new Error('Record not found') + } + + throw error + } } async function setRecordPublication(recordId, publicationInfo) { diff --git a/plugins/publish-to-udata/jobs/udata-sync-one/index.js b/plugins/publish-to-udata/jobs/udata-sync-one/index.js index b35b2c8..1c2e7b6 100644 --- a/plugins/publish-to-udata/jobs/udata-sync-one/index.js +++ b/plugins/publish-to-udata/jobs/udata-sync-one/index.js @@ -47,7 +47,7 @@ exports.handler = async function ({data}) { return publicationInfo.unpublish() } - if (removeIfTargetDatasetNotFound === true && error.message === 'Target dataset doesn\'t exist anymore' && action === 'update') { + if (removeIfTargetDatasetNotFound === true && error.message === 'Target dataset doesn’t exist anymore' && action === 'update') { debug(`${recordId}: Target dataset not found. Going to remove the publication info…`) return publicationInfo.removeAndNotify() } diff --git a/plugins/publish-to-udata/models/Dataset.js b/plugins/publish-to-udata/models/Dataset.js index 34ad0ad..e1da9ac 100644 --- a/plugins/publish-to-udata/models/Dataset.js +++ b/plugins/publish-to-udata/models/Dataset.js @@ -117,7 +117,7 @@ schema.method('update', async function (options = {}) { await this.transferTo(targetOrganization) } catch (error) { if (error.message === 'Dataset doesn’t exist') { - throw new Error('Target dataset doesnt exist anymore') + throw new Error('Target dataset doesn’t exist anymore') } throw error }