Alter entity spec version and add branchId, trunkVersion in form sche…

…ma (#1210)

* XML schema method to alter entity spec version and add branchId, trunkVersion

* Dont replace xml if entities-version doesnt match expected old version

* minor code improvements

* worker wip

* working with published and draft forms separately

* update version in a draft

* more tests

* More test and code refinement

* moving args around

* Migration and test

* change suffix and exclude deleted forms/projects

* moving code around

* changing code and tests

* responding to code review

lib/data/schema.js

@@ -17,7 +17,7 @@
 // 2 When walking the schema to do some later processing of data, this ordering
 //   allows us to iterate the fields in order and still understand the tree structure.
 
-const { always, last, map } = require('ramda');
+const { always, equals, last, map } = require('ramda');
 const ptr = last; // just rename to make it more relevant to our context.
 const hparser = require('htmlparser2');
 const parse = require('csv-parse/lib/sync');
@@ -582,12 +582,77 @@ const _versionSplicer = (replace) => (xml, insert) => new Promise((pass, fail) =
 const addVersionSuffix = _versionSplicer(false);
 const setVersion = _versionSplicer(true);
 
+// The following helper functions are for a form migration described in issue c#692.
+// Forms with entity spec version 2023.1.0 that support entity updates need to
+// be updated to spec version 2024.1.0 and have `branchId` and `trunkVersion`
+// included alongside the existing `baseVersion`.
+const _addBranchIdAndTrunkVersion = (xml) => new Promise((pass, fail) => {
+  const stack = [];
+  const parser = new hparser.Parser({
+    onopentag: (fullname) => {
+      stack.push(stripNamespacesFromPath(fullname));
+      if (equals(stack, ['html', 'head', 'model', 'instance', 'data', 'meta', 'entity'])) {
+        const idx = _findSplicePoint(xml, parser.endIndex);
+        parser.reset(); // stop parsing.
+        return pass(`${xml.slice(0, idx)} trunkVersion="" branchId=""${xml.slice(idx)}`);
+      }
+    },
+    onclosetag: () => {
+      stack.pop();
+    },
+    // If the entity tag can't be found (it should be found in the forms this will run on)
+    // or there is another xml parsing problem, just fail here. This error will be caught below
+    // by updateEntityForm.
+    onend: () => fail(Problem.internal.unknown())
+  }, { xmlMode: true, decodeEntities: true });
+
+  parser.write(xml);
+  parser.end();
+});
+
+const _updateEntityVersion = (xml, oldVersion, newVersion) => new Promise((pass, fail) => {
+  const stack = [];
+  const parser = new hparser.Parser({
+    onattribute: (name, value) => {
+      if ((stripNamespacesFromPath(name) === 'entities-version') && (value === oldVersion)
+        && (stack.length) === 2 && (stack[0] === 'html') && (stack[1] === 'head')) {
+        const idx = parser._tokenizer._index;
+        parser.reset();
+        return pass(`${xml.slice(0, idx - value.length)}${newVersion}${xml.slice(idx)}`);
+      }
+    },
+    // n.b. opentag happens AFTER all the attributes for that tag have been emitted!
+    onopentag: (fullname) => {
+      stack.push(stripNamespacesFromPath(fullname));
+    },
+    onclosetag: () => {
+      stack.pop();
+    },
+    // If the entities-version attribute can't be found or there is another
+    // xml parsing problem, just fail here. This error will be caught below
+    // by updateEntityForm.
+    onend: () => fail(Problem.internal.unknown())
+  }, { xmlMode: true, decodeEntities: true });
+
+  parser.write(xml);
+  parser.end();
+});
+
+// If there are any problems with updating the XML, this will just
+// return the unaltered XML which will then be a clue for the worker
+// to not change anything about the Form.
+const updateEntityForm = (xml, oldVersion, newVersion, suffix) =>
+  _updateEntityVersion(xml, oldVersion, newVersion)
+    .then(_addBranchIdAndTrunkVersion)
+    .then(x => addVersionSuffix(x, suffix))
+    .catch(() => xml);
 
 module.exports = {
   getFormFields,
   SchemaStack,
   sanitizeFieldsForOdata,
   expectedFormAttachments, merge, compare,
-  injectPublicKey, addVersionSuffix, setVersion
+  injectPublicKey, addVersionSuffix, setVersion,
+  updateEntityForm
 };
 

lib/model/migrations/20241010-01-schedule-entity-form-upgrade.js

@@ -0,0 +1,26 @@
+// Copyright 2024 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+const up = (db) => db.raw(`
+  INSERT INTO audits ("action", "acteeId", "loggedAt", "details")
+  SELECT 'upgrade.process.form.entities_version', forms."acteeId", clock_timestamp(),
+    '{"upgrade": "As part of upgrading Central to v2024.3, this form is being updated to the latest entities-version spec."}'
+  FROM forms
+  JOIN form_defs fd ON forms."id" = fd."formId"
+  JOIN dataset_form_defs dfd ON fd."id" = dfd."formDefId"
+  JOIN projects ON projects."id" = forms."projectId"
+  WHERE dfd."actions" @> '["update"]'
+  AND forms."deletedAt" IS NULL
+  AND projects."deletedAt" IS NULL
+  GROUP BY forms."acteeId";
+`);
+
+const down = () => {};
+
+module.exports = { up, down };

lib/model/query/audits.js

@@ -46,7 +46,7 @@ const actionCondition = (action) => {
   else if (action === 'project')
     return sql`action in ('project.create', 'project.update', 'project.delete')`;
   else if (action === 'form')
-    return sql`action in ('form.create', 'form.update', 'form.delete', 'form.restore', 'form.purge', 'form.attachment.update', 'form.submission.export', 'form.update.draft.set', 'form.update.draft.delete', 'form.update.publish')`;
+    return sql`action in ('form.create', 'form.update', 'form.delete', 'form.restore', 'form.purge', 'form.attachment.update', 'form.submission.export', 'form.update.draft.set', 'form.update.draft.delete', 'form.update.draft.replace', 'form.update.publish', 'upgrade.process.form.entities_version')`;
   else if (action === 'submission')
     return sql`action in ('submission.create', 'submission.update', 'submission.update.version', 'submission.attachment.update', 'submission.reprocess', 'submission.delete', 'submission.restore', 'submission.purge')`;
   else if (action === 'dataset')

lib/model/query/forms.js

@@ -201,7 +201,9 @@ const _createNewDef = (partial, form, publish, data) => ({ one }) =>
 // ===========
 // partial:     Partial form definition of the new version
 // form:        Form frame of existing form
-// publish:     set true if you want new version to be published (only used by setManagedKey, everywhere else calls publish() explicitly)
+// publish:     set true if you want new version to be published (used infrequently)
+//     One example where publish=true is in setManagedKey, which updates the form XML.
+//     Most other situations call publish() explicitly.
 // duplicating: set true if copying form definition from previously uploaded definition, in that cases we don't check for structural change
 //              as user has already been warned otherwise set false
 const createVersion = (partial, form, publish, duplicating = false) => async ({ Datasets, FormAttachments, Forms, Keys }) => {
@@ -290,7 +292,21 @@ createVersion.audit = (newDef, partial, form, publish) => (log) => ((publish ===
   ? log('form.update.publish', form, { oldDefId: form.currentDefId, newDefId: newDef.id })
   : log('form.update.draft.set', form, { oldDraftDefId: form.draftDefId, newDraftDefId: newDef.id }));
 createVersion.audit.withResult = true;
+createVersion.audit.logEvenIfAnonymous = true;
+
+// This is used in the rare case where we want to change and update a FormDef in place without
+// creating a new def. This is basically a wrapper around _updateDef that also logs an event.
+// eslint-disable-next-line no-unused-vars
+const replaceDef = (partial, form, details) => async ({ Forms }) => {
+  const { version, hash, sha, sha256 } = partial.def;
+  await Forms._updateDef(form.def, { xml: partial.xml, version, hash, sha, sha256 });
+  // all this does is changed updatedAt
+  await Forms._update(form, { updatedAt: (new Date()).toISOString() });
+};
 
+replaceDef.audit = (_, form, details) => (log) =>
+  log('form.update.draft.replace', form, details);
+replaceDef.audit.logEvenIfAnonymous = true;
 
 ////////////////////////////////////////////////////////////////////////////////
 // PUBLISHING MANAGEMENT
@@ -805,7 +821,7 @@ module.exports = {
   _insertFormFields,
   _createNew, createNew, _createNewDef, createVersion,
   publish, clearDraft,
-  _update, update, _updateDef, del, restore, purge,
+  _update, update, _updateDef, replaceDef, del, restore, purge,
   clearUnneededDrafts,
   setManagedKey,
   getByAuthForOpenRosa,

lib/worker/form.js

@@ -8,6 +8,7 @@
 // except according to the terms contained in the LICENSE file.
 
 const { Form } = require('../model/frames');
+const { updateEntityForm } = require('../data/schema');
 
 const pushDraftToEnketo = ({ Forms }, event) =>
   Forms.getByActeeIdForUpdate(event.acteeId, undefined, Form.DraftVersion)
@@ -42,5 +43,33 @@ const create = pushDraftToEnketo;
 const updateDraftSet = pushDraftToEnketo;
 const updatePublish = pushFormToEnketo;
 
-module.exports = { create, updateDraftSet, updatePublish };
+const _upgradeEntityVersion = async (form) => {
+  const xml = await updateEntityForm(form.xml, '2023.1.0', '2024.1.0', '[upgrade]');
+  // If the XML doesnt change (not the version in question, or a parsing error), don't return the new partial Form
+  if (xml === form.xml)
+    return null;
+  const partial = await Form.fromXml(xml);
+  return partial.withAux('xls', { xlsBlobId: form.def.xlsBlobId });
+};
+
+const updateEntitiesVersion = async ({ Forms }, event) => {
+  const { projectId, xmlFormId } = await Forms.getByActeeIdForUpdate(event.acteeId).then(o => o.get());
+  const publishedVersion = await Forms.getByProjectAndXmlFormId(projectId, xmlFormId, true, Form.PublishedVersion).then(o => o.get());
+  if (publishedVersion.currentDefId != null) {
+    const partial = await _upgradeEntityVersion(publishedVersion);
+    if (partial != null) {
+      await Forms.createVersion(partial, publishedVersion, true, true);
+    }
+  }
+
+  const draftVersion = await Forms.getByProjectAndXmlFormId(projectId, xmlFormId, true, Form.DraftVersion).then(o => o.get());
+  if (draftVersion.draftDefId != null) {
+    const partial = await _upgradeEntityVersion(draftVersion);
+    // update xml and version in place
+    if (partial != null)
+      await Forms.replaceDef(partial, draftVersion, { upgrade: 'Updated entities-version in form draft to 2024.1' });
+  }
+};
+
+module.exports = { create, updateDraftSet, updatePublish, updateEntitiesVersion };
 

lib/worker/jobs.js

@@ -27,6 +27,8 @@ const jobs = {
   'upgrade.process.form.draft': [ require('./form').updateDraftSet ],
   'upgrade.process.form': [ require('./form').updatePublish ],
 
+  'upgrade.process.form.entities_version': [ require('./form').updateEntitiesVersion ],
+
   'dataset.update': [ require('./dataset').createEntitiesFromPendingSubmissions ]
 };