feat: integration of js-ipfs-repo-migrations

Integration of js-ipfs-repo-migrations brings automatic repo migrations
to ipfs-repo (both in-browser and fs). It is possible to control the
automatic migration using either config's setting
'repoDisableAutoMigration' or IPFSRepo's option 'disableAutoMigration'.

License: MIT
Signed-off-by: Adam Uhlir <adam@uhlir.dev>

README.md

@@ -28,6 +28,7 @@ This is the implementation of the [IPFS repo spec](https://github.com/ipfs/specs
   - [Use in a browser Using a script tag](#use-in-a-browser-using-a-script-tag)
 - [Usage](#usage)
 - [API](#api)
+- [Notes](#notes)
 - [Contribute](#contribute)
 - [License](#license)
 
@@ -318,6 +319,11 @@ Returned promise resolves to a `boolean` indicating the existence of the lock.
 
 - [Explanation of how repo is structured](https://github.com/ipfs/js-ipfs-repo/pull/111#issuecomment-279948247)
 
+### Migrations
+
+When there is a new repo migration and the version of repo is increased, don't 
+forget to propagate the changes into the test repo (`test/test-repo`).
+
 ## Contribute
 
 There are some ways you can make this module better:

package.json

@@ -51,7 +51,8 @@
     "multihashes": "~0.4.14",
     "multihashing-async": "~0.7.0",
     "ncp": "^2.0.0",
-    "rimraf": "^2.6.3"
+    "rimraf": "^2.6.3",
+    "sinon": "^7.3.1"
   },
   "dependencies": {
     "base32.js": "~0.1.0",
@@ -64,6 +65,7 @@
     "err-code": "^1.1.2",
     "interface-datastore": "~0.7.0",
     "ipfs-block": "~0.8.1",
+    "ipfs-repo-migrations": "AuHau/js-ipfs-repo-migrations#dev",
     "just-safe-get": "^1.3.0",
     "just-safe-set": "^2.1.0",
     "lodash.has": "^4.5.2",
@@ -73,6 +75,7 @@
   },
   "license": "MIT",
   "contributors": [
+    "Adam Uhlir<adam@uhlir.dev>",
     "Alan Shaw <alan.shaw@protocol.ai>",
     "Alex Potsides <alex@achingbrain.net>",
     "Brian Hoffman <hoffmabc@users.noreply.github.com>",

src/default-options-browser.js

@@ -2,6 +2,7 @@
 
 // Default configuration for a repo in the browser
 module.exports = {
+  disableAutoMigration: false,
   lock: 'memory',
   storageBackends: {
     root: require('datastore-level'),

src/default-options.js

@@ -2,6 +2,7 @@
 
 // Default configuration for a repo in node.js
 module.exports = {
+  disableAutoMigration: false,
   lock: 'fs',
   storageBackends: {
     root: require('datastore-fs'),

src/errors/index.js

@@ -15,6 +15,21 @@ class NotFoundError extends Error {
 NotFoundError.code = 'ERR_NOT_FOUND'
 exports.NotFoundError = NotFoundError
 
+/**
+ * Error raised when version of the stored repo is not compatible with version of this package.
+ */
+class InvalidRepoVersionError extends Error {
+  constructor (message) {
+    super(message)
+    this.name = 'InvalidRepoVersionError'
+    this.code = 'ERR_INVALID_REPO_VERSION'
+    this.message = message
+  }
+}
+
+InvalidRepoVersionError.code = 'ERR_INVALID_REPO_VERSION'
+exports.InvalidRepoVersionError = InvalidRepoVersionError
+
 exports.ERR_REPO_NOT_INITIALIZED = 'ERR_REPO_NOT_INITIALIZED'
 exports.ERR_REPO_ALREADY_OPEN = 'ERR_REPO_ALREADY_OPEN'
 exports.ERR_REPO_ALREADY_CLOSED = 'ERR_REPO_ALREADY_CLOSED'

src/index.js

@@ -6,7 +6,9 @@ const path = require('path')
 const debug = require('debug')
 const Big = require('bignumber.js')
 const errcode = require('err-code')
+const migrator = require('ipfs-repo-migrations')
 
+const constants = require('./constants')
 const backends = require('./backends')
 const version = require('./version')
 const config = require('./config')
@@ -26,7 +28,6 @@ const lockers = {
   fs: require('./lock')
 }
 
-const repoVersion = require('./constants').repoVersion
 
 /**
  * IpfsRepo implements all required functionality to read and write to an ipfs repo.
@@ -64,7 +65,7 @@ class IpfsRepo {
     await this._openRoot()
     await this.config.set(buildConfig(config))
     await this.spec.set(buildDatastoreSpec(config))
-    await this.version.set(repoVersion)
+    await this.version.set(constants.repoVersion)
   }
 
   /**
@@ -92,6 +93,17 @@ class IpfsRepo {
       this.blocks = await blockstore(blocksBaseStore, this.options.storageBackendOptions.blocks)
       log('creating keystore')
       this.keys = backends.create('keys', path.join(this.path, 'keys'), this.options)
+
+      if (!await this.version.check(constants.repoVersion)) {
+        log('Something is fishy')
+        if (!this.options.disableAutoMigration) {
+          log('Let see what')
+          await this._migrate(constants.repoVersion)
+        } else {
+          throw new ERRORS.InvalidRepoVersionError('Incompatible repo versions. Automatic migrations disabled. Please migrate the repo manually.')
+        }
+      }
+
       this.closed = false
       log('all opened')
     } catch (err) {
@@ -176,7 +188,7 @@ class IpfsRepo {
       [config] = await Promise.all([
         this.config.exists(),
         this.spec.exists(),
-        this.version.check(repoVersion)
+        this.version.exists()
       ])
     } catch (err) {
       if (err.code === 'ERR_NOT_FOUND') {
@@ -239,7 +251,7 @@ class IpfsRepo {
    * @return {Object}
    */
   async stat (options) {
-    options = Object.assign({}, { human: false }, options)
+    options = Object.assign({}, {human: false}, options)
     let storageMax, blocks, version, datastore, keys
     [storageMax, blocks, version, datastore, keys] = await Promise.all([
       this._storageMaxStat(),
@@ -264,6 +276,40 @@ class IpfsRepo {
     }
   }
 
+  async _migrate (toVersion) {
+    let disableMigrationsConfig
+    try {
+      disableMigrationsConfig = await this.config.get('repoDisableAutoMigration')
+    } catch (e) {
+      if (e.code === ERRORS.NotFoundError.code) {
+        disableMigrationsConfig = false
+      } else {
+        throw e
+      }
+    }
+
+    if (disableMigrationsConfig) {
+      throw new ERRORS.InvalidRepoVersionError('Incompatible repo versions. Automatic migrations disabled. Please migrate the repo manually.')
+    }
+
+    const currentRepoVersion = await this.version.get()
+    log(currentRepoVersion)
+    if (currentRepoVersion >= toVersion) {
+      if (currentRepoVersion > toVersion) {
+        log('Your repo\'s version is higher then this version of js-ipfs-repo require! You should revert it.')
+      }
+
+      log('Nothing to migrate')
+      return
+    }
+
+    if (toVersion > migrator.getLatestMigrationVersion()) {
+      throw new Error('The ipfs-repo-migrations package does not have migration for version: ' + toVersion)
+    }
+
+    return migrator.migrate(this.path, {toVersion: toVersion, ignoreLock: true, repoOptions: this.options})
+  }
+
   async _storageMaxStat () {
     try {
       const max = await this.config.get('Datastore.StorageMax')
@@ -284,7 +330,7 @@ class IpfsRepo {
         .plus(block.key._buf.byteLength)
     }
 
-    return { count, size }
+    return {count, size}
   }
 }
 
@@ -298,7 +344,7 @@ async function getSize (queryFn) {
 }
 
 module.exports = IpfsRepo
-module.exports.repoVersion = repoVersion
+module.exports.repoVersion = constants.repoVersion
 module.exports.errors = ERRORS
 
 function buildOptions (_options) {

src/version.js

@@ -36,9 +36,9 @@ module.exports = (store) => {
       return store.put(versionKey, Buffer.from(String(version)))
     },
     /**
-     * Check the current version, and return an error on missmatch
+     * Check the current version, and returns true if versions matches
      * @param {number} expected
-     * @returns {void}
+     * @returns {boolean}
      */
     async check (expected) {
       const version = await this.get()
@@ -47,9 +47,7 @@ module.exports = (store) => {
       // TODO: Clean up the compatibility logic. Repo feature detection would be ideal, or a better version schema
       const compatibleVersion = (version === 6 && expected === 7) || (expected === 6 && version === 7)
 
-      if (version !== expected && !compatibleVersion) {
-        throw errcode(new Error(`ipfs repo needs migration: expected version v${expected}, found version v${version}`), 'ERR_INVALID_REPO_VERSION')
-      }
+      return version === expected || compatibleVersion
     }
   }
 }

test/browser.js

@@ -4,8 +4,29 @@
 
 const IPFSRepo = require('../src')
 
+
+async function createTempRepo ({dontOpen, opts}) {
+  const date = Date.now().toString()
+  const repoPath = 'test-repo-for-' + date
+
+  const repo = new IPFSRepo(repoPath, opts)
+  await repo.init({})
+
+  if (!dontOpen) {
+    await repo.open()
+  }
+
+  return {
+    path: repoPath,
+    instance: repo,
+    teardown: async () => {}
+  }
+}
+
 describe('IPFS Repo Tests on the Browser', () => {
   require('./options-test')
+  require('./migrations-test')(createTempRepo)
+
   const repo = new IPFSRepo('myrepo')
 
   before(async () => {