feat(xsnap): snapstore with compressed snapshots

packages/xsnap/package.json

@@ -7,7 +7,7 @@
   "parsers": {
     "js": "mjs"
   },
-  "main": "./src/xsnap.js",
+  "main": "./src/index.js",
   "bin": {
     "ava-xs": "./src/ava-xs.js",
     "xsrepl": "./src/xsrepl"

packages/xsnap/src/index.js

@@ -0,0 +1,2 @@
+export { xsnap } from './xsnap';
+export { makeSnapstore } from './snapStore';

packages/xsnap/src/snapStore.js

@@ -0,0 +1,129 @@
+// @ts-check
+import { createHash } from 'crypto';
+import { pipeline } from 'stream';
+import { createGzip, createGunzip } from 'zlib';
+import { assert, details as d } from '@agoric/assert';
+import { promisify } from 'util';
+
+const pipe = promisify(pipeline);
+
+const { freeze } = Object;
+
+/**
+ * @param {string} root
+ * @param {{
+ *   tmpName: typeof import('tmp').tmpName,
+ *   existsSync: typeof import('fs').existsSync
+ *   createReadStream: typeof import('fs').createReadStream,
+ *   createWriteStream: typeof import('fs').createWriteStream,
+ *   resolve: typeof import('path').resolve,
+ *   rename: typeof import('fs').promises.rename,
+ *   unlink: typeof import('fs').promises.unlink,
+ * }} io
+ */
+export function makeSnapstore(
+  root,
+  {
+    tmpName,
+    existsSync,
+    createReadStream,
+    createWriteStream,
+    resolve,
+    rename,
+    unlink,
+  },
+) {
+  /** @type {(opts: unknown) => Promise<string>} */
+  const ptmpName = promisify(tmpName);
+  const tmpOpts = { tmpdir: root, template: 'tmp-XXXXXX.xss' };
+  /**
+   * @param { (name: string) => Promise<T> } thunk
+   * @returns { Promise<T> }
+   * @template T
+   */
+  async function withTempName(thunk) {
+    const name = await ptmpName(tmpOpts);
+    let result;
+    try {
+      result = await thunk(name);
+    } finally {
+      try {
+        await unlink(name);
+      } catch (ignore) {
+        // ignore
+      }
+    }
+    return result;
+  }
+
+  /**
+   * @param {string} dest
+   * @param { (name: string) => Promise<T> } thunk
+   * @returns { Promise<T> }
+   * @template T
+   */
+  async function atomicWrite(dest, thunk) {
+    const tmp = await ptmpName(tmpOpts);
+    let result;
+    try {
+      result = await thunk(tmp);
+      await rename(tmp, resolve(root, dest));
+    } finally {
+      try {
+        await unlink(tmp);
+      } catch (ignore) {
+        // ignore
+      }
+    }
+    return result;
+  }
+
+  /** @type {(input: string, f: NodeJS.ReadWriteStream, output: string) => Promise<void>} */
+  async function filter(input, f, output) {
+    const source = createReadStream(input);
+    const destination = createWriteStream(output);
+    await pipe(source, f, destination);
+  }
+
+  /** @type {(filename: string) => Promise<string>} */
+  async function fileHash(filename) {
+    const hash = createHash('sha256');
+    const input = createReadStream(filename);
+    await pipe(input, hash);
+    return hash.digest('hex');
+  }
+
+  /**
+   * @param {(fn: string) => Promise<void>} saveRaw
+   * @returns { Promise<string> } sha256 hash of (uncompressed) snapshot
+   */
+  async function save(saveRaw) {
+    return withTempName(async snapFile => {
+      await saveRaw(snapFile);
+      const h = await fileHash(snapFile);
+      if (existsSync(`${h}.gz`)) return h;
+      await atomicWrite(`${h}.gz`, gztmp =>
+        filter(snapFile, createGzip(), gztmp),
+      );
+      return h;
+    });
+  }
+
+  /**
+   * @param {string} hash
+   * @param {(fn: string) => Promise<T>} loadRaw
+   * @template T
+   */
+  async function load(hash, loadRaw) {
+    return withTempName(async raw => {
+      await filter(resolve(root, `${hash}.gz`), createGunzip(), raw);
+      const actual = await fileHash(raw);
+      assert(actual === hash, d`actual hash ${actual} !== expected ${hash}`);
+      // be sure to await loadRaw before exiting withTempName
+      const result = await loadRaw(raw);
+      return result;
+    });
+  }
+
+  return freeze({ load, save });
+}

packages/xsnap/test/test-snapstore.js

@@ -0,0 +1,144 @@
+/* global __dirname, __filename */
+// @ts-check
+
+import '@agoric/install-ses';
+import { spawn } from 'child_process';
+import { type as osType } from 'os';
+import fs from 'fs';
+import path from 'path';
+
+// eslint-disable-next-line import/no-extraneous-dependencies
+import test from 'ava';
+// eslint-disable-next-line import/no-extraneous-dependencies
+import tmp from 'tmp';
+import { xsnap } from '../src/xsnap';
+import { makeSnapstore } from '../src/snapStore';
+
+const importModuleUrl = `file://${__filename}`;
+
+const asset = async (...segments) =>
+  fs.promises.readFile(
+    path.join(importModuleUrl.replace('file:/', ''), '..', ...segments),
+    'utf-8',
+  );
+
+/**
+ * @param {string} name
+ * @param {(request:Uint8Array) => Promise<Uint8Array>} handleCommand
+ */
+async function bootWorker(name, handleCommand) {
+  const worker = xsnap({
+    os: osType(),
+    spawn,
+    handleCommand,
+    name,
+    stdout: 'inherit',
+    stderr: 'inherit',
+    // debug: !!env.XSNAP_DEBUG,
+  });
+
+  const bootScript = await asset('..', 'dist', 'bundle-ses-boot.umd.js');
+  await worker.evaluate(bootScript);
+  return worker;
+}
+
+test('build temp file; compress to cache file', async t => {
+  const pool = path.resolve(__dirname, './fixture-snap-pool/');
+  await fs.promises.mkdir(pool, { recursive: true });
+  const store = makeSnapstore(pool, {
+    ...tmp,
+    ...path,
+    ...fs,
+    ...fs.promises,
+  });
+  let keepTmp = '';
+  const hash = await store.save(async fn => {
+    t.falsy(fs.existsSync(fn));
+    fs.writeFileSync(fn, 'abc');
+    keepTmp = fn;
+  });
+  t.is(
+    'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad',
+    hash,
+  );
+  t.falsy(
+    fs.existsSync(keepTmp),
+    'temp file should have been deleted after withTempName',
+  );
+  const dest = path.resolve(pool, `${hash}.gz`);
+  t.truthy(fs.existsSync(dest));
+  const gz = fs.readFileSync(dest);
+  t.is(gz.toString('hex'), '1f8b08000000000000034b4c4a0600c241243503000000');
+});
+
+test('bootstrap, save, compress', async t => {
+  const vat = await bootWorker('ses-boot1', async m => m);
+  t.teardown(() => vat.close());
+
+  const pool = path.resolve(__dirname, './fixture-snap-pool/');
+  await fs.promises.mkdir(pool, { recursive: true });
+
+  const store = makeSnapstore(pool, {
+    ...tmp,
+    ...path,
+    ...fs,
+    ...fs.promises,
+  });
+
+  await vat.evaluate('globalThis.x = harden({a: 1})');
+
+  /** @type {(fn: string) => number} */
+  const Kb = fn => Math.round(fs.statSync(fn).size / 1024);
+  /** @type {(fn: string, fullSize: number) => number} */
+  const relativeSize = (fn, fullSize) =>
+    Math.round((fs.statSync(fn).size / 1024 / fullSize) * 10) / 10;
+
+  const snapSize = {
+    raw: 857,
+    compression: 0.1,
+  };
+
+  const h = await store.save(async snapFile => {
+    await vat.snapshot(snapFile);
+    t.is(snapSize.raw, Kb(snapFile), 'raw snapshots are large-ish');
+  });
+
+  const zfile = path.resolve(pool, `${h}.gz`);
+  t.is(
+    relativeSize(zfile, snapSize.raw),
+    snapSize.compression,
+    'compressed snapshots are smaller',
+  );
+});
+
+test('create, save, restore, resume', async t => {
+  const pool = path.resolve(__dirname, './fixture-snap-pool/');
+  await fs.promises.mkdir(pool, { recursive: true });
+
+  const store = makeSnapstore(pool, {
+    ...tmp,
+    ...path,
+    ...fs,
+    ...fs.promises,
+  });
+
+  const vat0 = await bootWorker('ses-boot2', async m => m);
+  t.teardown(() => vat0.close());
+  await vat0.evaluate('globalThis.x = harden({a: 1})');
+  const h = await store.save(vat0.snapshot);
+
+  const worker = await store.load(h, async snapshot => {
+    const xs = xsnap({ name: 'ses-resume', snapshot, os: osType(), spawn });
+    await xs.evaluate('0');
+    return xs;
+  });
+  t.teardown(() => worker.close());
+  await worker.evaluate('x.a');
+  t.pass();
+});
+
+// see https://github.com/Agoric/agoric-sdk/issues/2776
+test.failing('xs snapshots should be deterministic', t => {
+  const h = 'abc';
+  t.is('66244b4bfe92ae9138d24a9b50b492d231f6a346db0cf63543d200860b423724', h);
+});