feat(swingset): support raw devices

"Raw devices" bypass the deviceSlots layer and allow device code direct
access to `syscall`, and the arguments arriving through the `dispatch` object
it must produce. This makes some patterns much easier to implement, such as
producing new device nodes as part of the device's API (e.g. one device node
per code bundle).

It also provides vatstoreGet/Set/Delete, so the device code can manage one
piece of state at a time, instead of doing an expensive read-modify-write
cycle on a single large aggregate state object.

A helper library named deviceTools.js was added to make it slightly easier to
write a raw device.

In the longer run (see #1346), we'd like these devices to support Promises
and plain object references. This change doesn't go that far. The remaining
limitations are:

* the deviceTools.js library refuses to handle exported objects, imported
foreign device nodes, or promises of any sort
* the outbound translator (deviceKeeper.js `mapDeviceSlotToKernelSlot`)
refuses to handle exported objects and exported promises
* the vat outbound translator (vatTranslator.js `translateCallNow`) refuses
to handle promises
* liveslots rejects promises in `D()` arguments

refs #1346

packages/SwingSet/docs/devices.md

@@ -255,6 +255,51 @@ However the inbound message pathway uses `dispatch.invoke(deviceNodeID,
 method, argsCapdata) -> resultCapdata`, and the outbound pathway uses
 `syscall.sendOnly`.
 
+## Raw Devices
+
+An alternate way to write a device is to use the "raw device API". In this
+mode, there is no deviceSlots layer, and no attempt to provide
+object-capability abstractions. Instead, the device code is given a `syscall`
+object, and is expected to provide a `dispatch` object, and everything else
+is left up to the device.
+
+This mode makes it possible to create new device nodes as part of the normal
+API, because the code can learn the raw device ref (dref) of the target
+device node on each inbound invocation, without needing a pre-registered
+table of JavaScript `Object` instances for every export.
+
+Raw devices have access to a per-device string/string key-value store whose
+API matches the `vatStore` available to vats:
+
+* `syscall.vatstoreGet(key)` -> `string`
+* `syscall.vatstoreSet(key, value)`
+* `syscall.vatstoreDelete(key)`
+
+The mode is enabled by exporting a function named `buildDevice` instead of
+`buildRootDeviceNode`.
+
+```js
+export function buildDevice(tools, endowments) {
+  const { syscall } = tools;
+  const dispatch = {
+    invoke: (dnid, method, argsCapdata) => {
+      ..
+    },
+  };
+  return dispatch;
+}
+```
+
+To make it easier to write a raw device, a helper library named "deviceTools"
+is available in `src/deviceTools.js`. This provides a marshalling layer that
+can parse the incoming `argsCapdata` into representations of different sorts
+of objects, and a reverse direction for serializing the returned results.
+Unlike liveslots and deviceslots, this library makes no attempt to present
+the parsed output as invokable objects. When it parses `o-4` into a
+"Presence", you cannot use `E()` on that presence. However, you can extract
+the `o-4` from it. The library is most useful for building the data structure
+of the return results without manual JSON hacking.
+
 ## Kernel Devices
 
 The kernel automatically configures devices for internal use. Most are paired

packages/SwingSet/src/deviceTools.js

@@ -0,0 +1,101 @@
+import { assert, details as X } from '@agoric/assert';
+import { makeMarshal, Far } from '@endo/marshal';
+import { parseVatSlot } from './parseVatSlots.js';
+
+// raw devices can use this to build a set of convenience tools for
+// serialization/unserialization
+
+export function buildSerializationTools(syscall, deviceName) {
+  // TODO: prevent our Presence/DeviceNode objects from being accidentally be
+  // marshal-serialized into persistent state
+
+  const presences = new WeakMap();
+  const myDeviceNodes = new WeakMap();
+
+  function slotFromPresence(p) {
+    return presences.get(p);
+  }
+  function presenceForSlot(slot) {
+    const { type, allocatedByVat } = parseVatSlot(slot);
+    assert.equal(type, 'object');
+    assert.equal(allocatedByVat, false);
+    const p = Far('presence', {
+      send(method, args) {
+        assert.typeof(method, 'string');
+        assert(Array.isArray(args), args);
+        // eslint-disable-next-line no-use-before-define
+        const capdata = serialize(args);
+        syscall.sendOnly(slot, method, capdata);
+      },
+    });
+    presences.set(p, slot);
+    return p;
+  }
+
+  function slotFromMyDeviceNode(dn) {
+    return myDeviceNodes.get(dn);
+  }
+  function deviceNodeForSlot(slot) {
+    const { type, allocatedByVat } = parseVatSlot(slot);
+    assert.equal(type, 'device');
+    assert.equal(allocatedByVat, true);
+    const dn = Far('device node', {});
+    myDeviceNodes.set(dn, slot);
+    return dn;
+  }
+
+  function convertSlotToVal(slot) {
+    const { type, allocatedByVat } = parseVatSlot(slot);
+    if (type === 'object') {
+      assert(!allocatedByVat, X`devices cannot yet allocate objects ${slot}`);
+      return presenceForSlot(slot);
+    } else if (type === 'device') {
+      assert(
+        allocatedByVat,
+        X`devices should yet not be given other devices '${slot}'`,
+      );
+      return deviceNodeForSlot(slot);
+    } else if (type === 'promise') {
+      assert.fail(X`devices should not yet be given promises '${slot}'`);
+    } else {
+      assert.fail(X`unrecognized slot type '${type}'`);
+    }
+  }
+
+  function convertValToSlot(val) {
+    const objSlot = slotFromPresence(val);
+    if (objSlot) {
+      return objSlot;
+    }
+    const devnodeSlot = slotFromMyDeviceNode(val);
+    if (devnodeSlot) {
+      return devnodeSlot;
+    }
+    throw Error(X`unable to convert value ${val}`);
+  }
+
+  const m = makeMarshal(convertValToSlot, convertSlotToVal, {
+    marshalName: `device:${deviceName}`,
+    // TODO Temporary hack.
+    // See https://github.com/Agoric/agoric-sdk/issues/2780
+    errorIdNum: 60000,
+  });
+
+  // for invoke(), these will unserialize the arguments, and serialize the
+  // response (into a vatresult with the 'ok' header)
+  const unserialize = capdata => m.unserialize(capdata);
+  const serialize = data => m.serialize(harden(data));
+  const returnFromInvoke = args => harden(['ok', serialize(args)]);
+
+  const tools = {
+    slotFromPresence,
+    presenceForSlot,
+    slotFromMyDeviceNode,
+    deviceNodeForSlot,
+    unserialize,
+    returnFromInvoke,
+  };
+
+  return harden(tools);
+}
+harden(buildSerializationTools);

packages/SwingSet/src/kernel/deviceManager.js

@@ -19,42 +19,64 @@ import '../types.js';
  * Produce an object that will serve as the kernel's handle onto a device.
  *
  * @param {string} deviceName  The device's name, for human readable diagnostics
- * @param {*} buildRootDeviceNode
+ * @param {*} deviceNamespace The module namespace object exported by the device bundle
  * @param {*} state  A get/set object for the device's persistent state
  * @param {Record<string, any>} endowments  The device's configured endowments
  * @param {*} testLog
  * @param {*} deviceParameters  Parameters from the device's config entry
+ * @param {*} deviceSyscallHandler
  */
 export default function makeDeviceManager(
   deviceName,
-  buildRootDeviceNode,
+  deviceNamespace,
   state,
   endowments,
   testLog,
   deviceParameters,
+  deviceSyscallHandler,
 ) {
-  let deviceSyscallHandler;
-  function setDeviceSyscallHandler(handler) {
-    deviceSyscallHandler = handler;
-  }
-
   const syscall = harden({
     sendOnly: (target, method, args) => {
       const dso = harden(['sendOnly', target, method, args]);
       deviceSyscallHandler(dso);
     },
+    vatstoreGet: key => {
+      const dso = harden(['vatstoreGet', key]);
+      return deviceSyscallHandler(dso);
+    },
+    vatstoreSet: (key, value) => {
+      const dso = harden(['vatstoreSet', key, value]);
+      deviceSyscallHandler(dso);
+    },
+    vatstoreDelete: key => {
+      const dso = harden(['vatstoreDelete', key]);
+      deviceSyscallHandler(dso);
+    },
   });
 
-  // Setting up the device runtime gives us back the device's dispatch object
-  const dispatch = makeDeviceSlots(
-    syscall,
-    state,
-    buildRootDeviceNode,
-    deviceName,
-    endowments,
-    testLog,
-    deviceParameters,
-  );
+  let dispatch;
+  if (typeof deviceNamespace.buildDevice === 'function') {
+    // raw device
+    const tools = { syscall };
+    // maybe add state utilities
+    dispatch = deviceNamespace.buildDevice(tools, endowments);
+  } else {
+    assert(
+      typeof deviceNamespace.buildRootDeviceNode === 'function',
+      `device ${deviceName} lacks buildRootDeviceNode`,
+    );
+
+    // Setting up the device runtime gives us back the device's dispatch object
+    dispatch = makeDeviceSlots(
+      syscall,
+      state,
+      deviceNamespace.buildRootDeviceNode,
+      deviceName,
+      endowments,
+      testLog,
+      deviceParameters,
+    );
+  }
 
   /**
    * Invoke a method on a device node.
@@ -84,7 +106,6 @@ export default function makeDeviceManager(
 
   const manager = {
     invoke,
-    setDeviceSyscallHandler,
   };
   return manager;
 }

packages/SwingSet/src/kernel/kernel.js

@@ -1019,52 +1019,6 @@ export default function buildKernel(
     return vatID;
   }
 
-  function buildDeviceManager(
-    deviceID,
-    name,
-    buildRootDeviceNode,
-    endowments,
-    deviceParameters,
-  ) {
-    const deviceKeeper = kernelKeeper.allocateDeviceKeeperIfNeeded(deviceID);
-    // Wrapper for state, to give to the device to access its state.
-    // Devices are allowed to get their state at startup, and set it anytime.
-    // They do not use orthogonal persistence or transcripts.
-    const state = harden({
-      get() {
-        return deviceKeeper.getDeviceState();
-      },
-      set(value) {
-        deviceKeeper.setDeviceState(value);
-      },
-    });
-    const manager = makeDeviceManager(
-      name,
-      buildRootDeviceNode,
-      state,
-      endowments,
-      testLog,
-      deviceParameters,
-    );
-    return manager;
-  }
-
-  // plug a new DeviceManager into the kernel
-  function addDeviceManager(deviceID, name, manager) {
-    const translators = makeDeviceTranslators(deviceID, name, kernelKeeper);
-    function deviceSyscallHandler(deviceSyscallObject) {
-      const ksc = translators.deviceSyscallToKernelSyscall(deviceSyscallObject);
-      // devices can only do syscall.sendOnly, which has no results
-      kernelSyscallHandler.doKernelSyscall(ksc);
-    }
-    manager.setDeviceSyscallHandler(deviceSyscallHandler);
-
-    ephemeral.devices.set(deviceID, {
-      translators,
-      manager,
-    });
-  }
-
   async function start() {
     if (started) {
       throw Error('kernel.start already called');
@@ -1102,24 +1056,47 @@ export default function buildKernel(
       assertKnownOptions(options, ['deviceParameters', 'unendowed']);
       const { deviceParameters = {}, unendowed } = options;
       const devConsole = makeConsole(`${debugPrefix}SwingSet:dev-${name}`);
+
       // eslint-disable-next-line no-await-in-loop
       const NS = await importBundle(source.bundle, {
         filePrefix: `dev-${name}/...`,
         endowments: harden({ ...vatEndowments, console: devConsole, assert }),
       });
-      assert(
-        typeof NS.buildRootDeviceNode === 'function',
-        `device ${name} lacks buildRootDeviceNode`,
-      );
+
       if (deviceEndowments[name] || unendowed) {
-        const manager = buildDeviceManager(
-          deviceID,
+        const translators = makeDeviceTranslators(deviceID, name, kernelKeeper);
+        function deviceSyscallHandler(deviceSyscallObject) {
+          const ksc = translators.deviceSyscallToKernelSyscall(
+            deviceSyscallObject,
+          );
+          const kres = kernelSyscallHandler.doKernelSyscall(ksc);
+          const dres = translators.kernelResultToDeviceResult(ksc[0], kres);
+          assert.equal(dres[0], 'ok');
+          return dres[1];
+        }
+
+        // Wrapper for state, to give to the device to access its state.
+        // Devices are allowed to get their state at startup, and set it anytime.
+        // They do not use orthogonal persistence or transcripts.
+        const state = harden({
+          get() {
+            return deviceKeeper.getDeviceState();
+          },
+          set(value) {
+            deviceKeeper.setDeviceState(value);
+          },
+        });
+
+        const manager = makeDeviceManager(
           name,
-          NS.buildRootDeviceNode,
+          NS,
+          state,
           deviceEndowments[name],
+          testLog,
           deviceParameters,
+          deviceSyscallHandler,
         );
-        addDeviceManager(deviceID, name, manager);
+        ephemeral.devices.set(deviceID, { translators, manager });
       } else {
         console.log(
           `WARNING: skipping device ${deviceID} (${name}) because it has no endowments`,