diff --git a/packages/SwingSet/docs/devices.md b/packages/SwingSet/docs/devices.md index f53e923dc99..fa67c55f6a7 100644 --- a/packages/SwingSet/docs/devices.md +++ b/packages/SwingSet/docs/devices.md @@ -255,6 +255,51 @@ However the inbound message pathway uses `dispatch.invoke(deviceNodeID, method, argsCapdata) -> resultCapdata`, and the outbound pathway uses `syscall.sendOnly`. +## Raw Devices + +An alternate way to write a device is to use the "raw device API". In this +mode, there is no deviceSlots layer, and no attempt to provide +object-capability abstractions. Instead, the device code is given a `syscall` +object, and is expected to provide a `dispatch` object, and everything else +is left up to the device. + +This mode makes it possible to create new device nodes as part of the normal +API, because the code can learn the raw device ref (dref) of the target +device node on each inbound invocation, without needing a pre-registered +table of JavaScript `Object` instances for every export. + +Raw devices have access to a per-device string/string key-value store whose +API matches the `vatStore` available to vats: + +* `syscall.vatstoreGet(key)` -> `string` +* `syscall.vatstoreSet(key, value)` +* `syscall.vatstoreDelete(key)` + +The mode is enabled by exporting a function named `buildDevice` instead of +`buildRootDeviceNode`. + +```js +export function buildDevice(tools, endowments) { + const { syscall } = tools; + const dispatch = { + invoke: (dnid, method, argsCapdata) => { + .. + }, + }; + return dispatch; +} +``` + +To make it easier to write a raw device, a helper library named "deviceTools" +is available in `src/deviceTools.js`. This provides a marshalling layer that +can parse the incoming `argsCapdata` into representations of different sorts +of objects, and a reverse direction for serializing the returned results. +Unlike liveslots and deviceslots, this library makes no attempt to present +the parsed output as invokable objects. When it parses `o-4` into a +"Presence", you cannot use `E()` on that presence. However, you can extract +the `o-4` from it. The library is most useful for building the data structure +of the return results without manual JSON hacking. + ## Kernel Devices The kernel automatically configures devices for internal use. Most are paired diff --git a/packages/SwingSet/src/deviceTools.js b/packages/SwingSet/src/deviceTools.js new file mode 100644 index 00000000000..b347272c3ef --- /dev/null +++ b/packages/SwingSet/src/deviceTools.js @@ -0,0 +1,101 @@ +import { assert, details as X } from '@agoric/assert'; +import { makeMarshal, Far } from '@endo/marshal'; +import { parseVatSlot } from './parseVatSlots.js'; + +// raw devices can use this to build a set of convenience tools for +// serialization/unserialization + +export function buildSerializationTools(syscall, deviceName) { + // TODO: prevent our Presence/DeviceNode objects from being accidentally be + // marshal-serialized into persistent state + + const presences = new WeakMap(); + const myDeviceNodes = new WeakMap(); + + function slotFromPresence(p) { + return presences.get(p); + } + function presenceForSlot(slot) { + const { type, allocatedByVat } = parseVatSlot(slot); + assert.equal(type, 'object'); + assert.equal(allocatedByVat, false); + const p = Far('presence', { + send(method, args) { + assert.typeof(method, 'string'); + assert(Array.isArray(args), args); + // eslint-disable-next-line no-use-before-define + const capdata = serialize(args); + syscall.sendOnly(slot, method, capdata); + }, + }); + presences.set(p, slot); + return p; + } + + function slotFromMyDeviceNode(dn) { + return myDeviceNodes.get(dn); + } + function deviceNodeForSlot(slot) { + const { type, allocatedByVat } = parseVatSlot(slot); + assert.equal(type, 'device'); + assert.equal(allocatedByVat, true); + const dn = Far('device node', {}); + myDeviceNodes.set(dn, slot); + return dn; + } + + function convertSlotToVal(slot) { + const { type, allocatedByVat } = parseVatSlot(slot); + if (type === 'object') { + assert(!allocatedByVat, X`devices cannot yet allocate objects ${slot}`); + return presenceForSlot(slot); + } else if (type === 'device') { + assert( + allocatedByVat, + X`devices should yet not be given other devices '${slot}'`, + ); + return deviceNodeForSlot(slot); + } else if (type === 'promise') { + assert.fail(X`devices should not yet be given promises '${slot}'`); + } else { + assert.fail(X`unrecognized slot type '${type}'`); + } + } + + function convertValToSlot(val) { + const objSlot = slotFromPresence(val); + if (objSlot) { + return objSlot; + } + const devnodeSlot = slotFromMyDeviceNode(val); + if (devnodeSlot) { + return devnodeSlot; + } + throw Error(X`unable to convert value ${val}`); + } + + const m = makeMarshal(convertValToSlot, convertSlotToVal, { + marshalName: `device:${deviceName}`, + // TODO Temporary hack. + // See https://github.com/Agoric/agoric-sdk/issues/2780 + errorIdNum: 60000, + }); + + // for invoke(), these will unserialize the arguments, and serialize the + // response (into a vatresult with the 'ok' header) + const unserialize = capdata => m.unserialize(capdata); + const serialize = data => m.serialize(harden(data)); + const returnFromInvoke = args => harden(['ok', serialize(args)]); + + const tools = { + slotFromPresence, + presenceForSlot, + slotFromMyDeviceNode, + deviceNodeForSlot, + unserialize, + returnFromInvoke, + }; + + return harden(tools); +} +harden(buildSerializationTools); diff --git a/packages/SwingSet/src/kernel/deviceManager.js b/packages/SwingSet/src/kernel/deviceManager.js index f4c02681430..21e2121b9e6 100644 --- a/packages/SwingSet/src/kernel/deviceManager.js +++ b/packages/SwingSet/src/kernel/deviceManager.js @@ -19,42 +19,64 @@ import '../types.js'; * Produce an object that will serve as the kernel's handle onto a device. * * @param {string} deviceName The device's name, for human readable diagnostics - * @param {*} buildRootDeviceNode + * @param {*} deviceNamespace The module namespace object exported by the device bundle * @param {*} state A get/set object for the device's persistent state * @param {Record} endowments The device's configured endowments * @param {*} testLog * @param {*} deviceParameters Parameters from the device's config entry + * @param {*} deviceSyscallHandler */ export default function makeDeviceManager( deviceName, - buildRootDeviceNode, + deviceNamespace, state, endowments, testLog, deviceParameters, + deviceSyscallHandler, ) { - let deviceSyscallHandler; - function setDeviceSyscallHandler(handler) { - deviceSyscallHandler = handler; - } - const syscall = harden({ sendOnly: (target, method, args) => { const dso = harden(['sendOnly', target, method, args]); deviceSyscallHandler(dso); }, + vatstoreGet: key => { + const dso = harden(['vatstoreGet', key]); + return deviceSyscallHandler(dso); + }, + vatstoreSet: (key, value) => { + const dso = harden(['vatstoreSet', key, value]); + deviceSyscallHandler(dso); + }, + vatstoreDelete: key => { + const dso = harden(['vatstoreDelete', key]); + deviceSyscallHandler(dso); + }, }); - // Setting up the device runtime gives us back the device's dispatch object - const dispatch = makeDeviceSlots( - syscall, - state, - buildRootDeviceNode, - deviceName, - endowments, - testLog, - deviceParameters, - ); + let dispatch; + if (typeof deviceNamespace.buildDevice === 'function') { + // raw device + const tools = { syscall }; + // maybe add state utilities + dispatch = deviceNamespace.buildDevice(tools, endowments); + } else { + assert( + typeof deviceNamespace.buildRootDeviceNode === 'function', + `device ${deviceName} lacks buildRootDeviceNode`, + ); + + // Setting up the device runtime gives us back the device's dispatch object + dispatch = makeDeviceSlots( + syscall, + state, + deviceNamespace.buildRootDeviceNode, + deviceName, + endowments, + testLog, + deviceParameters, + ); + } /** * Invoke a method on a device node. @@ -84,7 +106,6 @@ export default function makeDeviceManager( const manager = { invoke, - setDeviceSyscallHandler, }; return manager; } diff --git a/packages/SwingSet/src/kernel/kernel.js b/packages/SwingSet/src/kernel/kernel.js index 9501b9afc25..c23709589a2 100644 --- a/packages/SwingSet/src/kernel/kernel.js +++ b/packages/SwingSet/src/kernel/kernel.js @@ -1019,52 +1019,6 @@ export default function buildKernel( return vatID; } - function buildDeviceManager( - deviceID, - name, - buildRootDeviceNode, - endowments, - deviceParameters, - ) { - const deviceKeeper = kernelKeeper.allocateDeviceKeeperIfNeeded(deviceID); - // Wrapper for state, to give to the device to access its state. - // Devices are allowed to get their state at startup, and set it anytime. - // They do not use orthogonal persistence or transcripts. - const state = harden({ - get() { - return deviceKeeper.getDeviceState(); - }, - set(value) { - deviceKeeper.setDeviceState(value); - }, - }); - const manager = makeDeviceManager( - name, - buildRootDeviceNode, - state, - endowments, - testLog, - deviceParameters, - ); - return manager; - } - - // plug a new DeviceManager into the kernel - function addDeviceManager(deviceID, name, manager) { - const translators = makeDeviceTranslators(deviceID, name, kernelKeeper); - function deviceSyscallHandler(deviceSyscallObject) { - const ksc = translators.deviceSyscallToKernelSyscall(deviceSyscallObject); - // devices can only do syscall.sendOnly, which has no results - kernelSyscallHandler.doKernelSyscall(ksc); - } - manager.setDeviceSyscallHandler(deviceSyscallHandler); - - ephemeral.devices.set(deviceID, { - translators, - manager, - }); - } - async function start() { if (started) { throw Error('kernel.start already called'); @@ -1102,24 +1056,47 @@ export default function buildKernel( assertKnownOptions(options, ['deviceParameters', 'unendowed']); const { deviceParameters = {}, unendowed } = options; const devConsole = makeConsole(`${debugPrefix}SwingSet:dev-${name}`); + // eslint-disable-next-line no-await-in-loop const NS = await importBundle(source.bundle, { filePrefix: `dev-${name}/...`, endowments: harden({ ...vatEndowments, console: devConsole, assert }), }); - assert( - typeof NS.buildRootDeviceNode === 'function', - `device ${name} lacks buildRootDeviceNode`, - ); + if (deviceEndowments[name] || unendowed) { - const manager = buildDeviceManager( - deviceID, + const translators = makeDeviceTranslators(deviceID, name, kernelKeeper); + function deviceSyscallHandler(deviceSyscallObject) { + const ksc = translators.deviceSyscallToKernelSyscall( + deviceSyscallObject, + ); + const kres = kernelSyscallHandler.doKernelSyscall(ksc); + const dres = translators.kernelResultToDeviceResult(ksc[0], kres); + assert.equal(dres[0], 'ok'); + return dres[1]; + } + + // Wrapper for state, to give to the device to access its state. + // Devices are allowed to get their state at startup, and set it anytime. + // They do not use orthogonal persistence or transcripts. + const state = harden({ + get() { + return deviceKeeper.getDeviceState(); + }, + set(value) { + deviceKeeper.setDeviceState(value); + }, + }); + + const manager = makeDeviceManager( name, - NS.buildRootDeviceNode, + NS, + state, deviceEndowments[name], + testLog, deviceParameters, + deviceSyscallHandler, ); - addDeviceManager(deviceID, name, manager); + ephemeral.devices.set(deviceID, { translators, manager }); } else { console.log( `WARNING: skipping device ${deviceID} (${name}) because it has no endowments`, diff --git a/packages/SwingSet/test/devices/bootstrap-raw.js b/packages/SwingSet/test/devices/bootstrap-raw.js new file mode 100644 index 00000000000..310ee97b8af --- /dev/null +++ b/packages/SwingSet/test/devices/bootstrap-raw.js @@ -0,0 +1,66 @@ +import { Far } from '@endo/marshal'; +import { makePromiseKit } from '@agoric/promise-kit'; + +export function buildRootObject(vatPowers, _vatParameters) { + const { D } = vatPowers; + let devices; + return Far('root', { + bootstrap(vats, d0) { + devices = d0; + }, + + step1() { + return D(devices.dr).one(harden({ toPush: 'pushed', x: 4 })); + }, + + async step2() { + const pk1 = makePromiseKit(); + const pk2 = makePromiseKit(); + const got = []; + // give the device an object to return and do sendOnly + const target = Far('target', { + ping1(hello, p1) { + got.push(hello); // should be 'hi ping1' + got.push(p1 === target); + pk1.resolve(); + }, + ping2(hello, p2) { + got.push(hello); // should be 'hi ping2' + got.push(p2 === target); + pk2.resolve(); + }, + }); + const ret = D(devices.dr).two(target); // ['got', target] + got.push(ret[0]); + got.push(ret[1] === target); + await pk1.promise; + await pk2.promise; + return got; // ['got', true, 'hi ping1', true, 'hi ping2', true] + }, + + step3() { + const { dn1, dn2 } = D(devices.dr).three(); // returns new device nodes + const ret1 = D(dn1).threeplus(21, dn1, dn2); // ['dn1', 21, true, true] + const ret2 = D(dn2).threeplus(22, dn1, dn2); // ['dn2', 22, true, true] + return [ret1, ret2]; + }, + + step4() { + const got1 = D(devices.dr).fourGet(); + D(devices.dr).fourSet('value1'); + const got2 = D(devices.dr).fourGet(); + D(devices.dr).fourDelete(); + const got3 = D(devices.dr).fourGet(); + return [got1, got2, got3]; + }, + + step5() { + try { + D(devices.dr).fiveThrow(); + return false; + } catch (e) { + return e; + } + }, + }); +} diff --git a/packages/SwingSet/test/devices/device-raw-0.js b/packages/SwingSet/test/devices/device-raw-0.js new file mode 100644 index 00000000000..b81623b7ad1 --- /dev/null +++ b/packages/SwingSet/test/devices/device-raw-0.js @@ -0,0 +1,103 @@ +import { assert } from '@agoric/assert'; +import { buildSerializationTools } from '../../src/deviceTools.js'; + +export function buildDevice(tools, endowments) { + const { syscall } = tools; + const dtools = buildSerializationTools(syscall, 'dr0'); + const { unserialize, returnFromInvoke } = dtools; + const { slotFromPresence, presenceForSlot } = dtools; + const { deviceNodeForSlot, slotFromMyDeviceNode } = dtools; + + const ROOT = 'd+0'; + const DN1SLOT = 'd+1'; + const DN2SLOT = 'd+2'; + + // invoke() should use unserialize() and returnFromInvoke + // throwing errors or returning undefined will crash the kernel + + const dispatch = { + invoke: (dnid, method, argsCapdata) => { + const args = unserialize(argsCapdata); + + if (dnid === ROOT) { + if (method === 'one') { + // exercise basic invocation, args, return value + endowments.shared.push(args[0].toPush); + // need iserialize to make ['ok', capdata] + return returnFromInvoke({ a: args[0].x, b: [5, 6] }); + } + if (method === 'two') { + // exercise Presences, send + const pres1 = args[0]; + const slot1 = slotFromPresence(pres1); // should be o-1 + pres1.send('ping1', ['hi ping1', pres1]); + const pres2 = presenceForSlot(slot1); + pres2.send('ping2', ['hi ping2', pres2]); + return returnFromInvoke(['got', pres1]); + } + if (method === 'three') { + // create new device nodes + const dn1 = deviceNodeForSlot(DN1SLOT); + const dn2 = deviceNodeForSlot(DN2SLOT); + return returnFromInvoke({ dn1, dn2 }); + } + + // manage state through vatStore + if (method === 'fourGet') { + return returnFromInvoke([ + syscall.vatstoreGet('key1'), + syscall.vatstoreGet('key2'), + ]); + } + if (method === 'fourSet') { + assert.typeof(args[0], 'string'); + syscall.vatstoreSet('key1', args[0]); + return returnFromInvoke(); + } + if (method === 'fourDelete') { + syscall.vatstoreDelete('key1'); + return returnFromInvoke(); + } + + if (method === 'fiveThrow') { + throw Error('intentional device error'); + } + + throw TypeError(`target[${method}] does not exist`); + } + + if (dnid === DN1SLOT) { + // exercise new device nodes + if (method === 'threeplus') { + const [num, dn1b, dn2b] = args; + const ret = [ + 'dn1', + num, + slotFromMyDeviceNode(dn1b) === DN1SLOT, + slotFromMyDeviceNode(dn2b) === DN2SLOT, + ]; + return returnFromInvoke(ret); + } + throw TypeError(`dn1[${method}] does not exist`); + } + + if (dnid === DN2SLOT) { + // exercise new device nodes + if (method === 'threeplus') { + const [num, dn1b, dn2b] = args; + const ret = [ + 'dn2', + num, + slotFromMyDeviceNode(dn1b) === DN1SLOT, + slotFromMyDeviceNode(dn2b) === DN2SLOT, + ]; + return returnFromInvoke(ret); + } + throw TypeError(`dn2[${method}] does not exist`); + } + + throw TypeError(`target does not exist`); + }, + }; + return dispatch; +} diff --git a/packages/SwingSet/test/devices/test-raw-device.js b/packages/SwingSet/test/devices/test-raw-device.js new file mode 100644 index 00000000000..abf79f637a8 --- /dev/null +++ b/packages/SwingSet/test/devices/test-raw-device.js @@ -0,0 +1,98 @@ +// eslint-disable-next-line import/order +import { test } from '../../tools/prepare-test-env-ava.js'; + +// eslint-disable-next-line import/order +import bundleSource from '@endo/bundle-source'; +import { parse } from '@endo/marshal'; +import { provideHostStorage } from '../../src/hostStorage.js'; + +import { + initializeSwingset, + makeSwingsetController, + buildKernelBundles, +} from '../../src/index.js'; +import { capargs } from '../util.js'; + +function dfile(name) { + return new URL(`./${name}`, import.meta.url).pathname; +} + +test.before(async t => { + const kernelBundles = await buildKernelBundles(); + const bootstrapRaw = await bundleSource(dfile('bootstrap-raw.js')); + t.context.data = { + kernelBundles, + bootstrapRaw, + }; +}); + +test('d1', async t => { + const sharedArray = []; + const config = { + bootstrap: 'bootstrap', + defaultReapInterval: 'never', + vats: { + bootstrap: { + bundle: t.context.data.bootstrapRaw, + }, + }, + devices: { + dr: { + sourceSpec: dfile('device-raw-0'), + }, + }, + }; + const deviceEndowments = { + dr: { + shared: sharedArray, + }, + }; + + const hostStorage = provideHostStorage(); + await initializeSwingset(config, [], hostStorage, t.context.data); + const c = await makeSwingsetController(hostStorage, deviceEndowments); + c.pinVatRoot('bootstrap'); + await c.run(); + + // first, exercise plain arguments and return values + const r1 = c.queueToVatRoot('bootstrap', 'step1', capargs([])); + await c.run(); + t.deepEqual(JSON.parse(c.kpResolution(r1).body), { a: 4, b: [5, 6] }); + t.deepEqual(sharedArray, ['pushed']); + sharedArray.length = 0; + + // exercise giving objects to devices, getting them back, and the device's + // ability to do sendOnly to those objects + const r2 = c.queueToVatRoot('bootstrap', 'step2', capargs([])); + await c.run(); + const expected2 = ['got', true, 'hi ping1', true, 'hi ping2', true]; + t.deepEqual(JSON.parse(c.kpResolution(r2).body), expected2); + + // create and pass around new device nodes + const r3 = c.queueToVatRoot('bootstrap', 'step3', capargs([])); + await c.run(); + const expected3 = [ + ['dn1', 21, true, true], + ['dn2', 22, true, true], + ]; + t.deepEqual(JSON.parse(c.kpResolution(r3).body), expected3); + + // check that devices can manage state through vatstore + const r4 = c.queueToVatRoot('bootstrap', 'step4', capargs([])); + await c.run(); + const expected4 = [ + [undefined, undefined], + ['value1', undefined], + [undefined, undefined], + ]; + t.deepEqual(parse(c.kpResolution(r4).body), expected4); + + // check that device exceptions do not kill the device, calling vat, or kernel + const r5 = c.queueToVatRoot('bootstrap', 'step5', capargs([])); + await c.run(); + // body: '{"@qclass":"error","errorId":"error:liveSlots:v1#70001","message":"syscall.callNow failed: device.invoke failed, see logs for details","name":"Error"}', + const expected5 = Error( + 'syscall.callNow failed: device.invoke failed, see logs for details', + ); + t.deepEqual(parse(c.kpResolution(r5).body), expected5); +});