From a128e93803344d8a36140d53d3e7711bec5c2511 Mon Sep 17 00:00:00 2001 From: Chip Morningstar Date: Fri, 28 May 2021 00:46:42 -0700 Subject: [PATCH] feat: move transcripts out of key-value store and into stream stores --- .../bin/extract-transcript-from-kerneldb.js | 2 + .../SwingSet/src/kernel/initializeKernel.js | 4 +- packages/SwingSet/src/kernel/kernel.js | 8 +- .../SwingSet/src/kernel/state/kernelKeeper.js | 9 +- .../SwingSet/src/kernel/state/vatKeeper.js | 58 +++++--- packages/SwingSet/src/types.js | 2 +- .../test/metering/test-dynamic-vat-metered.js | 2 +- packages/SwingSet/test/test-clist.js | 5 +- packages/SwingSet/test/test-devices.js | 2 +- packages/SwingSet/test/test-gc-transcript.js | 1 + packages/SwingSet/test/test-state.js | 127 +++++++++++------- .../SwingSet/test/test-transcript-light.js | 55 ++++---- packages/SwingSet/test/test-transcript.js | 14 +- .../vat-admin/terminate/test-terminate.js | 10 +- .../SwingSet/test/vat-admin/test-replay.js | 8 +- .../swing-store-lmdb/src/lmdbSwingStore.js | 14 +- packages/swing-store-lmdb/test/test-state.js | 26 ++-- .../src/simpleSwingStore.js | 50 +++++-- .../swing-store-simple/test/test-state.js | 29 ++-- packages/swingset-runner/.gitignore | 1 + packages/swingset-runner/src/dumpstore.js | 60 +++++---- packages/swingset-runner/src/kerneldump.js | 2 +- packages/swingset-runner/src/main.js | 2 +- 23 files changed, 303 insertions(+), 188 deletions(-) diff --git a/packages/SwingSet/bin/extract-transcript-from-kerneldb.js b/packages/SwingSet/bin/extract-transcript-from-kerneldb.js index 19d17f01af2..eacf1594c1e 100644 --- a/packages/SwingSet/bin/extract-transcript-from-kerneldb.js +++ b/packages/SwingSet/bin/extract-transcript-from-kerneldb.js @@ -1,3 +1,5 @@ +// XXX this is wrong; it needs to use the swingstore instead of opening the LMDB +// file directly, then use stream store reads to get the transcript entries. import lmdb from 'node-lmdb'; import process from 'process'; import fs from 'fs'; diff --git a/packages/SwingSet/src/kernel/initializeKernel.js b/packages/SwingSet/src/kernel/initializeKernel.js index a25ac6139c1..161485417cf 100644 --- a/packages/SwingSet/src/kernel/initializeKernel.js +++ b/packages/SwingSet/src/kernel/initializeKernel.js @@ -17,11 +17,11 @@ function makeVatRootObjectSlot() { export function initializeKernel(config, hostStorage, verbose = false) { const logStartup = verbose ? console.debug : () => 0; - const { kvStore } = hostStorage; + const { kvStore, streamStore } = hostStorage; insistStorageAPI(kvStore); const { enhancedCrankBuffer, commitCrank } = wrapStorage(kvStore); - const kernelKeeper = makeKernelKeeper(enhancedCrankBuffer); + const kernelKeeper = makeKernelKeeper(enhancedCrankBuffer, streamStore); const wasInitialized = kernelKeeper.getInitialized(); assert(!wasInitialized); diff --git a/packages/SwingSet/src/kernel/kernel.js b/packages/SwingSet/src/kernel/kernel.js index e7a2c9b88e6..78f2113f84d 100644 --- a/packages/SwingSet/src/kernel/kernel.js +++ b/packages/SwingSet/src/kernel/kernel.js @@ -111,7 +111,7 @@ export default function buildKernel( const { verbose, defaultManagerType = 'local' } = kernelOptions; const logStartup = verbose ? console.debug : () => 0; - const { kvStore } = hostStorage; + const { kvStore, streamStore } = hostStorage; insistStorageAPI(kvStore); const { enhancedCrankBuffer, abortCrank, commitCrank } = wrapStorage(kvStore); @@ -119,7 +119,11 @@ export default function buildKernel( ? makeSlogger(slogCallbacks, writeSlogObject) : makeDummySlogger(slogCallbacks, makeConsole); - const kernelKeeper = makeKernelKeeper(enhancedCrankBuffer, kernelSlog); + const kernelKeeper = makeKernelKeeper( + enhancedCrankBuffer, + streamStore, + kernelSlog, + ); const meterManager = makeMeterManager(replaceGlobalMeter); diff --git a/packages/SwingSet/src/kernel/state/kernelKeeper.js b/packages/SwingSet/src/kernel/state/kernelKeeper.js index 036f4b809ec..0483ed87899 100644 --- a/packages/SwingSet/src/kernel/state/kernelKeeper.js +++ b/packages/SwingSet/src/kernel/state/kernelKeeper.js @@ -46,8 +46,7 @@ const enableKernelPromiseGC = true; // $R is 'R' when reachable, '_' when merely recognizable // $vatSlot is one of: o+$NN/o-$NN/p+$NN/p-$NN/d+$NN/d-$NN // v$NN.c.$vatSlot = $kernelSlot = ko$NN/kp$NN/kd$NN -// v$NN.t.$NN = JSON(transcript entry) -// v$NN.t.nextID = $NN +// v$NN.t.endPosition = $NN // v$NN.vs.$key = string // d$NN.o.nextID = $NN @@ -100,7 +99,7 @@ const FIRST_DEVNODE_ID = 30n; const FIRST_PROMISE_ID = 40n; const FIRST_CRANK_NUMBER = 0n; -export default function makeKernelKeeper(kvStore, kernelSlog) { +export default function makeKernelKeeper(kvStore, streamStore, kernelSlog) { insistEnhancedStorageAPI(kvStore); function getRequired(key) { @@ -668,11 +667,12 @@ export default function makeKernelKeeper(kvStore, kernelSlog) { function allocateVatKeeper(vatID) { insistVatID(vatID); if (!kvStore.has(`${vatID}.o.nextID`)) { - initializeVatState(kvStore, vatID); + initializeVatState(kvStore, streamStore, vatID); } assert(!ephemeral.vatKeepers.has(vatID), X`vatID ${vatID} already defined`); const vk = makeVatKeeper( kvStore, + streamStore, kernelSlog, vatID, addKernelObject, @@ -757,6 +757,7 @@ export default function makeKernelKeeper(kvStore, kernelSlog) { if (vk) { // TODO: find some way to expose the liveSlots internal tables, the // kernel doesn't see them + vk.closeTranscript(); const vatTable = { vatID, state: { transcript: Array.from(vk.getTranscript()) }, diff --git a/packages/SwingSet/src/kernel/state/vatKeeper.js b/packages/SwingSet/src/kernel/state/vatKeeper.js index 4afae0b76ed..20a9c160240 100644 --- a/packages/SwingSet/src/kernel/state/vatKeeper.js +++ b/packages/SwingSet/src/kernel/state/vatKeeper.js @@ -15,26 +15,30 @@ import { kdebug } from '../kdebug'; const FIRST_OBJECT_ID = 50n; const FIRST_PROMISE_ID = 60n; const FIRST_DEVICE_ID = 70n; -const FIRST_TRANSCRIPT_ID = 0n; /** * Establish a vat's state. * - * @param {*} kvStore The storage in which the persistent state will be kept + * @param {*} kvStore The key-value store in which the persistent state will be kept + * @param {*} streamStore Accompanying stream store * @param {string} vatID The vat ID string of the vat in question * TODO: consider making this part of makeVatKeeper */ -export function initializeVatState(kvStore, vatID) { +export function initializeVatState(kvStore, streamStore, vatID) { kvStore.set(`${vatID}.o.nextID`, `${FIRST_OBJECT_ID}`); kvStore.set(`${vatID}.p.nextID`, `${FIRST_PROMISE_ID}`); kvStore.set(`${vatID}.d.nextID`, `${FIRST_DEVICE_ID}`); - kvStore.set(`${vatID}.t.nextID`, `${FIRST_TRANSCRIPT_ID}`); + kvStore.set( + `${vatID}.t.endPosition`, + `${JSON.stringify(streamStore.STREAM_START)}`, + ); } /** * Produce a vat keeper for a vat. * - * @param {*} kvStore The storage in which the persistent state will be kept + * @param {*} kvStore The keyValue store in which the persistent state will be kept + * @param {*} streamStore Accompanying stream store, for the transcripts * @param {*} kernelSlog * @param {string} vatID The vat ID string of the vat in question * @param {*} addKernelObject Kernel function to add a new object to the kernel's @@ -50,6 +54,7 @@ export function initializeVatState(kvStore, vatID) { */ export function makeVatKeeper( kvStore, + streamStore, kernelSlog, vatID, addKernelObject, @@ -61,6 +66,7 @@ export function makeVatKeeper( getCrankNumber, ) { insistVatID(vatID); + const transcriptStream = `transcript-${vatID}`; function setSourceAndOptions(source, options) { // take care with API change @@ -292,22 +298,42 @@ export function makeVatKeeper( /** * Generator function to return the vat's transcript, one entry at a time. + * + * @param {Object?} startPos Optional position to begin reading from + * + * @yields {string} a stream of transcript entries */ - function* getTranscript() { - for (const value of kvStore.getPrefixedValues(`${vatID}.t.`)) { - yield JSON.parse(value); + function* getTranscript(startPos = streamStore.STREAM_START) { + const endPos = JSON.parse(kvStore.get(`${vatID}.t.endPosition`)); + for (const entry of streamStore.readStream( + transcriptStream, + startPos, + endPos, + )) { + yield JSON.parse(entry); } } /** - * Append a message to the vat's transcript. + * Append an entry to the vat's transcript. * - * @param {string} msg The message to append. + * @param {Object} entry The transcript entry to append. + */ + function addToTranscript(entry) { + const oldPos = JSON.parse(kvStore.get(`${vatID}.t.endPosition`)); + const newPos = streamStore.writeStreamItem( + transcriptStream, + JSON.stringify(entry), + oldPos, + ); + kvStore.set(`${vatID}.t.endPosition`, `${JSON.stringify(newPos)}`); + } + + /** + * Cease writing to the vat's transcript. */ - function addToTranscript(msg) { - const id = Nat(BigInt(kvStore.get(`${vatID}.t.nextID`))); - kvStore.set(`${vatID}.t.nextID`, `${id + 1n}`); - kvStore.set(`${vatID}.t.${id}`, JSON.stringify(msg)); + function closeTranscript() { + streamStore.closeStream(transcriptStream); } function vatStats() { @@ -319,7 +345,8 @@ export function makeVatKeeper( const objectCount = getCount(`${vatID}.o.nextID`, FIRST_OBJECT_ID); const promiseCount = getCount(`${vatID}.p.nextID`, FIRST_PROMISE_ID); const deviceCount = getCount(`${vatID}.d.nextID`, FIRST_DEVICE_ID); - const transcriptCount = getCount(`${vatID}.t.nextID`, FIRST_TRANSCRIPT_ID); + const transcriptCount = JSON.parse(kvStore.get(`${vatID}.t.endPosition`)) + .itemCount; // TODO: Fix the downstream JSON.stringify to allow the counts to be BigInts return harden({ @@ -363,6 +390,7 @@ export function makeVatKeeper( deleteCListEntriesForKernelSlots, getTranscript, addToTranscript, + closeTranscript, vatStats, dumpState, }); diff --git a/packages/SwingSet/src/types.js b/packages/SwingSet/src/types.js index 11de7e31fff..a970fdf3549 100644 --- a/packages/SwingSet/src/types.js +++ b/packages/SwingSet/src/types.js @@ -103,7 +103,7 @@ * * @typedef { { d: VatDeliveryObject, syscalls: VatSyscallObject[] } } TranscriptEntry * @typedef { { transcriptCount: number } } VatStats - * @typedef { { getTranscript: () => TranscriptEntry[], + * @typedef { { getTranscript: (startPos?: Object) => TranscriptEntry[], * vatStats: () => VatStats, * } } VatKeeper * @typedef { { getVatKeeper: (vatID: string) => VatKeeper } } KernelKeeper diff --git a/packages/SwingSet/test/metering/test-dynamic-vat-metered.js b/packages/SwingSet/test/metering/test-dynamic-vat-metered.js index 6648a2703fb..78fb0bf9b57 100644 --- a/packages/SwingSet/test/metering/test-dynamic-vat-metered.js +++ b/packages/SwingSet/test/metering/test-dynamic-vat-metered.js @@ -80,7 +80,7 @@ async function runOneTest(t, explosion, managerType) { await c.run(); t.is(JSON.parse(kvStore.get('vat.dynamicIDs')).length, 1); t.is(kvStore.get(`${root}.owner`), vatID); - t.is(Array.from(kvStore.getKeys(`${vatID}`, `${vatID}/`)).length, 12); + t.is(Array.from(kvStore.getKeys(`${vatID}`, `${vatID}/`)).length, 10); // neverKPID should still be unresolved t.is(kvStore.get(`${neverKPID}.state`), 'unresolved'); diff --git a/packages/SwingSet/test/test-clist.js b/packages/SwingSet/test/test-clist.js index f972367a978..d7bc7e47305 100644 --- a/packages/SwingSet/test/test-clist.js +++ b/packages/SwingSet/test/test-clist.js @@ -8,9 +8,10 @@ import { wrapStorage } from '../src/kernel/state/storageWrapper'; test(`clist reachability`, async t => { const slog = makeDummySlogger({}); - const { enhancedCrankBuffer: s } = wrapStorage(initSwingStore().kvStore); + const hostStorage = initSwingStore(); + const { enhancedCrankBuffer: s } = wrapStorage(hostStorage.kvStore); - const kk = makeKernelKeeper(s, slog); + const kk = makeKernelKeeper(s, hostStorage.streamStore, slog); kk.createStartingKernelState('local'); const vatID = kk.allocateUnusedVatID(); const vk = kk.allocateVatKeeper(vatID); diff --git a/packages/SwingSet/test/test-devices.js b/packages/SwingSet/test/test-devices.js index fdec235ec19..0a67c93af57 100644 --- a/packages/SwingSet/test/test-devices.js +++ b/packages/SwingSet/test/test-devices.js @@ -237,7 +237,7 @@ test.serial('device state', async t => { const d3 = c1.deviceNameToID('d3'); await c1.run(); t.deepEqual(c1.dump().log, ['undefined', 'w+r', 'called', 'got {"s":"new"}']); - const s = getAllState(hostStorage.kvStore); + const s = getAllState(hostStorage).kvStuff; t.deepEqual(JSON.parse(s[`${d3}.deviceState`]), capargs({ s: 'new' })); t.deepEqual(JSON.parse(s[`${d3}.o.nextID`]), 10); }); diff --git a/packages/SwingSet/test/test-gc-transcript.js b/packages/SwingSet/test/test-gc-transcript.js index f5195104aef..63782c76a97 100644 --- a/packages/SwingSet/test/test-gc-transcript.js +++ b/packages/SwingSet/test/test-gc-transcript.js @@ -20,6 +20,7 @@ function setup(storedTranscript = []) { getTranscript() { return storedTranscript; }, + closeTranscript() {}, }; const kernelKeeper = { getVatKeeper() { diff --git a/packages/SwingSet/test/test-state.js b/packages/SwingSet/test/test-state.js index 3184646d9f3..ed41c507045 100644 --- a/packages/SwingSet/test/test-state.js +++ b/packages/SwingSet/test/test-state.js @@ -68,14 +68,14 @@ function testStorage(t, s, getState, commit) { } test('storageInMemory', t => { - const { kvStore } = initSwingStore(); - testStorage(t, kvStore, () => getAllState(kvStore), null); + const store = initSwingStore(); + testStorage(t, store.kvStore, () => getAllState(store).kvStuff, null); }); function buildHostDBAndGetState() { - const { kvStore } = initSwingStore(); - const hostDB = buildHostDBInMemory(kvStore); - return { hostDB, getState: () => getAllState(kvStore) }; + const store = initSwingStore(); + const hostDB = buildHostDBInMemory(store.kvStore); + return { hostDB, getState: () => getAllState(store).kvStuff }; } test('hostDBInMemory', t => { @@ -119,15 +119,15 @@ test('blockBuffer fulfills storage API', t => { }); test('guardStorage fulfills storage API', t => { - const { kvStore } = initSwingStore(); - const guardedHostStorage = guardStorage(kvStore); - testStorage(t, guardedHostStorage, () => getAllState(kvStore), null); + const store = initSwingStore(); + const guardedHostStorage = guardStorage(store.kvStore); + testStorage(t, guardedHostStorage, () => getAllState(store).kvStuff, null); }); test('crankBuffer fulfills storage API', t => { - const { kvStore } = initSwingStore(); - const { crankBuffer, commitCrank } = buildCrankBuffer(kvStore); - testStorage(t, crankBuffer, () => getAllState(kvStore), commitCrank); + const store = initSwingStore(); + const { crankBuffer, commitCrank } = buildCrankBuffer(store.kvStore); + testStorage(t, crankBuffer, () => getAllState(store).kvStuff, commitCrank); }); test('crankBuffer can abortCrank', t => { @@ -186,8 +186,8 @@ test('crankBuffer can abortCrank', t => { }); test('storage helpers', t => { - const { kvStore } = initSwingStore(); - const s = addHelpers(kvStore); + const store = initSwingStore(); + const s = addHelpers(store.kvStore); s.set('foo.0', 'f0'); s.set('foo.1', 'f1'); @@ -195,7 +195,7 @@ test('storage helpers', t => { s.set('foo.3', 'f3'); // omit foo.4 s.set('foo.5', 'f5'); - checkState(t, () => getAllState(kvStore), [ + checkState(t, () => getAllState(store).kvStuff, [ ['foo.0', 'f0'], ['foo.1', 'f1'], ['foo.2', 'f2'], @@ -229,44 +229,52 @@ test('storage helpers', t => { t.falsy(s.has('foo.3')); t.falsy(s.has('foo.4')); t.truthy(s.has('foo.5')); - checkState(t, () => getAllState(kvStore), [ + checkState(t, () => getAllState(store).kvStuff, [ ['foo.0', 'f0'], ['foo.5', 'f5'], ]); }); function buildKeeperStorageInMemory() { - const { kvStore } = initSwingStore(); + const store = initSwingStore(); + const { kvStore, streamStore } = store; const { enhancedCrankBuffer, commitCrank } = wrapStorage(kvStore); return { - kstorage: enhancedCrankBuffer, - getState: () => getAllState(kvStore), + kvStore: enhancedCrankBuffer, + streamStore, + getState: () => getAllState(store).kvStuff, commitCrank, }; } function duplicateKeeper(getState) { - const { kvStore } = initSwingStore(); - setAllState(kvStore, getState()); + const store = initSwingStore(); + const { kvStore, streamStore } = store; + setAllState(store, { kvStuff: getState(), streamStuff: new Map() }); const { enhancedCrankBuffer } = wrapStorage(kvStore); - return makeKernelKeeper(enhancedCrankBuffer); + return makeKernelKeeper(enhancedCrankBuffer, streamStore); } test('hostStorage param guards', async t => { - const { kstorage } = buildKeeperStorageInMemory(); + const { kvStore } = buildKeeperStorageInMemory(); const exp = { message: /true must be a string/ }; - t.throws(() => kstorage.set('foo', true), exp); - t.throws(() => kstorage.set(true, 'foo'), exp); - t.throws(() => kstorage.has(true), exp); - t.throws(() => Array.from(kstorage.getKeys('foo', true)), exp); - t.throws(() => Array.from(kstorage.getKeys(true, 'foo')), exp); - t.throws(() => kstorage.get(true), exp); - t.throws(() => kstorage.delete(true), exp); + t.throws(() => kvStore.set('foo', true), exp); + t.throws(() => kvStore.set(true, 'foo'), exp); + t.throws(() => kvStore.has(true), exp); + t.throws(() => Array.from(kvStore.getKeys('foo', true)), exp); + t.throws(() => Array.from(kvStore.getKeys(true, 'foo')), exp); + t.throws(() => kvStore.get(true), exp); + t.throws(() => kvStore.delete(true), exp); }); test('kernel state', async t => { - const { kstorage, getState, commitCrank } = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(kstorage); + const { + kvStore, + streamStore, + getState, + commitCrank, + } = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(kvStore, streamStore); t.truthy(!k.getInitialized()); k.createStartingKernelState('local'); k.setInitialized(); @@ -289,8 +297,13 @@ test('kernel state', async t => { }); test('kernelKeeper vat names', async t => { - const { kstorage, getState, commitCrank } = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(kstorage); + const { + kvStore, + streamStore, + getState, + commitCrank, + } = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(kvStore, streamStore); k.createStartingKernelState('local'); const v1 = k.allocateVatIDForNameIfNeeded('vatname5'); @@ -331,8 +344,13 @@ test('kernelKeeper vat names', async t => { }); test('kernelKeeper device names', async t => { - const { kstorage, getState, commitCrank } = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(kstorage); + const { + kvStore, + streamStore, + getState, + commitCrank, + } = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(kvStore, streamStore); k.createStartingKernelState('local'); const d7 = k.allocateDeviceIDForNameIfNeeded('devicename5'); @@ -373,8 +391,13 @@ test('kernelKeeper device names', async t => { }); test('kernelKeeper runQueue', async t => { - const { kstorage, getState, commitCrank } = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(kstorage); + const { + kvStore, + streamStore, + getState, + commitCrank, + } = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(kvStore, streamStore); k.createStartingKernelState('local'); t.truthy(k.isRunQueueEmpty()); @@ -409,8 +432,13 @@ test('kernelKeeper runQueue', async t => { }); test('kernelKeeper promises', async t => { - const { kstorage, getState, commitCrank } = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(kstorage); + const { + kvStore, + streamStore, + getState, + commitCrank, + } = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(kvStore, streamStore); k.createStartingKernelState('local'); const p1 = k.addKernelPromiseForVat('v4'); @@ -524,8 +552,8 @@ test('kernelKeeper promises', async t => { }); test('kernelKeeper promise resolveToData', async t => { - const { kstorage } = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(kstorage); + const { kvStore, streamStore } = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(kvStore, streamStore); k.createStartingKernelState('local'); const p1 = k.addKernelPromiseForVat('v4'); @@ -545,8 +573,8 @@ test('kernelKeeper promise resolveToData', async t => { }); test('kernelKeeper promise reject', async t => { - const { kstorage } = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(kstorage); + const { kvStore, streamStore } = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(kvStore, streamStore); k.createStartingKernelState('local'); const p1 = k.addKernelPromiseForVat('v4'); @@ -566,8 +594,13 @@ test('kernelKeeper promise reject', async t => { }); test('vatKeeper', async t => { - const { kstorage, getState, commitCrank } = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(kstorage); + const { + kvStore, + streamStore, + getState, + commitCrank, + } = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(kvStore, streamStore); k.createStartingKernelState('local'); const v1 = k.allocateVatIDForNameIfNeeded('name1'); @@ -598,8 +631,8 @@ test('vatKeeper', async t => { }); test('XS vatKeeper defaultManagerType', async t => { - const { kstorage } = buildKeeperStorageInMemory(); - const k = makeKernelKeeper(kstorage); + const { kvStore, streamStore } = buildKeeperStorageInMemory(); + const k = makeKernelKeeper(kvStore, streamStore); k.createStartingKernelState('xs-worker'); t.is(k.getDefaultManagerType(), 'xs-worker'); }); diff --git a/packages/SwingSet/test/test-transcript-light.js b/packages/SwingSet/test/test-transcript-light.js index 25b10b0eac9..f08e89de2d3 100644 --- a/packages/SwingSet/test/test-transcript-light.js +++ b/packages/SwingSet/test/test-transcript-light.js @@ -13,95 +13,90 @@ test('transcript-light load', async t => { ); const hostStorage = provideHostStorage(); const c = await buildVatController(config, ['one'], { hostStorage }); - const kvStore = hostStorage.kvStore; - const state0 = getAllState(kvStore); - t.is(state0.initialized, 'true'); - t.not(state0.runQueue, '[]'); + const state0 = getAllState(hostStorage); + t.is(state0.kvStuff.initialized, 'true'); + t.not(state0.kvStuff.runQueue, '[]'); await c.step(); - const state1 = getAllState(kvStore); + const state1 = getAllState(hostStorage); await c.step(); - const state2 = getAllState(kvStore); + const state2 = getAllState(hostStorage); await c.step(); - const state3 = getAllState(kvStore); + const state3 = getAllState(hostStorage); await c.step(); - const state4 = getAllState(kvStore); + const state4 = getAllState(hostStorage); await c.step(); - const state5 = getAllState(kvStore); + const state5 = getAllState(hostStorage); // build from loaded state // Step 0 const cfg0 = await loadBasedir(path.resolve(__dirname, 'basedir-transcript')); const hostStorage0 = provideHostStorage(); - const kvStore0 = hostStorage0.kvStore; - // XXX TODO copy transcripts - setAllState(kvStore0, state0); + setAllState(hostStorage0, state0); const c0 = await buildVatController(cfg0, ['one'], { hostStorage: hostStorage0, }); await c0.step(); - t.deepEqual(state1, getAllState(kvStore0), `p1`); + t.deepEqual(state1, getAllState(hostStorage0), `p1`); await c0.step(); - t.deepEqual(state2, getAllState(kvStore0), `p2`); + t.deepEqual(state2, getAllState(hostStorage0), `p2`); await c0.step(); - t.deepEqual(state3, getAllState(kvStore0), `p3`); + t.deepEqual(state3, getAllState(hostStorage0), `p3`); await c0.step(); - t.deepEqual(state4, getAllState(kvStore0), `p4`); + t.deepEqual(state4, getAllState(hostStorage0), `p4`); await c0.step(); - t.deepEqual(state5, getAllState(kvStore0), `p5`); + t.deepEqual(state5, getAllState(hostStorage0), `p5`); // Step 1 const cfg1 = await loadBasedir(path.resolve(__dirname, 'basedir-transcript')); const hostStorage1 = provideHostStorage(); - const kvStore1 = hostStorage1.kvStore; - setAllState(kvStore1, state1); + setAllState(hostStorage1, state1); const c1 = await buildVatController(cfg1, ['one'], { hostStorage: hostStorage1, }); - t.deepEqual(state1, getAllState(kvStore1), `p6`); // actual, expected + t.deepEqual(state1, getAllState(hostStorage1), `p6`); // actual, expected await c1.step(); - t.deepEqual(state2, getAllState(kvStore1), `p7`); + t.deepEqual(state2, getAllState(hostStorage1), `p7`); await c1.step(); - t.deepEqual(state3, getAllState(kvStore1), `p8`); + t.deepEqual(state3, getAllState(hostStorage1), `p8`); await c1.step(); - t.deepEqual(state4, getAllState(kvStore1), `p9`); + t.deepEqual(state4, getAllState(hostStorage1), `p9`); await c1.step(); - t.deepEqual(state5, getAllState(kvStore1), `p10`); + t.deepEqual(state5, getAllState(hostStorage1), `p10`); // Step 2 const cfg2 = await loadBasedir(path.resolve(__dirname, 'basedir-transcript')); const hostStorage2 = provideHostStorage(); - const kvStore2 = hostStorage2.kvStore; - setAllState(kvStore2, state2); + setAllState(hostStorage2, state2); const c2 = await buildVatController(cfg2, ['one'], { hostStorage: hostStorage2, }); - t.deepEqual(state2, getAllState(kvStore2), `p11`); + t.deepEqual(state2, getAllState(hostStorage2), `p11`); await c2.step(); - t.deepEqual(state3, getAllState(kvStore2), `p12`); + t.deepEqual(state3, getAllState(hostStorage2), `p12`); await c2.step(); - t.deepEqual(state4, getAllState(kvStore2), `p13`); + t.deepEqual(state4, getAllState(hostStorage2), `p13`); await c2.step(); - t.deepEqual(state5, getAllState(kvStore2), `p14`); + t.deepEqual(state5, getAllState(hostStorage2), `p14`); }); diff --git a/packages/SwingSet/test/test-transcript.js b/packages/SwingSet/test/test-transcript.js index 1bbb4e105cd..fe316a6136c 100644 --- a/packages/SwingSet/test/test-transcript.js +++ b/packages/SwingSet/test/test-transcript.js @@ -9,14 +9,13 @@ import { provideHostStorage } from '../src/hostStorage'; import { buildVatController, loadBasedir } from '../src/index'; async function buildTrace(c, storage) { - // XXX TODO also copy transcript const states = []; while (c.dump().runQueue.length) { - states.push(getAllState(storage.kvStore)); + states.push(getAllState(storage)); // eslint-disable-next-line no-await-in-loop await c.step(); } - states.push(getAllState(storage.kvStore)); + states.push(getAllState(storage)); return states; } @@ -51,12 +50,13 @@ test('transcript-one save', async t => { // Instead, we just do simple comparison. Leave investigation to the // experts. const s2 = states2[i]; - const extra = new Set(Object.keys(s2)); - for (const k of Object.keys(s)) { - t.assert(s[k] === s2[k], `states[${i}][${k}] differs`); + const extra = new Set(Object.keys(s2.kvStuff)); + for (const k of Object.keys(s.kvStuff)) { + t.assert(s.kvStuff[k] === s2.kvStuff[k], `states[${i}][${k}] differs`); extra.delete(k); } t.deepEqual([...extra.keys()], [], `states2[${i}] has missing keys`); + t.deepEqual(s.streamStuff, s2.streamStuff); }); }); @@ -77,7 +77,7 @@ test('transcript-one load', async t => { path.resolve(__dirname, 'basedir-transcript'), ); const s = provideHostStorage(); - setAllState(s.kvStore, states[i]); + setAllState(s, states[i]); // eslint-disable-next-line no-await-in-loop const c = await buildVatController(cfg, ['one'], { hostStorage: s }); // eslint-disable-next-line no-await-in-loop diff --git a/packages/SwingSet/test/vat-admin/terminate/test-terminate.js b/packages/SwingSet/test/vat-admin/terminate/test-terminate.js index 3da4aed4779..96c27e4c472 100644 --- a/packages/SwingSet/test/vat-admin/terminate/test-terminate.js +++ b/packages/SwingSet/test/vat-admin/terminate/test-terminate.js @@ -133,10 +133,10 @@ test('dispatches to the dead do not harm kernel', async t => { 'done: Error: arbitrary reason', ]); } - const state1 = getAllState(hostStorage1.kvStore); + const state1 = getAllState(hostStorage1); const hostStorage2 = provideHostStorage(); // XXX TODO also copy transcripts - setAllState(hostStorage2.kvStore, state1); + setAllState(hostStorage2, state1); { const c2 = await buildVatController(config, [], { hostStorage: hostStorage2, @@ -176,10 +176,10 @@ test('replay does not resurrect dead vat', async t => { t.deepEqual(c1.dump().log, [`w: I ate'nt dead`]); } - const state1 = getAllState(hostStorage1.kvStore); + const state1 = getAllState(hostStorage1); const hostStorage2 = provideHostStorage(); // XXX TODO also copy transcripts - setAllState(hostStorage2.kvStore, state1); + setAllState(hostStorage2, state1); { const c2 = await buildVatController(config, [], { hostStorage: hostStorage2, @@ -208,7 +208,7 @@ test('dead vat state removed', async t => { const kvStore = hostStorage.kvStore; t.is(kvStore.get('vat.dynamicIDs'), '["v6"]'); t.is(kvStore.get('ko26.owner'), 'v6'); - t.is(Array.from(kvStore.getKeys('v6.', 'v6/')).length, 9); + t.is(Array.from(kvStore.getKeys('v6.', 'v6/')).length, 8); controller.queueToVatExport('bootstrap', 'o+0', 'phase2', capargs([])); await controller.run(); diff --git a/packages/SwingSet/test/vat-admin/test-replay.js b/packages/SwingSet/test/vat-admin/test-replay.js index 683233412e3..1bfd6128d04 100644 --- a/packages/SwingSet/test/vat-admin/test-replay.js +++ b/packages/SwingSet/test/vat-admin/test-replay.js @@ -60,9 +60,9 @@ test('replay bundleSource-based dynamic vat', async t => { // we could re-use the Storage object, but I'll be paranoid and create a // new one. - const state1 = getAllState(hostStorage1.kvStore); + const state1 = getAllState(hostStorage1); const hostStorage2 = provideHostStorage(); - setAllState(hostStorage2.kvStore, state1); + setAllState(hostStorage2, state1); { const c2 = await buildVatController(copy(config), [], { hostStorage: hostStorage2, @@ -108,9 +108,9 @@ test('replay bundleName-based dynamic vat', async t => { t.deepEqual(c1.kpResolution(r1), capargs('created')); } - const state1 = getAllState(hostStorage1.kvStore); + const state1 = getAllState(hostStorage1); const hostStorage2 = provideHostStorage(); - setAllState(hostStorage2.kvStore, state1); + setAllState(hostStorage2, state1); { const c2 = await buildVatController(copy(config), [], { hostStorage: hostStorage2, diff --git a/packages/swing-store-lmdb/src/lmdbSwingStore.js b/packages/swing-store-lmdb/src/lmdbSwingStore.js index 2105ff2568b..2f99d005df1 100644 --- a/packages/swing-store-lmdb/src/lmdbSwingStore.js +++ b/packages/swing-store-lmdb/src/lmdbSwingStore.js @@ -32,7 +32,7 @@ function makeSwingStore(dirPath, forceReset = false) { if (forceReset) { fs.rmdirSync(dirPath, { recursive: true }); } - fs.mkdirSync(dirPath, { recursive: true }); + fs.mkdirSync(`${dirPath}/streams`, { recursive: true }); let lmdbEnv = new lmdb.Env(); lmdbEnv.open({ @@ -241,7 +241,7 @@ function makeSwingStore(dirPath, forceReset = false) { assert(itemCount === 0); return []; } else { - const filePath = `${dirPath}/${streamName}`; + const filePath = `${dirPath}/streams/${streamName}.sss`; fs.truncateSync(filePath, endPosition.offset); const fd = fs.openSync(filePath, 'r'); streamFds.set(streamName, fd); @@ -280,9 +280,12 @@ function makeSwingStore(dirPath, forceReset = false) { X`can't read stream ${q(streamName)}, it's been closed`, ); const line = /** @type {string|false} */ (innerReader.next()); - if (line) { + // N.b.: since uncommitted writes may leave an overhang of data in + // the stream file, the itemCount is the true indicator of the end + // of the stream, not the point at which the line reader reaches the + // end-of-file. + if (line && itemCount > 0) { itemCount -= 1; - assert(itemCount >= 0); const result = line.toString(); if (skipCount > 0) { skipCount -= 1; @@ -290,6 +293,7 @@ function makeSwingStore(dirPath, forceReset = false) { yield result; } } else { + closefd(fd); break; } } @@ -323,7 +327,7 @@ function makeSwingStore(dirPath, forceReset = false) { let fd = streamFds.get(streamName); if (!fd) { - const filePath = `${dirPath}/${streamName}`; + const filePath = `${dirPath}/streams/${streamName}.sss`; const mode = fs.existsSync(filePath) ? 'r+' : 'w'; fd = fs.openSync(filePath, mode); streamFds.set(streamName, fd); diff --git a/packages/swing-store-lmdb/test/test-state.js b/packages/swing-store-lmdb/test/test-state.js index a3bd3721efd..51a99ff0304 100644 --- a/packages/swing-store-lmdb/test/test-state.js +++ b/packages/swing-store-lmdb/test/test-state.js @@ -15,7 +15,8 @@ import { isSwingStore, } from '../src/lmdbSwingStore'; -function testKVStore(t, kvStore) { +function testKVStore(t, store) { + const kvStore = store.kvStore; t.falsy(kvStore.has('missing')); t.is(kvStore.get('missing'), undefined); @@ -39,11 +40,14 @@ function testKVStore(t, kvStore) { t.deepEqual(Array.from(kvStore.getKeys('foo1', 'foo4')), ['foo1', 'foo3']); const reference = { - foo: 'f', - foo1: 'f1', - foo3: 'f3', + kvStuff: { + foo: 'f', + foo1: 'f1', + foo3: 'f3', + }, + streamStuff: new Map(), }; - t.deepEqual(getAllState(kvStore), reference, 'check state after changes'); + t.deepEqual(getAllState(store), reference, 'check state after changes'); } test('storageInLMDB under SES', t => { @@ -51,15 +55,17 @@ test('storageInLMDB under SES', t => { t.teardown(() => fs.rmdirSync(dbDir, { recursive: true })); fs.rmdirSync(dbDir, { recursive: true }); t.is(isSwingStore(dbDir), false); - const { kvStore, commit, close } = initSwingStore(dbDir); - testKVStore(t, kvStore); + const store = initSwingStore(dbDir); + const { commit, close } = store; + testKVStore(t, store); commit(); - const before = getAllState(kvStore); + const before = getAllState(store); close(); t.is(isSwingStore(dbDir), true); - const { kvStore: after, close: close2 } = openSwingStore(dbDir); - t.deepEqual(getAllState(after), before, 'check state after reread'); + const store2 = openSwingStore(dbDir); + const { close: close2 } = store2; + t.deepEqual(getAllState(store2), before, 'check state after reread'); t.is(isSwingStore(dbDir), true); close2(); }); diff --git a/packages/swing-store-simple/src/simpleSwingStore.js b/packages/swing-store-simple/src/simpleSwingStore.js index f5beb13e656..fc29d6cb9f5 100644 --- a/packages/swing-store-simple/src/simpleSwingStore.js +++ b/packages/swing-store-simple/src/simpleSwingStore.js @@ -156,6 +156,8 @@ function makeStorageInMemory() { return harden({ kvStore, state }); } +const streamPeek = new WeakMap(); // for tests to get raw access to the streams + /** * Do the work of `initSwingStore` and `openSwingStore`. * @@ -341,6 +343,8 @@ function makeSwingStore(dirPath, forceReset = false) { STREAM_START, }); + streamPeek.set(streamStore, streams); + return harden({ kvStore, streamStore, commit, close }); } @@ -389,19 +393,29 @@ export function openSwingStore(dirPath) { * stupidest possible way, hence it is likely to be a performance and memory * hog if you attempt to use it on anything real. * - * @param {KVStore} kvStore The swing storage whose state is to be extracted. + * @param {SwingStore} swingStore The swing storage whose state is to be extracted. * - * @returns {Record} an array representing all the current state in `kvStore`, one - * element of the form [key, value] per key/value pair. + * @returns {{ + * kvStuff: Record, + * streamStuff: Map>, + * }} A crude representation of all of the state of `swingStore` */ -export function getAllState(kvStore) { +export function getAllState(swingStore) { + const { kvStore, streamStore } = swingStore; /** @type { Record } */ - const stuff = {}; + const kvStuff = {}; for (const key of Array.from(kvStore.getKeys('', ''))) { // @ts-ignore get(key) of key from getKeys() is not undefined - stuff[key] = kvStore.get(key); + kvStuff[key] = kvStore.get(key); } - return stuff; + const streamStuff = new Map(); + const peek = streamPeek.get(streamStore); + if (peek) { + for (const [streamName, stream] of peek.entries()) { + streamStuff.set(streamName, Array.from(stream)); + } + } + return { kvStuff, streamStuff }; } /** @@ -411,12 +425,24 @@ export function getAllState(kvStore) { * general store initialization mechanism. In particular, note that it does * not bother to remove any existing state in the store that it is given. * - * @param {KVStore} kvStore The swing storage whose state is to be set. - * @param {Array<[string, string]>} stuff An array of key/value pairs, each element of the form [key, value] + * @param {SwingStore} swingStore The swing storage whose state is to be set. + * @param {{ + * kvStuff: Record, + * streamStuff: Map>, + * }} stuff The state to stuff into `swingStore` */ -export function setAllState(kvStore, stuff) { - for (const k of Object.getOwnPropertyNames(stuff)) { - kvStore.set(k, stuff[k]); +export function setAllState(swingStore, stuff) { + const { kvStore, streamStore } = swingStore; + const { kvStuff, streamStuff } = stuff; + for (const k of Object.getOwnPropertyNames(kvStuff)) { + kvStore.set(k, kvStuff[k]); + } + for (const [streamName, stream] of streamStuff.entries()) { + let pos = streamStore.STREAM_START; + for (const item of stream) { + pos = streamStore.writeStreamItem(streamName, item, pos); + } + streamStore.closeStream(streamName); } } diff --git a/packages/swing-store-simple/test/test-state.js b/packages/swing-store-simple/test/test-state.js index c267c6e6498..30cfd2fad5d 100644 --- a/packages/swing-store-simple/test/test-state.js +++ b/packages/swing-store-simple/test/test-state.js @@ -11,7 +11,8 @@ import { isSwingStore, } from '../src/simpleSwingStore'; -function testKVStore(t, kvStore) { +function testKVStore(t, store) { + const kvStore = store.kvStore; t.falsy(kvStore.has('missing')); t.is(kvStore.get('missing'), undefined); @@ -35,16 +36,19 @@ function testKVStore(t, kvStore) { t.deepEqual(Array.from(kvStore.getKeys('foo1', 'foo4')), ['foo1', 'foo3']); const reference = { - foo: 'f', - foo1: 'f1', - foo3: 'f3', + kvStuff: { + foo: 'f', + foo1: 'f1', + foo3: 'f3', + }, + streamStuff: new Map(), }; - t.deepEqual(getAllState(kvStore), reference, 'check state after changes'); + t.deepEqual(getAllState(store), reference, 'check state after changes'); } test('storageInMemory', t => { - const { kvStore } = initSwingStore(); - testKVStore(t, kvStore); + const store = initSwingStore(); + testKVStore(t, store); }); test('storageInFile', t => { @@ -52,15 +56,16 @@ test('storageInFile', t => { t.teardown(() => fs.rmdirSync(dbDir, { recursive: true })); fs.rmdirSync(dbDir, { recursive: true }); t.is(isSwingStore(dbDir), false); - const { kvStore, commit, close } = initSwingStore(dbDir); - testKVStore(t, kvStore); + const store = initSwingStore(dbDir); + const { commit, close } = store; + testKVStore(t, store); commit(); - const before = getAllState(kvStore); + const before = getAllState(store); close(); t.is(isSwingStore(dbDir), true); - const { kvStore: after } = openSwingStore(dbDir); - t.deepEqual(getAllState(after), before, 'check state after reread'); + const afterStore = openSwingStore(dbDir); + t.deepEqual(getAllState(afterStore), before, 'check state after reread'); t.is(isSwingStore(dbDir), true); }); diff --git a/packages/swingset-runner/.gitignore b/packages/swingset-runner/.gitignore index 7bd27b99788..940244846b3 100644 --- a/packages/swingset-runner/.gitignore +++ b/packages/swingset-runner/.gitignore @@ -1,5 +1,6 @@ # swingset persistence data *.mdb +*.sss *.jsonlines *.naive slog diff --git a/packages/swingset-runner/src/dumpstore.js b/packages/swingset-runner/src/dumpstore.js index a4998667a78..d18716700e2 100644 --- a/packages/swingset-runner/src/dumpstore.js +++ b/packages/swingset-runner/src/dumpstore.js @@ -2,7 +2,8 @@ import fs from 'fs'; import process from 'process'; /* eslint-disable no-use-before-define */ -export function dumpStore(store, outfile, rawMode) { +export function dumpStore(swingStore, outfile, rawMode) { + const streamStore = swingStore.streamStore; let out; if (outfile) { out = fs.createWriteStream(outfile); @@ -11,8 +12,8 @@ export function dumpStore(store, outfile, rawMode) { } const state = new Map(); - for (const key of store.getKeys('', '~')) { - const value = store.get(key); + for (const key of swingStore.kvStore.getKeys('', '~')) { + const value = swingStore.kvStore.get(key); if (rawMode) { pkv(key, value); } else { @@ -117,7 +118,7 @@ export function dumpStore(store, outfile, rawMode) { pgroup('kp'); gap(); - const transcript = []; + const vatInfo = []; let starting = true; for (const [vn, v] of vats.entries()) { if (starting) { @@ -131,37 +132,40 @@ export function dumpStore(store, outfile, rawMode) { popt(`${v}.d.nextID`); popt(`${v}.o.nextID`); popt(`${v}.p.nextID`); - popt(`${v}.t.nextID`); + const endPos = JSON.parse(popt(`${v}.t.endPosition`)); + vatInfo.push([v, vn, endPos]); for (const key of groupKeys(`${v}.c.kd`)) { const val = popt(key); - popt(`${v}.c.${val}`); + popt(`${v}.c.${val.slice(2)}`); } for (const key of groupKeys(`${v}.c.ko`)) { const val = popt(key); - popt(`${v}.c.${val}`); + popt(`${v}.c.${val.slice(2)}`); } for (const key of groupKeys(`${v}.c.kp`)) { const val = popt(key); - popt(`${v}.c.${val}`); + popt(`${v}.c.${val.slice(2)}`); } for (const key of groupKeys(`${v}.vs`)) { popt(key); } - for (const key of groupKeys(`${v}.t.`)) { - transcript.push([key, eat(key)]); - } } - - if (transcript.length > 0) { + for (const info of vatInfo) { + const [v, vn, endPos] = info; gap(); - p('// transcript'); - transcript.sort((a, b) => { - const aEntry = JSON.parse(a[1]); - const bEntry = JSON.parse(b[1]); - return aEntry.crankNumber - bEntry.crankNumber; - }); - for (let i = 0; i < transcript.length; i += 1) { - pkv(transcript[i][0], transcript[i][1]); + p(`// transcript of vat ${v} (${vn})`); + if (endPos) { + let idx = 1; + for (const item of streamStore.readStream( + `transcript-${v}`, + streamStore.STREAM_START, + endPos, + )) { + pkvBig('transcript', `${v}.${idx}`, item, 500); + idx += 1; + } + } else { + p(`// (no transcript)`); } } @@ -263,14 +267,18 @@ export function dumpStore(store, outfile, rawMode) { return value; } + function pkvBig(tag, key, value, maxWidth = 50) { + if (value.length > maxWidth) { + pkv(key, `<<${tag} ${value.length}>>`); + } else { + pkv(key, value); + } + } + function poptBig(tag, key) { const value = eat(key); if (value) { - if (value.length > 50) { - pkv(key, `<<${tag} ${value.length}>>`); - } else { - pkv(key, value); - } + pkvBig(tag, key, value); } } } diff --git a/packages/swingset-runner/src/kerneldump.js b/packages/swingset-runner/src/kerneldump.js index fb919e86a7f..290e520b9a2 100644 --- a/packages/swingset-runner/src/kerneldump.js +++ b/packages/swingset-runner/src/kerneldump.js @@ -135,7 +135,7 @@ export function main() { printMainStats(organizeMainStats(rawStats, cranks)); } else { if (doDump) { - dumpStore(swingStore.kvStore, outfile, rawMode); + dumpStore(swingStore, outfile, rawMode); } if (refCounts) { auditRefCounts(swingStore.kvStore); diff --git a/packages/swingset-runner/src/main.js b/packages/swingset-runner/src/main.js index d51ec3bf51c..3a74d6b8e3b 100644 --- a/packages/swingset-runner/src/main.js +++ b/packages/swingset-runner/src/main.js @@ -531,7 +531,7 @@ export async function main() { function kernelStateDump() { const dumpPath = `${dumpDir}/${dumpTag}${crankNumber}`; - dumpStore(swingStore.kvStore, dumpPath, rawMode); + dumpStore(swingStore, dumpPath, rawMode); } async function runBenchmark(rounds) {