capsite/toybox: a mount outlives its handler: the vat restarts on demand

A handler is a live reference, held by the VatAdmin DO; workerd evicts
an idle DO within seconds, and with it the handler, so a vat's page went
503 almost at once.

createVat now keeps a vat's params when they hold only data and
ingresses (an ingress by its key: the VatAdmin can make it again), and
which vat was given which ingress. A request for a mount with no handler
restarts that vat with its params; it rebuilds its root, serves again,
and the request is answered. A vat whose params hold another peer's
objects isn't restartable yet.

The test crashes its own toybox (a crash ends every session to it), and
the creator waits for the root's handoff before it disconnects, as
scripts/hello.js now does too.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

toybox/supervisor.js

@@ -2,7 +2,7 @@
 import "@endo/init";
 import { DurableObject, WorkerEntrypoint } from "cloudflare:workers";
 import { RpcTarget, newWebSocketRpcSession, newWorkersRpcResponse } from "capnweb";
-import { E, Far } from "@endo/far";
+import { E, Far, passStyleOf } from "@endo/far";
 import { makeClient, swissnumFromBytes } from "@endo/ocapn";
 import { mergeLabels, urlLabels } from "./labels.js";
 import { LOOKUP, MOUNT, MOUNT_LABELS, MOUNT_SCHEMA, OVERLAP, UNMOUNT, UNMOUNT_LABELS } from "./mounts-sql.js";
@@ -178,29 +178,136 @@ const encodeUtf8 = harden(TextEncoder.prototype.encode.bind(new TextEncoder()));
  */
 export class VatAdmin extends DurableObject {
 	#clients = new Map();
-	/** mount key → the handler serving it (a live reference, for now) */
+	/** mount key → the handler serving it: live, so lost when this object is evicted */
 	#handlers = new Map();
+	/** mount key → resolvers waiting for a handler */
+	#waiting = new Map();
+	/** mount key ↔ its ingress object, made again as needed */
+	#ingresses = new Map();
+	#ingressKeys = new WeakMap();
+	#reviving = new Map();
 	#ready;
 
 	constructor(ctx, env) {
 		super(ctx, env);
+		const { sql } = ctx.storage;
+		// a vat's params, encoded (see #encode); a vat absent here can't be restarted
+		sql.exec("CREATE TABLE IF NOT EXISTS vat_params (vat TEXT PRIMARY KEY, params TEXT)");
+		// which vat was given which ingress, so a request can restart it
+		sql.exec("CREATE TABLE IF NOT EXISTS ingress_vat (key TEXT PRIMARY KEY, vat TEXT)");
 		this.#ready = this.#boot();
 	}
 
+	/** @param {string} key */
+	#ingressFor(key) {
+		let ingress = this.#ingresses.get(key);
+		if (!ingress) {
+			ingress = Far("ingress", {
+				/** @param {{ handle: (request: object) => object }} handler */
+				serve: (handler) => {
+					this.#handlers.set(key, handler);
+					for (const resolve of this.#waiting.get(key) ?? []) resolve();
+					this.#waiting.delete(key);
+				},
+			});
+			this.#ingresses.set(key, ingress);
+			this.#ingressKeys.set(ingress, key);
+		}
+		return ingress;
+	}
+
+	/**
+	 * `params` as JSON, an ingress by its key; `null` if anything in it can't
+	 * be made again here (a remote object of another peer, say).
+	 */
+	#encode(params) {
+		const keys = [];
+		const walk = (v) => {
+			const style = passStyleOf(v);
+			if (style === "remotable") {
+				const key = this.#ingressKeys.get(v);
+				if (!key) throw new Error("not restartable");
+				keys.push(key);
+				return { "@ingress": key };
+			}
+			if (style === "copyArray") return v.map(walk);
+			if (style === "copyRecord") return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]));
+			if (["string", "number", "boolean", "null"].includes(style)) return v;
+			throw new Error("not restartable");
+		};
+		try {
+			return { json: JSON.stringify(walk(params)), keys };
+		} catch {
+			return null;
+		}
+	}
+
+	#decode(json) {
+		const walk = (v) => {
+			if (Array.isArray(v)) return v.map(walk);
+			if (v && typeof v === "object") {
+				if ("@ingress" in v) return this.#ingressFor(v["@ingress"]);
+				return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, walk(x)]));
+			}
+			return v;
+		};
+		return harden(walk(JSON.parse(json)));
+	}
+
+	/** (Re)start a vat's code with `params`: it rebuilds its root. */
+	async #startVat(vat, params) {
+		const vatBootstrap = await this.#enliven(vat, VAT_SWISSNUM);
+		return E(vatBootstrap).start(params);
+	}
+
+	/**
+	 * The handler for a mount whose handler went with an evicted instance:
+	 * restart the vat given its ingress, and wait for it to serve again.
+	 *
+	 * @param {string} key
+	 */
+	#revive(key) {
+		let reviving = this.#reviving.get(key);
+		if (!reviving) {
+			reviving = (async () => {
+				const [row] = this.ctx.storage.sql
+					.exec("SELECT i.vat, p.params FROM ingress_vat i JOIN vat_params p USING (vat) WHERE i.key = ?", key)
+					.toArray();
+				if (!row) return null;
+				const served = new Promise((resolve) => this.#waiting.set(key, [...(this.#waiting.get(key) ?? []), resolve]));
+				await this.#startVat(row.vat, this.#decode(row.params));
+				await Promise.race([served, new Promise((resolve) => setTimeout(resolve, 5000))]);
+				return this.#handlers.get(key) ?? null;
+			})().finally(() => this.#reviving.delete(key));
+			this.#reviving.set(key, reviving);
+		}
+		return reviving;
+	}
+
 	async #boot() {
 		const vatAdmin = Far("vatAdmin", {
 			/**
 			 * Create an OCapN vat running `src`; `params` (any passable,
 			 * remote objects included) reach its `buildRootObject`.
 			 *
+			 * If `params` holds only data and ingresses, they are kept, and the
+			 * vat is restarted with them when a request needs it after an
+			 * eviction or a restart.
+			 *
 			 * @param {string} src an ES module exporting `buildRootObject(vatPowers, params)`
 			 */
 			createVat: async (src, params = {}) => {
 				const id = this.env.VATS.newUniqueId();
+				const vat = id.toString();
 				await this.env.VATS.get(id).start(src, null, "ocapn");
-				const vatBootstrap = await this.#enliven(id.toString(), VAT_SWISSNUM);
-				const root = await E(vatBootstrap).start(params);
-				return harden({ id: id.toString(), root });
+				const encoded = this.#encode(params);
+				if (encoded) {
+					const { sql } = this.ctx.storage;
+					sql.exec("INSERT INTO vat_params VALUES (?, ?)", vat, encoded.json);
+					for (const key of encoded.keys) sql.exec("INSERT OR REPLACE INTO ingress_vat VALUES (?, ?)", key, vat);
+				}
+				const root = await this.#startVat(vat, params);
+				return harden({ id: vat, root });
 			},
 			/**
 			 * Reserve a mount at `labels` and return the ingress for it: whoever
@@ -212,10 +319,7 @@ export class VatAdmin extends DurableObject {
 			makeIngress: async (labels) => {
 				const key = `ingress:${crypto.randomUUID()}`;
 				await this.env.DIRECTORY.getByName("directory").mount(key, harden({ ...labels }));
-				return Far("ingress", {
-					/** @param {{ handle: (request: object) => object }} handler */
-					serve: (handler) => void this.#handlers.set(key, handler),
-				});
+				return this.#ingressFor(key);
 			},
 		});
 		// workerd runs FinalizationRegistry callbacks outside any request, where
@@ -266,7 +370,7 @@ export class VatAdmin extends DurableObject {
 	 */
 	async serveHttp(key, request) {
 		await this.#ready;
-		const handler = this.#handlers.get(key);
+		const handler = this.#handlers.get(key) ?? (await this.#revive(key));
 		return handler ? E(handler).handle(harden(request)) : null;
 	}