WIP: toybox: OCapN vats on workerd, with ingress and egress as capabilities #4

Draft
dckc wants to merge 13 commits from toybox into main
pull from: toybox
12 changed files with 401 additions and 0 deletions
Showing only changes of commit 88739d9495 - Show all commits

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
state/
admin.sock
+36
View File
@@ -0,0 +1,36 @@
# toybox
Vats whose code is installed at runtime: each vat is a Durable Object whose app runs as a [facet](https://developers.cloudflare.com/dynamic-workers/usage/durable-object-facets/) on code loaded through workerd's Worker Loader. Think `vatAdmin.createVat(bundle)`, with a supervisor per vat.
- [supervisor.js](supervisor.js): `Vat` stores code by SHA-256 and runs the app on it; `Public` and `Admin` are the two entrypoints.
- [config.capnp](config.capnp): the sockets have no addresses; whoever starts workerd hands them in, so holding a socket is the authority it grants.
| socket | reaches | what its holder can do |
|---|---|---|
| `http` | `Public` | `GET /<vat>/…`: talk to a vat's app |
| `admin` | `Admin` | `PUT /<vat>` with an ES module body: install it and (re)start the app; its storage carries over |
An app must export `class App extends DurableObject` (see [apps/](apps/)). It gets an empty `env` and no `fetch`.
## Run it
```sh
pnpm install
pnpm test # a real workerd: install, upgrade, isolation, no fetch, crash and restart
pnpm start # systemd-socket-activate on 127.0.0.1:8787 and ./admin.sock; state in ./state
```
Then, in another terminal:
```sh
curl -X PUT --unix-socket admin.sock --data-binary @apps/counter-v1.js http://toybox/alice
curl localhost:8787/alice # v1 count 1
curl -X PUT --unix-socket admin.sock --data-binary @apps/counter-v2.js http://toybox/alice
curl localhost:8787/alice # v2 count 2
```
## Under systemd
[systemd/](systemd/) has a socket and service pair, assuming this directory is installed at `/opt/toybox`. The service gets no network of its own (`PrivateNetwork=yes`): only the two sockets it is handed.
`PrivateNetwork=` needs the system manager; as a `systemd-run --user` unit, systemd logs "lack privileges for network namespace, proceeding without".
+8
View File
@@ -0,0 +1,8 @@
import { DurableObject } from "cloudflare:workers";
export class App extends DurableObject {
fetch() {
const n = (this.ctx.storage.kv.get("n") ?? 0) + 1;
this.ctx.storage.kv.put("n", n);
return new Response(`v1 count ${n}\n`);
}
}
+8
View File
@@ -0,0 +1,8 @@
import { DurableObject } from "cloudflare:workers";
export class App extends DurableObject {
fetch() {
const n = (this.ctx.storage.kv.get("n") ?? 0) + 1;
this.ctx.storage.kv.put("n", n);
return new Response(`v2 count ${n}\n`);
}
}
+7
View File
@@ -0,0 +1,7 @@
import { DurableObject } from "cloudflare:workers";
export class App extends DurableObject {
async fetch() {
try { await fetch("https://example.com/"); return new Response("fetch ALLOWED\n"); }
catch (e) { return new Response(`fetch refused: ${e.message}\n`); }
}
}
+32
View File
@@ -0,0 +1,32 @@
# toybox: vats whose code is installed at runtime, each a Durable Object facet.
#
# Sockets have no address here: whoever starts workerd hands them in
# (-S name=fd from systemd, or -s name=addr), so holding a socket is the
# authority it grants. `state` is likewise overridden with -d state=<dir>.
using Workerd = import "/workerd/workerd.capnp";
const config :Workerd.Config = (
services = [
(name = "main", worker = .supervisor),
(name = "nowhere", network = (allow = [])),
(name = "state", disk = (path = "state", writable = true)),
],
sockets = [
# reach a running vat by name
(name = "http", http = (), service = (name = "main", entrypoint = "Public")),
# install code into a vat and (re)start it
(name = "admin", http = (), service = (name = "main", entrypoint = "Admin")),
],
);
const supervisor :Workerd.Worker = (
compatibilityDate = "2026-10-01",
modules = [(name = "supervisor.js", esModule = embed "supervisor.js")],
durableObjectNamespaces = [(className = "Vat", uniqueKey = "toybox-vat", enableSql = true)],
durableObjectStorage = (localDisk = "state"),
bindings = [
(name = "VATS", durableObjectNamespace = "Vat"),
(name = "LOADER", workerLoader = ()),
],
globalOutbound = "nowhere",
);
+13
View File
@@ -0,0 +1,13 @@
{
"name": "toybox",
"version": "0.0.1",
"private": true,
"type": "module",
"scripts": {
"start": "mkdir -p state && rm -f admin.sock && systemd-socket-activate -l 127.0.0.1:8787 -l ./admin.sock -- node_modules/workerd/bin/workerd serve --experimental config.capnp -S http=3 -S admin=4 -d state=state",
"test": "node --test"
},
"devDependencies": {
"workerd": "1.20261006.1"
}
}
+75
View File
@@ -0,0 +1,75 @@
lockfileVersion: '9.0'
settings:
autoInstallPeers: true
excludeLinksFromLockfile: false
importers:
.:
devDependencies:
workerd:
specifier: 1.20261006.1
version: 1.20261006.1
packages:
'@cloudflare/workerd-darwin-64@1.20261006.1':
resolution: {integrity: sha512-lCX6avJO2It9AyDEYitjmOtJRVp/xmaW0TziyoLAnfxlzozyyRbSFfIIMMCiDx/sTP1++E4VLbV+7BHSD33f0Q==}
engines: {node: '>=16'}
cpu: [x64]
os: [darwin]
'@cloudflare/workerd-darwin-arm64@1.20261006.1':
resolution: {integrity: sha512-PyFGjOaldtppnLuKsdkoYWXfLVpDjuf8u+uoykX+1Xvxo2EPmCVViq0iDKm+U7c7PRno85ehS3/MPVXyk2HwHw==}
engines: {node: '>=16'}
cpu: [arm64]
os: [darwin]
'@cloudflare/workerd-linux-64@1.20261006.1':
resolution: {integrity: sha512-Ntf9S8eesmVufdjaT2kzWlz6Ng9XGYTc6cHQVRKAg59lFufwsIabGeMT6U4KkrpEeSIxgHq8aPtuLbZXE19tCA==}
engines: {node: '>=16'}
cpu: [x64]
os: [linux]
'@cloudflare/workerd-linux-arm64@1.20261006.1':
resolution: {integrity: sha512-JyeDBymVWw5tVRebSsVE8ojDYxd6UlZrAseHU8G+NpIyd5wF61tXlF/axLeCUqu0pOhBFHb8I5QKi0jwrCK41Q==}
engines: {node: '>=16'}
cpu: [arm64]
os: [linux]
'@cloudflare/workerd-windows-64@1.20261006.1':
resolution: {integrity: sha512-nWP0URNqOBN9fa/gZ7eh/w051DwjOXN8j4lDZDYVXyQPsux6sV3DfyYZT+CjR/rt9nKBSmJT9QE9/bWT18+84w==}
engines: {node: '>=16'}
cpu: [x64]
os: [win32]
workerd@1.20261006.1:
resolution: {integrity: sha512-8MadBuffZ5NdpLwZjUmsqT5QlaKXq4gjgT6wzcKeynZTICNXyePhc+5QVoKiK7+MMhFTLNrRO2KXLlpk8Q6BkA==}
engines: {node: '>=16'}
hasBin: true
snapshots:
'@cloudflare/workerd-darwin-64@1.20261006.1':
optional: true
'@cloudflare/workerd-darwin-arm64@1.20261006.1':
optional: true
'@cloudflare/workerd-linux-64@1.20261006.1':
optional: true
'@cloudflare/workerd-linux-arm64@1.20261006.1':
optional: true
'@cloudflare/workerd-windows-64@1.20261006.1':
optional: true
workerd@1.20261006.1:
optionalDependencies:
'@cloudflare/workerd-darwin-64': 1.20261006.1
'@cloudflare/workerd-darwin-arm64': 1.20261006.1
'@cloudflare/workerd-linux-64': 1.20261006.1
'@cloudflare/workerd-linux-arm64': 1.20261006.1
'@cloudflare/workerd-windows-64': 1.20261006.1
+72
View File
@@ -0,0 +1,72 @@
/** @file toybox supervisor: one Vat per name; its app runs as a facet on installed code. */
import { DurableObject, WorkerEntrypoint } from "cloudflare:workers";
const hex = (buf) => [...new Uint8Array(buf)].map((b) => b.toString(16).padStart(2, "0")).join("");
/**
* Holds the vat's installed code; the app facet holds the app's state.
*
* Code is stored by hash so the facet can be rebuilt after hibernation or a
* restart. The app gets an empty `env` and no `fetch`.
*/
export class Vat extends DurableObject {
constructor(ctx, env) {
super(ctx, env);
ctx.storage.sql.exec("CREATE TABLE IF NOT EXISTS bundle (hash TEXT PRIMARY KEY, src TEXT)");
ctx.storage.sql.exec("CREATE TABLE IF NOT EXISTS current (k INTEGER PRIMARY KEY CHECK (k = 0), hash TEXT)");
}
/**
* Install `src` and run the app on it; the app's storage carries over.
*
* @param {string} src an ES module exporting `class App extends DurableObject`
* @returns {Promise<string>} the code's SHA-256, hex
*/
async start(src) {
const hash = hex(await crypto.subtle.digest("SHA-256", new TextEncoder().encode(src)));
this.ctx.storage.sql.exec("INSERT OR IGNORE INTO bundle VALUES (?, ?)", hash, src);
this.ctx.storage.sql.exec("INSERT OR REPLACE INTO current VALUES (0, ?)", hash);
this.ctx.facets.abort("app", new Error("upgraded"));
return hash;
}
#app(hash, src) {
return this.ctx.facets.get("app", () => {
const worker = this.env.LOADER.get(hash, () => ({
compatibilityDate: "2026-10-01",
mainModule: "app.js",
modules: { "app.js": src },
env: {},
globalOutbound: null,
}));
return { class: worker.getDurableObjectClass("App") };
});
}
fetch(request) {
const [row] = this.ctx.storage.sql
.exec("SELECT b.hash, b.src FROM current c JOIN bundle b USING (hash)")
.toArray();
if (!row) return new Response("no code installed\n", { status: 404 });
return this.#app(row.hash, row.src).fetch(request);
}
}
const vatOf = (env, url) => env.VATS.getByName(new URL(url).pathname.split("/")[1]);
/** The http socket: `/<vat>/...` goes to that vat's app. */
export class Public extends WorkerEntrypoint {
fetch(request) {
return vatOf(this.env, request.url).fetch(request);
}
}
/** The admin socket: `PUT /<vat>` with an ES module body installs and starts it. */
export class Admin extends WorkerEntrypoint {
async fetch(request) {
if (request.method !== "PUT") return new Response("PUT code to /<vat>\n", { status: 405 });
return new Response((await vatOf(this.env, request.url).start(await request.text())) + "\n");
}
}
export default Public;
+16
View File
@@ -0,0 +1,16 @@
[Unit]
Description=toybox vats (workerd)
Requires=toybox.socket
[Service]
ExecStart=/opt/toybox/node_modules/workerd/bin/workerd serve --experimental /opt/toybox/config.capnp -S http=3 -S admin=4 -d state=/var/lib/toybox
DynamicUser=yes
StateDirectory=toybox
# no network but the sockets handed in
PrivateNetwork=yes
RestrictAddressFamilies=AF_UNIX
ProtectSystem=strict
ProtectHome=yes
PrivateTmp=yes
SystemCallFilter=@system-service
# not MemoryDenyWriteExecute: V8's JIT needs W+X
+13
View File
@@ -0,0 +1,13 @@
[Unit]
Description=toybox vats: public port and admin socket
[Socket]
# order sets the fds: http=3, admin=4 (see toybox.service)
ListenStream=127.0.0.1:8787
ListenStream=/run/toybox/admin.sock
# whoever can open admin.sock can install code
SocketUser=root
SocketMode=0600
[Install]
WantedBy=sockets.target
+119
View File
@@ -0,0 +1,119 @@
/** @file toybox end to end: a real workerd, an admin socket, a public port. */
import { spawn } from "node:child_process";
import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
import { request } from "node:http";
import { createServer } from "node:net";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { after, before, test } from "node:test";
import assert from "node:assert/strict";
const here = new URL("..", import.meta.url).pathname;
// the package binary, not the .bin shim: the shim forks it, so a kill would orphan workerd
const workerd = join(here, "node_modules/workerd/bin/workerd");
const app = (name) => readFile(join(here, "apps", name), "utf8");
const freePort = () =>
new Promise((resolve) => {
const s = createServer().listen(0, "127.0.0.1", () => {
const { port } = s.address();
s.close(() => resolve(port));
});
});
/** @param {string} socketPath the admin socket; holding it is the authority to install */
const makeAdmin = (socketPath) => ({
install: (vat, src) =>
new Promise((resolve, reject) => {
const req = request({ socketPath, path: `/${vat}`, method: "PUT", agent: false }, (res) => {
let body = "";
res.on("data", (c) => (body += c));
res.on("end", () => resolve(body.trim()));
});
req.on("error", reject);
req.end(src);
}),
});
/** @param {string} base the public origin */
const makePublic = (base) => ({
get: async (vat) => (await fetch(`${base}/${vat}`)).text(),
put: async (vat, src) => (await fetch(`${base}/${vat}`, { method: "PUT", body: src })).status,
});
/** Run workerd on `dir`'s state until `stop()`; resolves once the port answers. */
const startWorkerd = async (dir, port) => {
await rm(join(dir, "admin.sock"), { force: true }); // left behind by a crash; bind() would fail
const child = spawn(
workerd,
["serve", "--experimental", join(here, "config.capnp"),
"-s", `http=127.0.0.1:${port}`, "-s", `admin=unix:${join(dir, "admin.sock")}`,
"-d", `state=${join(dir, "state")}`],
{ stdio: ["ignore", "ignore", "pipe"] },
);
let log = "";
child.stderr.on("data", (c) => (log += c));
const exited = new Promise((resolve) => child.on("exit", resolve));
for (let i = 0; i < 100; i++) {
try {
await fetch(`http://127.0.0.1:${port}/`);
// SIGKILL: a crash is the harder durability test
return { stop: () => (child.kill("SIGKILL"), exited) };
} catch {
await new Promise((r) => setTimeout(r, 50));
}
}
child.kill("SIGKILL");
throw new Error(`workerd did not start: ${log}`);
};
let dir, port, server, admin, pub;
before(async () => {
dir = await mkdtemp(join(tmpdir(), "toybox-"));
await mkdir(join(dir, "state"));
port = await freePort();
server = await startWorkerd(dir, port);
admin = makeAdmin(join(dir, "admin.sock"));
pub = makePublic(`http://127.0.0.1:${port}`);
});
after(async () => {
await server?.stop();
await rm(dir, { recursive: true, force: true });
});
test("a vat with no code is 404", async () => {
assert.equal(await pub.get("nobody"), "no code installed\n");
});
test("install, run, upgrade: the app's state carries over", async () => {
assert.match(await admin.install("alice", await app("counter-v1.js")), /^[0-9a-f]{64}$/);
assert.equal(await pub.get("alice"), "v1 count 1\n");
assert.equal(await pub.get("alice"), "v1 count 2\n");
await admin.install("alice", await app("counter-v2.js"));
assert.equal(await pub.get("alice"), "v2 count 3\n");
});
test("vats don't share state", async () => {
await admin.install("carol", await app("counter-v1.js"));
assert.equal(await pub.get("carol"), "v1 count 1\n");
});
test("an app gets no fetch", async () => {
await admin.install("bob", await app("try-fetch.js"));
assert.match(await pub.get("bob"), /^fetch refused: This worker is not permitted to access the internet/);
});
test("the public port can't install code", async () => {
assert.equal(await pub.put("mallory", "export class App {}"), 404);
assert.equal(await pub.get("mallory"), "no code installed\n");
});
test("code and state survive a restart", async () => {
await admin.install("dave", await app("counter-v1.js"));
assert.equal(await pub.get("dave"), "v1 count 1\n");
await server.stop();
server = await startWorkerd(dir, port);
assert.equal(await pub.get("dave"), "v1 count 2\n");
});