From f0557d29d6b1dc47f624b494c94ad09f6c229413 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Tue, 13 Oct 2026 20:31:10 -0500 Subject: [PATCH] capsite/toybox: egress through Flower-style switchboard handles The switchboard now serves a Cap'n Web Handle on each of its sockets: public (no loopback or private addresses) and local. A socket's labels are fixed in config, so holding the socket's fd is holding that handle; constrain(labels) returns a narrower one, as Flower's Constrain returns a narrower fd. Labels only accumulate: a missing key matches anything, and a key both sides set must agree (labels.js, shared by both). The switchboard enforces the labels; toybox's Egress no longer checks URLs itself. It opens a session to its one switchboard socket, constrains the handle to the vat's grant, and forwards through it; a WebSocket crosses the session as a pair of message streams. A vat narrows its own grant with env.egress.constrain(). Grants are now labels as JSON. One stored before, a bare host[:port], reads as {host, port}. pnpm start and the systemd units give toybox the public handle; the switchboard unit binds the local one in its own 0700 runtime directory. The tests give their toybox the local handle to reach an echo server, and check that a toybox holding only the public one never reaches it. Co-Authored-By: Claude Opus 5.5 (1M context) --- toybox/.gitignore | 2 +- toybox/README.md | 26 ++++-- toybox/apps/egress-probe.js | 8 +- toybox/config.capnp | 10 +- toybox/labels.js | 52 +++++++++++ toybox/package.json | 7 +- toybox/pnpm-lock.yaml | 9 ++ toybox/supervisor.js | 68 ++++++++++++-- toybox/switchboard.capnp | 32 ++++--- toybox/switchboard.js | 89 +++++++++++++++--- toybox/systemd/toybox-switchboard.service | 10 +- toybox/systemd/toybox-switchboard.socket | 6 +- toybox/systemd/toybox.service | 4 +- toybox/test/vat.test.js | 106 ++++++++++++++++------ 14 files changed, 337 insertions(+), 92 deletions(-) create mode 100644 toybox/labels.js diff --git a/toybox/.gitignore b/toybox/.gitignore index 3de5902..4340523 100644 --- a/toybox/.gitignore +++ b/toybox/.gitignore @@ -1,3 +1,3 @@ state/ admin.sock -switchboard.sock +switchboard-*.sock diff --git a/toybox/README.md b/toybox/README.md index d7511f0..6df609a 100644 --- a/toybox/README.md +++ b/toybox/README.md @@ -3,24 +3,32 @@ Vats whose code is installed at runtime: each vat is a Durable Object whose app runs as a [facet](https://developers.cloudflare.com/dynamic-workers/usage/durable-object-facets/) on code loaded through workerd's Worker Loader. Think `vatAdmin.createVat(bundle)`, with a supervisor per vat. - [supervisor.js](supervisor.js): `Vat` stores code by SHA-256 and runs the app on it; `Public` and `Admin` are the two entrypoints; `Egress` is a grant of one host. -- [switchboard.js](switchboard.js): a separate workerd process, the vats' only way out: a forward proxy for HTTP and WebSockets. +- [switchboard.js](switchboard.js): a separate workerd process, the vats' only way out, after [Flower](https://github.com/NuxiNL/flower)'s switchboard: each of its sockets serves a Cap'n Web `Handle` with fixed labels, and `constrain(labels)` returns a narrower one. +- [labels.js](labels.js): Flower's label rule, shared by both: labels only accumulate, a missing key matches anything, and a key both sides set must agree. - [config.capnp](config.capnp): the sockets have no addresses; whoever starts workerd hands them in, so holding a socket is the authority it grants. | socket | reaches | what its holder can do | |---|---|---| | `http` | `Public` | `GET //…`: talk to a vat's app | -| `admin` | `Admin` | `PUT /` with an ES module body: install it and (re)start the app; its storage carries over. `?egress=` grants the app that one host. | +| `admin` | `Admin` | `PUT /` with an ES module body: install it and (re)start the app; its storage carries over. `?egress=` grants the app egress within those labels. | -An app must export `class App extends DurableObject` (see [apps/](apps/)). Its global `fetch` always throws: nothing in a vat is ambient. A granted host arrives as `env.egress`, an `Egress` with the host in `ctx.props`: its `fetch` forwards matching requests, WebSocket upgrades included, to the switchboard and refuses the rest with a 403. It is an object, so the app can pass it to whichever of its parts should have it. +An app must export `class App extends DurableObject` (see [apps/](apps/)). Its global `fetch` always throws: nothing in a vat is ambient. A grant arrives as `env.egress`, an `Egress` with the labels in `ctx.props`. Its `fetch`, WebSocket upgrades included, goes through a switchboard handle constrained to those labels; the switchboard refuses the rest, which comes back as a 403. `env.egress.constrain(labels)` returns a narrower `Egress`. It is an object, so the app can pass it to whichever of its parts should have it. -toybox reaches the switchboard over a unix socket (`-e switchboard=unix:…`), in HTTP proxy style so the full URL, scheme included, survives the hop. +| label | constrains | +|---|---| +| `scheme` | `http` or `https` | +| `host` | host name or address, no port | +| `port` | explicit, or the scheme's default | +| `network` | set by the switchboard socket: `public` or `local` | + +The switchboard has two sockets, so two root handles: `public` (no loopback or private addresses) and `local`. toybox holds one, as an external service (`-e switchboard=unix:…`): `pnpm start` and the systemd units give it `public`; the tests give theirs `local` to reach an echo server. ## Run it ```sh pnpm install -pnpm test # real workerds: install, upgrade, isolation, egress grants, fetch and WebSocket out, crash and restart -pnpm switchboard # systemd-socket-activate on ./switchboard.sock +pnpm test # real workerds: install, upgrade, isolation, egress grants and constrain, fetch and WebSocket out, crash and restart +pnpm switchboard # systemd-socket-activate on ./switchboard-public.sock and ./switchboard-local.sock pnpm start # likewise on 127.0.0.1:8787 and ./admin.sock; state in ./state ``` @@ -32,14 +40,14 @@ curl localhost:8787/alice # v1 count 1 curl -X PUT --unix-socket admin.sock --data-binary @apps/counter-v2.js http://toybox/alice curl localhost:8787/alice # v2 count 2 -curl -X PUT --unix-socket admin.sock --data-binary @apps/egress-probe.js 'http://toybox/erin?egress=example.com' +curl -g -X PUT --unix-socket admin.sock --data-binary @apps/egress-probe.js 'http://toybox/erin?egress={"host":"example.com"}' curl 'localhost:8787/erin?url=https://example.com/' # got: … -curl 'localhost:8787/erin?url=https://example.org/' # got: egress to example.org not granted +curl 'localhost:8787/erin?url=https://example.org/' # got: egress refused: egress to example.org not granted ``` ## Under systemd -[systemd/](systemd/) has socket and service pairs for toybox and its switchboard, assuming this directory is installed at `/opt/toybox`. toybox gets no network of its own (`PrivateNetwork=yes`): only the two sockets it is handed and the switchboard's socket, which only members of the `toybox` group can open. The switchboard has the network, fenced by `IPAddressDeny=`; `switchboard.capnp` allows `local` addresses only so the tests can reach a local echo server. +[systemd/](systemd/) has socket and service pairs for toybox and its switchboard, assuming this directory is installed at `/opt/toybox`. toybox gets no network of its own (`PrivateNetwork=yes`): only the two sockets it is handed and the switchboard's `public` socket, which only members of the `toybox` group can open. The switchboard binds its `local` socket in its own 0700 runtime directory, so no other unit holds that handle. Neither unit has been run under the system manager yet. diff --git a/toybox/apps/egress-probe.js b/toybox/apps/egress-probe.js index 3c09f5e..69e8fe1 100644 --- a/toybox/apps/egress-probe.js +++ b/toybox/apps/egress-probe.js @@ -1,13 +1,15 @@ import { DurableObject } from "cloudflare:workers"; -/** `?url=` fetches it through `env.egress`; with `&ws=1`, opens a WebSocket there and reports the echo of "hello". */ +/** `?url=` fetches it through `env.egress` (narrowed by `&constrain=`, if given); with `&ws=1`, opens a WebSocket there and reports the echo of "hello". */ export class App extends DurableObject { async fetch(request) { const params = new URL(request.url).searchParams; const target = params.get("url"); - const { egress } = this.env; - if (!egress) return new Response("no egress granted\n"); + if (!this.env.egress) return new Response("no egress granted\n"); try { + const egress = params.has("constrain") + ? await this.env.egress.constrain(JSON.parse(params.get("constrain"))) + : this.env.egress; if (!params.has("ws")) return new Response(`got: ${await (await egress.fetch(target)).text()}\n`); const resp = await egress.fetch(target, { headers: { Upgrade: "websocket" } }); const ws = resp.webSocket; diff --git a/toybox/config.capnp b/toybox/config.capnp index 47c58f0..37403b8 100644 --- a/toybox/config.capnp +++ b/toybox/config.capnp @@ -10,8 +10,8 @@ const config :Workerd.Config = ( services = [ (name = "main", worker = .supervisor), (name = "nowhere", network = (allow = [])), - # the way out: see switchboard.capnp - (name = "switchboard", external = (address = "unix:switchboard.sock", http = (style = proxy))), + # the way out: one switchboard socket, i.e. one handle (see switchboard.capnp) + (name = "switchboard", external = (address = "unix:switchboard.sock", http = ())), (name = "state", disk = (path = "state", writable = true)), ], sockets = [ @@ -24,7 +24,11 @@ const config :Workerd.Config = ( const supervisor :Workerd.Worker = ( compatibilityDate = "2026-10-01", - modules = [(name = "supervisor.js", esModule = embed "supervisor.js")], + modules = [ + (name = "supervisor.js", esModule = embed "supervisor.js"), + (name = "labels.js", esModule = embed "labels.js"), + (name = "capnweb", esModule = embed "node_modules/capnweb/dist/index-workers.js"), + ], durableObjectNamespaces = [(className = "Vat", uniqueKey = "toybox-vat", enableSql = true)], durableObjectStorage = (localDisk = "state"), bindings = [ diff --git a/toybox/labels.js b/toybox/labels.js new file mode 100644 index 0000000..75f36ad --- /dev/null +++ b/toybox/labels.js @@ -0,0 +1,52 @@ +/** + * @file Egress labels, after [Flower](https://github.com/NuxiNL/flower)'s + * switchboard: string key-value pairs that only accumulate. A missing key + * matches anything; a key present on both sides must agree. + * + * Keys that constrain a URL: `scheme` (`http`, `https`), `host` (name or + * address, no port), `port` (explicit or the scheme's default). The + * switchboard sets `network` per socket; it constrains no URL. + */ + +const DEFAULT_PORT = { http: "80", https: "443" }; + +/** + * Union of two label sets; throws on a key they give different values, + * naming the key but not the values (they may end up in logs). + * + * @param {Record} held + * @param {Record} more + */ +export const mergeLabels = (held, more) => { + const merged = { ...held }; + for (const [k, v] of Object.entries(more)) { + if (typeof v !== "string") throw new TypeError(`label ${k} is not a string`); + if (k in merged && merged[k] !== v) throw new Error(`label ${k} is already set to a different value`); + merged[k] = v; + } + return Object.freeze(merged); +}; + +/** + * The labels a URL carries. + * + * @param {string} url + */ +export const urlLabels = (url) => { + const u = new URL(url); + const scheme = u.protocol.replace(/:$/, ""); + return { scheme, host: u.hostname, port: u.port || DEFAULT_PORT[scheme] || "" }; +}; + +/** + * Throws unless `url` is within `labels`. + * + * @param {Record} labels + * @param {string} url + */ +export const checkUrl = (labels, url) => { + const actual = urlLabels(url); + for (const [k, v] of Object.entries(actual)) { + if (k in labels && labels[k] !== v) throw new Error(`egress to ${actual.host} not granted`); + } +}; diff --git a/toybox/package.json b/toybox/package.json index eb67ba4..dbfce98 100644 --- a/toybox/package.json +++ b/toybox/package.json @@ -4,12 +4,15 @@ "private": true, "type": "module", "scripts": { - "start": "mkdir -p state && rm -f admin.sock && systemd-socket-activate -l 127.0.0.1:8787 -l ./admin.sock -- node_modules/workerd/bin/workerd serve --experimental config.capnp -S http=3 -S admin=4 -d state=state -e switchboard=unix:./switchboard.sock", - "switchboard": "rm -f switchboard.sock && systemd-socket-activate -l ./switchboard.sock -- node_modules/workerd/bin/workerd serve --experimental switchboard.capnp -S http=3", + "start": "mkdir -p state && rm -f admin.sock && systemd-socket-activate -l 127.0.0.1:8787 -l ./admin.sock -- node_modules/workerd/bin/workerd serve --experimental config.capnp -S http=3 -S admin=4 -d state=state -e switchboard=unix:./switchboard-public.sock", + "switchboard": "rm -f switchboard-public.sock switchboard-local.sock && systemd-socket-activate -l ./switchboard-public.sock -l ./switchboard-local.sock -- node_modules/workerd/bin/workerd serve --experimental switchboard.capnp -S public=3 -S local=4", "test": "node --test" }, "devDependencies": { "workerd": "1.20261006.1", "ws": "8.22.0" + }, + "dependencies": { + "capnweb": "0.12.0" } } diff --git a/toybox/pnpm-lock.yaml b/toybox/pnpm-lock.yaml index 50e5b12..996a657 100644 --- a/toybox/pnpm-lock.yaml +++ b/toybox/pnpm-lock.yaml @@ -7,6 +7,10 @@ settings: importers: .: + dependencies: + capnweb: + specifier: 0.12.0 + version: 0.12.0 devDependencies: workerd: specifier: 1.20261006.1 @@ -47,6 +51,9 @@ packages: cpu: [x64] os: [win32] + capnweb@0.12.0: + resolution: {integrity: sha512-jgZ/LMtMVTi+RVlooIslH78Gg23XbDTdvcLghWx3oz17D6u5GuJARt+uhZk/wcTo+f81eeabfnow4d3zLBj+JQ==} + workerd@1.20261006.1: resolution: {integrity: sha512-8MadBuffZ5NdpLwZjUmsqT5QlaKXq4gjgT6wzcKeynZTICNXyePhc+5QVoKiK7+MMhFTLNrRO2KXLlpk8Q6BkA==} engines: {node: '>=16'} @@ -81,6 +88,8 @@ snapshots: '@cloudflare/workerd-windows-64@1.20261006.1': optional: true + capnweb@0.12.0: {} + workerd@1.20261006.1: optionalDependencies: '@cloudflare/workerd-darwin-64': 1.20261006.1 diff --git a/toybox/supervisor.js b/toybox/supervisor.js index 527b41a..0b543a7 100644 --- a/toybox/supervisor.js +++ b/toybox/supervisor.js @@ -1,5 +1,18 @@ /** @file toybox supervisor: one Vat per name; its app runs as a facet on installed code. */ import { DurableObject, WorkerEntrypoint } from "cloudflare:workers"; +import { newWebSocketRpcSession } from "capnweb"; +import { mergeLabels, urlLabels } from "./labels.js"; + +/** + * A stored grant: labels as JSON, or (before labels) a bare `host[:port]`. + * + * @param {string} stored + */ +const grantLabels = (stored) => { + if (stored.startsWith("{")) return JSON.parse(stored); + const { host, port } = urlLabels(`http://${stored}`); + return stored.includes(":") ? { host, port } : { host }; +}; const hex = (buf) => [...new Uint8Array(buf)].map((b) => b.toString(16).padStart(2, "0")).join(""); @@ -27,7 +40,7 @@ export class Vat extends DurableObject { * Install `src` and run the app on it; the app's storage carries over. * * @param {string} src an ES module exporting `class App extends DurableObject` - * @param {string | null} egress the one host (`name[:port]`) the app may fetch, or none + * @param {string | null} egress the app's grant: egress labels as JSON (see labels.js), or none * @returns {Promise} the code's SHA-256, hex */ async start(src, egress) { @@ -47,7 +60,7 @@ export class Vat extends DurableObject { compatibilityDate: "2026-10-01", mainModule: "app.js", modules: { "app.js": src }, - env: egress ? { egress: this.ctx.exports.Egress({ props: { host: egress } }) } : {}, + env: egress ? { egress: this.ctx.exports.Egress({ props: { labels: grantLabels(egress) } }) } : {}, globalOutbound: null, })); return { class: worker.getDurableObjectClass("App") }; @@ -67,14 +80,51 @@ export class Vat extends DurableObject { } /** - * A grant of one host: `fetch`, WebSocket upgrades included, reaches the - * switchboard only for the host in `ctx.props`, which no holder can change. + * A vat's egress grant: a switchboard handle constrained to the labels in + * `ctx.props`, which no holder can change. The switchboard enforces them; + * a refusal comes back as a 403. */ export class Egress extends WorkerEntrypoint { - fetch(request) { - const { host } = new URL(request.url); - if (host !== this.ctx.props.host) return new Response(`egress to ${host} not granted\n`, { status: 403 }); - return this.env.SWITCHBOARD.fetch(request); + /** A narrower grant: these labels too (conflicting ones throw). */ + constrain(more) { + return this.ctx.exports.Egress({ props: { labels: mergeLabels(this.ctx.props.labels, more) } }); + } + + async #handle() { + const resp = await this.env.SWITCHBOARD.fetch("http://switchboard/", { headers: { Upgrade: "websocket" } }); + const ws = resp.webSocket; + if (!ws) throw new Error(`switchboard refused a session: ${resp.status}`); + ws.accept(); + return newWebSocketRpcSession(ws).constrain(this.ctx.props.labels); + } + + async fetch(request) { + try { + const handle = await this.#handle(); + if (request.headers.get("Upgrade")?.toLowerCase() !== "websocket") return await handle.fetch(request); + const protocols = (request.headers.get("Sec-WebSocket-Protocol") ?? "").split(",").map((p) => p.trim()).filter(Boolean); + const { readable, writable, protocol } = await handle.webSocket(request.url, protocols); + return this.#bridge(readable, writable, protocol); + } catch (e) { + return new Response(`egress refused: ${e.message}\n`, { status: 403 }); + } + } + + /** A WebSocket for the vat whose messages travel the switchboard's streams. */ + #bridge(readable, writable, protocol) { + const [client, server] = Object.values(new WebSocketPair()); + server.accept(); + const writer = writable.getWriter(); + server.addEventListener("message", (e) => writer.write(e.data).catch(() => server.close(1011))); + server.addEventListener("close", () => writer.close().catch(() => {})); + this.ctx.waitUntil( + (async () => { + for await (const m of readable) server.send(m); + server.close(); + })().catch(() => server.close(1011)), + ); + const headers = protocol ? { "Sec-WebSocket-Protocol": protocol } : {}; + return new Response(null, { status: 101, webSocket: client, headers }); } } @@ -89,7 +139,7 @@ export class Public extends WorkerEntrypoint { /** * The admin socket: `PUT /` with an ES module body installs and starts - * it; `?egress=` grants it that one host. + * it; `?egress=` grants it egress within those labels. */ export class Admin extends WorkerEntrypoint { async fetch(request) { diff --git a/toybox/switchboard.capnp b/toybox/switchboard.capnp index 3d28a4d..fa569b0 100644 --- a/toybox/switchboard.capnp +++ b/toybox/switchboard.capnp @@ -1,26 +1,34 @@ -# switchboard: toybox's only way out. A forward proxy (HTTP and WebSocket) -# that toybox's Egress gateway reaches over a socket handed in with -S/-s. +# switchboard: toybox's only way out, after Flower's switchboard. # -# The network service is the switchboard's outer fence; under systemd, -# IPAddressAllow=/IPAddressDeny= narrow it further (see systemd/). +# Each socket is a handle with fixed labels: a WebSocket upgrade starts a +# Cap'n Web session whose main object is that handle (see switchboard.js). +# Whoever starts workerd hands the sockets in (-S/-s), so whoever holds a +# socket's fd holds that handle and no other. using Workerd = import "/workerd/workerd.capnp"; const config :Workerd.Config = ( services = [ (name = "main", worker = .switchboard), - (name = "internet", network = (allow = ["public", "local"], tlsOptions = (trustBrowserCas = true))), + (name = "nowhere", network = (allow = [])), + (name = "public-net", network = (allow = ["public"], tlsOptions = (trustBrowserCas = true))), + (name = "local-net", network = (allow = ["local"], tlsOptions = (trustBrowserCas = true))), ], sockets = [ - # proxy style: the request line carries the full URL, scheme included - (name = "http", http = (style = proxy), service = "main"), + (name = "public", http = (), service = (name = "main", props = (json = "{\"network\":\"public\"}"))), + (name = "local", http = (), service = (name = "main", props = (json = "{\"network\":\"local\"}"))), ], ); const switchboard :Workerd.Worker = ( compatibilityDate = "2026-10-01", - modules = [(name = "switchboard.js", esModule = embed "switchboard.js")], - durableObjectNamespaces = [(className = "Switchboard", uniqueKey = "toybox-switchboard")], - durableObjectStorage = (inMemory = void), - bindings = [(name = "BOARD", durableObjectNamespace = "Switchboard")], - globalOutbound = "internet", + modules = [ + (name = "switchboard.js", esModule = embed "switchboard.js"), + (name = "labels.js", esModule = embed "labels.js"), + (name = "capnweb", esModule = embed "node_modules/capnweb/dist/index-workers.js"), + ], + bindings = [ + (name = "PUBLIC", service = "public-net"), + (name = "LOCAL", service = "local-net"), + ], + globalOutbound = "nowhere", ); diff --git a/toybox/switchboard.js b/toybox/switchboard.js index f90d8e9..f81f1a2 100644 --- a/toybox/switchboard.js +++ b/toybox/switchboard.js @@ -1,22 +1,81 @@ -/** @file switchboard: forwards each proxied request, WebSocket upgrades included. */ -import { DurableObject } from "cloudflare:workers"; +/** @file switchboard: Flower-style egress handles, served over Cap'n Web. */ +import { WorkerEntrypoint } from "cloudflare:workers"; +import { RpcTarget, newWorkersRpcResponse } from "capnweb"; +import { checkUrl, mergeLabels } from "./labels.js"; /** - * The one place egress happens. A 101 response from `fetch` carries its - * `webSocket`; returning it passes the socket through to the caller. - * - * Which vat may reach which host is decided before a request gets here - * (toybox's Egress gateway); this process's network service and its - * systemd unit are the outer fence. + * One egress handle: reaches only URLs within its labels, through the + * network its `network` label names. Like a Flower switchboard handle, it + * can only be narrowed; unlike one, it is a Cap'n Web stub, not an fd. */ -export class Switchboard extends DurableObject { +export class Handle extends RpcTarget { + #labels; + #net; + + /** + * @param {Record} labels + * @param {Fetcher} net + */ + constructor(labels, net) { + super(); + this.#labels = labels; + this.#net = net; + } + + /** @param {Record} more */ + constrain(more) { + return new Handle(mergeLabels(this.#labels, more), this.#net); + } + + labels() { + return this.#labels; + } + + /** @param {Request} request */ fetch(request) { - return fetch(request); + checkUrl(this.#labels, request.url); + return this.#net.fetch(request); + } + + /** + * Open a WebSocket to `url`; its messages cross the session as streams. + * + * @param {string} url `http:` or `https:`, as for a `fetch` upgrade + * @param {string[]} [protocols] + */ + async webSocket(url, protocols = []) { + checkUrl(this.#labels, url); + const headers = { Upgrade: "websocket" }; + if (protocols.length) headers["Sec-WebSocket-Protocol"] = protocols.join(", "); + const resp = await this.#net.fetch(url, { headers }); + const ws = resp.webSocket; + if (!ws) throw new Error(`no websocket: ${resp.status}`); + ws.accept(); + const readable = new ReadableStream({ + start(controller) { + ws.addEventListener("message", (e) => controller.enqueue(e.data)); + ws.addEventListener("close", () => controller.close()); + ws.addEventListener("error", () => controller.error(new Error("websocket error"))); + }, + }); + const writable = new WritableStream({ + write: (chunk) => ws.send(chunk), + close: () => ws.close(), + abort: () => ws.close(1011), + }); + return { readable, writable, protocol: resp.headers.get("Sec-WebSocket-Protocol") ?? "" }; } } -export default { - fetch(request, env) { - return env.BOARD.getByName("switchboard").fetch(request); - }, -}; +/** + * A switchboard socket: a WebSocket upgrade here starts a Cap'n Web + * session whose main object is a {@link Handle} with this socket's labels. + */ +export default class Root extends WorkerEntrypoint { + fetch(request) { + const labels = Object.freeze({ ...this.ctx.props }); + const net = { public: this.env.PUBLIC, local: this.env.LOCAL }[labels.network]; + if (!net) return new Response("socket has no network\n", { status: 500 }); + return newWorkersRpcResponse(request, new Handle(labels, net)); + } +} diff --git a/toybox/systemd/toybox-switchboard.service b/toybox/systemd/toybox-switchboard.service index fdfb23b..a2d251e 100644 --- a/toybox/systemd/toybox-switchboard.service +++ b/toybox/systemd/toybox-switchboard.service @@ -1,12 +1,14 @@ [Unit] -Description=toybox switchboard (workerd forward proxy) +Description=toybox switchboard (workerd, Cap'n Web egress handles) Requires=toybox-switchboard.socket [Service] -ExecStart=/opt/toybox/node_modules/workerd/bin/workerd serve --experimental /opt/toybox/switchboard.capnp -S http=3 +# the local (localhost) handle goes where no other unit can open it; +# hand it out deliberately, if ever +ExecStart=/opt/toybox/node_modules/workerd/bin/workerd serve --experimental /opt/toybox/switchboard.capnp -S public=3 -s local=unix:/run/toybox-switchboard/local.sock +RuntimeDirectory=toybox-switchboard +RuntimeDirectoryMode=0700 DynamicUser=yes -# the outer fence: switchboard.capnp allows "local" for tests; deny it here -IPAddressDeny=localhost link-local multicast ProtectSystem=strict ProtectHome=yes PrivateTmp=yes diff --git a/toybox/systemd/toybox-switchboard.socket b/toybox/systemd/toybox-switchboard.socket index 5cd94c3..2915c03 100644 --- a/toybox/systemd/toybox-switchboard.socket +++ b/toybox/systemd/toybox-switchboard.socket @@ -1,9 +1,9 @@ [Unit] -Description=toybox switchboard: the vats' only way out +Description=toybox switchboard: the public egress handle [Socket] -ListenStream=/run/toybox/switchboard.sock -# only toybox may dial out: its service joins this group +ListenStream=/run/toybox/switchboard-public.sock +# only toybox holds this handle: its service joins this group SocketGroup=toybox SocketMode=0660 diff --git a/toybox/systemd/toybox.service b/toybox/systemd/toybox.service index a17b960..c9a1252 100644 --- a/toybox/systemd/toybox.service +++ b/toybox/systemd/toybox.service @@ -4,9 +4,9 @@ Requires=toybox.socket toybox-switchboard.socket After=toybox-switchboard.socket [Service] -ExecStart=/opt/toybox/node_modules/workerd/bin/workerd serve --experimental /opt/toybox/config.capnp -S http=3 -S admin=4 -d state=/var/lib/toybox -e switchboard=unix:/run/toybox/switchboard.sock +ExecStart=/opt/toybox/node_modules/workerd/bin/workerd serve --experimental /opt/toybox/config.capnp -S http=3 -S admin=4 -d state=/var/lib/toybox -e switchboard=unix:/run/toybox/switchboard-public.sock DynamicUser=yes -# to open the switchboard socket +# to open the public switchboard socket SupplementaryGroups=toybox StateDirectory=toybox # no network but the sockets handed in, and the switchboard's diff --git a/toybox/test/vat.test.js b/toybox/test/vat.test.js index 1811d23..63709c4 100644 --- a/toybox/test/vat.test.js +++ b/toybox/test/vat.test.js @@ -1,4 +1,4 @@ -/** @file toybox end to end: a real workerd, an admin socket, a public port, a switchboard. */ +/** @file toybox end to end: a real workerd, an admin socket, a public port, a switchboard with two handles. */ import { spawn } from "node:child_process"; import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; import { createServer as createHttpServer, request } from "node:http"; @@ -31,9 +31,10 @@ const accepting = (opts) => /** @param {string} socketPath the admin socket; holding it is the authority to install */ const makeAdmin = (socketPath) => ({ - install: (vat, src, egress) => + install: (vat, src, labels) => new Promise((resolve, reject) => { - const path = egress ? `/${vat}?egress=${encodeURIComponent(egress)}` : `/${vat}`; + const grant = typeof labels === "string" ? labels : JSON.stringify(labels); + const path = labels ? `/${vat}?egress=${encodeURIComponent(grant)}` : `/${vat}`; const req = request({ socketPath, path, method: "PUT", agent: false }, (res) => { let body = ""; res.on("data", (c) => (body += c)); @@ -75,44 +76,57 @@ const startWorkerd = async (config, args, listening, stale = []) => { throw new Error(`workerd did not start: ${log}`); }; -/** An HTTP server that answers GET with "plain ok" and echoes WebSocket messages. */ +/** An HTTP server that answers GET with "plain ok", echoes WebSocket messages, and counts requests. */ const startEcho = async () => { - const server = createHttpServer((_req, res) => res.end("plain ok")); + let hits = 0; + const server = createHttpServer((_req, res) => (hits++, res.end("plain ok"))); const wss = new WebSocketServer({ server }); wss.on("connection", (ws) => ws.on("message", (m) => ws.send(String(m)))); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); return { host: `127.0.0.1:${server.address().port}`, + hits: () => hits, stop: () => new Promise((resolve) => (wss.close(), server.closeAllConnections(), server.close(resolve))), }; }; -let dir, port, server, switchboard, echo, admin, pub; +let dir, port, server, switchboard, echo, echoLabels, admin, pub; -const startToybox = () => - startWorkerd( +/** + * A toybox in `dir/` that holds one switchboard handle. + * + * @param {string} name + * @param {number} port + * @param {"public" | "local"} handle which switchboard socket it gets + */ +const startToybox = async (name, port, handle) => { + const home = join(dir, name); + await mkdir(join(home, "state"), { recursive: true }); + return startWorkerd( "config.capnp", [ - "-s", `http=127.0.0.1:${port}`, "-s", `admin=unix:${join(dir, "admin.sock")}`, - "-d", `state=${join(dir, "state")}`, "-e", `switchboard=unix:${join(dir, "switchboard.sock")}`, + "-s", `http=127.0.0.1:${port}`, "-s", `admin=unix:${join(home, "admin.sock")}`, + "-d", `state=${join(home, "state")}`, "-e", `switchboard=unix:${join(dir, `sb-${handle}.sock`)}`, ], { port, host: "127.0.0.1" }, - [join(dir, "admin.sock")], + [join(home, "admin.sock")], ); +}; before(async () => { dir = await mkdtemp(join(tmpdir(), "toybox-")); - await mkdir(join(dir, "state")); port = await freePort(); echo = await startEcho(); + echoLabels = { host: "127.0.0.1", port: echo.host.split(":")[1] }; switchboard = await startWorkerd( "switchboard.capnp", - ["-s", `http=unix:${join(dir, "switchboard.sock")}`], - { path: join(dir, "switchboard.sock") }, - [join(dir, "switchboard.sock")], + ["-s", `public=unix:${join(dir, "sb-public.sock")}`, "-s", `local=unix:${join(dir, "sb-local.sock")}`], + { path: join(dir, "sb-local.sock") }, + [join(dir, "sb-public.sock"), join(dir, "sb-local.sock")], ); - server = await startToybox(); - admin = makeAdmin(join(dir, "admin.sock")); + // the tests' echo server is on localhost, so this toybox holds the local handle + server = await startToybox("main", port, "local"); + admin = makeAdmin(join(dir, "main", "admin.sock")); pub = makePublic(`http://127.0.0.1:${port}`); }); @@ -154,17 +168,21 @@ test("code and state survive a restart", async () => { await admin.install("dave", await app("counter-v1.js")); assert.equal(await pub.get("dave"), "v1 count 1\n"); await server.stop(); - server = await startToybox(); + server = await startToybox("main", port, "local"); assert.equal(await pub.get("dave"), "v1 count 2\n"); }); -const probe = (vat, target, ws) => - pub.get(`${vat}?url=${encodeURIComponent(`http://${target}/`)}${ws ? "&ws=1" : ""}`); +const probe = (vat, target, { ws, constrain } = {}) => + pub.get( + `${vat}?url=${encodeURIComponent(`http://${target}/`)}` + + (ws ? "&ws=1" : "") + + (constrain ? `&constrain=${encodeURIComponent(JSON.stringify(constrain))}` : ""), + ); test("a vat granted a host reaches it through the switchboard: fetch and WebSocket", async () => { - await admin.install("erin", await app("egress-probe.js"), echo.host); + await admin.install("erin", await app("egress-probe.js"), echoLabels); assert.equal(await probe("erin", echo.host), "got: plain ok\n"); - assert.equal(await probe("erin", echo.host, true), "echo: hello\n"); + assert.equal(await probe("erin", echo.host, { ws: true }), "echo: hello\n"); }); test("same code, no grant: no egress", async () => { @@ -173,18 +191,48 @@ test("same code, no grant: no egress", async () => { }); test("a grant is an object, not ambient: the global fetch still throws", async () => { - await admin.install("heidi", await app("try-fetch.js"), echo.host); + await admin.install("heidi", await app("try-fetch.js"), echoLabels); assert.match(await pub.get("heidi"), /^fetch refused: This worker is not permitted to access the internet/); }); -test("a grant covers one host only", async () => { - await admin.install("grace", await app("egress-probe.js"), "127.0.0.1:1"); - assert.equal(await probe("grace", echo.host), `got: egress to ${echo.host} not granted\n\n`); - assert.match(await probe("grace", echo.host, true), /^no websocket: 403 egress to .* not granted/); +test("the switchboard refuses what a grant doesn't cover", async () => { + await admin.install("grace", await app("egress-probe.js"), { host: "127.0.0.1", port: "1" }); + assert.match(await probe("grace", echo.host), /^got: egress refused: .*egress to 127\.0\.0\.1 not granted/); + assert.match(await probe("grace", echo.host, { ws: true }), /^no websocket: 403 egress refused: .*not granted/); +}); + +test("a vat can narrow its grant, but not contradict it", async () => { + await admin.install("ivan", await app("egress-probe.js"), { scheme: "http" }); + assert.equal(await probe("ivan", echo.host, { constrain: echoLabels }), "got: plain ok\n"); + assert.match( + await probe("ivan", echo.host, { constrain: { scheme: "https" } }), + /^refused: label scheme is already set to a different value/, + ); +}); + +test("a grant stored before labels, a bare host:port, still works", async () => { + await admin.install("kim", await app("egress-probe.js"), echo.host); + assert.equal(await probe("kim", echo.host), "got: plain ok\n"); }); test("a grant survives a restart", async () => { await server.stop(); - server = await startToybox(); - assert.equal(await probe("erin", echo.host, true), "echo: hello\n"); + server = await startToybox("main", port, "local"); + assert.equal(await probe("erin", echo.host, { ws: true }), "echo: hello\n"); +}); + +test("a toybox holding only the public handle can't reach localhost, whatever the grant", async () => { + const port2 = await freePort(); + const other = await startToybox("public-only", port2, "public"); + try { + const admin2 = makeAdmin(join(dir, "public-only", "admin.sock")); + await admin2.install("judy", await app("egress-probe.js"), echoLabels); + const before = echo.hits(); + const got = await (await fetch(`http://127.0.0.1:${port2}/judy?url=${encodeURIComponent(`http://${echo.host}/`)}`)).text(); + // workerd reports the public network's refusal (restrictPeers) to JS only as "internal error" + assert.match(got, /^got: egress refused: /); + assert.equal(echo.hits(), before); + } finally { + await other.stop(); + } });