From a2ca59a27eef9c744888099350e930e7de532943 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 9 Oct 2026 00:21:44 -0500 Subject: [PATCH] feat(good-fences): code-mode prompt; the model writes code to mow codeModeMessages turns what I say plus `about` into a prompt: a neighbor persona, rules for the reply (one async arrow function over `env`), and the lent things' types. Things are remote and calls on one are delivered in order, so a lamp example shows pipelining: await only for a value, or on a batch's last call. With only the rule, the model awaited every call; with the example, it pipelines. aboutMower gives the Mower type with Promise returns. The model test checks the code calls only Mower methods, pushes, and awaits less often than it calls; it saves the code to itest-output/ via a reporter, since the test runs in workerd, which can't write files. Model tests get 60s. Green: both model tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- good-fences/.gitignore | 1 + good-fences/itest/code-mode.test.ts | 10 +++-- good-fences/itest/save-annotations.ts | 21 +++++++++ good-fences/src/code-mode.ts | 63 +++++++++++++++++++++++++++ good-fences/src/mower.ts | 15 +++++++ good-fences/vitest.model.config.ts | 7 ++- 6 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 good-fences/.gitignore create mode 100644 good-fences/itest/save-annotations.ts create mode 100644 good-fences/src/code-mode.ts create mode 100644 good-fences/src/mower.ts diff --git a/good-fences/.gitignore b/good-fences/.gitignore new file mode 100644 index 0000000..bcaedf0 --- /dev/null +++ b/good-fences/.gitignore @@ -0,0 +1 @@ +itest-output/ diff --git a/good-fences/itest/code-mode.test.ts b/good-fences/itest/code-mode.test.ts index d1f50fa..b7794ff 100644 --- a/good-fences/itest/code-mode.test.ts +++ b/good-fences/itest/code-mode.test.ts @@ -6,16 +6,20 @@ import { codeModeMessages, extractCode } from "../src/code-mode.ts"; import { makeWorkersAiModel, MODEL } from "../src/model.ts"; import { aboutMower } from "../src/mower.ts"; -test("lent a mower, the neighbor writes code to mow", async () => { +test("lent a mower, the neighbor writes code to mow", async ({ annotate }) => { const model = makeWorkersAiModel(env.AI, MODEL); const messages = codeModeMessages({ say: "you can use this mower for the weekend", about: { mower: aboutMower }, }); const code = extractCode(await model.complete(messages)); - console.log(code); + await annotate("the code the model wrote", { body: code, contentType: "text/javascript" }); - const calls = [...code.matchAll(/env\.mower\.(\w+)\(/g)].map(([, method]) => method); + // `env.mower.push()`, or `mower.push()` after destructuring env + const calls = [...code.matchAll(/\bmower\.(\w+)\(/g)].map(([, method]) => method); expect(calls).toContain("push"); for (const method of calls) expect(["start", "push", "turn", "stop"]).toContain(method); + // calls pipeline: await only for a value, or once at the end + const awaits = code.match(/\bawait\b/g) ?? []; + expect(awaits.length).toBeLessThan(calls.length); }); diff --git a/good-fences/itest/save-annotations.ts b/good-fences/itest/save-annotations.ts new file mode 100644 index 0000000..b46d24f --- /dev/null +++ b/good-fences/itest/save-annotations.ts @@ -0,0 +1,21 @@ +/** @file A vitest reporter that saves annotation bodies (e.g. the code a model + * wrote) to `itest-output/`, one file per test per run; tests run in workerd, + * which can't write files. */ +import { mkdir, writeFile } from "node:fs/promises"; +import type { Reporter } from "vitest/reporters"; + +const outDir = new URL("../itest-output/", import.meta.url); +const extFor: Record = { "text/javascript": "js", "text/plain": "txt" }; +const slug = (text: string) => text.replace(/[^\w]+/g, "-").replace(/^-|-$/g, ""); + +export default class SaveAnnotations implements Reporter { + async onTestCaseAnnotate(...[testCase, annotation]: Parameters>) { + const { body, contentType = "text/plain" } = annotation.attachment ?? {}; + if (body === undefined) return; + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const file = new URL(`${slug(testCase.name)}-${stamp}.${extFor[contentType] ?? "txt"}`, outDir); + await mkdir(outDir, { recursive: true }); + await writeFile(file, body); + console.log(`saved ${annotation.message}: ${file.pathname}`); + } +} diff --git a/good-fences/src/code-mode.ts b/good-fences/src/code-mode.ts new file mode 100644 index 0000000..73431a3 --- /dev/null +++ b/good-fences/src/code-mode.ts @@ -0,0 +1,63 @@ +/** @file Code mode: the model answers with code against the things it was lent. */ +import type { Message } from "./model.ts"; + +/** what the neighbor knows about each thing: its TypeScript type */ +export type About = Readonly>>; + +/** a template literal with `~` for each backtick, as in Markdown's `~~~` + * fences; interpolated values are left as they are */ +const md = (strings: TemplateStringsArray, ...values: unknown[]) => { + const tick = (s: string) => s.replaceAll("~", "`"); + return tick(strings[0]) + values.map((value, i) => String(value) + tick(strings[i + 1])).join(""); +}; + +const persona = md` +You are my neighbor. You have 50 acres of lawn that need mowing. +You act by writing code. When I lend you things, your code gets them as ~env.~. +`.trim(); + +const rules = md` +Reply with exactly one JavaScript arrow function, ~async (env) => { ... }~, +in a ~~~js block, and nothing else. Things are remote: each call on one +returns a promise, and calls on a thing are delivered in order, so there's +no need to wait for one before making the next. Await only when you need a +value back, or on the last call of a batch, to wait for the whole batch. +Your code can use only ~env~: no network, timers, imports, or console. +The function returns a short string saying what it did. + +For example, lent ~env.lamp~ with ~on()~ and ~dim()~: + +~~~js +async (env) => { + const { lamp } = env; + lamp.on(); + lamp.dim(); + await lamp.dim(); // the whole batch + return 'dimmed the lamp twice'; +} +~~~ +`.trim(); + +/** the prompt for `say`, with the types of the things lent alongside it */ +export const codeModeMessages = ({ say, about }: { say: string; about: About }): Message[] => { + const types = Object.entries(about) + .map( + ([name, { ts }]) => md` +~env.${name}~: +~~~ts +${ts} +~~~ +`.trim(), + ) + .join("\n\n"); + return [ + { role: "system", content: `${persona}\n\n${rules}` }, + { role: "user", content: `${say}\n\nWhat I lent you:\n\n${types}` }, + ]; +}; + +/** the code in a reply: the first fenced block, or the whole reply if none */ +export const extractCode = (reply: string): string => { + const fenced = reply.match(/```(?:js|javascript)?\n([\s\S]*?)```/); + return (fenced?.[1] ?? reply).trim(); +}; diff --git a/good-fences/src/mower.ts b/good-fences/src/mower.ts new file mode 100644 index 0000000..2dec5b5 --- /dev/null +++ b/good-fences/src/mower.ts @@ -0,0 +1,15 @@ +/** @file The mower: Logo-turtle moves on a yard of cells. */ + +/** what a model is told about a mower: its type, with the moves explained */ +export const aboutMower = { + ts: `interface Mower { + /** put the mower down, facing north, and cut the cell under it */ + start(): Promise; + /** move forward one cell (10 yards), cutting the cell it enters */ + push(): Promise; + /** turn right 90 degrees, in place */ + turn(): Promise; + /** park the mower */ + stop(): Promise; +}`, +}; diff --git a/good-fences/vitest.model.config.ts b/good-fences/vitest.model.config.ts index c08a44d..a3d6aed 100644 --- a/good-fences/vitest.model.config.ts +++ b/good-fences/vitest.model.config.ts @@ -4,5 +4,10 @@ import { cloudflareTest } from "@cloudflare/vitest-plugin"; /** opt-in (`pnpm test:model`): calls Workers AI, which needs `wrangler login` and costs money */ export default defineProject({ plugins: [cloudflareTest({ wrangler: { configPath: "./wrangler.model-tests.jsonc" } })], - test: { include: ["itest/**/*.test.ts"] }, + test: { + include: ["itest/**/*.test.ts"], + testTimeout: 60_000, + // what the model wrote, saved per run to itest-output/ (gitignored) + reporters: ["default", "./itest/save-annotations.ts"], + }, });