diff --git a/good-fences/.gitignore b/good-fences/.gitignore new file mode 100644 index 0000000..bcaedf0 --- /dev/null +++ b/good-fences/.gitignore @@ -0,0 +1 @@ +itest-output/ diff --git a/good-fences/itest/code-mode.test.ts b/good-fences/itest/code-mode.test.ts index d1f50fa..b7794ff 100644 --- a/good-fences/itest/code-mode.test.ts +++ b/good-fences/itest/code-mode.test.ts @@ -6,16 +6,20 @@ import { codeModeMessages, extractCode } from "../src/code-mode.ts"; import { makeWorkersAiModel, MODEL } from "../src/model.ts"; import { aboutMower } from "../src/mower.ts"; -test("lent a mower, the neighbor writes code to mow", async () => { +test("lent a mower, the neighbor writes code to mow", async ({ annotate }) => { const model = makeWorkersAiModel(env.AI, MODEL); const messages = codeModeMessages({ say: "you can use this mower for the weekend", about: { mower: aboutMower }, }); const code = extractCode(await model.complete(messages)); - console.log(code); + await annotate("the code the model wrote", { body: code, contentType: "text/javascript" }); - const calls = [...code.matchAll(/env\.mower\.(\w+)\(/g)].map(([, method]) => method); + // `env.mower.push()`, or `mower.push()` after destructuring env + const calls = [...code.matchAll(/\bmower\.(\w+)\(/g)].map(([, method]) => method); expect(calls).toContain("push"); for (const method of calls) expect(["start", "push", "turn", "stop"]).toContain(method); + // calls pipeline: await only for a value, or once at the end + const awaits = code.match(/\bawait\b/g) ?? []; + expect(awaits.length).toBeLessThan(calls.length); }); diff --git a/good-fences/itest/save-annotations.ts b/good-fences/itest/save-annotations.ts new file mode 100644 index 0000000..b46d24f --- /dev/null +++ b/good-fences/itest/save-annotations.ts @@ -0,0 +1,21 @@ +/** @file A vitest reporter that saves annotation bodies (e.g. the code a model + * wrote) to `itest-output/`, one file per test per run; tests run in workerd, + * which can't write files. */ +import { mkdir, writeFile } from "node:fs/promises"; +import type { Reporter } from "vitest/reporters"; + +const outDir = new URL("../itest-output/", import.meta.url); +const extFor: Record = { "text/javascript": "js", "text/plain": "txt" }; +const slug = (text: string) => text.replace(/[^\w]+/g, "-").replace(/^-|-$/g, ""); + +export default class SaveAnnotations implements Reporter { + async onTestCaseAnnotate(...[testCase, annotation]: Parameters>) { + const { body, contentType = "text/plain" } = annotation.attachment ?? {}; + if (body === undefined) return; + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const file = new URL(`${slug(testCase.name)}-${stamp}.${extFor[contentType] ?? "txt"}`, outDir); + await mkdir(outDir, { recursive: true }); + await writeFile(file, body); + console.log(`saved ${annotation.message}: ${file.pathname}`); + } +} diff --git a/good-fences/src/code-mode.ts b/good-fences/src/code-mode.ts new file mode 100644 index 0000000..73431a3 --- /dev/null +++ b/good-fences/src/code-mode.ts @@ -0,0 +1,63 @@ +/** @file Code mode: the model answers with code against the things it was lent. */ +import type { Message } from "./model.ts"; + +/** what the neighbor knows about each thing: its TypeScript type */ +export type About = Readonly>>; + +/** a template literal with `~` for each backtick, as in Markdown's `~~~` + * fences; interpolated values are left as they are */ +const md = (strings: TemplateStringsArray, ...values: unknown[]) => { + const tick = (s: string) => s.replaceAll("~", "`"); + return tick(strings[0]) + values.map((value, i) => String(value) + tick(strings[i + 1])).join(""); +}; + +const persona = md` +You are my neighbor. You have 50 acres of lawn that need mowing. +You act by writing code. When I lend you things, your code gets them as ~env.~. +`.trim(); + +const rules = md` +Reply with exactly one JavaScript arrow function, ~async (env) => { ... }~, +in a ~~~js block, and nothing else. Things are remote: each call on one +returns a promise, and calls on a thing are delivered in order, so there's +no need to wait for one before making the next. Await only when you need a +value back, or on the last call of a batch, to wait for the whole batch. +Your code can use only ~env~: no network, timers, imports, or console. +The function returns a short string saying what it did. + +For example, lent ~env.lamp~ with ~on()~ and ~dim()~: + +~~~js +async (env) => { + const { lamp } = env; + lamp.on(); + lamp.dim(); + await lamp.dim(); // the whole batch + return 'dimmed the lamp twice'; +} +~~~ +`.trim(); + +/** the prompt for `say`, with the types of the things lent alongside it */ +export const codeModeMessages = ({ say, about }: { say: string; about: About }): Message[] => { + const types = Object.entries(about) + .map( + ([name, { ts }]) => md` +~env.${name}~: +~~~ts +${ts} +~~~ +`.trim(), + ) + .join("\n\n"); + return [ + { role: "system", content: `${persona}\n\n${rules}` }, + { role: "user", content: `${say}\n\nWhat I lent you:\n\n${types}` }, + ]; +}; + +/** the code in a reply: the first fenced block, or the whole reply if none */ +export const extractCode = (reply: string): string => { + const fenced = reply.match(/```(?:js|javascript)?\n([\s\S]*?)```/); + return (fenced?.[1] ?? reply).trim(); +}; diff --git a/good-fences/src/mower.ts b/good-fences/src/mower.ts new file mode 100644 index 0000000..2dec5b5 --- /dev/null +++ b/good-fences/src/mower.ts @@ -0,0 +1,15 @@ +/** @file The mower: Logo-turtle moves on a yard of cells. */ + +/** what a model is told about a mower: its type, with the moves explained */ +export const aboutMower = { + ts: `interface Mower { + /** put the mower down, facing north, and cut the cell under it */ + start(): Promise; + /** move forward one cell (10 yards), cutting the cell it enters */ + push(): Promise; + /** turn right 90 degrees, in place */ + turn(): Promise; + /** park the mower */ + stop(): Promise; +}`, +}; diff --git a/good-fences/vitest.model.config.ts b/good-fences/vitest.model.config.ts index c08a44d..a3d6aed 100644 --- a/good-fences/vitest.model.config.ts +++ b/good-fences/vitest.model.config.ts @@ -4,5 +4,10 @@ import { cloudflareTest } from "@cloudflare/vitest-plugin"; /** opt-in (`pnpm test:model`): calls Workers AI, which needs `wrangler login` and costs money */ export default defineProject({ plugins: [cloudflareTest({ wrangler: { configPath: "./wrangler.model-tests.jsonc" } })], - test: { include: ["itest/**/*.test.ts"] }, + test: { + include: ["itest/**/*.test.ts"], + testTimeout: 60_000, + // what the model wrote, saved per run to itest-output/ (gitignored) + reporters: ["default", "./itest/save-annotations.ts"], + }, });