From cf5ccce4e2a3b0a7cce44db93ae4fc2c15579407 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 7 Oct 2026 04:02:22 -0500 Subject: [PATCH] diigo-cache: fill job, designed as skipped specs (goal 1e) A button that grabs every page into a cache: owner-only POST /_fill (cache id in the body; target private|public; action start|stop), advanced by DO alarms a few pages per step, progress stored in the DO (new, additive state) so it survives eviction, cached pages skipped, a rerun rejoins the running job, failures recorded and logged once. The created page gets the button; the public page shows progress. startFill/fillStatus/stopFill on PagedCache are JSDoc'd stubs; 13 specs are describe.skip until it lands (alarms driven by runDurableObjectAlarm, eviction by evictDurableObject). Generated-by: Claude Opus 5.5 --- diigo-cache/src/cache.ts | 39 ++++++++ diigo-cache/tests/fill.test.ts | 170 +++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 diigo-cache/tests/fill.test.ts diff --git a/diigo-cache/src/cache.ts b/diigo-cache/src/cache.ts index bc7c2bf..b45ab01 100644 --- a/diigo-cache/src/cache.ts +++ b/diigo-cache/src/cache.ts @@ -53,6 +53,25 @@ export type CachePage = { const MS_PER_CALL = 500; +/** + * a fill job's progress (goal 1e, **not yet implemented**: the design is + * pinned by the skipped suite in tests/fill.test.ts). Stored in the DO + * (new state, additive), so it survives eviction; advanced by alarms. + */ +export type FillStatus = { + state: "running" | "done" | "stopped" | "failed"; + /** the next diigo page to fetch (pages already cached are skipped) */ + nextPage: number; + /** diigo's page count, known after page 1 */ + pages: number | null; + /** pages fetched from diigo by this job (cache hits not counted) */ + fetched: number; + startedAt: string; + updatedAt: string; + /** why it failed: safe to show and log (no ids, no header values) */ + error?: string; +}; + function sleep(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } @@ -104,6 +123,26 @@ abstract class PagedCache extends DurableObject { /** which cache this is, for logs (never the id: that's the bearer key) */ protected abstract readonly kind: "private" | "public"; + /** + * start (or rejoin) the fill job: walk every diigo page into this + * cache, `pagesPerStep` per alarm (default 20, 1–50). Starting while a + * job runs returns that job unchanged. Goal 1e; not yet implemented. + */ + async startFill(opts: { pagesPerStep?: number } = {}): Promise { + void opts; + throw new Error("startFill: not implemented (goal 1e)"); + } + + /** the fill job's progress, or undefined if none was ever started */ + async fillStatus(): Promise { + throw new Error("fillStatus: not implemented (goal 1e)"); + } + + /** stop the fill job after its current step; cached pages stay */ + async stopFill(): Promise { + throw new Error("stopFill: not implemented (goal 1e)"); + } + /** * read page `pageNum` (`count` per page, diigo's updated-desc order): * storage first; on miss, fetch through the webapp instance (etching diff --git a/diigo-cache/tests/fill.test.ts b/diigo-cache/tests/fill.test.ts new file mode 100644 index 0000000..b3ba2dd --- /dev/null +++ b/diigo-cache/tests/fill.test.ts @@ -0,0 +1,170 @@ +import { env } from "cloudflare:workers"; +import { evictDurableObject, runDurableObjectAlarm, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it, vi } from "vitest"; +import worker from "../src/index.js"; +import { toWebkey } from "../src/webkeys.js"; + +// goal 1e (capsite checklist): a job that grabs every page. The design as +// tests; unskip as it lands. Fake accounts: tests/fake-webapp.js +// ("fake-webapp-id": 30 items = 2 pages of 24, every 3rd private). + +type TestEnv = { + CACHE_DO: DurableObjectNamespace; + PUBLIC_DO: DurableObjectNamespace; + WEBAPP: { calls(): Promise }; +}; +const testEnv = env as unknown as TestEnv; + +const wired = async (webappId = "fake-webapp-id") => { + const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId()); + await stub.setup({ webappId }); + return stub; +}; + +/** run alarms until none is scheduled (bounded, so a runaway job fails) */ +const drain = async (stub: DurableObjectStub, max = 20) => { + let ran = 0; + while (ran < max && (await runDurableObjectAlarm(stub))) ran += 1; + expect(ran).toBeLessThan(max); + return ran; +}; + +describe.skip("fill job (goal 1e)", () => { + it("fills every page; afterwards every page is a hit with no webapp call", async () => { + const stub = await wired(); + expect((await stub.startFill()).state).toBe("running"); + await drain(stub); + const status = await stub.fillStatus(); + expect(status).toMatchObject({ state: "done", pages: 2, fetched: 2 }); + + const before = await testEnv.WEBAPP.calls(); + for (const n of [1, 2]) expect((await stub.page(n, 24)).cached).toBe(true); + expect(await testEnv.WEBAPP.calls()).toBe(before); + }); + + it("advances pagesPerStep pages per alarm, with progress readable between steps", async () => { + const stub = await wired(); + await stub.startFill({ pagesPerStep: 1 }); + expect(await runDurableObjectAlarm(stub)).toBe(true); + expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2, pages: 2 }); + await drain(stub); + expect((await stub.fillStatus())?.state).toBe("done"); + }); + + it("survives eviction: progress is in storage, not memory", async () => { + const stub = await wired(); + await stub.startFill({ pagesPerStep: 1 }); + await runDurableObjectAlarm(stub); + await evictDurableObject(stub); + expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2 }); + await drain(stub); + expect((await stub.fillStatus())?.state).toBe("done"); + }); + + it("starting again while running rejoins the same job", async () => { + const stub = await wired(); + const first = await stub.startFill({ pagesPerStep: 1 }); + await runDurableObjectAlarm(stub); + const again = await stub.startFill({ pagesPerStep: 1 }); + expect(again.startedAt).toBe(first.startedAt); + expect(again.nextPage).toBe(2); + }); + + it("skips pages already cached", async () => { + const stub = await wired(); + await stub.page(1, 24); + await stub.startFill(); + await drain(stub); + expect(await stub.fillStatus()).toMatchObject({ state: "done", fetched: 1 }); + }); + + it("stops when asked; cached pages stay", async () => { + const stub = await wired(); + await stub.startFill({ pagesPerStep: 1 }); + await runDurableObjectAlarm(stub); + expect((await stub.stopFill()).state).toBe("stopped"); + expect(await runDurableObjectAlarm(stub)).toBe(false); + expect((await stub.page(1, 24)).cached).toBe(true); + }); + + it("a webapp failure fails the job loudly, once: status says why, the log too", async () => { + const spy = vi.spyOn(console, "error"); + try { + const stub = await wired("no-such-webapp"); + await stub.startFill(); + await drain(stub); + const status = await stub.fillStatus(); + expect(status?.state).toBe("failed"); + expect(status?.error).toMatch(/unknown id/); + expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.event === "fill")).not.toHaveLength(0); + } finally { + spy.mockRestore(); + } + }); + + it("a public cache's fill stores no private rows", async () => { + const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId()); + await stub.setup({ webappId: "fake-webapp-id" }); + await stub.startFill(); + await drain(stub); + await runInDurableObject(stub, (_obj, state) => { + const rows = state.storage.sql.exec<{ n: number }>("select count(*) as n from items where private = 1").toArray(); + expect(rows[0].n).toBe(0); + }); + }); +}); + +describe.skip("fill routes (goal 1e)", () => { + const call = (path: string, init?: RequestInit) => + worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never); + const post = (body: Record) => + call("/_fill", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) }); + + it("POST /_fill starts the owner's job (cache id in the body) and answers 202 with its status", async () => { + const id = testEnv.CACHE_DO.newUniqueId(); + await testEnv.CACHE_DO.get(id).setup({ webappId: "fake-webapp-id" }); + const res = await post({ cacheId: toWebkey(id) }); + expect(res.status).toBe(202); + expect(await res.json()).toMatchObject({ state: "running" }); + }); + + it("target=public fills the cache's public companion; action=stop stops", async () => { + const wiredRes = await call("/_wire", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ webappId: "fake-webapp-id", mode: "public" }), + }); + const { cacheId } = (await wiredRes.json()) as { cacheId: string }; + expect((await post({ cacheId, target: "public" })).status).toBe(202); + const stopped = await post({ cacheId, target: "public", action: "stop" }); + expect(await stopped.json()).toMatchObject({ state: "stopped" }); + }); + + it("refuses an unknown cache id (404) and never takes the id from the URL", async () => { + expect((await post({ cacheId: toWebkey(testEnv.CACHE_DO.newUniqueId()) })).status).toBe(404); + expect((await call(`/_fill/${toWebkey(testEnv.CACHE_DO.newUniqueId())}`, { method: "POST" })).status).toBe(404); + }); + + it("the created page offers a fetch-everything button, the id in a hidden field", async () => { + const res = await call("/_wire", { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ webappId: "fake-webapp-id", mode: "private" }), + }); + const html = await res.text(); + expect(html).toMatch(/
/); + expect(html).toMatch(/ { + const wiredRes = await call("/_wire", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ webappId: "fake-webapp-id", mode: "public" }), + }); + const { cacheId, username } = (await wiredRes.json()) as { cacheId: string; username: string }; + await post({ cacheId, target: "public" }); + const html = await (await call(`/public/${username}`)).text(); + expect(html).toMatch(/fill(ed|ing)[^<]*\d+ of \d+ pages/); + }); +});