From 31ded9735b2ae689ceca1552820388d0fa9de431 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 7 Oct 2026 04:25:38 -0500 Subject: [PATCH] =?UTF-8?q?diigo-cache:=20fill=20job=20=E2=80=94=20a=20but?= =?UTF-8?q?ton=20that=20grabs=20every=20page=20(goal=201e)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /_fill (cache id in the body; target private|public; action start|stop) starts or stops a background walk of every diigo page into the cache. DO alarms advance it pagesPerStep pages per step (default 20), a second apart; progress (FillStatus) is stored in the DO after each page, so it survives eviction. Cached pages are skipped; starting while running rejoins the job; a stop mid-step wins; a failure is recorded with its reason and logged once. The created page gets fill buttons; the public page shows "filling: n of m pages". Fill state is new, additive DO storage (fill, fillStep): no existing state changes meaning. The 13 fill specs are unskipped and green, plus one: a fill makes one diigo call per page fetched. Generated-by: Claude Opus 5.5 --- diigo-cache/README.md | 5 ++ diigo-cache/src/cache.ts | 92 +++++++++++++++++++++++++++++++--- diigo-cache/src/index.ts | 87 +++++++++++++++++++++++++++++++- diigo-cache/tests/fill.test.ts | 13 +++-- 4 files changed, 182 insertions(+), 15 deletions(-) diff --git a/diigo-cache/README.md b/diigo-cache/README.md index cfc4cae..a833ebe 100644 --- a/diigo-cache/README.md +++ b/diigo-cache/README.md @@ -33,6 +33,11 @@ goal 2 preamble). outcome}`. - **Publish later**: `POST /_publish` (cache id in the body) wires a public companion to a private-only cache. +- **Fill job**: `POST /_fill` (cache id in the body; `target=private|public`, + `action=start|stop`) walks every diigo page into a cache in the + background — DO alarms, a few pages per step, progress stored in the + DO (`startFill`/`fillStatus`/`stopFill`). The created page has the + buttons; the public page shows progress. - The `DiigoBridge` entrypoint for other workers (its cache-id source is the goal-2 wiring TODO). diff --git a/diigo-cache/src/cache.ts b/diigo-cache/src/cache.ts index 1ccdd2b..c2ffdf5 100644 --- a/diigo-cache/src/cache.ts +++ b/diigo-cache/src/cache.ts @@ -53,9 +53,15 @@ export type CachePage = { const MS_PER_CALL = 500; +/** items per diigo page, for the fill job and the pages it etches */ +const PAGE_SIZE = 24; + +/** the pause before each fill step's alarm: a step already paces its + * pages at MS_PER_CALL; this keeps steps from running back to back */ +const FILL_STEP_DELAY_MS = 1000; + /** - * a fill job's progress (goal 1e, **not yet implemented**: the design is - * pinned by the skipped suite in tests/fill.test.ts). Stored in the DO + * a fill job's progress (goal 1e; tests/fill.test.ts). Stored in the DO * (new state, additive), so it survives eviction; advanced by alarms. */ export type FillStatus = { @@ -132,21 +138,91 @@ abstract class PagedCache extends DurableObject { /** * start (or rejoin) the fill job: walk every diigo page into this * cache, `pagesPerStep` per alarm (default 20, 1–50). Starting while a - * job runs returns that job unchanged. Goal 1e; not yet implemented. + * job runs returns that job unchanged; a finished, stopped, or failed + * job is replaced by a fresh one (cached pages are skipped anyway). */ async startFill(opts: { pagesPerStep?: number } = {}): Promise { - void opts; - throw new Error("startFill: not implemented (goal 1e)"); + const current = await this.fillStatus(); + if (current?.state === "running") return current; + const step = Math.trunc(opts.pagesPerStep ?? 20); + if (!(step >= 1 && step <= 50)) throw new Error("pagesPerStep: expected 1–50"); + const now = new Date().toISOString(); + const status: FillStatus = { + state: "running", + nextPage: 1, + pages: this.#knownPages(), + fetched: 0, + startedAt: now, + updatedAt: now, + }; + await this.ctx.storage.put({ fill: status, fillStep: step }); + await this.ctx.storage.setAlarm(Date.now() + FILL_STEP_DELAY_MS); + console.log({ event: "fill", kind: this.kind, state: status.state, pagesPerStep: step }); + return status; } - /** the fill job's progress, or undefined if none was ever started */ + /** the fill job's progress, or undefined if none was ever started; + * the page count comes from any cached page until the job learns it */ async fillStatus(): Promise { - throw new Error("fillStatus: not implemented (goal 1e)"); + const fill = await this.ctx.storage.get("fill"); + return fill && fill.pages === null ? { ...fill, pages: this.#knownPages() } : fill; } /** stop the fill job after its current step; cached pages stay */ async stopFill(): Promise { - throw new Error("stopFill: not implemented (goal 1e)"); + const current = await this.fillStatus(); + if (!current) throw new Error("no fill job to stop"); + if (current.state !== "running") return current; + const stopped: FillStatus = { ...current, state: "stopped", updatedAt: new Date().toISOString() }; + await this.ctx.storage.put("fill", stopped); + await this.ctx.storage.deleteAlarm(); + console.log({ event: "fill", kind: this.kind, state: "stopped", nextPage: stopped.nextPage, pages: stopped.pages }); + return stopped; + } + + /** one fill step: up to `fillStep` pages, progress saved after each, + * then the next step is scheduled (or the job ends) */ + async alarm(): Promise { + const step = (await this.ctx.storage.get("fillStep")) ?? 20; + let status = await this.fillStatus(); + for (let i = 0; i < step && status?.state === "running"; i += 1) { + if (status.pages !== null && status.nextPage > status.pages) { + status = { ...status, state: "done" }; + break; + } + try { + const page = await this.page(status.nextPage, PAGE_SIZE); + status = { + ...status, + nextPage: status.nextPage + 1, + pages: Math.max(1, Math.ceil(page.total / PAGE_SIZE)), + fetched: status.fetched + (page.cached ? 0 : 1), + updatedAt: new Date().toISOString(), + }; + } catch (err) { + status = { ...status, state: "failed", error: String(err), updatedAt: new Date().toISOString() }; + console.error({ event: "fill", kind: this.kind, state: "failed", page: status.nextPage, error: status.error }); + } + // a stop that landed mid-step wins + if ((await this.fillStatus())?.state === "stopped") return; + await this.ctx.storage.put("fill", status); + } + if (!status) return; + if (status.state === "running" && status.pages !== null && status.nextPage > status.pages) { + status = { ...status, state: "done", updatedAt: new Date().toISOString() }; + } + await this.ctx.storage.put("fill", status); + console.log({ event: "fill", kind: this.kind, state: status.state, nextPage: status.nextPage, pages: status.pages, fetched: status.fetched }); + if (status.state === "running") await this.ctx.storage.setAlarm(Date.now() + FILL_STEP_DELAY_MS); + } + + /** diigo's page count, if any cached page recorded `total` */ + #knownPages(): number | null { + this.ctx.storage.sql.exec(SCHEMA_V1); + const row = this.ctx.storage.sql + .exec<{ total: number | null }>("select total from pages where total is not null order by etched_at desc limit 1") + .toArray(); + return row.length > 0 && row[0].total !== null ? Math.max(1, Math.ceil(row[0].total / PAGE_SIZE)) : null; } /** diff --git a/diigo-cache/src/index.ts b/diigo-cache/src/index.ts index 224da4d..1a9da86 100644 --- a/diigo-cache/src/index.ts +++ b/diigo-cache/src/index.ts @@ -14,6 +14,7 @@ import { WorkerEntrypoint } from "cloudflare:workers"; import { DiigoCache, PublicCache } from "./cache.js"; +import type { FillStatus } from "./cache.js"; import type { Item } from "./cache.js"; import { updatedISO, usernameOf } from "./items.js"; import { toWebkey, idFromWebkey, isWebkey, shashKey } from "./webkeys.js"; @@ -95,6 +96,11 @@ export default { return publishCache(request, env); } + // the fill job (goal 1e): owner-only, cache id in the body + if (url.pathname === "/_fill" && request.method === "POST") { + return fillRoute(request, env); + } + if (url.pathname !== "/") return new Response("not found", { status: 404 }); if (request.method === "POST") { @@ -268,7 +274,7 @@ async function wire(request: Request, env: Env): Promise { await pubStub.setup({ webappId }); await index.register(username, publicId); if (wantsJson) return Response.json({ cacheId, publicId, username }); - return wiredPage(cacheId, `public cache live at /public/${esc(username)}`); + return wiredPage(cacheId, `public cache live at /public/${esc(username)}`, true); } if (wantsJson) { return Response.json({ @@ -283,11 +289,24 @@ async function wire(request: Request, env: Env): Promise { (or POST /_publish with cacheId in the body)`, + false, ); } +/** the fill job's buttons: one per cache the holder can fill (the id + * rides a hidden field, never the URL) */ +function fillForms(cacheId: string, hasPublic: boolean): string { + const form = (target: string, label: string) => `
+ + +
`; + return `

fetch every page now, in the background (a few minutes for thousands of bookmarks):

+${form("private", "fill the private cache")} +${hasPublic ? form("public", "fill the public cache") : ""}`; +} + /** the created-cache page: ids + save/copy buttons + the pointer home */ -async function wiredPage(cacheId: string, note: string): Promise { +async function wiredPage(cacheId: string, note: string, hasPublic: boolean): Promise { const fingerprint = await nick(cacheId); const body = ` diigo cache: created @@ -300,6 +319,7 @@ async function wiredPage(cacheId: string, note: string): Promise {

${note}

+${fillForms(cacheId, hasPublic)}

the cache id is the only way back to this cache — saved (above) it lives with the rest of your credentials. it is a webkey: it never travels in URLs (query strings leak to history/Referer/logs); the dump POST carries @@ -311,6 +331,67 @@ hold it separately only if you want another cache off the same session.

return new Response(body, { headers: { "content-type": "text/html;charset=utf-8" } }); } +/** a fill job's progress, as one line of page text */ +function fillLine(fill: FillStatus): string { + const of = `${Math.max(0, fill.nextPage - 1)} of ${fill.pages ?? "?"} pages`; + const said = { + running: `filling: ${of}`, + done: `filled: ${of}`, + stopped: `fill stopped: ${of}`, + failed: `fill failed at page ${fill.nextPage}: ${fill.error ?? "unknown error"}`, + }[fill.state]; + return `

${esc(said)}

`; +} + +/** POST /_fill: start or stop the fill job of the holder's private cache + * (target=private, the default) or of its public companion + * (target=public). JSON in → JSON out; a form post gets a small page. */ +async function fillRoute(request: Request, env: Env): Promise { + const ct = request.headers.get("content-type") ?? ""; + const wantsJson = ct.includes("application/json"); + let fields: Record; + try { + fields = wantsJson + ? Object.fromEntries(Object.entries((await request.json()) as Record).map(([k, v]) => [k, String(v ?? "")])) + : Object.fromEntries([...(await request.formData()).entries()].map(([k, v]) => [k, String(v)])); + } catch (err) { + return bad(`bad input: ${String(err)}`); + } + const { cacheId = "", target = "private", action = "start" } = fields; + if (!isWebkey(cacheId)) return bad("no cache id supplied (POST body: cacheId)"); + if (target !== "private" && target !== "public") return bad("target: expected private or public"); + if (action !== "start" && action !== "stop") return bad("action: expected start or stop"); + + const cache = env.CACHE_DO.get(env.CACHE_DO.idFromString(idFromWebkey(cacheId))); + const wiring = await cache.wiring(); + if (!wiring) return Response.json({ error: "unknown cache id" }, { status: 404 }); + let stub: { startFill(): Promise; stopFill(): Promise } = cache; + if (target === "public") { + if (!wiring.publicId) return bad("this cache has no public companion; publish it first"); + stub = env.PUBLIC_DO.get(env.PUBLIC_DO.idFromString(idFromWebkey(wiring.publicId))); + } + let status: FillStatus; + try { + status = action === "start" ? await stub.startFill() : await stub.stopFill(); + } catch (err) { + console.error({ event: "fill", target, action, error: String(err) }); + return bad(`could not ${action} the fill: ${String(err)}`); + } + const code = action === "start" ? 202 : 200; + if (wantsJson) return Response.json(status, { status: code }); + return new Response( + ` +diigo cache: fill + +

fill (${esc(target)} cache)

+${fillLine(status)} +

it runs in the background; the progress shows on the public page, or post this form again to check.

+← home +`, + { status: code, headers: { "content-type": "text/html;charset=utf-8" } }, + ); +} + /** publish, later: the private cache's wiring already holds the webapp * id, so the public companion needs no re-mint; the username comes from * the items already in the cache */ @@ -434,6 +515,7 @@ async function publicPage(request: Request, env: Env, username: string): Promise // page n with the private rows dropped const pages = Math.max(1, Math.ceil(page.total / PUBLIC_PER_PAGE)); const items = page.items.map((j) => JSON.parse(j) as Item); + const fill = await cache.fillStatus(); const accept = request.headers.get("accept") ?? ""; if (accept.includes("application/json") && !accept.includes("text/html")) { @@ -469,6 +551,7 @@ ${items

${esc(username)}'s shared bookmarks

${pages} page${pages === 1 ? "" : "s"} (diigo's, ${PUBLIC_PER_PAGE} bookmarks each; private ones are left out, so a page may show fewer).

cache: ${page.cached ? "HIT (from storage)" : "MISS (fetched from diigo now)"} — page ${valid ? pageN : 1}, ${items.length} shared

+${fill ? fillLine(fill) : ""}

${nav(pageN - 1, "← prev")} diff --git a/diigo-cache/tests/fill.test.ts b/diigo-cache/tests/fill.test.ts index b3ba2dd..3d3b76a 100644 --- a/diigo-cache/tests/fill.test.ts +++ b/diigo-cache/tests/fill.test.ts @@ -4,9 +4,9 @@ import { describe, expect, it, vi } from "vitest"; import worker from "../src/index.js"; import { toWebkey } from "../src/webkeys.js"; -// goal 1e (capsite checklist): a job that grabs every page. The design as -// tests; unskip as it lands. Fake accounts: tests/fake-webapp.js -// ("fake-webapp-id": 30 items = 2 pages of 24, every 3rd private). +// goal 1e (capsite checklist): a job that grabs every page. Fake +// accounts: tests/fake-webapp.js ("fake-webapp-id": 30 items = 2 pages +// of 24, every 3rd private). type TestEnv = { CACHE_DO: DurableObjectNamespace; @@ -29,13 +29,16 @@ const drain = async (stub: DurableObjectStub, max = 20) => { return ran; }; -describe.skip("fill job (goal 1e)", () => { +describe("fill job (goal 1e)", () => { it("fills every page; afterwards every page is a hit with no webapp call", async () => { const stub = await wired(); + const atStart = await testEnv.WEBAPP.calls(); expect((await stub.startFill()).state).toBe("running"); await drain(stub); const status = await stub.fillStatus(); expect(status).toMatchObject({ state: "done", pages: 2, fetched: 2 }); + // one diigo call per page fetched (no extra page-1 fetch per miss) + expect((await testEnv.WEBAPP.calls()) - atStart).toBe(2); const before = await testEnv.WEBAPP.calls(); for (const n of [1, 2]) expect((await stub.page(n, 24)).cached).toBe(true); @@ -114,7 +117,7 @@ describe.skip("fill job (goal 1e)", () => { }); }); -describe.skip("fill routes (goal 1e)", () => { +describe("fill routes (goal 1e)", () => { const call = (path: string, init?: RequestInit) => worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never); const post = (body: Record) =>