diigo-cache: fill job — a button that grabs every page (goal 1e)

POST /_fill (cache id in the body; target private|public; action
start|stop) starts or stops a background walk of every diigo page into
the cache. DO alarms advance it pagesPerStep pages per step (default
20), a second apart; progress (FillStatus) is stored in the DO after
each page, so it survives eviction. Cached pages are skipped; starting
while running rejoins the job; a stop mid-step wins; a failure is
recorded with its reason and logged once. The created page gets fill
buttons; the public page shows "filling: n of m pages".

Fill state is new, additive DO storage (fill, fillStep): no existing
state changes meaning.

The 13 fill specs are unskipped and green, plus one: a fill makes one
diigo call per page fetched.

Generated-by: Claude Opus 5.5

diigo-cache/README.md

@@ -33,6 +33,11 @@ goal 2 preamble).
   outcome}`.
 - **Publish later**: `POST /_publish` (cache id in the body) wires a
   public companion to a private-only cache.
+- **Fill job**: `POST /_fill` (cache id in the body; `target=private|public`,
+  `action=start|stop`) walks every diigo page into a cache in the
+  background — DO alarms, a few pages per step, progress stored in the
+  DO (`startFill`/`fillStatus`/`stopFill`). The created page has the
+  buttons; the public page shows progress.
 - The `DiigoBridge` entrypoint for other workers (its cache-id source is
   the goal-2 wiring TODO).
 

diigo-cache/src/cache.ts

@@ -53,9 +53,15 @@ export type CachePage = {
 
 const MS_PER_CALL = 500;
 
+/** items per diigo page, for the fill job and the pages it etches */
+const PAGE_SIZE = 24;
+
+/** the pause before each fill step's alarm: a step already paces its
+ * pages at MS_PER_CALL; this keeps steps from running back to back */
+const FILL_STEP_DELAY_MS = 1000;
+
 /**
- * a fill job's progress (goal 1e, **not yet implemented**: the design is
+ * a fill job's progress (goal 1e; tests/fill.test.ts). Stored in the DO
- * pinned by the skipped suite in tests/fill.test.ts). Stored in the DO
  * (new state, additive), so it survives eviction; advanced by alarms.
  */
 export type FillStatus = {
@@ -132,21 +138,91 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
 	/**
 	 * start (or rejoin) the fill job: walk every diigo page into this
 	 * cache, `pagesPerStep` per alarm (default 20, 1–50). Starting while a
-	 * job runs returns that job unchanged. Goal 1e; not yet implemented.
+	 * job runs returns that job unchanged; a finished, stopped, or failed
+	 * job is replaced by a fresh one (cached pages are skipped anyway).
 	 */
 	async startFill(opts: { pagesPerStep?: number } = {}): Promise<FillStatus> {
-		void opts;
+		const current = await this.fillStatus();
-		throw new Error("startFill: not implemented (goal 1e)");
+		if (current?.state === "running") return current;
+		const step = Math.trunc(opts.pagesPerStep ?? 20);
+		if (!(step >= 1 && step <= 50)) throw new Error("pagesPerStep: expected 1–50");
+		const now = new Date().toISOString();
+		const status: FillStatus = {
+			state: "running",
+			nextPage: 1,
+			pages: this.#knownPages(),
+			fetched: 0,
+			startedAt: now,
+			updatedAt: now,
+		};
+		await this.ctx.storage.put({ fill: status, fillStep: step });
+		await this.ctx.storage.setAlarm(Date.now() + FILL_STEP_DELAY_MS);
+		console.log({ event: "fill", kind: this.kind, state: status.state, pagesPerStep: step });
+		return status;
 	}
 
-	/** the fill job's progress, or undefined if none was ever started */
+	/** the fill job's progress, or undefined if none was ever started;
+	 * the page count comes from any cached page until the job learns it */
 	async fillStatus(): Promise<FillStatus | undefined> {
-		throw new Error("fillStatus: not implemented (goal 1e)");
+		const fill = await this.ctx.storage.get<FillStatus>("fill");
+		return fill && fill.pages === null ? { ...fill, pages: this.#knownPages() } : fill;
 	}
 
 	/** stop the fill job after its current step; cached pages stay */
 	async stopFill(): Promise<FillStatus> {
-		throw new Error("stopFill: not implemented (goal 1e)");
+		const current = await this.fillStatus();
+		if (!current) throw new Error("no fill job to stop");
+		if (current.state !== "running") return current;
+		const stopped: FillStatus = { ...current, state: "stopped", updatedAt: new Date().toISOString() };
+		await this.ctx.storage.put("fill", stopped);
+		await this.ctx.storage.deleteAlarm();
+		console.log({ event: "fill", kind: this.kind, state: "stopped", nextPage: stopped.nextPage, pages: stopped.pages });
+		return stopped;
+	}
+
+	/** one fill step: up to `fillStep` pages, progress saved after each,
+	 * then the next step is scheduled (or the job ends) */
+	async alarm(): Promise<void> {
+		const step = (await this.ctx.storage.get<number>("fillStep")) ?? 20;
+		let status = await this.fillStatus();
+		for (let i = 0; i < step && status?.state === "running"; i += 1) {
+			if (status.pages !== null && status.nextPage > status.pages) {
+				status = { ...status, state: "done" };
+				break;
+			}
+			try {
+				const page = await this.page(status.nextPage, PAGE_SIZE);
+				status = {
+					...status,
+					nextPage: status.nextPage + 1,
+					pages: Math.max(1, Math.ceil(page.total / PAGE_SIZE)),
+					fetched: status.fetched + (page.cached ? 0 : 1),
+					updatedAt: new Date().toISOString(),
+				};
+			} catch (err) {
+				status = { ...status, state: "failed", error: String(err), updatedAt: new Date().toISOString() };
+				console.error({ event: "fill", kind: this.kind, state: "failed", page: status.nextPage, error: status.error });
+			}
+			// a stop that landed mid-step wins
+			if ((await this.fillStatus())?.state === "stopped") return;
+			await this.ctx.storage.put("fill", status);
+		}
+		if (!status) return;
+		if (status.state === "running" && status.pages !== null && status.nextPage > status.pages) {
+			status = { ...status, state: "done", updatedAt: new Date().toISOString() };
+		}
+		await this.ctx.storage.put("fill", status);
+		console.log({ event: "fill", kind: this.kind, state: status.state, nextPage: status.nextPage, pages: status.pages, fetched: status.fetched });
+		if (status.state === "running") await this.ctx.storage.setAlarm(Date.now() + FILL_STEP_DELAY_MS);
+	}
+
+	/** diigo's page count, if any cached page recorded `total` */
+	#knownPages(): number | null {
+		this.ctx.storage.sql.exec(SCHEMA_V1);
+		const row = this.ctx.storage.sql
+			.exec<{ total: number | null }>("select total from pages where total is not null order by etched_at desc limit 1")
+			.toArray();
+		return row.length > 0 && row[0].total !== null ? Math.max(1, Math.ceil(row[0].total / PAGE_SIZE)) : null;
 	}
 
 	/**

diigo-cache/src/index.ts

@@ -14,6 +14,7 @@
 
 import { WorkerEntrypoint } from "cloudflare:workers";
 import { DiigoCache, PublicCache } from "./cache.js";
+import type { FillStatus } from "./cache.js";
 import type { Item } from "./cache.js";
 import { updatedISO, usernameOf } from "./items.js";
 import { toWebkey, idFromWebkey, isWebkey, shashKey } from "./webkeys.js";
@@ -95,6 +96,11 @@ export default {
 			return publishCache(request, env);
 		}
 
+		// the fill job (goal 1e): owner-only, cache id in the body
+		if (url.pathname === "/_fill" && request.method === "POST") {
+			return fillRoute(request, env);
+		}
+
 		if (url.pathname !== "/") return new Response("not found", { status: 404 });
 
 		if (request.method === "POST") {
@@ -268,7 +274,7 @@ async function wire(request: Request, env: Env): Promise<Response> {
 		await pubStub.setup({ webappId });
 		await index.register(username, publicId);
 		if (wantsJson) return Response.json({ cacheId, publicId, username });
-		return wiredPage(cacheId, `public cache live at <a href="/public/${esc(encodeURIComponent(username))}">/public/${esc(username)}</a>`);
+		return wiredPage(cacheId, `public cache live at <a href="/public/${esc(encodeURIComponent(username))}">/public/${esc(username)}</a>`, true);
 	}
 	if (wantsJson) {
 		return Response.json({
@@ -283,11 +289,24 @@ async function wire(request: Request, env: Env): Promise<Response> {
 <input type="hidden" name="cacheId" value="${esc(cacheId)}">
 <button type="submit">publish</button></form>
 (or <code>POST /_publish</code> with <code>cacheId</code> in the body)`,
+		false,
 	);
 }
 
+/** the fill job's buttons: one per cache the holder can fill (the id
+ * rides a hidden field, never the URL) */
+function fillForms(cacheId: string, hasPublic: boolean): string {
+	const form = (target: string, label: string) => `<form method="post" action="/_fill">
+<input type="hidden" name="cacheId" value="${esc(cacheId)}">
+<input type="hidden" name="target" value="${target}">
+<button type="submit">${label}</button></form>`;
+	return `<p>fetch every page now, in the background (a few minutes for thousands of bookmarks):</p>
+${form("private", "fill the private cache")}
+${hasPublic ? form("public", "fill the public cache") : ""}`;
+}
+
 /** the created-cache page: ids + save/copy buttons + the pointer home */
-async function wiredPage(cacheId: string, note: string): Promise<Response> {
+async function wiredPage(cacheId: string, note: string, hasPublic: boolean): Promise<Response> {
 	const fingerprint = await nick(cacheId);
 	const body = `<!doctype html>
 <html><head><meta charset="utf-8"><title>diigo cache: created</title></head>
@@ -300,6 +319,7 @@ async function wiredPage(cacheId: string, note: string): Promise<Response> {
 <script>${PETNAME_TIME_JS}</script>
 <p><button type="button" onclick='${pmSaveClick(cacheId)}'>save to password manager</button></p>
 <p>${note}</p>
+${fillForms(cacheId, hasPublic)}
 <p>the cache id is the only way back to this cache — saved (above) it lives
 with the rest of your credentials. it is a <b>webkey</b>: it never travels
 in URLs (query strings leak to history/Referer/logs); the dump POST carries
@@ -311,6 +331,67 @@ hold it separately only if you want another cache off the same session.</p>
 	return new Response(body, { headers: { "content-type": "text/html;charset=utf-8" } });
 }
 
+/** a fill job's progress, as one line of page text */
+function fillLine(fill: FillStatus): string {
+	const of = `${Math.max(0, fill.nextPage - 1)} of ${fill.pages ?? "?"} pages`;
+	const said = {
+		running: `filling: ${of}`,
+		done: `filled: ${of}`,
+		stopped: `fill stopped: ${of}`,
+		failed: `fill failed at page ${fill.nextPage}: ${fill.error ?? "unknown error"}`,
+	}[fill.state];
+	return `<p>${esc(said)}</p>`;
+}
+
+/** POST /_fill: start or stop the fill job of the holder's private cache
+ * (target=private, the default) or of its public companion
+ * (target=public). JSON in → JSON out; a form post gets a small page. */
+async function fillRoute(request: Request, env: Env): Promise<Response> {
+	const ct = request.headers.get("content-type") ?? "";
+	const wantsJson = ct.includes("application/json");
+	let fields: Record<string, string>;
+	try {
+		fields = wantsJson
+			? Object.fromEntries(Object.entries((await request.json()) as Record<string, unknown>).map(([k, v]) => [k, String(v ?? "")]))
+			: Object.fromEntries([...(await request.formData()).entries()].map(([k, v]) => [k, String(v)]));
+	} catch (err) {
+		return bad(`bad input: ${String(err)}`);
+	}
+	const { cacheId = "", target = "private", action = "start" } = fields;
+	if (!isWebkey(cacheId)) return bad("no cache id supplied (POST body: cacheId)");
+	if (target !== "private" && target !== "public") return bad("target: expected private or public");
+	if (action !== "start" && action !== "stop") return bad("action: expected start or stop");
+
+	const cache = env.CACHE_DO.get(env.CACHE_DO.idFromString(idFromWebkey(cacheId)));
+	const wiring = await cache.wiring();
+	if (!wiring) return Response.json({ error: "unknown cache id" }, { status: 404 });
+	let stub: { startFill(): Promise<FillStatus>; stopFill(): Promise<FillStatus> } = cache;
+	if (target === "public") {
+		if (!wiring.publicId) return bad("this cache has no public companion; publish it first");
+		stub = env.PUBLIC_DO.get(env.PUBLIC_DO.idFromString(idFromWebkey(wiring.publicId)));
+	}
+	let status: FillStatus;
+	try {
+		status = action === "start" ? await stub.startFill() : await stub.stopFill();
+	} catch (err) {
+		console.error({ event: "fill", target, action, error: String(err) });
+		return bad(`could not ${action} the fill: ${String(err)}`);
+	}
+	const code = action === "start" ? 202 : 200;
+	if (wantsJson) return Response.json(status, { status: code });
+	return new Response(
+		`<!doctype html>
+<html><head><meta charset="utf-8"><title>diigo cache: fill</title></head>
+<body>
+<h1>fill (${esc(target)} cache)</h1>
+${fillLine(status)}
+<p>it runs in the background; the progress shows on the public page, or post this form again to check.</p>
+<a href="/">← home</a>
+</body></html>`,
+		{ status: code, headers: { "content-type": "text/html;charset=utf-8" } },
+	);
+}
+
 /** publish, later: the private cache's wiring already holds the webapp
  * id, so the public companion needs no re-mint; the username comes from
  * the items already in the cache */
@@ -434,6 +515,7 @@ async function publicPage(request: Request, env: Env, username: string): Promise
 	// page n with the private rows dropped
 	const pages = Math.max(1, Math.ceil(page.total / PUBLIC_PER_PAGE));
 	const items = page.items.map((j) => JSON.parse(j) as Item);
+	const fill = await cache.fillStatus();
 
 	const accept = request.headers.get("accept") ?? "";
 	if (accept.includes("application/json") && !accept.includes("text/html")) {
@@ -469,6 +551,7 @@ ${items
 <h1>${esc(username)}'s shared bookmarks</h1>
 <p>${pages} page${pages === 1 ? "" : "s"} (diigo's, ${PUBLIC_PER_PAGE} bookmarks each; private ones are left out, so a page may show fewer).</p>
 <p><strong>cache: ${page.cached ? "HIT (from storage)" : "MISS (fetched from diigo now)"}</strong> — page ${valid ? pageN : 1}, ${items.length} shared</p>
+${fill ? fillLine(fill) : ""}
 <form method="get" action="${here}">
 <p>${nav(pageN - 1, "← prev")}
 <label>page <input type="number" name="page" min="1" value="${Number.isNaN(pageN) ? 1 : pageN}"></label>

diigo-cache/tests/fill.test.ts

@@ -4,9 +4,9 @@ import { describe, expect, it, vi } from "vitest";
 import worker from "../src/index.js";
 import { toWebkey } from "../src/webkeys.js";
 
-// goal 1e (capsite checklist): a job that grabs every page. The design as
+// goal 1e (capsite checklist): a job that grabs every page. Fake
-// tests; unskip as it lands. Fake accounts: tests/fake-webapp.js
+// accounts: tests/fake-webapp.js ("fake-webapp-id": 30 items = 2 pages
-// ("fake-webapp-id": 30 items = 2 pages of 24, every 3rd private).
+// of 24, every 3rd private).
 
 type TestEnv = {
 	CACHE_DO: DurableObjectNamespace<import("../src/cache.js").DiigoCache>;
@@ -29,13 +29,16 @@ const drain = async (stub: DurableObjectStub, max = 20) => {
 	return ran;
 };
 
-describe.skip("fill job (goal 1e)", () => {
+describe("fill job (goal 1e)", () => {
 	it("fills every page; afterwards every page is a hit with no webapp call", async () => {
 		const stub = await wired();
+		const atStart = await testEnv.WEBAPP.calls();
 		expect((await stub.startFill()).state).toBe("running");
 		await drain(stub);
 		const status = await stub.fillStatus();
 		expect(status).toMatchObject({ state: "done", pages: 2, fetched: 2 });
+		// one diigo call per page fetched (no extra page-1 fetch per miss)
+		expect((await testEnv.WEBAPP.calls()) - atStart).toBe(2);
 
 		const before = await testEnv.WEBAPP.calls();
 		for (const n of [1, 2]) expect((await stub.page(n, 24)).cached).toBe(true);
@@ -114,7 +117,7 @@ describe.skip("fill job (goal 1e)", () => {
 	});
 });
 
-describe.skip("fill routes (goal 1e)", () => {
+describe("fill routes (goal 1e)", () => {
 	const call = (path: string, init?: RequestInit) =>
 		worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
 	const post = (body: Record<string, string>) =>