diigo-cache: fill job — a button that grabs every page (goal 1e)

POST /_fill (cache id in the body; target private|public; action
start|stop) starts or stops a background walk of every diigo page into
the cache. DO alarms advance it pagesPerStep pages per step (default
20), a second apart; progress (FillStatus) is stored in the DO after
each page, so it survives eviction. Cached pages are skipped; starting
while running rejoins the job; a stop mid-step wins; a failure is
recorded with its reason and logged once. The created page gets fill
buttons; the public page shows "filling: n of m pages".

Fill state is new, additive DO storage (fill, fillStep): no existing
state changes meaning.

The 13 fill specs are unskipped and green, plus one: a fill makes one
diigo call per page fetched.

Generated-by: Claude Opus 5.5

diigo-cache/src/index.ts

@@ -14,6 +14,7 @@
 
 import { WorkerEntrypoint } from "cloudflare:workers";
 import { DiigoCache, PublicCache } from "./cache.js";
+import type { FillStatus } from "./cache.js";
 import type { Item } from "./cache.js";
 import { updatedISO, usernameOf } from "./items.js";
 import { toWebkey, idFromWebkey, isWebkey, shashKey } from "./webkeys.js";
@@ -95,6 +96,11 @@ export default {
 			return publishCache(request, env);
 		}
 
+		// the fill job (goal 1e): owner-only, cache id in the body
+		if (url.pathname === "/_fill" && request.method === "POST") {
+			return fillRoute(request, env);
+		}
+
 		if (url.pathname !== "/") return new Response("not found", { status: 404 });
 
 		if (request.method === "POST") {
@@ -268,7 +274,7 @@ async function wire(request: Request, env: Env): Promise<Response> {
 		await pubStub.setup({ webappId });
 		await index.register(username, publicId);
 		if (wantsJson) return Response.json({ cacheId, publicId, username });
-		return wiredPage(cacheId, `public cache live at <a href="/public/${esc(encodeURIComponent(username))}">/public/${esc(username)}</a>`);
+		return wiredPage(cacheId, `public cache live at <a href="/public/${esc(encodeURIComponent(username))}">/public/${esc(username)}</a>`, true);
 	}
 	if (wantsJson) {
 		return Response.json({
@@ -283,11 +289,24 @@ async function wire(request: Request, env: Env): Promise<Response> {
 <input type="hidden" name="cacheId" value="${esc(cacheId)}">
 <button type="submit">publish</button></form>
 (or <code>POST /_publish</code> with <code>cacheId</code> in the body)`,
+		false,
 	);
 }
 
+/** the fill job's buttons: one per cache the holder can fill (the id
+ * rides a hidden field, never the URL) */
+function fillForms(cacheId: string, hasPublic: boolean): string {
+	const form = (target: string, label: string) => `<form method="post" action="/_fill">
+<input type="hidden" name="cacheId" value="${esc(cacheId)}">
+<input type="hidden" name="target" value="${target}">
+<button type="submit">${label}</button></form>`;
+	return `<p>fetch every page now, in the background (a few minutes for thousands of bookmarks):</p>
+${form("private", "fill the private cache")}
+${hasPublic ? form("public", "fill the public cache") : ""}`;
+}
+
 /** the created-cache page: ids + save/copy buttons + the pointer home */
-async function wiredPage(cacheId: string, note: string): Promise<Response> {
+async function wiredPage(cacheId: string, note: string, hasPublic: boolean): Promise<Response> {
 	const fingerprint = await nick(cacheId);
 	const body = `<!doctype html>
 <html><head><meta charset="utf-8"><title>diigo cache: created</title></head>
@@ -300,6 +319,7 @@ async function wiredPage(cacheId: string, note: string): Promise<Response> {
 <script>${PETNAME_TIME_JS}</script>
 <p><button type="button" onclick='${pmSaveClick(cacheId)}'>save to password manager</button></p>
 <p>${note}</p>
+${fillForms(cacheId, hasPublic)}
 <p>the cache id is the only way back to this cache — saved (above) it lives
 with the rest of your credentials. it is a <b>webkey</b>: it never travels
 in URLs (query strings leak to history/Referer/logs); the dump POST carries
@@ -311,6 +331,67 @@ hold it separately only if you want another cache off the same session.</p>
 	return new Response(body, { headers: { "content-type": "text/html;charset=utf-8" } });
 }
 
+/** a fill job's progress, as one line of page text */
+function fillLine(fill: FillStatus): string {
+	const of = `${Math.max(0, fill.nextPage - 1)} of ${fill.pages ?? "?"} pages`;
+	const said = {
+		running: `filling: ${of}`,
+		done: `filled: ${of}`,
+		stopped: `fill stopped: ${of}`,
+		failed: `fill failed at page ${fill.nextPage}: ${fill.error ?? "unknown error"}`,
+	}[fill.state];
+	return `<p>${esc(said)}</p>`;
+}
+
+/** POST /_fill: start or stop the fill job of the holder's private cache
+ * (target=private, the default) or of its public companion
+ * (target=public). JSON in → JSON out; a form post gets a small page. */
+async function fillRoute(request: Request, env: Env): Promise<Response> {
+	const ct = request.headers.get("content-type") ?? "";
+	const wantsJson = ct.includes("application/json");
+	let fields: Record<string, string>;
+	try {
+		fields = wantsJson
+			? Object.fromEntries(Object.entries((await request.json()) as Record<string, unknown>).map(([k, v]) => [k, String(v ?? "")]))
+			: Object.fromEntries([...(await request.formData()).entries()].map(([k, v]) => [k, String(v)]));
+	} catch (err) {
+		return bad(`bad input: ${String(err)}`);
+	}
+	const { cacheId = "", target = "private", action = "start" } = fields;
+	if (!isWebkey(cacheId)) return bad("no cache id supplied (POST body: cacheId)");
+	if (target !== "private" && target !== "public") return bad("target: expected private or public");
+	if (action !== "start" && action !== "stop") return bad("action: expected start or stop");
+
+	const cache = env.CACHE_DO.get(env.CACHE_DO.idFromString(idFromWebkey(cacheId)));
+	const wiring = await cache.wiring();
+	if (!wiring) return Response.json({ error: "unknown cache id" }, { status: 404 });
+	let stub: { startFill(): Promise<FillStatus>; stopFill(): Promise<FillStatus> } = cache;
+	if (target === "public") {
+		if (!wiring.publicId) return bad("this cache has no public companion; publish it first");
+		stub = env.PUBLIC_DO.get(env.PUBLIC_DO.idFromString(idFromWebkey(wiring.publicId)));
+	}
+	let status: FillStatus;
+	try {
+		status = action === "start" ? await stub.startFill() : await stub.stopFill();
+	} catch (err) {
+		console.error({ event: "fill", target, action, error: String(err) });
+		return bad(`could not ${action} the fill: ${String(err)}`);
+	}
+	const code = action === "start" ? 202 : 200;
+	if (wantsJson) return Response.json(status, { status: code });
+	return new Response(
+		`<!doctype html>
+<html><head><meta charset="utf-8"><title>diigo cache: fill</title></head>
+<body>
+<h1>fill (${esc(target)} cache)</h1>
+${fillLine(status)}
+<p>it runs in the background; the progress shows on the public page, or post this form again to check.</p>
+<a href="/">← home</a>
+</body></html>`,
+		{ status: code, headers: { "content-type": "text/html;charset=utf-8" } },
+	);
+}
+
 /** publish, later: the private cache's wiring already holds the webapp
  * id, so the public companion needs no re-mint; the username comes from
  * the items already in the cache */
@@ -434,6 +515,7 @@ async function publicPage(request: Request, env: Env, username: string): Promise
 	// page n with the private rows dropped
 	const pages = Math.max(1, Math.ceil(page.total / PUBLIC_PER_PAGE));
 	const items = page.items.map((j) => JSON.parse(j) as Item);
+	const fill = await cache.fillStatus();
 
 	const accept = request.headers.get("accept") ?? "";
 	if (accept.includes("application/json") && !accept.includes("text/html")) {
@@ -469,6 +551,7 @@ ${items
 <h1>${esc(username)}'s shared bookmarks</h1>
 <p>${pages} page${pages === 1 ? "" : "s"} (diigo's, ${PUBLIC_PER_PAGE} bookmarks each; private ones are left out, so a page may show fewer).</p>
 <p><strong>cache: ${page.cached ? "HIT (from storage)" : "MISS (fetched from diigo now)"}</strong> — page ${valid ? pageN : 1}, ${items.length} shared</p>
+${fill ? fillLine(fill) : ""}
 <form method="get" action="${here}">
 <p>${nav(pageN - 1, "← prev")}
 <label>page <input type="number" name="page" min="1" value="${Number.isNaN(pageN) ? 1 : pageN}"></label>