diigo-cache: page size 100, diigo's cap; fill jobs record their page size

Asked for 128, diigo returned 100 (the short-page guard caught it on the
first real fill), so PAGE_SIZE = 100: ~168 pages for 16.7k bookmarks.

The same run showed a fill job outliving a page-size change: progress
counted in 24s ("page 25 of 698") resumed as if counted in 128s. Stored
state reinterpreted, so: FillStatus records pageSize (additive; jobs
saved without it counted in 24), and a job whose size differs from
PAGE_SIZE fails with "page size changed …; start the fill again"
instead of resuming; starting again replaces it. Tested with the old
shape in storage.

Generated-by: Claude Opus 5.5

diigo-cache/src/cache.ts

@@ -60,9 +60,14 @@ const MS_PER_CALL = 500;
  * items per diigo page — the one size the worker asks for everywhere
  * (fill, public pages, dumps), so a cache never mixes sizes. Pages are
  * cached by (page number, size): changing this turns every cached page
- * into a miss (items survive; pages refetch).
+ * into a miss (items survive; pages refetch), and fails any fill job
+ * counting in the old size. 100 is diigo's cap: asked for 128, it
+ * returned 100 (observed 2026-10-07).
  */
-export const PAGE_SIZE = 128;
+export const PAGE_SIZE = 100;
+
+/** the size fill jobs counted in before FillStatus recorded it */
+const LEGACY_FILL_PAGE_SIZE = 24;
 
 /** the pause before each fill step's alarm: a step already paces its
  * pages at MS_PER_CALL; this keeps steps from running back to back */
@@ -78,6 +83,9 @@ export type FillStatus = {
 	nextPage: number;
 	/** diigo's page count, known after page 1 */
 	pages: number | null;
+	/** the page size `nextPage` and `pages` count in; absent on jobs
+	 * saved before this field existed (they counted in 24s) */
+	pageSize?: number;
 	/** pages fetched from diigo by this job (cache hits not counted) */
 	fetched: number;
 	startedAt: string;
@@ -86,6 +94,11 @@ export type FillStatus = {
 	error?: string;
 };
 
+/** the page size a fill job counts in (old jobs didn't record it) */
+function fillPageSize(status: FillStatus): number {
+	return status.pageSize ?? LEGACY_FILL_PAGE_SIZE;
+}
+
 /** release an RPC stub, if it is one (plain objects are left alone) */
 function disposeStub(stub: unknown): void {
 	const dispose = (stub as { [Symbol.dispose]?: () => void } | null)?.[Symbol.dispose];
@@ -169,7 +182,7 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
 	 */
 	async startFill(opts: { pagesPerStep?: number } = {}): Promise<FillStatus> {
 		const current = await this.fillStatus();
-		if (current?.state === "running") return current;
+		if (current?.state === "running" && fillPageSize(current) === PAGE_SIZE) return current;
 		const step = Math.trunc(opts.pagesPerStep ?? 20);
 		if (!(step >= 1 && step <= 50)) throw new Error("pagesPerStep: expected 1–50");
 		const now = this.#iso();
@@ -177,6 +190,7 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
 			state: "running",
 			nextPage: 1,
 			pages: this.#knownPages(),
+			pageSize: PAGE_SIZE,
 			fetched: 0,
 			startedAt: now,
 			updatedAt: now,
@@ -211,6 +225,15 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
 	async alarm(): Promise<void> {
 		const step = (await this.ctx.storage.get<number>("fillStep")) ?? 20;
 		let status = await this.fillStatus();
+		// a job saved under another page size counts its pages differently:
+		// resuming it would skip or repeat items, so it fails instead
+		if (status?.state === "running" && fillPageSize(status) !== PAGE_SIZE) {
+			const error = `page size changed (from ${fillPageSize(status)} to ${PAGE_SIZE}); start the fill again`;
+			status = { ...status, state: "failed", error, updatedAt: this.#iso() };
+			console.error({ event: "fill", kind: this.kind, state: "failed", error });
+			await this.ctx.storage.put("fill", status);
+			return;
+		}
 		for (let i = 0; i < step && status?.state === "running"; i += 1) {
 			if (status.pages !== null && status.nextPage > status.pages) {
 				status = { ...status, state: "done" };

diigo-cache/tests/caches.test.ts

@@ -26,7 +26,7 @@ describe("PublicCache", () => {
 		const before = await testEnv.WEBAPP.calls();
 		const miss = await stub.page(1, 24);
 		expect(miss.cached).toBe(false);
-		expect(miss.total).toBe(300);
+		expect(miss.total).toBe(250);
 		// diigo page 1 = items 1..24, of which 8 are private
 		expect(parse(miss.items).map((i) => i.private)).toEqual(Array(16).fill(false));
 		const afterMiss = await testEnv.WEBAPP.calls();
@@ -78,7 +78,7 @@ describe("page size", () => {
 			// settle the RPC promise ourselves: expect().rejects on a DO stub's
 			// promise leaves the rejection unhandled
 			const failure = () => stub.page(1, PAGE_SIZE).then(() => null, (e: unknown) => String(e));
-			expect(await failure()).toMatch(/returned 100 items for page 1 of 3 \(asked for 128\)/);
+			expect(await failure()).toMatch(/returned 64 items for page 1 of 3 \(asked for 100\)/);
 			expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.outcome === "short-page")).toHaveLength(1);
 			// nothing was stored: asking again refuses again
 			expect(await failure()).toMatch(/page size not honored/);
@@ -90,7 +90,7 @@ describe("page size", () => {
 	it("the last page may be short", async () => {
 		const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
 		await stub.setup({ webappId: WEBAPP_ID });
-		expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(300 - 2 * PAGE_SIZE);
+		expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(250 - 2 * PAGE_SIZE);
 	});
 });
 
@@ -153,7 +153,7 @@ describe("worker routes", () => {
 		expect(html).toContain("<strong>cache: MISS");
 		const again = await (await call(`/public/${username}?page=2`)).text();
 		expect(again).toContain("<strong>cache: HIT");
-		// PAGE_SIZE 128: page 2 is bookmarks 129..256
+		// PAGE_SIZE 100: page 2 is bookmarks 101..200
 		expect(html).toContain(">bookmark 130<");
 		// the date comes first, in toISOString() form
 		expect(html).toMatch(/<li><time datetime="\d{4}-\d\d-\d\dT[\d:.]+Z">[^<]+<\/time> <a /);
@@ -163,7 +163,7 @@ describe("worker routes", () => {
 			await call(`/public/${username}?page=2`, { headers: { accept: "application/json" } })
 		).json()) as { pages: number; items: unknown[] };
 		expect(json.pages).toBe(3);
-		expect(json.items).toHaveLength(85); // 129..256 minus its 43 multiples of 3
+		expect(json.items).toHaveLength(67); // 101..200 minus its 33 multiples of 3
 	});
 
 	it("publishes a private cache later, without the cache id in the URL", async () => {
@@ -278,7 +278,7 @@ describe.skip("public page filters (goal 1d)", () => {
 			body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
 		});
 		expect(res.status).toBe(200);
-		// deep_dee: 47 diigo pages of 128; a search matching nothing must walk them all
+		// deep_dee: 60 diigo pages of 100; a search matching nothing must walk them all
 		const before = await testEnv.WEBAPP.calls();
 		const first = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
 		expect(await first.json()).toMatchObject({ complete: false, items: [] });
@@ -305,8 +305,8 @@ describe.skip("public page filters (goal 1d)", () => {
 			body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
 		});
 		expect(res.status).toBe(200);
-		// deep_dee: 6000 bookmarks, one a day, 47 diigo pages of 128; a week
+		// deep_dee: 6000 bookmarks, one a day, 60 diigo pages of 100; a week
-		// ~14 years back sits near page 40 — a walk from page 1 would cost ~40
+		// ~14 years back sits near page 51 — a walk from page 1 would cost ~51
 		const longAgo = new Date(Date.now() - 5100 * 24 * 60 * 60 * 1000);
 		const week = isoWeek(longAgo);
 		const before = await testEnv.WEBAPP.calls();

diigo-cache/tests/fake-webapp.js

@@ -1,17 +1,18 @@
 /**
  * @file a stand-in for webapp's `WebAppFactory` entrypoint, for tests: an
  * auxiliary worker (plain JS — the vitest pool doesn't transform aux
- * workers) serving diigo-shaped `load_user_items` pages. 300 items (3
+ * workers) serving diigo-shaped `load_user_items` pages. 250 items (3
- * pages of 128), every third private; a few webapp ids are known. Counts its diigo calls so tests
+ * pages of 100, the last short), every third private; a few webapp ids
+ * are known. Counts its diigo calls so tests
  * can tell a cache hit (no call) from a miss.
  */
 import { RpcTarget, WorkerEntrypoint } from "cloudflare:workers";
 
 const KNOWN_WEBAPP = "fake-webapp-id";
-const TOTAL = 300;
+const TOTAL = 250;
-/** the plain account, but diigo-style capped at 100 items per page */
+/** the plain account, capped below PAGE_SIZE (as diigo caps at 100) */
 const CAPPED_WEBAPP = "fake-capped-webapp";
-const CAP = 100;
+const CAP = 64;
 
 /** a second account, for date filters (goal 1d): 60 bookmarks, item n
  * updated n-1 days + 1 hour before the call, in diigo's updated-desc
@@ -19,7 +20,7 @@ const CAP = 100;
  * 5th an old bookmark edited recently (created 100 days before updated) */
 const DATED_WEBAPP = "fake-dated-webapp";
 const DATED_TOTAL = 60;
-/** the same shape, 6000 deep (~16 years, 47 pages of 128): for walk cost */
+/** the same shape, 6000 deep (~16 years, 60 pages of 100): for walk cost */
 const DEEP_WEBAPP = "fake-deep-webapp";
 const DEEP_TOTAL = 6000;
 const HOUR = 60 * 60 * 1000;
@@ -41,7 +42,8 @@ function datedItem(n, now, user = "dated_dan") {
 
 /** item n (1-based, in diigo's order): diigo's field names, made-up
  * values. link_id and updated_at are scrambled (distinct; 7 and 11 are
- * coprime to 307, a prime above TOTAL), so neither column sorts back into diigo's order — a
+ * coprime to 307, a prime above TOTAL), so neither column sorts back
+ * into diigo's order — a
  * cache that replays by column rather than by fetch order is caught. */
 function item(n) {
 	return {

diigo-cache/tests/fill.test.ts

@@ -9,8 +9,8 @@ import { FIXED_NOW } from "./test-worker.js";
 import { PAGE_SIZE } from "../src/cache.js";
 
 // goal 1e (capsite checklist): a job that grabs every page. Fake
-// accounts: tests/fake-webapp.js ("fake-webapp-id": 300 items = 3 pages
+// accounts: tests/fake-webapp.js ("fake-webapp-id": 250 items = 3 pages
-// of PAGE_SIZE 128, every 3rd private).
+// of PAGE_SIZE 100, every 3rd private).
 
 type TestEnv = {
 	CACHE_DO: DurableObjectNamespace<import("../src/index.js").DiigoCache>;
@@ -131,6 +131,21 @@ describe("fill job (goal 1e)", () => {
 		expect(status?.error).toMatch(/page size not honored/);
 	});
 
+	it("a job saved under another page size fails instead of resuming; a new start replaces it", async () => {
+		const stub = await wired();
+		// the shape a 24-per-page job left in storage (no pageSize field)
+		await runInDurableObject(stub, async (_obj, state) => {
+			const at = new Date().toISOString();
+			await state.storage.put("fill", { state: "running", nextPage: 25, pages: 698, fetched: 23, startedAt: at, updatedAt: at });
+			await state.storage.setAlarm(Date.now() + 60_000);
+		});
+		await drain(stub);
+		const stale = await stub.fillStatus();
+		expect(stale?.state).toBe("failed");
+		expect(stale?.error).toMatch(/page size changed \(from 24 to 100\)/);
+		expect(await stub.startFill()).toMatchObject({ state: "running", nextPage: 1, pageSize: PAGE_SIZE });
+	});
+
 	it("a public cache's fill stores no private rows", async () => {
 		const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
 		await stub.setup({ webappId: "fake-webapp-id" });