diigo-cache: fetch 128 items per diigo page; refuse short pages

PAGE_SIZE = 128 (was 24), one constant for the fill job, public pages,
dumps, and DiigoBridge, so a cache never mixes sizes: ~131 pages instead
of ~698 for 16.7k bookmarks.

Cache invalidation, agreed beforehand: pages are cached by (page number,
size), so pages cached at 24 become misses and refetch at 128. Items are
keyed by link_id and just re-upsert; nothing is lost.

Unknown whether diigo honors count=128, so a short page that isn't the
last is refused loudly (logged, thrown; a fill fails with the reason)
rather than silently skipping items. The fake gains a capped account to
test it; the plain account grows to 300 (3 pages), the deep one to 6000.

Generated-by: Claude Opus 5.5

diigo-cache/src/cache.ts

@@ -56,8 +56,13 @@ export type CachePage = {
 
 const MS_PER_CALL = 500;
 
-/** items per diigo page, for the fill job and the pages it etches */
-const PAGE_SIZE = 24;
+/**
+ * items per diigo page — the one size the worker asks for everywhere
+ * (fill, public pages, dumps), so a cache never mixes sizes. Pages are
+ * cached by (page number, size): changing this turns every cached page
+ * into a miss (items survive; pages refetch).
+ */
+export const PAGE_SIZE = 128;
 
 /** the pause before each fill step's alarm: a step already paces its
  * pages at MS_PER_CALL; this keeps steps from running back to back */
@@ -284,6 +289,14 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
 		} finally {
 			for (const stub of [queried, joined, app]) disposeStub(stub);
 		}
+		// diigo may cap the page size below what we asked: a short page that
+		// isn't the last would silently skip items, so refuse it loudly
+		const lastPage = Math.max(1, Math.ceil(data.total / count));
+		if (data.items.length < count && pageNum < lastPage) {
+			const error = `diigo returned ${data.items.length} items for page ${pageNum} of ${lastPage} (asked for ${count}): page size not honored?`;
+			console.error({ event: "cache", kind: this.kind, page: pageNum, outcome: "short-page", fetched: data.items.length, asked: count });
+			throw new Error(error);
+		}
 		const kept = this.keep(toRows(data.items, console));
 		// fetched vs kept: the public cache's drop of private rows shows here
 		console.log({ event: "cache", kind: this.kind, page: pageNum, outcome: "miss", fetched: data.items.length, kept: kept.length });

diigo-cache/src/index.ts

@@ -18,6 +18,7 @@ import { CacheApi } from "./api.js";
 import { CAPNWEB_BROWSER_JS } from "./capnweb-browser.js";
 import { CACHE_PAGE_HTML } from "./cache-page.js";
 import { DiigoCacheLogic, PublicCacheLogic } from "./cache.js";
+import { PAGE_SIZE } from "./cache.js";
 import type { CacheEnv, Clock, FillStatus } from "./cache.js";
 import type { Item } from "./cache.js";
 import { updatedISO, usernameOf } from "./items.js";
@@ -69,7 +70,7 @@ const DIIGO_INTERACT = "https://www.diigo.com/interact_api/";
 export class DiigoBridge extends WorkerEntrypoint {
 	/** bulk dump from the caller's address space: `pages * count` bookmarks
 	 * through the cache (cache-hits are storage reads) */
-	async dumpNdjson(pages: number, count = 24): Promise<string> {
+	async dumpNdjson(pages: number, count = PAGE_SIZE): Promise<string> {
 		const ns = (this.env as unknown as Env).CACHE_DO;
 		const cacheId = (this.env as unknown as Env).DIIGO_CACHE_ID;
 		if (!cacheId) throw new Error("no cache id supplied (caller config error)");
@@ -285,7 +286,7 @@ async function wire(request: Request, env: Env): Promise<Response> {
 	const cacheId = toWebkey(env.CACHE_DO.newUniqueId());
 	const cacheStub = env.CACHE_DO.get(env.CACHE_DO.idFromString(idFromWebkey(cacheId)));
 	await cacheStub.setup({ webappId, publicId });
-	const first = await cacheStub.page(1, 24);
+	const first = await cacheStub.page(1, PAGE_SIZE);
 
 	if (publicId) {
 		const firstItem = first.items[0] ? (JSON.parse(first.items[0]) as Item) : undefined;
@@ -401,7 +402,7 @@ async function publishCache(request: Request, env: Env): Promise<Response> {
 		// seed the projection *now*, while the cache is still private-only:
 		// only rows already cached and marked shared can enter the public
 		// side; the username rides the same rows
-		const firstPage = await cache.page(1, 24);
+		const firstPage = await cache.page(1, PAGE_SIZE);
 		const firstItem = firstPage.items[0] ? (JSON.parse(firstPage.items[0]) as Item) : undefined;
 		const username = usernameOf(firstItem);
 		if (!username) {
@@ -452,7 +453,7 @@ async function dump(request: Request, env: Env): Promise<Response> {
 	try {
 		const ns = env.CACHE_DO;
 		const cache = ns.get(ns.idFromString(idFromWebkey(cacheId)));
-		const page = await cache.page(pageN, 24);
+		const page = await cache.page(pageN, PAGE_SIZE);
 		const data = `total: ${page.total}\nitems: ${page.items.length}\ncache: ${page.cached ? "HIT (from storage)" : "MISS (etched now)"}` +
 			`\n\n${page.items.join("\n")}`;
 		return home(env, null, data, cacheId);
@@ -462,7 +463,7 @@ async function dump(request: Request, env: Env): Promise<Response> {
 	}
 }
 
-const PUBLIC_PER_PAGE = 24;
+const PUBLIC_PER_PAGE = PAGE_SIZE;
 
 /** the public page for a username: an HTML page with a page-number form
  * (GET — nothing secret here, so `?page=` in the URL is fine, unlike the

diigo-cache/tests/caches.test.ts

@@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest";
 import worker from "../src/index.js";
 import { toWebkey } from "../src/webkeys.js";
 import { FETCH_BUDGET } from "../src/query.js";
+import { PAGE_SIZE } from "../src/cache.js";
 
 // matches tests/fake-webapp.js: 30 items, every third private
 const WEBAPP_ID = "fake-webapp-id";
@@ -25,7 +26,7 @@ describe("PublicCache", () => {
 		const before = await testEnv.WEBAPP.calls();
 		const miss = await stub.page(1, 24);
 		expect(miss.cached).toBe(false);
-		expect(miss.total).toBe(30);
+		expect(miss.total).toBe(300);
 		// diigo page 1 = items 1..24, of which 8 are private
 		expect(parse(miss.items).map((i) => i.private)).toEqual(Array(16).fill(false));
 		const afterMiss = await testEnv.WEBAPP.calls();
@@ -68,6 +69,31 @@ describe("PublicCache", () => {
 	});
 });
 
+describe("page size", () => {
+	it("refuses, loudly, a short page that isn't the last (diigo capping the size would skip items)", async () => {
+		const spy = vi.spyOn(console, "error");
+		try {
+			const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
+			await stub.setup({ webappId: "fake-capped-webapp" });
+			// settle the RPC promise ourselves: expect().rejects on a DO stub's
+			// promise leaves the rejection unhandled
+			const failure = () => stub.page(1, PAGE_SIZE).then(() => null, (e: unknown) => String(e));
+			expect(await failure()).toMatch(/returned 100 items for page 1 of 3 \(asked for 128\)/);
+			expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.outcome === "short-page")).toHaveLength(1);
+			// nothing was stored: asking again refuses again
+			expect(await failure()).toMatch(/page size not honored/);
+		} finally {
+			spy.mockRestore();
+		}
+	});
+
+	it("the last page may be short", async () => {
+		const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
+		await stub.setup({ webappId: WEBAPP_ID });
+		expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(300 - 2 * PAGE_SIZE);
+	});
+});
+
 describe("eviction", () => {
 	it("a cache miss leaves no undisposed RPC stubs to block eviction", async () => {
 		const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
@@ -127,16 +153,17 @@ describe("worker routes", () => {
 		expect(html).toContain("<strong>cache: MISS");
 		const again = await (await call(`/public/${username}?page=2`)).text();
 		expect(again).toContain("<strong>cache: HIT");
-		expect(html).toContain("bookmark 25");
+		// PAGE_SIZE 128: page 2 is bookmarks 129..256
+		expect(html).toContain(">bookmark 130<");
 		// the date comes first, in toISOString() form
 		expect(html).toMatch(/<li><time datetime="\d{4}-\d\d-\d\dT[\d:.]+Z">[^<]+<\/time> <a /);
-		expect(html).not.toContain("bookmark 27"); // private (27 % 3 === 0)
+		expect(html).not.toContain(">bookmark 129<"); // private (129 % 3 === 0)
 
 		const json = (await (
 			await call(`/public/${username}?page=2`, { headers: { accept: "application/json" } })
 		).json()) as { pages: number; items: unknown[] };
-		expect(json.pages).toBe(2);
-		expect(json.items).toHaveLength(4); // 25..30 minus 27, 30
+		expect(json.pages).toBe(3);
+		expect(json.items).toHaveLength(85); // 129..256 minus its 43 multiples of 3
 	});
 
 	it("publishes a private cache later, without the cache id in the URL", async () => {
@@ -204,14 +231,23 @@ describe.skip("public page filters (goal 1d)", () => {
 		expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 42, 49, 56, 35));
 	});
 
-	it("?days=30 pages the matches 24 at a time; links keep the filter", async () => {
-		const page = await wireDated();
-		const first = await (await call(`${page}?days=30`)).text();
-		const want = Array.from({ length: 30 }, (_, k) => k + 1).filter((n) => n % 10 !== 0);
-		expect(titlesIn(first)).toEqual(dated(...want.slice(0, 24)));
-		expect(first).toContain(`href="${page}?days=30&amp;page=2"`);
-		const second = await (await call(`${page}?days=30&page=2`)).text();
-		expect(titlesIn(second)).toEqual(dated(...want.slice(24)));
+	it("?days=300 pages the matches PAGE_SIZE at a time; links keep the filter", async () => {
+		const res = await call("/_wire", {
+			method: "POST",
+			headers: { "content-type": "application/json" },
+			body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
+		});
+		expect(res.status).toBe(200);
+		const page = "/public/deep_dee";
+		// deep_dee: one a day; days=300 holds 1..300, minus every 10th (private)
+		const want = Array.from({ length: 300 }, (_, k) => k + 1).filter((n) => n % 10 !== 0);
+		const first = await (await call(`${page}?days=300`)).text();
+		expect(titlesIn(first)).toEqual(dated(...want.slice(0, PAGE_SIZE)));
+		expect(first).toContain(`href="${page}?days=300&amp;page=2"`);
+		const second = await (await call(`${page}?days=300&page=2`)).text();
+		expect(titlesIn(second)).toEqual(dated(...want.slice(PAGE_SIZE, 2 * PAGE_SIZE)));
+		const third = await (await call(`${page}?days=300&page=3`)).text();
+		expect(titlesIn(third)).toEqual(dated(...want.slice(2 * PAGE_SIZE)));
 	});
 
 	it("?after=&before= selects a date range", async () => {
@@ -242,7 +278,7 @@ describe.skip("public page filters (goal 1d)", () => {
 			body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
 		});
 		expect(res.status).toBe(200);
-		// deep_dee: 50 diigo pages; a search matching nothing must walk them all
+		// deep_dee: 47 diigo pages of 128; a search matching nothing must walk them all
 		const before = await testEnv.WEBAPP.calls();
 		const first = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
 		expect(await first.json()).toMatchObject({ complete: false, items: [] });
@@ -269,10 +305,10 @@ describe.skip("public page filters (goal 1d)", () => {
 			body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
 		});
 		expect(res.status).toBe(200);
-		// deep_dee: 1200 bookmarks, one a day, 50 diigo pages; a week ~2.5
-		// years back sits near page 38 — a walk from page 1 would cost ~38
-		const twoAndAHalfYearsAgo = new Date(Date.now() - 900 * 24 * 60 * 60 * 1000);
-		const week = isoWeek(twoAndAHalfYearsAgo);
+		// deep_dee: 6000 bookmarks, one a day, 47 diigo pages of 128; a week
+		// ~14 years back sits near page 40 — a walk from page 1 would cost ~40
+		const longAgo = new Date(Date.now() - 5100 * 24 * 60 * 60 * 1000);
+		const week = isoWeek(longAgo);
 		const before = await testEnv.WEBAPP.calls();
 		const html = await (await call(`/public/deep_dee?in=${week}`)).text();
 		// one a day; every 10th is private

diigo-cache/tests/fake-webapp.js

@@ -1,14 +1,17 @@
 /**
  * @file a stand-in for webapp's `WebAppFactory` entrypoint, for tests: an
  * auxiliary worker (plain JS — the vitest pool doesn't transform aux
- * workers) serving diigo-shaped `load_user_items` pages. 30 items, every
- * third private; one webapp id is known. Counts its diigo calls so tests
+ * workers) serving diigo-shaped `load_user_items` pages. 300 items (3
+ * pages of 128), every third private; a few webapp ids are known. Counts its diigo calls so tests
  * can tell a cache hit (no call) from a miss.
  */
 import { RpcTarget, WorkerEntrypoint } from "cloudflare:workers";
 
 const KNOWN_WEBAPP = "fake-webapp-id";
-const TOTAL = 30;
+const TOTAL = 300;
+/** the plain account, but diigo-style capped at 100 items per page */
+const CAPPED_WEBAPP = "fake-capped-webapp";
+const CAP = 100;
 
 /** a second account, for date filters (goal 1d): 60 bookmarks, item n
  * updated n-1 days + 1 hour before the call, in diigo's updated-desc
@@ -16,9 +19,9 @@ const TOTAL = 30;
  * 5th an old bookmark edited recently (created 100 days before updated) */
 const DATED_WEBAPP = "fake-dated-webapp";
 const DATED_TOTAL = 60;
-/** the same shape, 1200 deep (~3.3 years, 50 pages): for walk cost */
+/** the same shape, 6000 deep (~16 years, 47 pages of 128): for walk cost */
 const DEEP_WEBAPP = "fake-deep-webapp";
-const DEEP_TOTAL = 1200;
+const DEEP_TOTAL = 6000;
 const HOUR = 60 * 60 * 1000;
 
 function datedItem(n, now, user = "dated_dan") {
@@ -38,11 +41,11 @@ function datedItem(n, now, user = "dated_dan") {
 
 /** item n (1-based, in diigo's order): diigo's field names, made-up
  * values. link_id and updated_at are scrambled (distinct; 7 and 11 are
- * coprime to 31), so neither column sorts back into diigo's order — a
+ * coprime to 307, a prime above TOTAL), so neither column sorts back into diigo's order — a
  * cache that replays by column rather than by fetch order is caught. */
 function item(n) {
 	return {
-		link_id: 1000 + ((n * 7) % 31),
+		link_id: 1000 + ((n * 7) % 307),
 		url: `https://example.com/${n}`,
 		title: `bookmark ${n}`,
 		description: "",
@@ -52,7 +55,7 @@ function item(n) {
 		user_id: 4242,
 		tags: "",
 		created_at: 1700000000,
-		updated_at: 1700001000 + ((n * 11) % 31),
+		updated_at: 1700001000 + ((n * 11) % 307),
 	};
 }
 
@@ -61,7 +64,7 @@ let calls = 0;
 class FakeGet extends RpcTarget {
 	#path;
 	#params;
-	/** null (the plain account), "dated", or "deep" */
+	/** null (the plain account), "capped", "dated", or "deep" */
 	#dated;
 	constructor(path, params, dated) {
 		super();
@@ -78,13 +81,14 @@ class FakeGet extends RpcTarget {
 	async get() {
 		calls += 1;
 		if (this.#path !== "load_user_items") throw new Error(`fake webapp: no route ${this.#path}`);
-		const count = Number(this.#params.count);
+		const count = Math.min(Number(this.#params.count), this.#dated === "capped" ? CAP : Infinity);
 		const page = Number(this.#params.page_num);
-		const total = this.#dated === "deep" ? DEEP_TOTAL : this.#dated ? DATED_TOTAL : TOTAL;
+		const dated = this.#dated === "dated" || this.#dated === "deep";
+		const total = this.#dated === "deep" ? DEEP_TOTAL : dated ? DATED_TOTAL : TOTAL;
 		const now = Date.now();
 		const items = [];
 		for (let n = (page - 1) * count + 1; n <= Math.min(page * count, total); n += 1) {
-			items.push(this.#dated ? datedItem(n, now, this.#dated === "deep" ? "deep_dee" : "dated_dan") : item(n));
+			items.push(dated ? datedItem(n, now, this.#dated === "deep" ? "deep_dee" : "dated_dan") : item(n));
 		}
 		return JSON.stringify({ total, items });
 	}
@@ -92,7 +96,7 @@ class FakeGet extends RpcTarget {
 
 export class WebAppFactory extends WorkerEntrypoint {
 	async at(id) {
-		const kind = { [KNOWN_WEBAPP]: null, [DATED_WEBAPP]: "dated", [DEEP_WEBAPP]: "deep" }[id];
+		const kind = { [KNOWN_WEBAPP]: null, [CAPPED_WEBAPP]: "capped", [DATED_WEBAPP]: "dated", [DEEP_WEBAPP]: "deep" }[id];
 		if (kind === undefined) throw new Error("fake webapp: unknown id");
 		return new FakeGet("", {}, kind);
 	}

diigo-cache/tests/fill.test.ts

@@ -6,10 +6,11 @@ import { newHttpBatchRpcSession } from "capnweb";
 import type { CacheApi } from "../src/api.js";
 import { toWebkey } from "../src/webkeys.js";
 import { FIXED_NOW } from "./test-worker.js";
+import { PAGE_SIZE } from "../src/cache.js";
 
 // goal 1e (capsite checklist): a job that grabs every page. Fake
-// accounts: tests/fake-webapp.js ("fake-webapp-id": 30 items = 2 pages
-// of 24, every 3rd private).
+// accounts: tests/fake-webapp.js ("fake-webapp-id": 300 items = 3 pages
+// of PAGE_SIZE 128, every 3rd private).
 
 type TestEnv = {
 	CACHE_DO: DurableObjectNamespace<import("../src/index.js").DiigoCache>;
@@ -40,12 +41,12 @@ describe("fill job (goal 1e)", () => {
 		expect((await stub.startFill()).state).toBe("running");
 		await drain(stub);
 		const status = await stub.fillStatus();
-		expect(status).toMatchObject({ state: "done", pages: 2, fetched: 2 });
+		expect(status).toMatchObject({ state: "done", pages: 3, fetched: 3 });
 		// one diigo call per page fetched (no extra page-1 fetch per miss)
-		expect((await testEnv.WEBAPP.calls()) - atStart).toBe(2);
+		expect((await testEnv.WEBAPP.calls()) - atStart).toBe(3);
 
 		const before = await testEnv.WEBAPP.calls();
-		for (const n of [1, 2]) expect((await stub.page(n, 24)).cached).toBe(true);
+		for (const n of [1, 2, 3]) expect((await stub.page(n, PAGE_SIZE)).cached).toBe(true);
 		expect(await testEnv.WEBAPP.calls()).toBe(before);
 	});
 
@@ -53,7 +54,7 @@ describe("fill job (goal 1e)", () => {
 		const stub = await wired();
 		await stub.startFill({ pagesPerStep: 1 });
 		expect(await runDurableObjectAlarm(stub)).toBe(true);
-		expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2, pages: 2 });
+		expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2, pages: 3 });
 		await drain(stub);
 		expect((await stub.fillStatus())?.state).toBe("done");
 	});
@@ -79,10 +80,10 @@ describe("fill job (goal 1e)", () => {
 
 	it("skips pages already cached", async () => {
 		const stub = await wired();
-		await stub.page(1, 24);
+		await stub.page(1, PAGE_SIZE);
 		await stub.startFill();
 		await drain(stub);
-		expect(await stub.fillStatus()).toMatchObject({ state: "done", fetched: 1 });
+		expect(await stub.fillStatus()).toMatchObject({ state: "done", fetched: 2 });
 	});
 
 	it("stops when asked; cached pages stay", async () => {
@@ -91,7 +92,7 @@ describe("fill job (goal 1e)", () => {
 		await runDurableObjectAlarm(stub);
 		expect((await stub.stopFill()).state).toBe("stopped");
 		expect(await runDurableObjectAlarm(stub)).toBe(false);
-		expect((await stub.page(1, 24)).cached).toBe(true);
+		expect((await stub.page(1, PAGE_SIZE)).cached).toBe(true);
 	});
 
 	it("a webapp failure fails the job loudly, once: status says why, the log too", async () => {
@@ -121,6 +122,15 @@ describe("fill job (goal 1e)", () => {
 		expect(await stub.fillStatus()).toMatchObject({ state: "done", updatedAt: at });
 	});
 
+	it("a capped page size fails the fill with the reason, instead of skipping items", async () => {
+		const stub = await wired("fake-capped-webapp");
+		await stub.startFill();
+		await drain(stub);
+		const status = await stub.fillStatus();
+		expect(status?.state).toBe("failed");
+		expect(status?.error).toMatch(/page size not honored/);
+	});
+
 	it("a public cache's fill stores no private rows", async () => {
 		const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
 		await stub.setup({ webappId: "fake-webapp-id" });