diigo-cache: fetch 128 items per diigo page; refuse short pages

PAGE_SIZE = 128 (was 24), one constant for the fill job, public pages,
dumps, and DiigoBridge, so a cache never mixes sizes: ~131 pages instead
of ~698 for 16.7k bookmarks.

Cache invalidation, agreed beforehand: pages are cached by (page number,
size), so pages cached at 24 become misses and refetch at 128. Items are
keyed by link_id and just re-upsert; nothing is lost.

Unknown whether diigo honors count=128, so a short page that isn't the
last is refused loudly (logged, thrown; a fill fails with the reason)
rather than silently skipping items. The fake gains a capped account to
test it; the plain account grows to 300 (3 pages), the deep one to 6000.

Generated-by: Claude Opus 5.5

diigo-cache/src/cache.ts

@@ -56,8 +56,13 @@ export type CachePage = {
 
 const MS_PER_CALL = 500;
 
-/** items per diigo page, for the fill job and the pages it etches */
-const PAGE_SIZE = 24;
+/**
+ * items per diigo page — the one size the worker asks for everywhere
+ * (fill, public pages, dumps), so a cache never mixes sizes. Pages are
+ * cached by (page number, size): changing this turns every cached page
+ * into a miss (items survive; pages refetch).
+ */
+export const PAGE_SIZE = 128;
 
 /** the pause before each fill step's alarm: a step already paces its
  * pages at MS_PER_CALL; this keeps steps from running back to back */
@@ -284,6 +289,14 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
 		} finally {
 			for (const stub of [queried, joined, app]) disposeStub(stub);
 		}
+		// diigo may cap the page size below what we asked: a short page that
+		// isn't the last would silently skip items, so refuse it loudly
+		const lastPage = Math.max(1, Math.ceil(data.total / count));
+		if (data.items.length < count && pageNum < lastPage) {
+			const error = `diigo returned ${data.items.length} items for page ${pageNum} of ${lastPage} (asked for ${count}): page size not honored?`;
+			console.error({ event: "cache", kind: this.kind, page: pageNum, outcome: "short-page", fetched: data.items.length, asked: count });
+			throw new Error(error);
+		}
 		const kept = this.keep(toRows(data.items, console));
 		// fetched vs kept: the public cache's drop of private rows shows here
 		console.log({ event: "cache", kind: this.kind, page: pageNum, outcome: "miss", fetched: data.items.length, kept: kept.length });

diigo-cache/src/index.ts

@@ -18,6 +18,7 @@ import { CacheApi } from "./api.js";
 import { CAPNWEB_BROWSER_JS } from "./capnweb-browser.js";
 import { CACHE_PAGE_HTML } from "./cache-page.js";
 import { DiigoCacheLogic, PublicCacheLogic } from "./cache.js";
+import { PAGE_SIZE } from "./cache.js";
 import type { CacheEnv, Clock, FillStatus } from "./cache.js";
 import type { Item } from "./cache.js";
 import { updatedISO, usernameOf } from "./items.js";
@@ -69,7 +70,7 @@ const DIIGO_INTERACT = "https://www.diigo.com/interact_api/";
 export class DiigoBridge extends WorkerEntrypoint {
 	/** bulk dump from the caller's address space: `pages * count` bookmarks
 	 * through the cache (cache-hits are storage reads) */
-	async dumpNdjson(pages: number, count = 24): Promise<string> {
+	async dumpNdjson(pages: number, count = PAGE_SIZE): Promise<string> {
 		const ns = (this.env as unknown as Env).CACHE_DO;
 		const cacheId = (this.env as unknown as Env).DIIGO_CACHE_ID;
 		if (!cacheId) throw new Error("no cache id supplied (caller config error)");
@@ -285,7 +286,7 @@ async function wire(request: Request, env: Env): Promise<Response> {
 	const cacheId = toWebkey(env.CACHE_DO.newUniqueId());
 	const cacheStub = env.CACHE_DO.get(env.CACHE_DO.idFromString(idFromWebkey(cacheId)));
 	await cacheStub.setup({ webappId, publicId });
-	const first = await cacheStub.page(1, 24);
+	const first = await cacheStub.page(1, PAGE_SIZE);
 
 	if (publicId) {
 		const firstItem = first.items[0] ? (JSON.parse(first.items[0]) as Item) : undefined;
@@ -401,7 +402,7 @@ async function publishCache(request: Request, env: Env): Promise<Response> {
 		// seed the projection *now*, while the cache is still private-only:
 		// only rows already cached and marked shared can enter the public
 		// side; the username rides the same rows
-		const firstPage = await cache.page(1, 24);
+		const firstPage = await cache.page(1, PAGE_SIZE);
 		const firstItem = firstPage.items[0] ? (JSON.parse(firstPage.items[0]) as Item) : undefined;
 		const username = usernameOf(firstItem);
 		if (!username) {
@@ -452,7 +453,7 @@ async function dump(request: Request, env: Env): Promise<Response> {
 	try {
 		const ns = env.CACHE_DO;
 		const cache = ns.get(ns.idFromString(idFromWebkey(cacheId)));
-		const page = await cache.page(pageN, 24);
+		const page = await cache.page(pageN, PAGE_SIZE);
 		const data = `total: ${page.total}\nitems: ${page.items.length}\ncache: ${page.cached ? "HIT (from storage)" : "MISS (etched now)"}` +
 			`\n\n${page.items.join("\n")}`;
 		return home(env, null, data, cacheId);
@@ -462,7 +463,7 @@ async function dump(request: Request, env: Env): Promise<Response> {
 	}
 }
 
-const PUBLIC_PER_PAGE = 24;
+const PUBLIC_PER_PAGE = PAGE_SIZE;
 
 /** the public page for a username: an HTML page with a page-number form
  * (GET — nothing secret here, so `?page=` in the URL is fine, unlike the