diigo-cache: a short page before the stated end peeks at the next page

The real fill failed at "diigo returned 44 items for page 167 of 168":
diigo's `total` overstates what load_user_items returns, so the true
last page looked short and the guard took it for a capped page size.

A short page that isn't the last by `total` now fetches the next page
(paced like any diigo call): empty means the data really ends there —
accepted, with a "total-overstated" notice logging total vs actual;
anything else means items would be skipped — refused, the message now
giving both counts. The fake gains an account whose total claims 60
more than it has; a fill on it runs to done.

Generated-by: Claude Opus 5.5

diigo-cache/src/cache.ts

@@ -303,22 +303,23 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
 
 		// RPC stubs (the app and each pipelined step) are disposed once the
 		// page is in hand: undisposed stubs keep this DO from being evicted
-		const app = await this.env.WEBAPP.at(init.webappId);
-		const joined = app.pathjoin("load_user_items");
-		const queried = joined.query({ sort: "updated", count, page_num: pageNum });
-		let data: { total: number; items: Item[] };
-		try {
-			data = JSON.parse(await queried.get()) as { total: number; items: Item[] };
-		} finally {
-			for (const stub of [queried, joined, app]) disposeStub(stub);
-		}
+		const data = await this.#fetchPage(init.webappId, pageNum, count);
 		// diigo may cap the page size below what we asked: a short page that
-		// isn't the last would silently skip items, so refuse it loudly
+		// isn't the last would silently skip items. But diigo's `total` can
+		// also overstate what it returns, making the true last page look
+		// short. The next page tells them apart: empty means the data ends
+		// here; anything else means items would be skipped — refuse loudly.
 		const lastPage = Math.max(1, Math.ceil(data.total / count));
 		if (data.items.length < count && pageNum < lastPage) {
-			const error = `diigo returned ${data.items.length} items for page ${pageNum} of ${lastPage} (asked for ${count}): page size not honored?`;
-			console.error({ event: "cache", kind: this.kind, page: pageNum, outcome: "short-page", fetched: data.items.length, asked: count });
-			throw new Error(error);
+			await this.#clock.sleep(MS_PER_CALL);
+			const next = await this.#fetchPage(init.webappId, pageNum + 1, count);
+			if (next.items.length > 0) {
+				const error = `diigo returned ${data.items.length} items for page ${pageNum} of ${lastPage} (asked for ${count}), then ${next.items.length} for page ${pageNum + 1}: page size not honored?`;
+				console.error({ event: "cache", kind: this.kind, page: pageNum, outcome: "short-page", fetched: data.items.length, asked: count, nextFetched: next.items.length });
+				throw new Error(error);
+			}
+			const actual = (pageNum - 1) * count + data.items.length;
+			console.log({ event: "cache", kind: this.kind, page: pageNum, outcome: "total-overstated", total: data.total, actual });
 		}
 		const kept = this.keep(toRows(data.items, console));
 		// fetched vs kept: the public cache's drop of private rows shows here
@@ -326,6 +327,20 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
 		return this.#etch(pageNum, count, data.total, kept);
 	}
 
+	/** one diigo page, raw, through the webapp instance; the RPC stubs (the
+	 * app and each pipelined step) are disposed once the page is in hand —
+	 * undisposed stubs keep this DO from being evicted */
+	async #fetchPage(webappId: string, pageNum: number, count: number): Promise<{ total: number; items: Item[] }> {
+		const app = await this.env.WEBAPP.at(webappId);
+		const joined = app.pathjoin("load_user_items");
+		const queried = joined.query({ sort: "updated", count, page_num: pageNum });
+		try {
+			return JSON.parse(await queried.get()) as { total: number; items: Item[] };
+		} finally {
+			for (const stub of [queried, joined, app]) disposeStub(stub);
+		}
+	}
+
 	/** etch a page (one transaction): upsert items, mark the page, keep
 	 * the fetch-time order; returns the cache-page shape */
 	#etch(pageNum: number, count: number, total: number, rows: ItemRow[]): CachePage {