From 563cb83995259ebf9795ea621ea89219ab8b4a0e Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 7 Oct 2026 04:59:39 -0500 Subject: [PATCH] diigo-cache: page size 100, diigo's cap; fill jobs record their page size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Asked for 128, diigo returned 100 (the short-page guard caught it on the first real fill), so PAGE_SIZE = 100: ~168 pages for 16.7k bookmarks. The same run showed a fill job outliving a page-size change: progress counted in 24s ("page 25 of 698") resumed as if counted in 128s. Stored state reinterpreted, so: FillStatus records pageSize (additive; jobs saved without it counted in 24), and a job whose size differs from PAGE_SIZE fails with "page size changed …; start the fill again" instead of resuming; starting again replaces it. Tested with the old shape in storage. Generated-by: Claude Opus 5.5 --- diigo-cache/src/cache.ts | 29 ++++++++++++++++++++++++++--- diigo-cache/tests/caches.test.ts | 16 ++++++++-------- diigo-cache/tests/fake-webapp.js | 16 +++++++++------- diigo-cache/tests/fill.test.ts | 19 +++++++++++++++++-- 4 files changed, 60 insertions(+), 20 deletions(-) diff --git a/diigo-cache/src/cache.ts b/diigo-cache/src/cache.ts index fbb3b52..87693e3 100644 --- a/diigo-cache/src/cache.ts +++ b/diigo-cache/src/cache.ts @@ -60,9 +60,14 @@ const MS_PER_CALL = 500; * items per diigo page — the one size the worker asks for everywhere * (fill, public pages, dumps), so a cache never mixes sizes. Pages are * cached by (page number, size): changing this turns every cached page - * into a miss (items survive; pages refetch). + * into a miss (items survive; pages refetch), and fails any fill job + * counting in the old size. 100 is diigo's cap: asked for 128, it + * returned 100 (observed 2026-10-07). */ -export const PAGE_SIZE = 128; +export const PAGE_SIZE = 100; + +/** the size fill jobs counted in before FillStatus recorded it */ +const LEGACY_FILL_PAGE_SIZE = 24; /** the pause before each fill step's alarm: a step already paces its * pages at MS_PER_CALL; this keeps steps from running back to back */ @@ -78,6 +83,9 @@ export type FillStatus = { nextPage: number; /** diigo's page count, known after page 1 */ pages: number | null; + /** the page size `nextPage` and `pages` count in; absent on jobs + * saved before this field existed (they counted in 24s) */ + pageSize?: number; /** pages fetched from diigo by this job (cache hits not counted) */ fetched: number; startedAt: string; @@ -86,6 +94,11 @@ export type FillStatus = { error?: string; }; +/** the page size a fill job counts in (old jobs didn't record it) */ +function fillPageSize(status: FillStatus): number { + return status.pageSize ?? LEGACY_FILL_PAGE_SIZE; +} + /** release an RPC stub, if it is one (plain objects are left alone) */ function disposeStub(stub: unknown): void { const dispose = (stub as { [Symbol.dispose]?: () => void } | null)?.[Symbol.dispose]; @@ -169,7 +182,7 @@ abstract class PagedCache extends DurableObject { */ async startFill(opts: { pagesPerStep?: number } = {}): Promise { const current = await this.fillStatus(); - if (current?.state === "running") return current; + if (current?.state === "running" && fillPageSize(current) === PAGE_SIZE) return current; const step = Math.trunc(opts.pagesPerStep ?? 20); if (!(step >= 1 && step <= 50)) throw new Error("pagesPerStep: expected 1–50"); const now = this.#iso(); @@ -177,6 +190,7 @@ abstract class PagedCache extends DurableObject { state: "running", nextPage: 1, pages: this.#knownPages(), + pageSize: PAGE_SIZE, fetched: 0, startedAt: now, updatedAt: now, @@ -211,6 +225,15 @@ abstract class PagedCache extends DurableObject { async alarm(): Promise { const step = (await this.ctx.storage.get("fillStep")) ?? 20; let status = await this.fillStatus(); + // a job saved under another page size counts its pages differently: + // resuming it would skip or repeat items, so it fails instead + if (status?.state === "running" && fillPageSize(status) !== PAGE_SIZE) { + const error = `page size changed (from ${fillPageSize(status)} to ${PAGE_SIZE}); start the fill again`; + status = { ...status, state: "failed", error, updatedAt: this.#iso() }; + console.error({ event: "fill", kind: this.kind, state: "failed", error }); + await this.ctx.storage.put("fill", status); + return; + } for (let i = 0; i < step && status?.state === "running"; i += 1) { if (status.pages !== null && status.nextPage > status.pages) { status = { ...status, state: "done" }; diff --git a/diigo-cache/tests/caches.test.ts b/diigo-cache/tests/caches.test.ts index 6978681..a2dd670 100644 --- a/diigo-cache/tests/caches.test.ts +++ b/diigo-cache/tests/caches.test.ts @@ -26,7 +26,7 @@ describe("PublicCache", () => { const before = await testEnv.WEBAPP.calls(); const miss = await stub.page(1, 24); expect(miss.cached).toBe(false); - expect(miss.total).toBe(300); + expect(miss.total).toBe(250); // diigo page 1 = items 1..24, of which 8 are private expect(parse(miss.items).map((i) => i.private)).toEqual(Array(16).fill(false)); const afterMiss = await testEnv.WEBAPP.calls(); @@ -78,7 +78,7 @@ describe("page size", () => { // settle the RPC promise ourselves: expect().rejects on a DO stub's // promise leaves the rejection unhandled const failure = () => stub.page(1, PAGE_SIZE).then(() => null, (e: unknown) => String(e)); - expect(await failure()).toMatch(/returned 100 items for page 1 of 3 \(asked for 128\)/); + expect(await failure()).toMatch(/returned 64 items for page 1 of 3 \(asked for 100\)/); expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.outcome === "short-page")).toHaveLength(1); // nothing was stored: asking again refuses again expect(await failure()).toMatch(/page size not honored/); @@ -90,7 +90,7 @@ describe("page size", () => { it("the last page may be short", async () => { const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId()); await stub.setup({ webappId: WEBAPP_ID }); - expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(300 - 2 * PAGE_SIZE); + expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(250 - 2 * PAGE_SIZE); }); }); @@ -153,7 +153,7 @@ describe("worker routes", () => { expect(html).toContain("cache: MISS"); const again = await (await call(`/public/${username}?page=2`)).text(); expect(again).toContain("cache: HIT"); - // PAGE_SIZE 128: page 2 is bookmarks 129..256 + // PAGE_SIZE 100: page 2 is bookmarks 101..200 expect(html).toContain(">bookmark 130<"); // the date comes first, in toISOString() form expect(html).toMatch(/