From 4034fba76b0179c2e9b01e0f34eed0392753b529 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 7 Oct 2026 05:08:12 -0500 Subject: [PATCH] diigo-cache: a fill ends at the first page past the data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seen live: "filled: 168 of 1 pages". diigo's total overstated by 100 (16744 vs 16644), so the walk went on to page 168, which diigo answered empty with a total ≤ 100; the fill recomputed its page count from that, ceil(total/100) = 1. Now a page whose own total says the data ended before it ends the walk, with the count set to the last page with items ("167 of 167"). Judged by total, not by items, since the public cache may keep none of a page that isn't the end. The known-pages estimate ignores zero totals. The fake answers past-the-end pages as diigo does (no items, total 0). Generated-by: Claude Opus 5.5 --- diigo-cache/src/cache.ts | 18 ++++++++++++++---- diigo-cache/tests/fake-webapp.js | 3 +++ diigo-cache/tests/fill.test.ts | 6 +++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/diigo-cache/src/cache.ts b/diigo-cache/src/cache.ts index a4a9996..2c1c4d2 100644 --- a/diigo-cache/src/cache.ts +++ b/diigo-cache/src/cache.ts @@ -241,11 +241,20 @@ abstract class PagedCache extends DurableObject { } try { const page = await this.page(status.nextPage, PAGE_SIZE); + const fetched = status.fetched + (page.cached ? 0 : 1); + // a page whose own total says the data ended before it (diigo + // answers total 0 past the end) ends the walk; the count is the + // last page with items. Judged by total, not by items: the + // public cache may keep none of a page that isn't the end. + if (page.total <= (status.nextPage - 1) * PAGE_SIZE) { + status = { ...status, state: "done", pages: status.nextPage - 1, fetched, updatedAt: this.#iso() }; + break; + } status = { ...status, nextPage: status.nextPage + 1, - pages: Math.max(1, Math.ceil(page.total / PAGE_SIZE)), - fetched: status.fetched + (page.cached ? 0 : 1), + pages: Math.ceil(page.total / PAGE_SIZE), + fetched, updatedAt: this.#iso(), }; } catch (err) { @@ -265,11 +274,12 @@ abstract class PagedCache extends DurableObject { if (status.state === "running") await this.ctx.storage.setAlarm(this.#clock.now() + FILL_STEP_DELAY_MS); } - /** diigo's page count, if any cached page recorded `total` */ + /** diigo's page count, if any cached page recorded `total` (pages past + * the end record 0, so they don't count) */ #knownPages(): number | null { this.ctx.storage.sql.exec(SCHEMA_V1); const row = this.ctx.storage.sql - .exec<{ total: number | null }>("select total from pages where total is not null order by etched_at desc limit 1") + .exec<{ total: number | null }>("select total from pages where total > 0 order by etched_at desc limit 1") .toArray(); return row.length > 0 && row[0].total !== null ? Math.max(1, Math.ceil(row[0].total / PAGE_SIZE)) : null; } diff --git a/diigo-cache/tests/fake-webapp.js b/diigo-cache/tests/fake-webapp.js index ef4eb74..31adc8c 100644 --- a/diigo-cache/tests/fake-webapp.js +++ b/diigo-cache/tests/fake-webapp.js @@ -96,6 +96,9 @@ class FakeGet extends RpcTarget { for (let n = (page - 1) * count + 1; n <= Math.min(page * count, total); n += 1) { items.push(dated ? datedItem(n, now, this.#dated === "deep" ? "deep_dee" : "dated_dan") : item(n)); } + // past the end, diigo answers no items and a total of 0 (observed + // 2026-10-07: the page after the last reported total ≤ 100) + if (items.length === 0) return JSON.stringify({ total: 0, items }); return JSON.stringify({ total: this.#dated === "overstated" ? total + OVERSTATEMENT : total, items }); } } diff --git a/diigo-cache/tests/fill.test.ts b/diigo-cache/tests/fill.test.ts index ed738e7..e1cde42 100644 --- a/diigo-cache/tests/fill.test.ts +++ b/diigo-cache/tests/fill.test.ts @@ -135,7 +135,11 @@ describe("fill job (goal 1e)", () => { const stub = await wired("fake-overstated-webapp"); await stub.startFill(); await drain(stub); - expect(await stub.fillStatus()).toMatchObject({ state: "done" }); + // 250 items claimed as 310: the walk reaches page 4 (empty, total 0) + // and ends there; the count is the last page with items, not 4, and + // not the empty page's ceil(0/100) = 1 + const status = await stub.fillStatus(); + expect(status).toMatchObject({ state: "done", nextPage: 4, pages: 3 }); expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(50); });