diff --git a/diigo-cache/src/cache.ts b/diigo-cache/src/cache.ts index a4a9996..2c1c4d2 100644 --- a/diigo-cache/src/cache.ts +++ b/diigo-cache/src/cache.ts @@ -241,11 +241,20 @@ abstract class PagedCache extends DurableObject { } try { const page = await this.page(status.nextPage, PAGE_SIZE); + const fetched = status.fetched + (page.cached ? 0 : 1); + // a page whose own total says the data ended before it (diigo + // answers total 0 past the end) ends the walk; the count is the + // last page with items. Judged by total, not by items: the + // public cache may keep none of a page that isn't the end. + if (page.total <= (status.nextPage - 1) * PAGE_SIZE) { + status = { ...status, state: "done", pages: status.nextPage - 1, fetched, updatedAt: this.#iso() }; + break; + } status = { ...status, nextPage: status.nextPage + 1, - pages: Math.max(1, Math.ceil(page.total / PAGE_SIZE)), - fetched: status.fetched + (page.cached ? 0 : 1), + pages: Math.ceil(page.total / PAGE_SIZE), + fetched, updatedAt: this.#iso(), }; } catch (err) { @@ -265,11 +274,12 @@ abstract class PagedCache extends DurableObject { if (status.state === "running") await this.ctx.storage.setAlarm(this.#clock.now() + FILL_STEP_DELAY_MS); } - /** diigo's page count, if any cached page recorded `total` */ + /** diigo's page count, if any cached page recorded `total` (pages past + * the end record 0, so they don't count) */ #knownPages(): number | null { this.ctx.storage.sql.exec(SCHEMA_V1); const row = this.ctx.storage.sql - .exec<{ total: number | null }>("select total from pages where total is not null order by etched_at desc limit 1") + .exec<{ total: number | null }>("select total from pages where total > 0 order by etched_at desc limit 1") .toArray(); return row.length > 0 && row[0].total !== null ? Math.max(1, Math.ceil(row[0].total / PAGE_SIZE)) : null; } diff --git a/diigo-cache/tests/fake-webapp.js b/diigo-cache/tests/fake-webapp.js index ef4eb74..31adc8c 100644 --- a/diigo-cache/tests/fake-webapp.js +++ b/diigo-cache/tests/fake-webapp.js @@ -96,6 +96,9 @@ class FakeGet extends RpcTarget { for (let n = (page - 1) * count + 1; n <= Math.min(page * count, total); n += 1) { items.push(dated ? datedItem(n, now, this.#dated === "deep" ? "deep_dee" : "dated_dan") : item(n)); } + // past the end, diigo answers no items and a total of 0 (observed + // 2026-10-07: the page after the last reported total ≤ 100) + if (items.length === 0) return JSON.stringify({ total: 0, items }); return JSON.stringify({ total: this.#dated === "overstated" ? total + OVERSTATEMENT : total, items }); } } diff --git a/diigo-cache/tests/fill.test.ts b/diigo-cache/tests/fill.test.ts index ed738e7..e1cde42 100644 --- a/diigo-cache/tests/fill.test.ts +++ b/diigo-cache/tests/fill.test.ts @@ -135,7 +135,11 @@ describe("fill job (goal 1e)", () => { const stub = await wired("fake-overstated-webapp"); await stub.startFill(); await drain(stub); - expect(await stub.fillStatus()).toMatchObject({ state: "done" }); + // 250 items claimed as 310: the walk reaches page 4 (empty, total 0) + // and ends there; the count is the last page with items, not 4, and + // not the empty page's ceil(0/100) = 1 + const status = await stub.fillStatus(); + expect(status).toMatchObject({ state: "done", nextPage: 4, pages: 3 }); expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(50); });