diigo-cache: page size 100, diigo's cap; fill jobs record their page size

Asked for 128, diigo returned 100 (the short-page guard caught it on the
first real fill), so PAGE_SIZE = 100: ~168 pages for 16.7k bookmarks.

The same run showed a fill job outliving a page-size change: progress
counted in 24s ("page 25 of 698") resumed as if counted in 128s. Stored
state reinterpreted, so: FillStatus records pageSize (additive; jobs
saved without it counted in 24), and a job whose size differs from
PAGE_SIZE fails with "page size changed …; start the fill again"
instead of resuming; starting again replaces it. Tested with the old
shape in storage.

Generated-by: Claude Opus 5.5
This commit is contained in:
dckc committed 2026-10-07 04:59:39 -05:00
1 parent 3baffcab4c
commit 563cb83995
4 files changed
+60 -20

No files matched your search

+8 -8
View File
@@ -26,7 +26,7 @@ describe("PublicCache", () => {
const before = await testEnv.WEBAPP.calls();
const miss = await stub.page(1, 24);
expect(miss.cached).toBe(false);
expect(miss.total).toBe(300);
expect(miss.total).toBe(250);
// diigo page 1 = items 1..24, of which 8 are private
expect(parse(miss.items).map((i) => i.private)).toEqual(Array(16).fill(false));
const afterMiss = await testEnv.WEBAPP.calls();
@@ -78,7 +78,7 @@ describe("page size", () => {
// settle the RPC promise ourselves: expect().rejects on a DO stub's
// promise leaves the rejection unhandled
const failure = () => stub.page(1, PAGE_SIZE).then(() => null, (e: unknown) => String(e));
expect(await failure()).toMatch(/returned 100 items for page 1 of 3 \(asked for 128\)/);
expect(await failure()).toMatch(/returned 64 items for page 1 of 3 \(asked for 100\)/);
expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.outcome === "short-page")).toHaveLength(1);
// nothing was stored: asking again refuses again
expect(await failure()).toMatch(/page size not honored/);
@@ -90,7 +90,7 @@ describe("page size", () => {
it("the last page may be short", async () => {
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
await stub.setup({ webappId: WEBAPP_ID });
expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(300 - 2 * PAGE_SIZE);
expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(250 - 2 * PAGE_SIZE);
});
});
@@ -153,7 +153,7 @@ describe("worker routes", () => {
expect(html).toContain("<strong>cache: MISS");
const again = await (await call(`/public/${username}?page=2`)).text();
expect(again).toContain("<strong>cache: HIT");
// PAGE_SIZE 128: page 2 is bookmarks 129..256
// PAGE_SIZE 100: page 2 is bookmarks 101..200
expect(html).toContain(">bookmark 130<");
// the date comes first, in toISOString() form
expect(html).toMatch(/<li><time datetime="\d{4}-\d\d-\d\dT[\d:.]+Z">[^<]+<\/time> <a /);
@@ -163,7 +163,7 @@ describe("worker routes", () => {
await call(`/public/${username}?page=2`, { headers: { accept: "application/json" } })
).json()) as { pages: number; items: unknown[] };
expect(json.pages).toBe(3);
expect(json.items).toHaveLength(85); // 129..256 minus its 43 multiples of 3
expect(json.items).toHaveLength(67); // 101..200 minus its 33 multiples of 3
});
it("publishes a private cache later, without the cache id in the URL", async () => {
@@ -278,7 +278,7 @@ describe.skip("public page filters (goal 1d)", () => {
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
});
expect(res.status).toBe(200);
// deep_dee: 47 diigo pages of 128; a search matching nothing must walk them all
// deep_dee: 60 diigo pages of 100; a search matching nothing must walk them all
const before = await testEnv.WEBAPP.calls();
const first = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
expect(await first.json()).toMatchObject({ complete: false, items: [] });
@@ -305,8 +305,8 @@ describe.skip("public page filters (goal 1d)", () => {
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
});
expect(res.status).toBe(200);
// deep_dee: 6000 bookmarks, one a day, 47 diigo pages of 128; a week
// ~14 years back sits near page 40 — a walk from page 1 would cost ~40
// deep_dee: 6000 bookmarks, one a day, 60 diigo pages of 100; a week
// ~14 years back sits near page 51 — a walk from page 1 would cost ~51
const longAgo = new Date(Date.now() - 5100 * 24 * 60 * 60 * 1000);
const week = isoWeek(longAgo);
const before = await testEnv.WEBAPP.calls();