diigo-cache: fetch 128 items per diigo page; refuse short pages

PAGE_SIZE = 128 (was 24), one constant for the fill job, public pages,
dumps, and DiigoBridge, so a cache never mixes sizes: ~131 pages instead
of ~698 for 16.7k bookmarks.

Cache invalidation, agreed beforehand: pages are cached by (page number,
size), so pages cached at 24 become misses and refetch at 128. Items are
keyed by link_id and just re-upsert; nothing is lost.

Unknown whether diigo honors count=128, so a short page that isn't the
last is refused loudly (logged, thrown; a fill fails with the reason)
rather than silently skipping items. The fake gains a capped account to
test it; the plain account grows to 300 (3 pages), the deep one to 6000.

Generated-by: Claude Opus 5.5
This commit is contained in:
dckc committed 2026-10-07 04:56:56 -05:00
1 parent efaf74023d
commit 826153e35f
5 files changed
+111 -47

No files matched your search

+19 -9
View File
@@ -6,10 +6,11 @@ import { newHttpBatchRpcSession } from "capnweb";
import type { CacheApi } from "../src/api.js";
import { toWebkey } from "../src/webkeys.js";
import { FIXED_NOW } from "./test-worker.js";
import { PAGE_SIZE } from "../src/cache.js";
// goal 1e (capsite checklist): a job that grabs every page. Fake
// accounts: tests/fake-webapp.js ("fake-webapp-id": 30 items = 2 pages
// of 24, every 3rd private).
// accounts: tests/fake-webapp.js ("fake-webapp-id": 300 items = 3 pages
// of PAGE_SIZE 128, every 3rd private).
type TestEnv = {
CACHE_DO: DurableObjectNamespace<import("../src/index.js").DiigoCache>;
@@ -40,12 +41,12 @@ describe("fill job (goal 1e)", () => {
expect((await stub.startFill()).state).toBe("running");
await drain(stub);
const status = await stub.fillStatus();
expect(status).toMatchObject({ state: "done", pages: 2, fetched: 2 });
expect(status).toMatchObject({ state: "done", pages: 3, fetched: 3 });
// one diigo call per page fetched (no extra page-1 fetch per miss)
expect((await testEnv.WEBAPP.calls()) - atStart).toBe(2);
expect((await testEnv.WEBAPP.calls()) - atStart).toBe(3);
const before = await testEnv.WEBAPP.calls();
for (const n of [1, 2]) expect((await stub.page(n, 24)).cached).toBe(true);
for (const n of [1, 2, 3]) expect((await stub.page(n, PAGE_SIZE)).cached).toBe(true);
expect(await testEnv.WEBAPP.calls()).toBe(before);
});
@@ -53,7 +54,7 @@ describe("fill job (goal 1e)", () => {
const stub = await wired();
await stub.startFill({ pagesPerStep: 1 });
expect(await runDurableObjectAlarm(stub)).toBe(true);
expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2, pages: 2 });
expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2, pages: 3 });
await drain(stub);
expect((await stub.fillStatus())?.state).toBe("done");
});
@@ -79,10 +80,10 @@ describe("fill job (goal 1e)", () => {
it("skips pages already cached", async () => {
const stub = await wired();
await stub.page(1, 24);
await stub.page(1, PAGE_SIZE);
await stub.startFill();
await drain(stub);
expect(await stub.fillStatus()).toMatchObject({ state: "done", fetched: 1 });
expect(await stub.fillStatus()).toMatchObject({ state: "done", fetched: 2 });
});
it("stops when asked; cached pages stay", async () => {
@@ -91,7 +92,7 @@ describe("fill job (goal 1e)", () => {
await runDurableObjectAlarm(stub);
expect((await stub.stopFill()).state).toBe("stopped");
expect(await runDurableObjectAlarm(stub)).toBe(false);
expect((await stub.page(1, 24)).cached).toBe(true);
expect((await stub.page(1, PAGE_SIZE)).cached).toBe(true);
});
it("a webapp failure fails the job loudly, once: status says why, the log too", async () => {
@@ -121,6 +122,15 @@ describe("fill job (goal 1e)", () => {
expect(await stub.fillStatus()).toMatchObject({ state: "done", updatedAt: at });
});
it("a capped page size fails the fill with the reason, instead of skipping items", async () => {
const stub = await wired("fake-capped-webapp");
await stub.startFill();
await drain(stub);
const status = await stub.fillStatus();
expect(status?.state).toBe("failed");
expect(status?.error).toMatch(/page size not honored/);
});
it("a public cache's fill stores no private rows", async () => {
const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
await stub.setup({ webappId: "fake-webapp-id" });