Files
captoys/diigo-cache/tests/caches.test.ts
T
dckc a17d287e1d diigo-cache: SQL in .sql files, behind a read-only facet
No SQL text in the cache logic any more. Statements live in two files,
parsed by sqlStatements (`-- name:` sections, as in finquick's
brcal/gcbook.js) and imported as text (wrangler Text rule for *.sql):

- cache-read.sql: selects only, checked when the module loads
- cache-write.sql: schema, the v2 renumbering, etch, drops

store.ts turns them into two frozen facets with domain methods:
makeReader(sql) runs only the read statements (page, knownTotal,
deepestPageSince, pagesThrough); makeWriter(storage) only the write
ones (migrate, etch, dropPagesThrough), each change one transaction.

The cache migrates in its constructor under blockConcurrencyWhile, as
Cloudflare recommends, so readers never meet missing tables; the
migration test plants v1 storage and evicts to get a fresh activation.
A viewer() facet (PageViewer: stored pages and fill progress, from the
reader alone) is added for the public page.

Pure refactor: stored state and behavior unchanged.

Generated-by: Claude Opus 5.5
2026-10-07 09:24:33 -05:00

429 lines
19 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { env } from "cloudflare:workers";
import { evictDurableObject, runInDurableObject } from "cloudflare:test";
import { describe, expect, it, vi } from "vitest";
import worker from "../src/index.js";
import { toWebkey } from "../src/webkeys.js";
import { FETCH_BUDGET } from "../src/query.js";
import { PAGE_SIZE } from "../src/cache.js";
// matches tests/fake-webapp.js: 30 items, every third private
const WEBAPP_ID = "fake-webapp-id";
type TestEnv = {
CACHE_DO: DurableObjectNamespace<import("../src/index.js").DiigoCache>;
PUBLIC_DO: DurableObjectNamespace<import("../src/index.js").PublicCache>;
CLOCKED_DO: DurableObjectNamespace<import("./test-worker.js").FixedClockCache>;
WEBAPP: { calls(): Promise<number> };
};
const testEnv = env as unknown as TestEnv;
const parse = (items: string[]) => items.map((j) => JSON.parse(j) as Record<string, unknown>);
describe("PublicCache", () => {
it("on a miss asks the webapp and keeps only shared items; a hit makes no call", async () => {
const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
await stub.setup({ webappId: WEBAPP_ID });
const before = await testEnv.WEBAPP.calls();
const miss = await stub.page(1, 24);
expect(miss.cached).toBe(false);
expect(miss.total).toBe(250);
// diigo page 1 = items 1..24, of which 8 are private
expect(parse(miss.items).map((i) => i.private)).toEqual(Array(16).fill(false));
const afterMiss = await testEnv.WEBAPP.calls();
expect(afterMiss).toBeGreaterThan(before);
const hit = await stub.page(1, 24);
expect(hit.cached).toBe(true);
expect(hit.items).toEqual(miss.items);
expect(await testEnv.WEBAPP.calls()).toBe(afterMiss);
});
it("logs each page read as a hit or a miss (fetched vs kept), without the cache id", async () => {
const spy = vi.spyOn(console, "log");
try {
const id = testEnv.PUBLIC_DO.newUniqueId();
const stub = testEnv.PUBLIC_DO.get(id);
await stub.setup({ webappId: WEBAPP_ID });
await stub.page(1, 24);
await stub.page(1, 24);
const cacheLogs = spy.mock.calls.map((c) => c[0]).filter((e) => e?.event === "cache");
expect(cacheLogs).toEqual([
{ event: "cache", kind: "public", page: 1, outcome: "miss", fetched: 24, kept: 16 },
{ event: "cache", kind: "public", page: 1, outcome: "hit", items: 16 },
]);
expect(JSON.stringify(spy.mock.calls)).not.toContain(id.toString());
} finally {
spy.mockRestore();
}
});
it("stores no private rows at all", async () => {
const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
await stub.setup({ webappId: WEBAPP_ID });
await stub.page(1, 24);
await stub.page(2, 24);
await runInDurableObject(stub, (_obj, state) => {
const rows = state.storage.sql.exec<{ n: number }>("select count(*) as n from items where private = 1").toArray();
expect(rows[0].n).toBe(0);
});
});
});
describe("page size", () => {
it("refuses, loudly, a short page that isn't the last (diigo capping the size would skip items)", async () => {
const spy = vi.spyOn(console, "error");
try {
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
await stub.setup({ webappId: "fake-capped-webapp" });
// settle the RPC promise ourselves: expect().rejects on a DO stub's
// promise leaves the rejection unhandled
const failure = () => stub.page(1, PAGE_SIZE).then(() => null, (e: unknown) => String(e));
expect(await failure()).toMatch(/returned 64 items for page 1 of 3 \(asked for 100\), then 64 for page 2/);
expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.outcome === "short-page")).toHaveLength(1);
// nothing was stored: asking again refuses again
expect(await failure()).toMatch(/page size not honored/);
} finally {
spy.mockRestore();
}
});
it("a short page whose next page is empty is the real end: an overstated total is noted, not refused", async () => {
const spy = vi.spyOn(console, "log");
try {
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
await stub.setup({ webappId: "fake-overstated-webapp" });
// 250 items claimed as 310: page 3 holds the last 50, "of 4"
const page = await stub.page(3, PAGE_SIZE);
expect(page.items).toHaveLength(50);
const notes = spy.mock.calls.map((c) => c[0]).filter((e) => e?.outcome === "total-overstated");
expect(notes).toEqual([expect.objectContaining({ page: 3, total: 310, actual: 250 })]);
} finally {
spy.mockRestore();
}
});
it("the last page may be short", async () => {
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
await stub.setup({ webappId: WEBAPP_ID });
expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(250 - 2 * PAGE_SIZE);
});
});
describe("pacing", () => {
it("paces diigo calls only: a cache hit doesn't wait", async () => {
const stub = testEnv.CLOCKED_DO.get(testEnv.CLOCKED_DO.newUniqueId());
await stub.setup({ webappId: WEBAPP_ID });
await stub.page(1, PAGE_SIZE);
const afterMiss = (await stub.sleeps()).length;
expect(afterMiss).toBeGreaterThan(0);
for (let i = 0; i < 3; i += 1) expect((await stub.page(1, PAGE_SIZE)).cached).toBe(true);
expect(await stub.sleeps()).toHaveLength(afterMiss);
});
});
describe("page numbering (schema v2)", () => {
it("asks diigo for page_num n-1: page 1 is the newest page", async () => {
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
await stub.setup({ webappId: WEBAPP_ID });
const first = (await stub.page(1, PAGE_SIZE)).items.map((j) => (JSON.parse(j) as { title: string }).title);
expect(first[0]).toBe("bookmark 1");
});
it("migrates v1 storage: a stored 'page n' was really page n+1, so it is renumbered, not refetched", async () => {
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
await stub.setup({ webappId: WEBAPP_ID });
// the shape v1 left: a page 1 holding what diigo's page_num=1
// returned (really page 2), and no schema marker; then a fresh
// activation, whose constructor migrates
await runInDurableObject(stub, (_obj, state) => {
const sql = state.storage.sql;
sql.exec("insert into items values ('a', 'https://a.example/', 'old page one, first', 0, '{\"title\":\"A\"}', '1')");
sql.exec("insert into items values ('b', 'https://b.example/', 'old page one, second', 0, '{\"title\":\"B\"}', '1')");
sql.exec("insert into pages values (1, 100, 250, '2026-10-07T00:00:00.000Z')");
sql.exec("insert into page_members values (1, 1, 'a'), (1, 2, 'b')");
state.storage.kv.delete("schema");
});
await evictDurableObject(stub);
const before = await testEnv.WEBAPP.calls();
const two = await stub.page(2, PAGE_SIZE);
expect(two.cached).toBe(true);
expect(two.items.map((j) => (JSON.parse(j) as { title: string }).title)).toEqual(["A", "B"]);
expect(await testEnv.WEBAPP.calls()).toBe(before);
// page 1 was never fetched under v1: now it is a miss
expect((await stub.page(1, PAGE_SIZE)).cached).toBe(false);
await runInDurableObject(stub, (_obj, state) => {
expect(state.storage.kv.get("schema")).toBe(2);
});
});
});
describe("invalidateRecent", () => {
it("drops the pages holding anything updated in the last N days, and only those", async () => {
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
// deep_dee: one bookmark a day, newest first; page 1 = days 0–99,
// page 2 = days 100–199, page 3 = days 200–299
await stub.setup({ webappId: "fake-deep-webapp" });
for (const n of [1, 2, 3]) await stub.page(n, PAGE_SIZE);
expect(await stub.invalidateRecent(150)).toEqual({ pages: 2 });
expect((await stub.page(3, PAGE_SIZE)).cached).toBe(true);
expect((await stub.page(2, PAGE_SIZE)).cached).toBe(false);
expect((await stub.page(1, PAGE_SIZE)).cached).toBe(false);
});
it("drops nothing when nothing cached is that recent, and refuses a bad day count", async () => {
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
await stub.setup({ webappId: WEBAPP_ID }); // the plain account's dates are in 2023
await stub.page(1, PAGE_SIZE);
expect(await stub.invalidateRecent(7)).toEqual({ pages: 0 });
expect((await stub.page(1, PAGE_SIZE)).cached).toBe(true);
const refused = await stub.invalidateRecent(0).then(() => "", (e: unknown) => String(e));
expect(refused).toMatch(/days/);
});
});
describe("eviction", () => {
it("a cache miss leaves no undisposed RPC stubs to block eviction", async () => {
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
await stub.setup({ webappId: WEBAPP_ID });
await stub.page(1, 24);
await evictDurableObject(stub);
expect((await stub.page(1, 24)).cached).toBe(true);
}, 3000);
});
describe("DiigoCache", () => {
it("keeps every item of a page, each under its own link_id", async () => {
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
await stub.setup({ webappId: WEBAPP_ID });
const page = await stub.page(1, 24);
const ids = parse(page.items).map((i) => i.link_id);
expect(ids).toHaveLength(24);
expect(new Set(ids).size).toBe(24);
expect(parse(page.items).filter((i) => i.private)).toHaveLength(8);
});
});
describe("page replay order", () => {
it("a hit replays diigo's fetch-time order (page_members), not a column sort", async () => {
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
await stub.setup({ webappId: WEBAPP_ID });
const titles = (items: string[]) => parse(items).map((i) => i.title);
const expected = Array.from({ length: 24 }, (_, k) => `bookmark ${k + 1}`);
const miss = await stub.page(1, 24);
expect(titles(miss.items)).toEqual(expected);
// page 2's etch upserts other rows; page 1's replay must not shift
await stub.page(2, 24);
const hit = await stub.page(1, 24);
expect(hit.cached).toBe(true);
expect(titles(hit.items)).toEqual(expected);
});
});
describe("worker routes", () => {
const call = (path: string, init?: RequestInit) =>
worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
it("wires a public cache whose page fills itself on a miss", async () => {
const wired = await call("/_wire", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ webappId: WEBAPP_ID, mode: "public" }),
});
expect(wired.status).toBe(200);
const { username } = (await wired.json()) as { username: string };
expect(username).toBe("alice_99");
// page 2 was never fetched by anyone: the public cache asks the webapp
const html = await (await call(`/public/${username}?page=2`)).text();
expect(html).toContain('<form method="get"');
expect(html).toContain("<strong>cache: MISS");
const again = await (await call(`/public/${username}?page=2`)).text();
expect(again).toContain("<strong>cache: HIT");
// PAGE_SIZE 100: page 2 is bookmarks 101..200
expect(html).toContain(">bookmark 130<");
// the date comes first, in toISOString() form
expect(html).toMatch(/<li><time datetime="\d{4}-\d\d-\d\dT[\d:.]+Z">[^<]+<\/time> <a /);
expect(html).not.toContain(">bookmark 129<"); // private (129 % 3 === 0)
const json = (await (
await call(`/public/${username}?page=2`, { headers: { accept: "application/json" } })
).json()) as { pages: number; items: unknown[] };
expect(json.pages).toBe(3);
expect(json.items).toHaveLength(67); // 101..200 minus its 33 multiples of 3
});
it("publishes a private cache later, without the cache id in the URL", async () => {
const cacheId = toWebkey(testEnv.CACHE_DO.newUniqueId());
const res = await call("/_publish", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ cacheId }),
});
// never wired: refused, not crashed
expect(res.status).toBe(404);
});
});
/** `yyyy-Www` for the ISO 8601 week holding `d` (UTC) */
function isoWeek(d: Date): string {
const t = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
const day = t.getUTCDay() || 7;
t.setUTCDate(t.getUTCDate() + 4 - day); // the week's Thursday names its year
const year = t.getUTCFullYear();
const week = Math.ceil(((t.getTime() - Date.UTC(year, 0, 1)) / 86400000 + 1) / 7);
return `${year}-W${String(week).padStart(2, "0")}`;
}
describe.skip("public page filters (goal 1d)", () => {
// the fake's dated account: "dated n" updated n-1 days + 1h ago;
// #weekly on every 7th; every 10th private; every 5th created 100
// days before it was updated
const call = (path: string, init?: RequestInit) =>
worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
const wireDated = async () => {
const res = await call("/_wire", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ webappId: "fake-dated-webapp", mode: "public" }),
});
expect(res.status).toBe(200);
return "/public/dated_dan";
};
const titlesIn = (html: string) => [...html.matchAll(/>(dated \d+)<\/a>/g)].map((m) => m[1]);
const dated = (...ns: number[]) => ns.map((n) => `dated ${n}`);
it("?days=7 lists the last week, walking only the diigo pages it needs", async () => {
const page = await wireDated();
const before = await testEnv.WEBAPP.calls();
const html = await (await call(`${page}?days=7`)).text();
expect(titlesIn(html)).toEqual(dated(1, 2, 3, 4, 5, 6, 7));
expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(1);
});
it("?by=created windows, lists, and dates by when it was bookmarked", async () => {
const page = await wireDated();
const html = await (await call(`${page}?days=7&by=created`)).text();
// 5 was updated this week but bookmarked 100 days earlier
expect(titlesIn(html)).toEqual(dated(1, 2, 3, 4, 6, 7));
const updated = await (await call(`${page}?days=7`)).text();
expect(titlesIn(updated)).toContain("dated 5");
});
it("?by=created over a longer window sorts by created, newest first", async () => {
const page = await wireDated();
const html = await (await call(`${page}?days=150&by=created&q=${encodeURIComponent("#weekly")}`)).text();
// weekly: 7,14,21,28,35,42,49,56; 35 was created ~134 days ago
// (100 days before its update), so it sorts after 56 (~55 days)
expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 42, 49, 56, 35));
});
it("?days=300 pages the matches PAGE_SIZE at a time; links keep the filter", async () => {
const res = await call("/_wire", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
});
expect(res.status).toBe(200);
const page = "/public/deep_dee";
// deep_dee: one a day; days=300 holds 1..300, minus every 10th (private)
const want = Array.from({ length: 300 }, (_, k) => k + 1).filter((n) => n % 10 !== 0);
const first = await (await call(`${page}?days=300`)).text();
expect(titlesIn(first)).toEqual(dated(...want.slice(0, PAGE_SIZE)));
expect(first).toContain(`href="${page}?days=300&amp;page=2"`);
const second = await (await call(`${page}?days=300&page=2`)).text();
expect(titlesIn(second)).toEqual(dated(...want.slice(PAGE_SIZE, 2 * PAGE_SIZE)));
const third = await (await call(`${page}?days=300&page=3`)).text();
expect(titlesIn(third)).toEqual(dated(...want.slice(2 * PAGE_SIZE)));
});
it("?after=&before= selects a date range", async () => {
const page = await wireDated();
const day = (n: number) => new Date(Date.now() - (n - 1) * 24 * 60 * 60 * 1000 - 60 * 60 * 1000).toISOString().slice(0, 10);
// items are 24h apart, so each has its own UTC day: from dated 12's
// day (inclusive) up to dated 9's day (exclusive); 10 is private
const html = await (await call(`${page}?after=${day(12)}&before=${day(9)}`)).text();
expect(titlesIn(html)).toEqual(dated(11, 12));
});
it("?q=#weekly&days=60 filters by tag; private items stay out", async () => {
const page = await wireDated();
const html = await (await call(`${page}?days=60&q=${encodeURIComponent("#weekly")}`)).text();
expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 35, 42, 49, 56));
});
it("?q= alone searches all time", async () => {
const page = await wireDated();
const html = await (await call(`${page}?q=fragments`)).text();
expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 35, 42, 49, 56));
});
it("a request spends at most FETCH_BUDGET diigo fetches; incomplete results say so, and a reload continues", async () => {
const res = await call("/_wire", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
});
expect(res.status).toBe(200);
// deep_dee: 60 diigo pages of 100; a search matching nothing must walk them all
const before = await testEnv.WEBAPP.calls();
const first = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
expect(await first.json()).toMatchObject({ complete: false, items: [] });
expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(FETCH_BUDGET + 1);
const html = await (await call("/public/deep_dee?q=nomatch")).text();
expect(html).toMatch(/complete|no matches/);
const again = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
expect(await again.json()).toMatchObject({ complete: true, items: [] });
});
it("JSON answers carry the canonical filter, so devs see what was understood", async () => {
const page = await wireDated();
const res = await call(`${page}?days=7&q=${encodeURIComponent("#Weekly")}`, { headers: { accept: "application/json" } });
expect(await res.json()).toMatchObject({
filter: { by: "updated", tags: [{ text: "weekly", negated: false }] },
complete: true,
});
});
it("?in= an old week costs O(log pages) diigo fetches with by=updated, not a walk from page 1", async () => {
const res = await call("/_wire", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
});
expect(res.status).toBe(200);
// deep_dee: 6000 bookmarks, one a day, 60 diigo pages of 100; a week
// ~14 years back sits near page 51 — a walk from page 1 would cost ~51
const longAgo = new Date(Date.now() - 5100 * 24 * 60 * 60 * 1000);
const week = isoWeek(longAgo);
const before = await testEnv.WEBAPP.calls();
const html = await (await call(`/public/deep_dee?in=${week}`)).text();
// one a day; every 10th is private
expect(titlesIn(html).length).toBeGreaterThanOrEqual(6);
expect(titlesIn(html).length).toBeLessThanOrEqual(7);
expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(12);
});
it("a malformed filter is a 400 with a message, not an empty list", async () => {
const page = await wireDated();
const res = await call(`${page}?days=7x`);
expect(res.status).toBe(400);
expect(await res.text()).toMatch(/days/);
});
it("the form offers the day presets, date inputs, and search, echoing the current filter", async () => {
const page = await wireDated();
const html = await (await call(`${page}?days=30&q=ocap`)).text();
expect(html).toMatch(/<select name="days">/);
expect(html).toMatch(/<select name="by">/);
for (const n of [7, 30, 90, 365]) expect(html).toContain(`<option value="${n}"`);
expect(html).toContain('<option value="30" selected>');
expect(html).toMatch(/<input type="date" name="after"/);
expect(html).toMatch(/<input type="date" name="before"/);
expect(html).toMatch(/<input[^>]* name="q"[^>]* value="ocap"/);
// times are UTC, and the page says so
expect(html).toMatch(/UTC/);
});
});