dckc
530942f5d9
Two bugs in page()'s miss path, found while building the fill job: - the webapp stubs (app, pathjoin, query) were never disposed, which kept the DO from being evicted (evictDurableObject hung after a single miss); they are now disposed once the page is in hand - a miss on page n > 1 also fetched page 1 and threw it away, doubling diigo calls; every page carries `total`, so only page n is fetched Regression test: a miss, then eviction, then a cache hit. tsconfig adds the ESNext.Disposable lib for Symbol.dispose. Generated-by: Claude Opus 5.5
305 lines
13 KiB
TypeScript
305 lines
13 KiB
TypeScript
import { env } from "cloudflare:workers";
|
|
import { evictDurableObject, runInDurableObject } from "cloudflare:test";
|
|
import { describe, expect, it, vi } from "vitest";
|
|
import worker from "../src/index.js";
|
|
import { toWebkey } from "../src/webkeys.js";
|
|
import { FETCH_BUDGET } from "../src/query.js";
|
|
|
|
// matches tests/fake-webapp.js: 30 items, every third private
|
|
const WEBAPP_ID = "fake-webapp-id";
|
|
|
|
type TestEnv = {
|
|
CACHE_DO: DurableObjectNamespace<import("../src/cache.js").DiigoCache>;
|
|
PUBLIC_DO: DurableObjectNamespace<import("../src/cache.js").PublicCache>;
|
|
WEBAPP: { calls(): Promise<number> };
|
|
};
|
|
const testEnv = env as unknown as TestEnv;
|
|
|
|
const parse = (items: string[]) => items.map((j) => JSON.parse(j) as Record<string, unknown>);
|
|
|
|
describe("PublicCache", () => {
|
|
it("on a miss asks the webapp and keeps only shared items; a hit makes no call", async () => {
|
|
const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
|
|
await stub.setup({ webappId: WEBAPP_ID });
|
|
|
|
const before = await testEnv.WEBAPP.calls();
|
|
const miss = await stub.page(1, 24);
|
|
expect(miss.cached).toBe(false);
|
|
expect(miss.total).toBe(30);
|
|
// diigo page 1 = items 1..24, of which 8 are private
|
|
expect(parse(miss.items).map((i) => i.private)).toEqual(Array(16).fill(false));
|
|
const afterMiss = await testEnv.WEBAPP.calls();
|
|
expect(afterMiss).toBeGreaterThan(before);
|
|
|
|
const hit = await stub.page(1, 24);
|
|
expect(hit.cached).toBe(true);
|
|
expect(hit.items).toEqual(miss.items);
|
|
expect(await testEnv.WEBAPP.calls()).toBe(afterMiss);
|
|
});
|
|
|
|
it("logs each page read as a hit or a miss (fetched vs kept), without the cache id", async () => {
|
|
const spy = vi.spyOn(console, "log");
|
|
try {
|
|
const id = testEnv.PUBLIC_DO.newUniqueId();
|
|
const stub = testEnv.PUBLIC_DO.get(id);
|
|
await stub.setup({ webappId: WEBAPP_ID });
|
|
await stub.page(1, 24);
|
|
await stub.page(1, 24);
|
|
const cacheLogs = spy.mock.calls.map((c) => c[0]).filter((e) => e?.event === "cache");
|
|
expect(cacheLogs).toEqual([
|
|
{ event: "cache", kind: "public", page: 1, outcome: "miss", fetched: 24, kept: 16 },
|
|
{ event: "cache", kind: "public", page: 1, outcome: "hit", items: 16 },
|
|
]);
|
|
expect(JSON.stringify(spy.mock.calls)).not.toContain(id.toString());
|
|
} finally {
|
|
spy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("stores no private rows at all", async () => {
|
|
const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
|
|
await stub.setup({ webappId: WEBAPP_ID });
|
|
await stub.page(1, 24);
|
|
await stub.page(2, 24);
|
|
await runInDurableObject(stub, (_obj, state) => {
|
|
const rows = state.storage.sql.exec<{ n: number }>("select count(*) as n from items where private = 1").toArray();
|
|
expect(rows[0].n).toBe(0);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("eviction", () => {
|
|
it("a cache miss leaves no undisposed RPC stubs to block eviction", async () => {
|
|
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
|
|
await stub.setup({ webappId: WEBAPP_ID });
|
|
await stub.page(1, 24);
|
|
await evictDurableObject(stub);
|
|
expect((await stub.page(1, 24)).cached).toBe(true);
|
|
}, 3000);
|
|
});
|
|
|
|
describe("DiigoCache", () => {
|
|
it("keeps every item of a page, each under its own link_id", async () => {
|
|
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
|
|
await stub.setup({ webappId: WEBAPP_ID });
|
|
const page = await stub.page(1, 24);
|
|
const ids = parse(page.items).map((i) => i.link_id);
|
|
expect(ids).toHaveLength(24);
|
|
expect(new Set(ids).size).toBe(24);
|
|
expect(parse(page.items).filter((i) => i.private)).toHaveLength(8);
|
|
});
|
|
});
|
|
|
|
describe("page replay order", () => {
|
|
it("a hit replays diigo's fetch-time order (page_members), not a column sort", async () => {
|
|
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
|
|
await stub.setup({ webappId: WEBAPP_ID });
|
|
const titles = (items: string[]) => parse(items).map((i) => i.title);
|
|
const expected = Array.from({ length: 24 }, (_, k) => `bookmark ${k + 1}`);
|
|
|
|
const miss = await stub.page(1, 24);
|
|
expect(titles(miss.items)).toEqual(expected);
|
|
// page 2's etch upserts other rows; page 1's replay must not shift
|
|
await stub.page(2, 24);
|
|
const hit = await stub.page(1, 24);
|
|
expect(hit.cached).toBe(true);
|
|
expect(titles(hit.items)).toEqual(expected);
|
|
});
|
|
});
|
|
|
|
describe("worker routes", () => {
|
|
const call = (path: string, init?: RequestInit) =>
|
|
worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
|
|
|
|
it("wires a public cache whose page fills itself on a miss", async () => {
|
|
const wired = await call("/_wire", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ webappId: WEBAPP_ID, mode: "public" }),
|
|
});
|
|
expect(wired.status).toBe(200);
|
|
const { username } = (await wired.json()) as { username: string };
|
|
expect(username).toBe("alice_99");
|
|
|
|
// page 2 was never fetched by anyone: the public cache asks the webapp
|
|
const html = await (await call(`/public/${username}?page=2`)).text();
|
|
expect(html).toContain('<form method="get"');
|
|
expect(html).toContain("<strong>cache: MISS");
|
|
const again = await (await call(`/public/${username}?page=2`)).text();
|
|
expect(again).toContain("<strong>cache: HIT");
|
|
expect(html).toContain("bookmark 25");
|
|
// the date comes first, in toISOString() form
|
|
expect(html).toMatch(/<li><time datetime="\d{4}-\d\d-\d\dT[\d:.]+Z">[^<]+<\/time> <a /);
|
|
expect(html).not.toContain("bookmark 27"); // private (27 % 3 === 0)
|
|
|
|
const json = (await (
|
|
await call(`/public/${username}?page=2`, { headers: { accept: "application/json" } })
|
|
).json()) as { pages: number; items: unknown[] };
|
|
expect(json.pages).toBe(2);
|
|
expect(json.items).toHaveLength(4); // 25..30 minus 27, 30
|
|
});
|
|
|
|
it("publishes a private cache later, without the cache id in the URL", async () => {
|
|
const cacheId = toWebkey(testEnv.CACHE_DO.newUniqueId());
|
|
const res = await call("/_publish", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ cacheId }),
|
|
});
|
|
// never wired: refused, not crashed
|
|
expect(res.status).toBe(404);
|
|
});
|
|
});
|
|
|
|
/** `yyyy-Www` for the ISO 8601 week holding `d` (UTC) */
|
|
function isoWeek(d: Date): string {
|
|
const t = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
|
|
const day = t.getUTCDay() || 7;
|
|
t.setUTCDate(t.getUTCDate() + 4 - day); // the week's Thursday names its year
|
|
const year = t.getUTCFullYear();
|
|
const week = Math.ceil(((t.getTime() - Date.UTC(year, 0, 1)) / 86400000 + 1) / 7);
|
|
return `${year}-W${String(week).padStart(2, "0")}`;
|
|
}
|
|
|
|
describe.skip("public page filters (goal 1d)", () => {
|
|
// the fake's dated account: "dated n" updated n-1 days + 1h ago;
|
|
// #weekly on every 7th; every 10th private; every 5th created 100
|
|
// days before it was updated
|
|
const call = (path: string, init?: RequestInit) =>
|
|
worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
|
|
const wireDated = async () => {
|
|
const res = await call("/_wire", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ webappId: "fake-dated-webapp", mode: "public" }),
|
|
});
|
|
expect(res.status).toBe(200);
|
|
return "/public/dated_dan";
|
|
};
|
|
const titlesIn = (html: string) => [...html.matchAll(/>(dated \d+)<\/a>/g)].map((m) => m[1]);
|
|
const dated = (...ns: number[]) => ns.map((n) => `dated ${n}`);
|
|
|
|
it("?days=7 lists the last week, walking only the diigo pages it needs", async () => {
|
|
const page = await wireDated();
|
|
const before = await testEnv.WEBAPP.calls();
|
|
const html = await (await call(`${page}?days=7`)).text();
|
|
expect(titlesIn(html)).toEqual(dated(1, 2, 3, 4, 5, 6, 7));
|
|
expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(1);
|
|
});
|
|
|
|
it("?by=created windows, lists, and dates by when it was bookmarked", async () => {
|
|
const page = await wireDated();
|
|
const html = await (await call(`${page}?days=7&by=created`)).text();
|
|
// 5 was updated this week but bookmarked 100 days earlier
|
|
expect(titlesIn(html)).toEqual(dated(1, 2, 3, 4, 6, 7));
|
|
const updated = await (await call(`${page}?days=7`)).text();
|
|
expect(titlesIn(updated)).toContain("dated 5");
|
|
});
|
|
|
|
it("?by=created over a longer window sorts by created, newest first", async () => {
|
|
const page = await wireDated();
|
|
const html = await (await call(`${page}?days=150&by=created&q=${encodeURIComponent("#weekly")}`)).text();
|
|
// weekly: 7,14,21,28,35,42,49,56; 35 was created ~134 days ago
|
|
// (100 days before its update), so it sorts after 56 (~55 days)
|
|
expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 42, 49, 56, 35));
|
|
});
|
|
|
|
it("?days=30 pages the matches 24 at a time; links keep the filter", async () => {
|
|
const page = await wireDated();
|
|
const first = await (await call(`${page}?days=30`)).text();
|
|
const want = Array.from({ length: 30 }, (_, k) => k + 1).filter((n) => n % 10 !== 0);
|
|
expect(titlesIn(first)).toEqual(dated(...want.slice(0, 24)));
|
|
expect(first).toContain(`href="${page}?days=30&page=2"`);
|
|
const second = await (await call(`${page}?days=30&page=2`)).text();
|
|
expect(titlesIn(second)).toEqual(dated(...want.slice(24)));
|
|
});
|
|
|
|
it("?after=&before= selects a date range", async () => {
|
|
const page = await wireDated();
|
|
const day = (n: number) => new Date(Date.now() - (n - 1) * 24 * 60 * 60 * 1000 - 60 * 60 * 1000).toISOString().slice(0, 10);
|
|
// items are 24h apart, so each has its own UTC day: from dated 12's
|
|
// day (inclusive) up to dated 9's day (exclusive); 10 is private
|
|
const html = await (await call(`${page}?after=${day(12)}&before=${day(9)}`)).text();
|
|
expect(titlesIn(html)).toEqual(dated(11, 12));
|
|
});
|
|
|
|
it("?q=#weekly&days=60 filters by tag; private items stay out", async () => {
|
|
const page = await wireDated();
|
|
const html = await (await call(`${page}?days=60&q=${encodeURIComponent("#weekly")}`)).text();
|
|
expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 35, 42, 49, 56));
|
|
});
|
|
|
|
it("?q= alone searches all time", async () => {
|
|
const page = await wireDated();
|
|
const html = await (await call(`${page}?q=fragments`)).text();
|
|
expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 35, 42, 49, 56));
|
|
});
|
|
|
|
it("a request spends at most FETCH_BUDGET diigo fetches; incomplete results say so, and a reload continues", async () => {
|
|
const res = await call("/_wire", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
|
|
});
|
|
expect(res.status).toBe(200);
|
|
// deep_dee: 50 diigo pages; a search matching nothing must walk them all
|
|
const before = await testEnv.WEBAPP.calls();
|
|
const first = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
|
|
expect(await first.json()).toMatchObject({ complete: false, items: [] });
|
|
expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(FETCH_BUDGET + 1);
|
|
const html = await (await call("/public/deep_dee?q=nomatch")).text();
|
|
expect(html).toMatch(/complete|no matches/);
|
|
const again = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
|
|
expect(await again.json()).toMatchObject({ complete: true, items: [] });
|
|
});
|
|
|
|
it("JSON answers carry the canonical filter, so devs see what was understood", async () => {
|
|
const page = await wireDated();
|
|
const res = await call(`${page}?days=7&q=${encodeURIComponent("#Weekly")}`, { headers: { accept: "application/json" } });
|
|
expect(await res.json()).toMatchObject({
|
|
filter: { by: "updated", tags: [{ text: "weekly", negated: false }] },
|
|
complete: true,
|
|
});
|
|
});
|
|
|
|
it("?in= an old week costs O(log pages) diigo fetches with by=updated, not a walk from page 1", async () => {
|
|
const res = await call("/_wire", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
|
|
});
|
|
expect(res.status).toBe(200);
|
|
// deep_dee: 1200 bookmarks, one a day, 50 diigo pages; a week ~2.5
|
|
// years back sits near page 38 — a walk from page 1 would cost ~38
|
|
const twoAndAHalfYearsAgo = new Date(Date.now() - 900 * 24 * 60 * 60 * 1000);
|
|
const week = isoWeek(twoAndAHalfYearsAgo);
|
|
const before = await testEnv.WEBAPP.calls();
|
|
const html = await (await call(`/public/deep_dee?in=${week}`)).text();
|
|
// one a day; every 10th is private
|
|
expect(titlesIn(html).length).toBeGreaterThanOrEqual(6);
|
|
expect(titlesIn(html).length).toBeLessThanOrEqual(7);
|
|
expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(12);
|
|
});
|
|
|
|
it("a malformed filter is a 400 with a message, not an empty list", async () => {
|
|
const page = await wireDated();
|
|
const res = await call(`${page}?days=7x`);
|
|
expect(res.status).toBe(400);
|
|
expect(await res.text()).toMatch(/days/);
|
|
});
|
|
|
|
it("the form offers the day presets, date inputs, and search, echoing the current filter", async () => {
|
|
const page = await wireDated();
|
|
const html = await (await call(`${page}?days=30&q=ocap`)).text();
|
|
expect(html).toMatch(/<select name="days">/);
|
|
expect(html).toMatch(/<select name="by">/);
|
|
for (const n of [7, 30, 90, 365]) expect(html).toContain(`<option value="${n}"`);
|
|
expect(html).toContain('<option value="30" selected>');
|
|
expect(html).toMatch(/<input type="date" name="after"/);
|
|
expect(html).toMatch(/<input type="date" name="before"/);
|
|
expect(html).toMatch(/<input[^>]* name="q"[^>]* value="ocap"/);
|
|
// times are UTC, and the page says so
|
|
expect(html).toMatch(/UTC/);
|
|
});
|
|
});
|