dckc
4034fba76b
Seen live: "filled: 168 of 1 pages". diigo's total overstated by 100
(16744 vs 16644), so the walk went on to page 168, which diigo answered
empty with a total ≤ 100; the fill recomputed its page count from that,
ceil(total/100) = 1.
Now a page whose own total says the data ended before it ends the walk,
with the count set to the last page with items ("167 of 167"). Judged
by total, not by items, since the public cache may keep none of a page
that isn't the end. The known-pages estimate ignores zero totals. The
fake answers past-the-end pages as diigo does (no items, total 0).
Generated-by: Claude Opus 5.5
321 lines
12 KiB
TypeScript
321 lines
12 KiB
TypeScript
import { env } from "cloudflare:workers";
|
|
import { evictDurableObject, runDurableObjectAlarm, runInDurableObject } from "cloudflare:test";
|
|
import { describe, expect, it, vi } from "vitest";
|
|
import worker from "../src/index.js";
|
|
import { newHttpBatchRpcSession } from "capnweb";
|
|
import type { CacheApi } from "../src/api.js";
|
|
import { toWebkey } from "../src/webkeys.js";
|
|
import { FIXED_NOW } from "./test-worker.js";
|
|
import { PAGE_SIZE } from "../src/cache.js";
|
|
|
|
// goal 1e (capsite checklist): a job that grabs every page. Fake
|
|
// accounts: tests/fake-webapp.js ("fake-webapp-id": 250 items = 3 pages
|
|
// of PAGE_SIZE 100, every 3rd private).
|
|
|
|
type TestEnv = {
|
|
CACHE_DO: DurableObjectNamespace<import("../src/index.js").DiigoCache>;
|
|
CLOCKED_DO: DurableObjectNamespace<import("./test-worker.js").FixedClockCache>;
|
|
PUBLIC_DO: DurableObjectNamespace<import("../src/index.js").PublicCache>;
|
|
WEBAPP: { calls(): Promise<number> };
|
|
};
|
|
const testEnv = env as unknown as TestEnv;
|
|
|
|
const wired = async (webappId = "fake-webapp-id") => {
|
|
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
|
|
await stub.setup({ webappId });
|
|
return stub;
|
|
};
|
|
|
|
/** run alarms until none is scheduled (bounded, so a runaway job fails) */
|
|
const drain = async (stub: DurableObjectStub, max = 20) => {
|
|
let ran = 0;
|
|
while (ran < max && (await runDurableObjectAlarm(stub))) ran += 1;
|
|
expect(ran).toBeLessThan(max);
|
|
return ran;
|
|
};
|
|
|
|
describe("fill job (goal 1e)", () => {
|
|
it("fills every page; afterwards every page is a hit with no webapp call", async () => {
|
|
const stub = await wired();
|
|
const atStart = await testEnv.WEBAPP.calls();
|
|
expect((await stub.startFill()).state).toBe("running");
|
|
await drain(stub);
|
|
const status = await stub.fillStatus();
|
|
expect(status).toMatchObject({ state: "done", pages: 3, fetched: 3 });
|
|
// one diigo call per page fetched (no extra page-1 fetch per miss)
|
|
expect((await testEnv.WEBAPP.calls()) - atStart).toBe(3);
|
|
|
|
const before = await testEnv.WEBAPP.calls();
|
|
for (const n of [1, 2, 3]) expect((await stub.page(n, PAGE_SIZE)).cached).toBe(true);
|
|
expect(await testEnv.WEBAPP.calls()).toBe(before);
|
|
});
|
|
|
|
it("advances pagesPerStep pages per alarm, with progress readable between steps", async () => {
|
|
const stub = await wired();
|
|
await stub.startFill({ pagesPerStep: 1 });
|
|
expect(await runDurableObjectAlarm(stub)).toBe(true);
|
|
expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2, pages: 3 });
|
|
await drain(stub);
|
|
expect((await stub.fillStatus())?.state).toBe("done");
|
|
});
|
|
|
|
it("survives eviction: progress is in storage, not memory", async () => {
|
|
const stub = await wired();
|
|
await stub.startFill({ pagesPerStep: 1 });
|
|
await runDurableObjectAlarm(stub);
|
|
await evictDurableObject(stub);
|
|
expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2 });
|
|
await drain(stub);
|
|
expect((await stub.fillStatus())?.state).toBe("done");
|
|
});
|
|
|
|
it("starting again while running rejoins the same job", async () => {
|
|
const stub = await wired();
|
|
const first = await stub.startFill({ pagesPerStep: 1 });
|
|
await runDurableObjectAlarm(stub);
|
|
const again = await stub.startFill({ pagesPerStep: 1 });
|
|
expect(again.startedAt).toBe(first.startedAt);
|
|
expect(again.nextPage).toBe(2);
|
|
});
|
|
|
|
it("skips pages already cached", async () => {
|
|
const stub = await wired();
|
|
await stub.page(1, PAGE_SIZE);
|
|
await stub.startFill();
|
|
await drain(stub);
|
|
expect(await stub.fillStatus()).toMatchObject({ state: "done", fetched: 2 });
|
|
});
|
|
|
|
it("stops when asked; cached pages stay", async () => {
|
|
const stub = await wired();
|
|
await stub.startFill({ pagesPerStep: 1 });
|
|
await runDurableObjectAlarm(stub);
|
|
expect((await stub.stopFill()).state).toBe("stopped");
|
|
expect(await runDurableObjectAlarm(stub)).toBe(false);
|
|
expect((await stub.page(1, PAGE_SIZE)).cached).toBe(true);
|
|
});
|
|
|
|
it("a webapp failure fails the job loudly, once: status says why, the log too", async () => {
|
|
const spy = vi.spyOn(console, "error");
|
|
try {
|
|
const stub = await wired("no-such-webapp");
|
|
await stub.startFill();
|
|
await drain(stub);
|
|
const status = await stub.fillStatus();
|
|
expect(status?.state).toBe("failed");
|
|
expect(status?.error).toMatch(/unknown id/);
|
|
expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.event === "fill")).not.toHaveLength(0);
|
|
} finally {
|
|
spy.mockRestore();
|
|
}
|
|
});
|
|
|
|
it("reads time only through its clock: a fixed clock pins the timestamps and the alarm", async () => {
|
|
const stub = testEnv.CLOCKED_DO.get(testEnv.CLOCKED_DO.newUniqueId());
|
|
await stub.setup({ webappId: "fake-webapp-id" });
|
|
const at = new Date(FIXED_NOW).toISOString();
|
|
expect(await stub.startFill()).toMatchObject({ startedAt: at, updatedAt: at });
|
|
await runInDurableObject(stub, async (_obj, state) => {
|
|
expect(await state.storage.getAlarm()).toBe(FIXED_NOW + 1000);
|
|
});
|
|
await drain(stub);
|
|
expect(await stub.fillStatus()).toMatchObject({ state: "done", updatedAt: at });
|
|
});
|
|
|
|
it("a capped page size fails the fill with the reason, instead of skipping items", async () => {
|
|
const stub = await wired("fake-capped-webapp");
|
|
await stub.startFill();
|
|
await drain(stub);
|
|
const status = await stub.fillStatus();
|
|
expect(status?.state).toBe("failed");
|
|
expect(status?.error).toMatch(/page size not honored/);
|
|
});
|
|
|
|
it("fills to the real end when diigo's total overstates its items", async () => {
|
|
const stub = await wired("fake-overstated-webapp");
|
|
await stub.startFill();
|
|
await drain(stub);
|
|
// 250 items claimed as 310: the walk reaches page 4 (empty, total 0)
|
|
// and ends there; the count is the last page with items, not 4, and
|
|
// not the empty page's ceil(0/100) = 1
|
|
const status = await stub.fillStatus();
|
|
expect(status).toMatchObject({ state: "done", nextPage: 4, pages: 3 });
|
|
expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(50);
|
|
});
|
|
|
|
it("a job saved under another page size fails instead of resuming; a new start replaces it", async () => {
|
|
const stub = await wired();
|
|
// the shape a 24-per-page job left in storage (no pageSize field)
|
|
await runInDurableObject(stub, async (_obj, state) => {
|
|
const at = new Date().toISOString();
|
|
await state.storage.put("fill", { state: "running", nextPage: 25, pages: 698, fetched: 23, startedAt: at, updatedAt: at });
|
|
await state.storage.setAlarm(Date.now() + 60_000);
|
|
});
|
|
await drain(stub);
|
|
const stale = await stub.fillStatus();
|
|
expect(stale?.state).toBe("failed");
|
|
expect(stale?.error).toMatch(/page size changed \(from 24 to 100\)/);
|
|
expect(await stub.startFill()).toMatchObject({ state: "running", nextPage: 1, pageSize: PAGE_SIZE });
|
|
});
|
|
|
|
it("a public cache's fill stores no private rows", async () => {
|
|
const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
|
|
await stub.setup({ webappId: "fake-webapp-id" });
|
|
await stub.startFill();
|
|
await drain(stub);
|
|
await runInDurableObject(stub, (_obj, state) => {
|
|
const rows = state.storage.sql.exec<{ n: number }>("select count(*) as n from items where private = 1").toArray();
|
|
expect(rows[0].n).toBe(0);
|
|
});
|
|
});
|
|
});
|
|
|
|
describe("fill via webkey and Cap'n Web (goal 1e)", () => {
|
|
const ORIGIN = "https://diigo-cache.test";
|
|
const call = (path: string, init?: RequestInit) => worker.fetch(new Request(`${ORIGIN}${path}`, init), env as never);
|
|
|
|
/** a Cap'n Web batch session whose fetch goes to the worker; counts round trips */
|
|
const session = () => {
|
|
const trips = { n: 0 };
|
|
const fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
|
|
trips.n += 1;
|
|
return worker.fetch(new Request(input as RequestInfo, init), env as never);
|
|
});
|
|
const api = newHttpBatchRpcSession<CacheApi>(`${ORIGIN}/api`);
|
|
return { api, trips, done: () => fetchSpy.mockRestore() };
|
|
};
|
|
|
|
const wireJson = async (mode: "private" | "public") => {
|
|
const res = await call("/_wire", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({ webappId: "fake-webapp-id", mode }),
|
|
});
|
|
return (await res.json()) as { cacheId: string; username?: string };
|
|
};
|
|
|
|
it("trading the key and starting the fill pipeline into one round trip", async () => {
|
|
const { cacheId } = await wireJson("private");
|
|
const { api, trips, done } = session();
|
|
try {
|
|
const status = await api.cache(cacheId).startFill();
|
|
expect(status).toMatchObject({ state: "running" });
|
|
expect(trips.n).toBe(1);
|
|
} finally {
|
|
done();
|
|
}
|
|
});
|
|
|
|
it("the capability is a facet: fill and status, never the DO's wiring", async () => {
|
|
const { cacheId } = await wireJson("private");
|
|
const { api, done } = session();
|
|
try {
|
|
const cap = api.cache(cacheId) as unknown as Record<string, () => Promise<unknown>>;
|
|
await expect(cap.setup()).rejects.toThrow();
|
|
} finally {
|
|
done();
|
|
}
|
|
});
|
|
|
|
it("unknown and malformed keys get one refusal that never echoes the key", async () => {
|
|
const unknown = toWebkey(testEnv.CACHE_DO.newUniqueId());
|
|
for (const key of [unknown, "not-a-key", unknown.slice(0, 26)]) {
|
|
const { api, done } = session();
|
|
try {
|
|
const err = await api.cache(key).fillStatus().then(() => null, (e: unknown) => e);
|
|
expect(String(err)).toMatch(/no such cache/);
|
|
expect(String(err)).not.toContain(key);
|
|
} finally {
|
|
done();
|
|
}
|
|
}
|
|
});
|
|
|
|
it("publicCache() reaches the public companion's fill; refused when never published", async () => {
|
|
const pub = await wireJson("public");
|
|
const s1 = session();
|
|
try {
|
|
expect(await s1.api.cache(pub.cacheId).hasPublic()).toBe(true);
|
|
} finally {
|
|
s1.done();
|
|
}
|
|
const s2 = session();
|
|
try {
|
|
expect(await s2.api.cache(pub.cacheId).publicCache().startFill()).toMatchObject({ state: "running" });
|
|
} finally {
|
|
s2.done();
|
|
}
|
|
const s3 = session();
|
|
try {
|
|
expect(await s3.api.cache(pub.cacheId).publicCache().stopFill()).toMatchObject({ state: "stopped" });
|
|
} finally {
|
|
s3.done();
|
|
}
|
|
const priv = await wireJson("private");
|
|
const s4 = session();
|
|
try {
|
|
expect(await s4.api.cache(priv.cacheId).hasPublic()).toBe(false);
|
|
} finally {
|
|
s4.done();
|
|
}
|
|
// a batch session ends after its round trip: a fresh one per await
|
|
const s5 = session();
|
|
try {
|
|
await expect(s5.api.cache(priv.cacheId).publicCache().fillStatus()).rejects.toThrow(/no public cache/);
|
|
} finally {
|
|
s5.done();
|
|
}
|
|
});
|
|
|
|
it("the webkey page is a skeleton: the same for every key, which it reads from the fragment", async () => {
|
|
const res = await call("/cache/");
|
|
expect(res.status).toBe(200);
|
|
const html = await res.text();
|
|
expect(html).toContain("location.hash");
|
|
expect(html).toContain('from "/_capnweb.js"');
|
|
expect(html).toContain('newHttpBatchRpcSession("/api")');
|
|
expect(html).toContain('<meta name="referrer" content="no-referrer">');
|
|
});
|
|
|
|
it("without a fragment, the webkey page is a login form a password manager can fill, which posts nothing", async () => {
|
|
const html = await (await call("/cache/")).text();
|
|
const form = html.match(/<form id="login"[^]*?<\/form>/)?.[0] ?? "";
|
|
expect(form).toMatch(/<input[^>]* autocomplete="username"/);
|
|
expect(form).toMatch(/<input[^>]* type="password"[^>]* autocomplete="current-password"/);
|
|
// no name attributes: a native submit carries no key anywhere
|
|
expect(form).not.toMatch(/\bname=/);
|
|
expect(form).not.toMatch(/\baction=/);
|
|
expect(html).toContain("event.preventDefault()");
|
|
});
|
|
|
|
it("serves capnweb's browser build itself (no third-party script beside the key)", async () => {
|
|
const res = await call("/_capnweb.js");
|
|
expect(res.headers.get("content-type")).toMatch(/javascript/);
|
|
expect(await res.text()).toMatch(/export \{[^}]*newHttpBatchRpcSession/);
|
|
});
|
|
|
|
it("the created page links its webkey (key in the fragment); no cache id rides a form to /_fill", async () => {
|
|
const res = await call("/_wire", {
|
|
method: "POST",
|
|
headers: { "content-type": "application/x-www-form-urlencoded" },
|
|
body: new URLSearchParams({ webappId: "fake-webapp-id", mode: "private" }),
|
|
});
|
|
const html = await res.text();
|
|
expect(html).toMatch(/<a href="\/cache\/#[a-z2-7]{52}">/);
|
|
expect(html).not.toContain("/_fill");
|
|
expect((await call("/_fill", { method: "POST" })).status).toBe(404);
|
|
});
|
|
|
|
it("the public page shows a running fill's progress", async () => {
|
|
const { cacheId, username } = await wireJson("public");
|
|
const { api, done } = session();
|
|
try {
|
|
await api.cache(cacheId).publicCache().startFill();
|
|
} finally {
|
|
done();
|
|
}
|
|
const html = await (await call(`/public/${username}`)).text();
|
|
expect(html).toMatch(/fill(ed|ing)[^<]*\d+ of \d+ pages/);
|
|
});
|
|
});
|