diigo-cache: fill job, designed as skipped specs (goal 1e)
A button that grabs every page into a cache: owner-only POST /_fill (cache id in the body; target private|public; action start|stop), advanced by DO alarms a few pages per step, progress stored in the DO (new, additive state) so it survives eviction, cached pages skipped, a rerun rejoins the running job, failures recorded and logged once. The created page gets the button; the public page shows progress. startFill/fillStatus/stopFill on PagedCache are JSDoc'd stubs; 13 specs are describe.skip until it lands (alarms driven by runDurableObjectAlarm, eviction by evictDurableObject). Generated-by: Claude Opus 5.5
This commit is contained in:
dckc committed
1 parent
3e5e139dea
commit
cf5ccce4e2
2 files changed
+209
No files matched your search
@@ -53,6 +53,25 @@ export type CachePage = {
|
||||
|
||||
const MS_PER_CALL = 500;
|
||||
|
||||
/**
|
||||
* a fill job's progress (goal 1e, **not yet implemented**: the design is
|
||||
* pinned by the skipped suite in tests/fill.test.ts). Stored in the DO
|
||||
* (new state, additive), so it survives eviction; advanced by alarms.
|
||||
*/
|
||||
export type FillStatus = {
|
||||
state: "running" | "done" | "stopped" | "failed";
|
||||
/** the next diigo page to fetch (pages already cached are skipped) */
|
||||
nextPage: number;
|
||||
/** diigo's page count, known after page 1 */
|
||||
pages: number | null;
|
||||
/** pages fetched from diigo by this job (cache hits not counted) */
|
||||
fetched: number;
|
||||
startedAt: string;
|
||||
updatedAt: string;
|
||||
/** why it failed: safe to show and log (no ids, no header values) */
|
||||
error?: string;
|
||||
};
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((r) => setTimeout(r, ms));
|
||||
}
|
||||
@@ -104,6 +123,26 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
|
||||
/** which cache this is, for logs (never the id: that's the bearer key) */
|
||||
protected abstract readonly kind: "private" | "public";
|
||||
|
||||
/**
|
||||
* start (or rejoin) the fill job: walk every diigo page into this
|
||||
* cache, `pagesPerStep` per alarm (default 20, 1–50). Starting while a
|
||||
* job runs returns that job unchanged. Goal 1e; not yet implemented.
|
||||
*/
|
||||
async startFill(opts: { pagesPerStep?: number } = {}): Promise<FillStatus> {
|
||||
void opts;
|
||||
throw new Error("startFill: not implemented (goal 1e)");
|
||||
}
|
||||
|
||||
/** the fill job's progress, or undefined if none was ever started */
|
||||
async fillStatus(): Promise<FillStatus | undefined> {
|
||||
throw new Error("fillStatus: not implemented (goal 1e)");
|
||||
}
|
||||
|
||||
/** stop the fill job after its current step; cached pages stay */
|
||||
async stopFill(): Promise<FillStatus> {
|
||||
throw new Error("stopFill: not implemented (goal 1e)");
|
||||
}
|
||||
|
||||
/**
|
||||
* read page `pageNum` (`count` per page, diigo's updated-desc order):
|
||||
* storage first; on miss, fetch through the webapp instance (etching
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
import { env } from "cloudflare:workers";
|
||||
import { evictDurableObject, runDurableObjectAlarm, runInDurableObject } from "cloudflare:test";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import worker from "../src/index.js";
|
||||
import { toWebkey } from "../src/webkeys.js";
|
||||
|
||||
// goal 1e (capsite checklist): a job that grabs every page. The design as
|
||||
// tests; unskip as it lands. Fake accounts: tests/fake-webapp.js
|
||||
// ("fake-webapp-id": 30 items = 2 pages of 24, every 3rd private).
|
||||
|
||||
type TestEnv = {
|
||||
CACHE_DO: DurableObjectNamespace<import("../src/cache.js").DiigoCache>;
|
||||
PUBLIC_DO: DurableObjectNamespace<import("../src/cache.js").PublicCache>;
|
||||
WEBAPP: { calls(): Promise<number> };
|
||||
};
|
||||
const testEnv = env as unknown as TestEnv;
|
||||
|
||||
const wired = async (webappId = "fake-webapp-id") => {
|
||||
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
|
||||
await stub.setup({ webappId });
|
||||
return stub;
|
||||
};
|
||||
|
||||
/** run alarms until none is scheduled (bounded, so a runaway job fails) */
|
||||
const drain = async (stub: DurableObjectStub, max = 20) => {
|
||||
let ran = 0;
|
||||
while (ran < max && (await runDurableObjectAlarm(stub))) ran += 1;
|
||||
expect(ran).toBeLessThan(max);
|
||||
return ran;
|
||||
};
|
||||
|
||||
describe.skip("fill job (goal 1e)", () => {
|
||||
it("fills every page; afterwards every page is a hit with no webapp call", async () => {
|
||||
const stub = await wired();
|
||||
expect((await stub.startFill()).state).toBe("running");
|
||||
await drain(stub);
|
||||
const status = await stub.fillStatus();
|
||||
expect(status).toMatchObject({ state: "done", pages: 2, fetched: 2 });
|
||||
|
||||
const before = await testEnv.WEBAPP.calls();
|
||||
for (const n of [1, 2]) expect((await stub.page(n, 24)).cached).toBe(true);
|
||||
expect(await testEnv.WEBAPP.calls()).toBe(before);
|
||||
});
|
||||
|
||||
it("advances pagesPerStep pages per alarm, with progress readable between steps", async () => {
|
||||
const stub = await wired();
|
||||
await stub.startFill({ pagesPerStep: 1 });
|
||||
expect(await runDurableObjectAlarm(stub)).toBe(true);
|
||||
expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2, pages: 2 });
|
||||
await drain(stub);
|
||||
expect((await stub.fillStatus())?.state).toBe("done");
|
||||
});
|
||||
|
||||
it("survives eviction: progress is in storage, not memory", async () => {
|
||||
const stub = await wired();
|
||||
await stub.startFill({ pagesPerStep: 1 });
|
||||
await runDurableObjectAlarm(stub);
|
||||
await evictDurableObject(stub);
|
||||
expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2 });
|
||||
await drain(stub);
|
||||
expect((await stub.fillStatus())?.state).toBe("done");
|
||||
});
|
||||
|
||||
it("starting again while running rejoins the same job", async () => {
|
||||
const stub = await wired();
|
||||
const first = await stub.startFill({ pagesPerStep: 1 });
|
||||
await runDurableObjectAlarm(stub);
|
||||
const again = await stub.startFill({ pagesPerStep: 1 });
|
||||
expect(again.startedAt).toBe(first.startedAt);
|
||||
expect(again.nextPage).toBe(2);
|
||||
});
|
||||
|
||||
it("skips pages already cached", async () => {
|
||||
const stub = await wired();
|
||||
await stub.page(1, 24);
|
||||
await stub.startFill();
|
||||
await drain(stub);
|
||||
expect(await stub.fillStatus()).toMatchObject({ state: "done", fetched: 1 });
|
||||
});
|
||||
|
||||
it("stops when asked; cached pages stay", async () => {
|
||||
const stub = await wired();
|
||||
await stub.startFill({ pagesPerStep: 1 });
|
||||
await runDurableObjectAlarm(stub);
|
||||
expect((await stub.stopFill()).state).toBe("stopped");
|
||||
expect(await runDurableObjectAlarm(stub)).toBe(false);
|
||||
expect((await stub.page(1, 24)).cached).toBe(true);
|
||||
});
|
||||
|
||||
it("a webapp failure fails the job loudly, once: status says why, the log too", async () => {
|
||||
const spy = vi.spyOn(console, "error");
|
||||
try {
|
||||
const stub = await wired("no-such-webapp");
|
||||
await stub.startFill();
|
||||
await drain(stub);
|
||||
const status = await stub.fillStatus();
|
||||
expect(status?.state).toBe("failed");
|
||||
expect(status?.error).toMatch(/unknown id/);
|
||||
expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.event === "fill")).not.toHaveLength(0);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("a public cache's fill stores no private rows", async () => {
|
||||
const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
|
||||
await stub.setup({ webappId: "fake-webapp-id" });
|
||||
await stub.startFill();
|
||||
await drain(stub);
|
||||
await runInDurableObject(stub, (_obj, state) => {
|
||||
const rows = state.storage.sql.exec<{ n: number }>("select count(*) as n from items where private = 1").toArray();
|
||||
expect(rows[0].n).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe.skip("fill routes (goal 1e)", () => {
|
||||
const call = (path: string, init?: RequestInit) =>
|
||||
worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
|
||||
const post = (body: Record<string, string>) =>
|
||||
call("/_fill", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
|
||||
|
||||
it("POST /_fill starts the owner's job (cache id in the body) and answers 202 with its status", async () => {
|
||||
const id = testEnv.CACHE_DO.newUniqueId();
|
||||
await testEnv.CACHE_DO.get(id).setup({ webappId: "fake-webapp-id" });
|
||||
const res = await post({ cacheId: toWebkey(id) });
|
||||
expect(res.status).toBe(202);
|
||||
expect(await res.json()).toMatchObject({ state: "running" });
|
||||
});
|
||||
|
||||
it("target=public fills the cache's public companion; action=stop stops", async () => {
|
||||
const wiredRes = await call("/_wire", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ webappId: "fake-webapp-id", mode: "public" }),
|
||||
});
|
||||
const { cacheId } = (await wiredRes.json()) as { cacheId: string };
|
||||
expect((await post({ cacheId, target: "public" })).status).toBe(202);
|
||||
const stopped = await post({ cacheId, target: "public", action: "stop" });
|
||||
expect(await stopped.json()).toMatchObject({ state: "stopped" });
|
||||
});
|
||||
|
||||
it("refuses an unknown cache id (404) and never takes the id from the URL", async () => {
|
||||
expect((await post({ cacheId: toWebkey(testEnv.CACHE_DO.newUniqueId()) })).status).toBe(404);
|
||||
expect((await call(`/_fill/${toWebkey(testEnv.CACHE_DO.newUniqueId())}`, { method: "POST" })).status).toBe(404);
|
||||
});
|
||||
|
||||
it("the created page offers a fetch-everything button, the id in a hidden field", async () => {
|
||||
const res = await call("/_wire", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/x-www-form-urlencoded" },
|
||||
body: new URLSearchParams({ webappId: "fake-webapp-id", mode: "private" }),
|
||||
});
|
||||
const html = await res.text();
|
||||
expect(html).toMatch(/<form method="post" action="\/_fill">/);
|
||||
expect(html).toMatch(/<input type="hidden" name="cacheId"/);
|
||||
});
|
||||
|
||||
it("the public page shows a running fill's progress", async () => {
|
||||
const wiredRes = await call("/_wire", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ webappId: "fake-webapp-id", mode: "public" }),
|
||||
});
|
||||
const { cacheId, username } = (await wiredRes.json()) as { cacheId: string; username: string };
|
||||
await post({ cacheId, target: "public" });
|
||||
const html = await (await call(`/public/${username}`)).text();
|
||||
expect(html).toMatch(/fill(ed|ing)[^<]*\d+ of \d+ pages/);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user