captoys/diigo-cache/tests/fill.test.ts

import { env } from "cloudflare:workers";
import { evictDurableObject, runDurableObjectAlarm, runInDurableObject } from "cloudflare:test";
import { describe, expect, it, vi } from "vitest";
import worker from "../src/index.js";
import { newHttpBatchRpcSession } from "capnweb";
import type { CacheApi } from "../src/api.js";
import { toWebkey } from "../src/webkeys.js";
import { FIXED_NOW } from "./test-worker.js";
import { PAGE_SIZE } from "../src/cache.js";

// goal 1e (capsite checklist): a job that grabs every page. Fake
// accounts: tests/fake-webapp.js ("fake-webapp-id": 250 items = 3 pages
// of PAGE_SIZE 100, every 3rd private).

type TestEnv = {
	CACHE_DO: DurableObjectNamespace<import("../src/index.js").DiigoCache>;
	CLOCKED_DO: DurableObjectNamespace<import("./test-worker.js").FixedClockCache>;
	PUBLIC_DO: DurableObjectNamespace<import("../src/index.js").PublicCache>;
	WEBAPP: { calls(): Promise<number> };
};
const testEnv = env as unknown as TestEnv;

const wired = async (webappId = "fake-webapp-id") => {
	const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
	await stub.setup({ webappId });
	return stub;
};

/** run alarms until none is scheduled (bounded, so a runaway job fails) */
const drain = async (stub: DurableObjectStub, max = 20) => {
	let ran = 0;
	while (ran < max && (await runDurableObjectAlarm(stub))) ran += 1;
	expect(ran).toBeLessThan(max);
	return ran;
};

describe("fill job (goal 1e)", () => {
	it("fills every page; afterwards every page is a hit with no webapp call", async () => {
		const stub = await wired();
		const atStart = await testEnv.WEBAPP.calls();
		expect((await stub.startFill()).state).toBe("running");
		await drain(stub);
		const status = await stub.fillStatus();
		expect(status).toMatchObject({ state: "done", pages: 3, fetched: 3 });
		// one diigo call per page fetched (no extra page-1 fetch per miss)
		expect((await testEnv.WEBAPP.calls()) - atStart).toBe(3);

		const before = await testEnv.WEBAPP.calls();
		for (const n of [1, 2, 3]) expect((await stub.page(n, PAGE_SIZE)).cached).toBe(true);
		expect(await testEnv.WEBAPP.calls()).toBe(before);
	});

	it("advances pagesPerStep pages per alarm, with progress readable between steps", async () => {
		const stub = await wired();
		await stub.startFill({ pagesPerStep: 1 });
		expect(await runDurableObjectAlarm(stub)).toBe(true);
		expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2, pages: 3 });
		await drain(stub);
		expect((await stub.fillStatus())?.state).toBe("done");
	});

	it("survives eviction: progress is in storage, not memory", async () => {
		const stub = await wired();
		await stub.startFill({ pagesPerStep: 1 });
		await runDurableObjectAlarm(stub);
		await evictDurableObject(stub);
		expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2 });
		await drain(stub);
		expect((await stub.fillStatus())?.state).toBe("done");
	});

	it("starting again while running rejoins the same job", async () => {
		const stub = await wired();
		const first = await stub.startFill({ pagesPerStep: 1 });
		await runDurableObjectAlarm(stub);
		const again = await stub.startFill({ pagesPerStep: 1 });
		expect(again.startedAt).toBe(first.startedAt);
		expect(again.nextPage).toBe(2);
	});

	it("skips pages already cached", async () => {
		const stub = await wired();
		await stub.page(1, PAGE_SIZE);
		await stub.startFill();
		await drain(stub);
		expect(await stub.fillStatus()).toMatchObject({ state: "done", fetched: 2 });
	});

	it("stops when asked; cached pages stay", async () => {
		const stub = await wired();
		await stub.startFill({ pagesPerStep: 1 });
		await runDurableObjectAlarm(stub);
		expect((await stub.stopFill()).state).toBe("stopped");
		expect(await runDurableObjectAlarm(stub)).toBe(false);
		expect((await stub.page(1, PAGE_SIZE)).cached).toBe(true);
	});

	it("a webapp failure fails the job loudly, once: status says why, the log too", async () => {
		const spy = vi.spyOn(console, "error");
		try {
			const stub = await wired("no-such-webapp");
			await stub.startFill();
			await drain(stub);
			const status = await stub.fillStatus();
			expect(status?.state).toBe("failed");
			expect(status?.error).toMatch(/unknown id/);
			expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.event === "fill")).not.toHaveLength(0);
		} finally {
			spy.mockRestore();
		}
	});

	it("reads time only through its clock: a fixed clock pins the timestamps and the alarm", async () => {
		const stub = testEnv.CLOCKED_DO.get(testEnv.CLOCKED_DO.newUniqueId());
		await stub.setup({ webappId: "fake-webapp-id" });
		const at = new Date(FIXED_NOW).toISOString();
		expect(await stub.startFill()).toMatchObject({ startedAt: at, updatedAt: at });
		await runInDurableObject(stub, async (_obj, state) => {
			expect(await state.storage.getAlarm()).toBe(FIXED_NOW + 1000);
		});
		await drain(stub);
		expect(await stub.fillStatus()).toMatchObject({ state: "done", updatedAt: at });
	});

	it("a capped page size fails the fill with the reason, instead of skipping items", async () => {
		const stub = await wired("fake-capped-webapp");
		await stub.startFill();
		await drain(stub);
		const status = await stub.fillStatus();
		expect(status?.state).toBe("failed");
		expect(status?.error).toMatch(/page size not honored/);
	});

	it("fills to the real end when diigo's total overstates its items", async () => {
		const stub = await wired("fake-overstated-webapp");
		await stub.startFill();
		await drain(stub);
		expect(await stub.fillStatus()).toMatchObject({ state: "done" });
		expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(50);
	});

	it("a job saved under another page size fails instead of resuming; a new start replaces it", async () => {
		const stub = await wired();
		// the shape a 24-per-page job left in storage (no pageSize field)
		await runInDurableObject(stub, async (_obj, state) => {
			const at = new Date().toISOString();
			await state.storage.put("fill", { state: "running", nextPage: 25, pages: 698, fetched: 23, startedAt: at, updatedAt: at });
			await state.storage.setAlarm(Date.now() + 60_000);
		});
		await drain(stub);
		const stale = await stub.fillStatus();
		expect(stale?.state).toBe("failed");
		expect(stale?.error).toMatch(/page size changed \(from 24 to 100\)/);
		expect(await stub.startFill()).toMatchObject({ state: "running", nextPage: 1, pageSize: PAGE_SIZE });
	});

	it("a public cache's fill stores no private rows", async () => {
		const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
		await stub.setup({ webappId: "fake-webapp-id" });
		await stub.startFill();
		await drain(stub);
		await runInDurableObject(stub, (_obj, state) => {
			const rows = state.storage.sql.exec<{ n: number }>("select count(*) as n from items where private = 1").toArray();
			expect(rows[0].n).toBe(0);
		});
	});
});

describe("fill via webkey and Cap'n Web (goal 1e)", () => {
	const ORIGIN = "https://diigo-cache.test";
	const call = (path: string, init?: RequestInit) => worker.fetch(new Request(`${ORIGIN}${path}`, init), env as never);

	/** a Cap'n Web batch session whose fetch goes to the worker; counts round trips */
	const session = () => {
		const trips = { n: 0 };
		const fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
			trips.n += 1;
			return worker.fetch(new Request(input as RequestInfo, init), env as never);
		});
		const api = newHttpBatchRpcSession<CacheApi>(`${ORIGIN}/api`);
		return { api, trips, done: () => fetchSpy.mockRestore() };
	};

	const wireJson = async (mode: "private" | "public") => {
		const res = await call("/_wire", {
			method: "POST",
			headers: { "content-type": "application/json" },
			body: JSON.stringify({ webappId: "fake-webapp-id", mode }),
		});
		return (await res.json()) as { cacheId: string; username?: string };
	};

	it("trading the key and starting the fill pipeline into one round trip", async () => {
		const { cacheId } = await wireJson("private");
		const { api, trips, done } = session();
		try {
			const status = await api.cache(cacheId).startFill();
			expect(status).toMatchObject({ state: "running" });
			expect(trips.n).toBe(1);
		} finally {
			done();
		}
	});

	it("the capability is a facet: fill and status, never the DO's wiring", async () => {
		const { cacheId } = await wireJson("private");
		const { api, done } = session();
		try {
			const cap = api.cache(cacheId) as unknown as Record<string, () => Promise<unknown>>;
			await expect(cap.setup()).rejects.toThrow();
		} finally {
			done();
		}
	});

	it("unknown and malformed keys get one refusal that never echoes the key", async () => {
		const unknown = toWebkey(testEnv.CACHE_DO.newUniqueId());
		for (const key of [unknown, "not-a-key", unknown.slice(0, 26)]) {
			const { api, done } = session();
			try {
				const err = await api.cache(key).fillStatus().then(() => null, (e: unknown) => e);
				expect(String(err)).toMatch(/no such cache/);
				expect(String(err)).not.toContain(key);
			} finally {
				done();
			}
		}
	});

	it("publicCache() reaches the public companion's fill; refused when never published", async () => {
		const pub = await wireJson("public");
		const s1 = session();
		try {
			expect(await s1.api.cache(pub.cacheId).hasPublic()).toBe(true);
		} finally {
			s1.done();
		}
		const s2 = session();
		try {
			expect(await s2.api.cache(pub.cacheId).publicCache().startFill()).toMatchObject({ state: "running" });
		} finally {
			s2.done();
		}
		const s3 = session();
		try {
			expect(await s3.api.cache(pub.cacheId).publicCache().stopFill()).toMatchObject({ state: "stopped" });
		} finally {
			s3.done();
		}
		const priv = await wireJson("private");
		const s4 = session();
		try {
			expect(await s4.api.cache(priv.cacheId).hasPublic()).toBe(false);
		} finally {
			s4.done();
		}
		// a batch session ends after its round trip: a fresh one per await
		const s5 = session();
		try {
			await expect(s5.api.cache(priv.cacheId).publicCache().fillStatus()).rejects.toThrow(/no public cache/);
		} finally {
			s5.done();
		}
	});

	it("the webkey page is a skeleton: the same for every key, which it reads from the fragment", async () => {
		const res = await call("/cache/");
		expect(res.status).toBe(200);
		const html = await res.text();
		expect(html).toContain("location.hash");
		expect(html).toContain('from "/_capnweb.js"');
		expect(html).toContain('newHttpBatchRpcSession("/api")');
		expect(html).toContain('<meta name="referrer" content="no-referrer">');
	});

	it("without a fragment, the webkey page is a login form a password manager can fill, which posts nothing", async () => {
		const html = await (await call("/cache/")).text();
		const form = html.match(/<form id="login"[^]*?<\/form>/)?.[0] ?? "";
		expect(form).toMatch(/<input[^>]* autocomplete="username"/);
		expect(form).toMatch(/<input[^>]* type="password"[^>]* autocomplete="current-password"/);
		// no name attributes: a native submit carries no key anywhere
		expect(form).not.toMatch(/\bname=/);
		expect(form).not.toMatch(/\baction=/);
		expect(html).toContain("event.preventDefault()");
	});

	it("serves capnweb's browser build itself (no third-party script beside the key)", async () => {
		const res = await call("/_capnweb.js");
		expect(res.headers.get("content-type")).toMatch(/javascript/);
		expect(await res.text()).toMatch(/export \{[^}]*newHttpBatchRpcSession/);
	});

	it("the created page links its webkey (key in the fragment); no cache id rides a form to /_fill", async () => {
		const res = await call("/_wire", {
			method: "POST",
			headers: { "content-type": "application/x-www-form-urlencoded" },
			body: new URLSearchParams({ webappId: "fake-webapp-id", mode: "private" }),
		});
		const html = await res.text();
		expect(html).toMatch(/<a href="\/cache\/#[a-z2-7]{52}">/);
		expect(html).not.toContain("/_fill");
		expect((await call("/_fill", { method: "POST" })).status).toBe(404);
	});

	it("the public page shows a running fill's progress", async () => {
		const { cacheId, username } = await wireJson("public");
		const { api, done } = session();
		try {
			await api.cache(cacheId).publicCache().startFill();
		} finally {
			done();
		}
		const html = await (await call(`/public/${username}`)).text();
		expect(html).toMatch(/fill(ed|ing)[^<]*\d+ of \d+ pages/);
	});
});