captoys/diigo-cache/tests/caches.test.ts

import { env } from "cloudflare:workers";
import { evictDurableObject, runInDurableObject } from "cloudflare:test";
import { describe, expect, it, vi } from "vitest";
import worker from "../src/index.js";
import { toWebkey } from "../src/webkeys.js";
import { FETCH_BUDGET } from "../src/query.js";
import { PAGE_SIZE } from "../src/cache.js";

// matches tests/fake-webapp.js: 30 items, every third private
const WEBAPP_ID = "fake-webapp-id";

type TestEnv = {
	CACHE_DO: DurableObjectNamespace<import("../src/index.js").DiigoCache>;
	PUBLIC_DO: DurableObjectNamespace<import("../src/index.js").PublicCache>;
	CLOCKED_DO: DurableObjectNamespace<import("./test-worker.js").FixedClockCache>;
	WEBAPP: { calls(): Promise<number> };
};
const testEnv = env as unknown as TestEnv;

const parse = (items: string[]) => items.map((j) => JSON.parse(j) as Record<string, unknown>);

describe("PublicCache", () => {
	it("on a miss asks the webapp and keeps only shared items; a hit makes no call", async () => {
		const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
		await stub.setup({ webappId: WEBAPP_ID });

		const before = await testEnv.WEBAPP.calls();
		const miss = await stub.page(1, 24);
		expect(miss.cached).toBe(false);
		expect(miss.total).toBe(250);
		// diigo page 1 = items 1..24, of which 8 are private
		expect(parse(miss.items).map((i) => i.private)).toEqual(Array(16).fill(false));
		const afterMiss = await testEnv.WEBAPP.calls();
		expect(afterMiss).toBeGreaterThan(before);

		const hit = await stub.page(1, 24);
		expect(hit.cached).toBe(true);
		expect(hit.items).toEqual(miss.items);
		expect(await testEnv.WEBAPP.calls()).toBe(afterMiss);
	});

	it("logs each page read as a hit or a miss (fetched vs kept), without the cache id", async () => {
		const spy = vi.spyOn(console, "log");
		try {
			const id = testEnv.PUBLIC_DO.newUniqueId();
			const stub = testEnv.PUBLIC_DO.get(id);
			await stub.setup({ webappId: WEBAPP_ID });
			await stub.page(1, 24);
			await stub.page(1, 24);
			const cacheLogs = spy.mock.calls.map((c) => c[0]).filter((e) => e?.event === "cache");
			expect(cacheLogs).toEqual([
				{ event: "cache", kind: "public", page: 1, outcome: "miss", fetched: 24, kept: 16 },
				{ event: "cache", kind: "public", page: 1, outcome: "hit", items: 16 },
			]);
			expect(JSON.stringify(spy.mock.calls)).not.toContain(id.toString());
		} finally {
			spy.mockRestore();
		}
	});

	it("stores no private rows at all", async () => {
		const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
		await stub.setup({ webappId: WEBAPP_ID });
		await stub.page(1, 24);
		await stub.page(2, 24);
		await runInDurableObject(stub, (_obj, state) => {
			const rows = state.storage.sql.exec<{ n: number }>("select count(*) as n from items where private = 1").toArray();
			expect(rows[0].n).toBe(0);
		});
	});
});

describe("page size", () => {
	it("refuses, loudly, a short page that isn't the last (diigo capping the size would skip items)", async () => {
		const spy = vi.spyOn(console, "error");
		try {
			const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
			await stub.setup({ webappId: "fake-capped-webapp" });
			// settle the RPC promise ourselves: expect().rejects on a DO stub's
			// promise leaves the rejection unhandled
			const failure = () => stub.page(1, PAGE_SIZE).then(() => null, (e: unknown) => String(e));
			expect(await failure()).toMatch(/returned 64 items for page 1 of 3 \(asked for 100\), then 64 for page 2/);
			expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.outcome === "short-page")).toHaveLength(1);
			// nothing was stored: asking again refuses again
			expect(await failure()).toMatch(/page size not honored/);
		} finally {
			spy.mockRestore();
		}
	});

	it("a short page whose next page is empty is the real end: an overstated total is noted, not refused", async () => {
		const spy = vi.spyOn(console, "log");
		try {
			const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
			await stub.setup({ webappId: "fake-overstated-webapp" });
			// 250 items claimed as 310: page 3 holds the last 50, "of 4"
			const page = await stub.page(3, PAGE_SIZE);
			expect(page.items).toHaveLength(50);
			const notes = spy.mock.calls.map((c) => c[0]).filter((e) => e?.outcome === "total-overstated");
			expect(notes).toEqual([expect.objectContaining({ page: 3, total: 310, actual: 250 })]);
		} finally {
			spy.mockRestore();
		}
	});

	it("the last page may be short", async () => {
		const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
		await stub.setup({ webappId: WEBAPP_ID });
		expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(250 - 2 * PAGE_SIZE);
	});
});

describe("pacing", () => {
	it("paces diigo calls only: a cache hit doesn't wait", async () => {
		const stub = testEnv.CLOCKED_DO.get(testEnv.CLOCKED_DO.newUniqueId());
		await stub.setup({ webappId: WEBAPP_ID });
		await stub.page(1, PAGE_SIZE);
		const afterMiss = (await stub.sleeps()).length;
		expect(afterMiss).toBeGreaterThan(0);
		for (let i = 0; i < 3; i += 1) expect((await stub.page(1, PAGE_SIZE)).cached).toBe(true);
		expect(await stub.sleeps()).toHaveLength(afterMiss);
	});
});

describe("page numbering (schema v2)", () => {
	it("asks diigo for page_num n-1: page 1 is the newest page", async () => {
		const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
		await stub.setup({ webappId: WEBAPP_ID });
		const first = (await stub.page(1, PAGE_SIZE)).items.map((j) => (JSON.parse(j) as { title: string }).title);
		expect(first[0]).toBe("bookmark 1");
	});

	it("migrates v1 storage: a stored 'page n' was really page n+1, so it is renumbered, not refetched", async () => {
		const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
		await stub.setup({ webappId: WEBAPP_ID });
		// the shape v1 left: tables, a page 1 holding what diigo's
		// page_num=1 returned (really page 2), and no schema marker
		await runInDurableObject(stub, (_obj, state) => {
			const sql = state.storage.sql;
			sql.exec(`create table items (item_id text primary key, url text not null, title text not null,
				private integer not null, json text not null, updated text not null)`);
			sql.exec("create table pages (page_num integer primary key, count integer not null, total integer, etched_at text not null)");
			sql.exec("create table page_members (page_num integer not null, pos integer not null, item_id text not null, primary key (page_num, pos))");
			sql.exec("insert into items values ('a', 'https://a.example/', 'old page one, first', 0, '{\"title\":\"A\"}', '1')");
			sql.exec("insert into items values ('b', 'https://b.example/', 'old page one, second', 0, '{\"title\":\"B\"}', '1')");
			sql.exec("insert into pages values (1, 100, 250, '2026-10-07T00:00:00.000Z')");
			sql.exec("insert into page_members values (1, 1, 'a'), (1, 2, 'b')");
		});
		const before = await testEnv.WEBAPP.calls();
		const two = await stub.page(2, PAGE_SIZE);
		expect(two.cached).toBe(true);
		expect(two.items.map((j) => (JSON.parse(j) as { title: string }).title)).toEqual(["A", "B"]);
		expect(await testEnv.WEBAPP.calls()).toBe(before);
		// page 1 was never fetched under v1: now it is a miss
		expect((await stub.page(1, PAGE_SIZE)).cached).toBe(false);
		await runInDurableObject(stub, (_obj, state) => {
			expect(state.storage.kv.get("schema")).toBe(2);
		});
	});
});

describe("eviction", () => {
	it("a cache miss leaves no undisposed RPC stubs to block eviction", async () => {
		const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
		await stub.setup({ webappId: WEBAPP_ID });
		await stub.page(1, 24);
		await evictDurableObject(stub);
		expect((await stub.page(1, 24)).cached).toBe(true);
	}, 3000);
});

describe("DiigoCache", () => {
	it("keeps every item of a page, each under its own link_id", async () => {
		const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
		await stub.setup({ webappId: WEBAPP_ID });
		const page = await stub.page(1, 24);
		const ids = parse(page.items).map((i) => i.link_id);
		expect(ids).toHaveLength(24);
		expect(new Set(ids).size).toBe(24);
		expect(parse(page.items).filter((i) => i.private)).toHaveLength(8);
	});
});

describe("page replay order", () => {
	it("a hit replays diigo's fetch-time order (page_members), not a column sort", async () => {
		const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
		await stub.setup({ webappId: WEBAPP_ID });
		const titles = (items: string[]) => parse(items).map((i) => i.title);
		const expected = Array.from({ length: 24 }, (_, k) => `bookmark ${k + 1}`);

		const miss = await stub.page(1, 24);
		expect(titles(miss.items)).toEqual(expected);
		// page 2's etch upserts other rows; page 1's replay must not shift
		await stub.page(2, 24);
		const hit = await stub.page(1, 24);
		expect(hit.cached).toBe(true);
		expect(titles(hit.items)).toEqual(expected);
	});
});

describe("worker routes", () => {
	const call = (path: string, init?: RequestInit) =>
		worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);

	it("wires a public cache whose page fills itself on a miss", async () => {
		const wired = await call("/_wire", {
			method: "POST",
			headers: { "content-type": "application/json" },
			body: JSON.stringify({ webappId: WEBAPP_ID, mode: "public" }),
		});
		expect(wired.status).toBe(200);
		const { username } = (await wired.json()) as { username: string };
		expect(username).toBe("alice_99");

		// page 2 was never fetched by anyone: the public cache asks the webapp
		const html = await (await call(`/public/${username}?page=2`)).text();
		expect(html).toContain('<form method="get"');
		expect(html).toContain("<strong>cache: MISS");
		const again = await (await call(`/public/${username}?page=2`)).text();
		expect(again).toContain("<strong>cache: HIT");
		// PAGE_SIZE 100: page 2 is bookmarks 101..200
		expect(html).toContain(">bookmark 130<");
		// the date comes first, in toISOString() form
		expect(html).toMatch(/<li><time datetime="\d{4}-\d\d-\d\dT[\d:.]+Z">[^<]+<\/time> <a /);
		expect(html).not.toContain(">bookmark 129<"); // private (129 % 3 === 0)

		const json = (await (
			await call(`/public/${username}?page=2`, { headers: { accept: "application/json" } })
		).json()) as { pages: number; items: unknown[] };
		expect(json.pages).toBe(3);
		expect(json.items).toHaveLength(67); // 101..200 minus its 33 multiples of 3
	});

	it("publishes a private cache later, without the cache id in the URL", async () => {
		const cacheId = toWebkey(testEnv.CACHE_DO.newUniqueId());
		const res = await call("/_publish", {
			method: "POST",
			headers: { "content-type": "application/json" },
			body: JSON.stringify({ cacheId }),
		});
		// never wired: refused, not crashed
		expect(res.status).toBe(404);
	});
});

/** `yyyy-Www` for the ISO 8601 week holding `d` (UTC) */
function isoWeek(d: Date): string {
	const t = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
	const day = t.getUTCDay() || 7;
	t.setUTCDate(t.getUTCDate() + 4 - day); // the week's Thursday names its year
	const year = t.getUTCFullYear();
	const week = Math.ceil(((t.getTime() - Date.UTC(year, 0, 1)) / 86400000 + 1) / 7);
	return `${year}-W${String(week).padStart(2, "0")}`;
}

describe.skip("public page filters (goal 1d)", () => {
	// the fake's dated account: "dated n" updated n-1 days + 1h ago;
	// #weekly on every 7th; every 10th private; every 5th created 100
	// days before it was updated
	const call = (path: string, init?: RequestInit) =>
		worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
	const wireDated = async () => {
		const res = await call("/_wire", {
			method: "POST",
			headers: { "content-type": "application/json" },
			body: JSON.stringify({ webappId: "fake-dated-webapp", mode: "public" }),
		});
		expect(res.status).toBe(200);
		return "/public/dated_dan";
	};
	const titlesIn = (html: string) => [...html.matchAll(/>(dated \d+)<\/a>/g)].map((m) => m[1]);
	const dated = (...ns: number[]) => ns.map((n) => `dated ${n}`);

	it("?days=7 lists the last week, walking only the diigo pages it needs", async () => {
		const page = await wireDated();
		const before = await testEnv.WEBAPP.calls();
		const html = await (await call(`${page}?days=7`)).text();
		expect(titlesIn(html)).toEqual(dated(1, 2, 3, 4, 5, 6, 7));
		expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(1);
	});

	it("?by=created windows, lists, and dates by when it was bookmarked", async () => {
		const page = await wireDated();
		const html = await (await call(`${page}?days=7&by=created`)).text();
		// 5 was updated this week but bookmarked 100 days earlier
		expect(titlesIn(html)).toEqual(dated(1, 2, 3, 4, 6, 7));
		const updated = await (await call(`${page}?days=7`)).text();
		expect(titlesIn(updated)).toContain("dated 5");
	});

	it("?by=created over a longer window sorts by created, newest first", async () => {
		const page = await wireDated();
		const html = await (await call(`${page}?days=150&by=created&q=${encodeURIComponent("#weekly")}`)).text();
		// weekly: 7,14,21,28,35,42,49,56; 35 was created ~134 days ago
		// (100 days before its update), so it sorts after 56 (~55 days)
		expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 42, 49, 56, 35));
	});

	it("?days=300 pages the matches PAGE_SIZE at a time; links keep the filter", async () => {
		const res = await call("/_wire", {
			method: "POST",
			headers: { "content-type": "application/json" },
			body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
		});
		expect(res.status).toBe(200);
		const page = "/public/deep_dee";
		// deep_dee: one a day; days=300 holds 1..300, minus every 10th (private)
		const want = Array.from({ length: 300 }, (_, k) => k + 1).filter((n) => n % 10 !== 0);
		const first = await (await call(`${page}?days=300`)).text();
		expect(titlesIn(first)).toEqual(dated(...want.slice(0, PAGE_SIZE)));
		expect(first).toContain(`href="${page}?days=300&amp;page=2"`);
		const second = await (await call(`${page}?days=300&page=2`)).text();
		expect(titlesIn(second)).toEqual(dated(...want.slice(PAGE_SIZE, 2 * PAGE_SIZE)));
		const third = await (await call(`${page}?days=300&page=3`)).text();
		expect(titlesIn(third)).toEqual(dated(...want.slice(2 * PAGE_SIZE)));
	});

	it("?after=&before= selects a date range", async () => {
		const page = await wireDated();
		const day = (n: number) => new Date(Date.now() - (n - 1) * 24 * 60 * 60 * 1000 - 60 * 60 * 1000).toISOString().slice(0, 10);
		// items are 24h apart, so each has its own UTC day: from dated 12's
		// day (inclusive) up to dated 9's day (exclusive); 10 is private
		const html = await (await call(`${page}?after=${day(12)}&before=${day(9)}`)).text();
		expect(titlesIn(html)).toEqual(dated(11, 12));
	});

	it("?q=#weekly&days=60 filters by tag; private items stay out", async () => {
		const page = await wireDated();
		const html = await (await call(`${page}?days=60&q=${encodeURIComponent("#weekly")}`)).text();
		expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 35, 42, 49, 56));
	});

	it("?q= alone searches all time", async () => {
		const page = await wireDated();
		const html = await (await call(`${page}?q=fragments`)).text();
		expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 35, 42, 49, 56));
	});

	it("a request spends at most FETCH_BUDGET diigo fetches; incomplete results say so, and a reload continues", async () => {
		const res = await call("/_wire", {
			method: "POST",
			headers: { "content-type": "application/json" },
			body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
		});
		expect(res.status).toBe(200);
		// deep_dee: 60 diigo pages of 100; a search matching nothing must walk them all
		const before = await testEnv.WEBAPP.calls();
		const first = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
		expect(await first.json()).toMatchObject({ complete: false, items: [] });
		expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(FETCH_BUDGET + 1);
		const html = await (await call("/public/deep_dee?q=nomatch")).text();
		expect(html).toMatch(/complete|no matches/);
		const again = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
		expect(await again.json()).toMatchObject({ complete: true, items: [] });
	});

	it("JSON answers carry the canonical filter, so devs see what was understood", async () => {
		const page = await wireDated();
		const res = await call(`${page}?days=7&q=${encodeURIComponent("#Weekly")}`, { headers: { accept: "application/json" } });
		expect(await res.json()).toMatchObject({
			filter: { by: "updated", tags: [{ text: "weekly", negated: false }] },
			complete: true,
		});
	});

	it("?in= an old week costs O(log pages) diigo fetches with by=updated, not a walk from page 1", async () => {
		const res = await call("/_wire", {
			method: "POST",
			headers: { "content-type": "application/json" },
			body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
		});
		expect(res.status).toBe(200);
		// deep_dee: 6000 bookmarks, one a day, 60 diigo pages of 100; a week
		// ~14 years back sits near page 51 — a walk from page 1 would cost ~51
		const longAgo = new Date(Date.now() - 5100 * 24 * 60 * 60 * 1000);
		const week = isoWeek(longAgo);
		const before = await testEnv.WEBAPP.calls();
		const html = await (await call(`/public/deep_dee?in=${week}`)).text();
		// one a day; every 10th is private
		expect(titlesIn(html).length).toBeGreaterThanOrEqual(6);
		expect(titlesIn(html).length).toBeLessThanOrEqual(7);
		expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(12);
	});

	it("a malformed filter is a 400 with a message, not an empty list", async () => {
		const page = await wireDated();
		const res = await call(`${page}?days=7x`);
		expect(res.status).toBe(400);
		expect(await res.text()).toMatch(/days/);
	});

	it("the form offers the day presets, date inputs, and search, echoing the current filter", async () => {
		const page = await wireDated();
		const html = await (await call(`${page}?days=30&q=ocap`)).text();
		expect(html).toMatch(/<select name="days">/);
		expect(html).toMatch(/<select name="by">/);
		for (const n of [7, 30, 90, 365]) expect(html).toContain(`<option value="${n}"`);
		expect(html).toContain('<option value="30" selected>');
		expect(html).toMatch(/<input type="date" name="after"/);
		expect(html).toMatch(/<input type="date" name="before"/);
		expect(html).toMatch(/<input[^>]* name="q"[^>]* value="ocap"/);
		// times are UTC, and the page says so
		expect(html).toMatch(/UTC/);
	});
});