diigo-cache: bookmark cache in Durable Objects (goal 1c)
The dump worker becomes a page cache: a hit is served from DO SQLite storage with no diigo call; a miss asks the webapp instance in the cache's wiring and etches the page. - two independent caches, one behavior (PagedCache): DiigoCache keeps every item; PublicCache keeps shared items only, dropping private rows before they are stored; served at /public/<username> (HTML page form, or JSON) - create flow on one port: /_connect relays webapp's mint form, /_mint proxies the mint, /_wire wires a private or public cache; /_publish adds a public cache later - ids are webkeys: kept out of URLs (POST bodies), shown and logged only as a hash fingerprint; saved to the password manager under the holder's petname (never sent to the server) - items keyed on diigo's link_id; username from u_name; unkeyable items and every failure path are logged; each page read logs hit/miss - tests run the real worker + DOs in the vitest pool against a fake webapp (tests/fake-webapp.js) Generated-by: opencode (GLM) Generated-by: Claude Opus 5.5
This commit is contained in:
dckc committed
1 parent
c5e433cc9c
commit
6f409fc83a
15 files changed
+1664
-211
No files matched your search
@@ -0,0 +1,122 @@
|
||||
import { env } from "cloudflare:workers";
|
||||
import { runInDurableObject } from "cloudflare:test";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import worker from "../src/index.js";
|
||||
import { toWebkey } from "../src/webkeys.js";
|
||||
|
||||
// matches tests/fake-webapp.js: 30 items, every third private
|
||||
const WEBAPP_ID = "fake-webapp-id";
|
||||
|
||||
type TestEnv = {
|
||||
CACHE_DO: DurableObjectNamespace<import("../src/cache.js").DiigoCache>;
|
||||
PUBLIC_DO: DurableObjectNamespace<import("../src/cache.js").PublicCache>;
|
||||
WEBAPP: { calls(): Promise<number> };
|
||||
};
|
||||
const testEnv = env as unknown as TestEnv;
|
||||
|
||||
const parse = (items: string[]) => items.map((j) => JSON.parse(j) as Record<string, unknown>);
|
||||
|
||||
describe("PublicCache", () => {
|
||||
it("on a miss asks the webapp and keeps only shared items; a hit makes no call", async () => {
|
||||
const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
|
||||
await stub.setup({ webappId: WEBAPP_ID });
|
||||
|
||||
const before = await testEnv.WEBAPP.calls();
|
||||
const miss = await stub.page(1, 24);
|
||||
expect(miss.cached).toBe(false);
|
||||
expect(miss.total).toBe(30);
|
||||
// diigo page 1 = items 1..24, of which 8 are private
|
||||
expect(parse(miss.items).map((i) => i.private)).toEqual(Array(16).fill(false));
|
||||
const afterMiss = await testEnv.WEBAPP.calls();
|
||||
expect(afterMiss).toBeGreaterThan(before);
|
||||
|
||||
const hit = await stub.page(1, 24);
|
||||
expect(hit.cached).toBe(true);
|
||||
expect(hit.items).toEqual(miss.items);
|
||||
expect(await testEnv.WEBAPP.calls()).toBe(afterMiss);
|
||||
});
|
||||
|
||||
it("logs each page read as a hit or a miss (fetched vs kept), without the cache id", async () => {
|
||||
const spy = vi.spyOn(console, "log");
|
||||
try {
|
||||
const id = testEnv.PUBLIC_DO.newUniqueId();
|
||||
const stub = testEnv.PUBLIC_DO.get(id);
|
||||
await stub.setup({ webappId: WEBAPP_ID });
|
||||
await stub.page(1, 24);
|
||||
await stub.page(1, 24);
|
||||
const cacheLogs = spy.mock.calls.map((c) => c[0]).filter((e) => e?.event === "cache");
|
||||
expect(cacheLogs).toEqual([
|
||||
{ event: "cache", kind: "public", page: 1, outcome: "miss", fetched: 24, kept: 16 },
|
||||
{ event: "cache", kind: "public", page: 1, outcome: "hit", items: 16 },
|
||||
]);
|
||||
expect(JSON.stringify(spy.mock.calls)).not.toContain(id.toString());
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("stores no private rows at all", async () => {
|
||||
const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
|
||||
await stub.setup({ webappId: WEBAPP_ID });
|
||||
await stub.page(1, 24);
|
||||
await stub.page(2, 24);
|
||||
await runInDurableObject(stub, (_obj, state) => {
|
||||
const rows = state.storage.sql.exec<{ n: number }>("select count(*) as n from items where private = 1").toArray();
|
||||
expect(rows[0].n).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("DiigoCache", () => {
|
||||
it("keeps every item of a page, each under its own link_id", async () => {
|
||||
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
|
||||
await stub.setup({ webappId: WEBAPP_ID });
|
||||
const page = await stub.page(1, 24);
|
||||
const ids = parse(page.items).map((i) => i.link_id);
|
||||
expect(ids).toHaveLength(24);
|
||||
expect(new Set(ids).size).toBe(24);
|
||||
expect(parse(page.items).filter((i) => i.private)).toHaveLength(8);
|
||||
});
|
||||
});
|
||||
|
||||
describe("worker routes", () => {
|
||||
const call = (path: string, init?: RequestInit) =>
|
||||
worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
|
||||
|
||||
it("wires a public cache whose page fills itself on a miss", async () => {
|
||||
const wired = await call("/_wire", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ webappId: WEBAPP_ID, mode: "public" }),
|
||||
});
|
||||
expect(wired.status).toBe(200);
|
||||
const { username } = (await wired.json()) as { username: string };
|
||||
expect(username).toBe("alice_99");
|
||||
|
||||
// page 2 was never fetched by anyone: the public cache asks the webapp
|
||||
const html = await (await call(`/public/${username}?page=2`)).text();
|
||||
expect(html).toContain('<form method="get"');
|
||||
expect(html).toContain("<strong>cache: MISS");
|
||||
const again = await (await call(`/public/${username}?page=2`)).text();
|
||||
expect(again).toContain("<strong>cache: HIT");
|
||||
expect(html).toContain("bookmark 25");
|
||||
expect(html).not.toContain("bookmark 27"); // private (27 % 3 === 0)
|
||||
|
||||
const json = (await (
|
||||
await call(`/public/${username}?page=2`, { headers: { accept: "application/json" } })
|
||||
).json()) as { pages: number; items: unknown[] };
|
||||
expect(json.pages).toBe(2);
|
||||
expect(json.items).toHaveLength(4); // 25..30 minus 27, 30
|
||||
});
|
||||
|
||||
it("publishes a private cache later, without the cache id in the URL", async () => {
|
||||
const cacheId = toWebkey(testEnv.CACHE_DO.newUniqueId());
|
||||
const res = await call("/_publish", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ cacheId }),
|
||||
});
|
||||
// never wired: refused, not crashed
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* @file a stand-in for webapp's `WebAppFactory` entrypoint, for tests: an
|
||||
* auxiliary worker (plain JS — the vitest pool doesn't transform aux
|
||||
* workers) serving diigo-shaped `load_user_items` pages. 30 items, every
|
||||
* third private; one webapp id is known. Counts its diigo calls so tests
|
||||
* can tell a cache hit (no call) from a miss.
|
||||
*/
|
||||
import { RpcTarget, WorkerEntrypoint } from "cloudflare:workers";
|
||||
|
||||
const KNOWN_WEBAPP = "fake-webapp-id";
|
||||
const TOTAL = 30;
|
||||
|
||||
/** item n (1-based, updated-desc order): diigo's field names, made-up values */
|
||||
function item(n) {
|
||||
return {
|
||||
link_id: 1000 + n,
|
||||
url: `https://example.com/${n}`,
|
||||
title: `bookmark ${n}`,
|
||||
description: "",
|
||||
private: n % 3 === 0,
|
||||
u_name: "alice_99",
|
||||
first_by: "someone-else",
|
||||
user_id: 4242,
|
||||
tags: "",
|
||||
created_at: 1700000000,
|
||||
updated_at: 1700001000 - n,
|
||||
};
|
||||
}
|
||||
|
||||
let calls = 0;
|
||||
|
||||
class FakeGet extends RpcTarget {
|
||||
#path;
|
||||
#params;
|
||||
constructor(path, params) {
|
||||
super();
|
||||
this.#path = path;
|
||||
this.#params = params;
|
||||
}
|
||||
pathjoin(ref) {
|
||||
return new FakeGet(ref, this.#params);
|
||||
}
|
||||
query(params) {
|
||||
return new FakeGet(this.#path, { ...this.#params, ...params });
|
||||
}
|
||||
async get() {
|
||||
calls += 1;
|
||||
if (this.#path !== "load_user_items") throw new Error(`fake webapp: no route ${this.#path}`);
|
||||
const count = Number(this.#params.count);
|
||||
const page = Number(this.#params.page_num);
|
||||
const items = [];
|
||||
for (let n = (page - 1) * count + 1; n <= Math.min(page * count, TOTAL); n += 1) items.push(item(n));
|
||||
return JSON.stringify({ total: TOTAL, items });
|
||||
}
|
||||
}
|
||||
|
||||
export class WebAppFactory extends WorkerEntrypoint {
|
||||
async at(id) {
|
||||
if (id !== KNOWN_WEBAPP) throw new Error("fake webapp: unknown id");
|
||||
return new FakeGet("", {});
|
||||
}
|
||||
/** how many diigo calls so far (test hook) */
|
||||
async calls() {
|
||||
return calls;
|
||||
}
|
||||
}
|
||||
|
||||
export default {
|
||||
fetch() {
|
||||
return new Response("fake webapp", { status: 404 });
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,72 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { publicOnly, toRows, usernameOf } from "../src/items.js";
|
||||
import type { Item } from "../src/items.js";
|
||||
|
||||
/** a load_user_items row: diigo's actual field names (observed
|
||||
* 2026-10-07), made-up values */
|
||||
function diigoItem(linkId: number, url: string): Item {
|
||||
return {
|
||||
link_id: linkId,
|
||||
url,
|
||||
title: `title ${linkId}`,
|
||||
description: "",
|
||||
private: false,
|
||||
u_name: "alice_99",
|
||||
first_by: "bob-the-first",
|
||||
user_id: 4242,
|
||||
real_name: "Alice A.",
|
||||
tags: "",
|
||||
created_at: 1700000000,
|
||||
updated_at: 1700000100 + linkId,
|
||||
annotations: [],
|
||||
comments: [],
|
||||
};
|
||||
}
|
||||
|
||||
function makeLog() {
|
||||
const entries: { level: string; args: unknown[] }[] = [];
|
||||
const at = (level: string) => (...args: unknown[]) => entries.push({ level, args });
|
||||
return { log: { warn: at("warn") }, entries };
|
||||
}
|
||||
|
||||
describe("toRows", () => {
|
||||
it("keys each item on diigo's link_id, so a page of distinct items stays distinct", () => {
|
||||
const { log } = makeLog();
|
||||
const rows = toRows([diigoItem(11, "https://a.example/"), diigoItem(12, "https://b.example/")], log);
|
||||
expect(rows.map((r) => r.itemId)).toEqual(["11", "12"]);
|
||||
});
|
||||
|
||||
it("logs, rather than silently drops or merges, items it cannot key", () => {
|
||||
const { log, entries } = makeLog();
|
||||
const noId = diigoItem(13, "https://c.example/");
|
||||
delete noId.link_id;
|
||||
const noUrl = diigoItem(14, "");
|
||||
const rows = toRows([noId, noUrl, diigoItem(15, "https://d.example/")], log);
|
||||
expect(rows.map((r) => r.itemId)).toEqual(["15"]);
|
||||
expect(entries.map((e) => e.level)).toEqual(["warn", "warn"]);
|
||||
// diagnostics name the fields present, never their values
|
||||
expect(JSON.stringify(entries)).not.toContain("https://c.example/");
|
||||
});
|
||||
});
|
||||
|
||||
describe("usernameOf", () => {
|
||||
it("reads the owner's username from diigo's u_name", () => {
|
||||
expect(usernameOf(diigoItem(1, "https://a.example/"))).toBe("alice_99");
|
||||
});
|
||||
|
||||
it("answers empty for an item without one", () => {
|
||||
expect(usernameOf(undefined)).toBe("");
|
||||
expect(usernameOf({ u_name: "has spaces" })).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("publicOnly", () => {
|
||||
it("keeps shared rows and drops private ones, before anything is stored", () => {
|
||||
const { log } = makeLog();
|
||||
const shared = diigoItem(21, "https://shared.example/");
|
||||
const secret = { ...diigoItem(22, "https://secret.example/"), private: true };
|
||||
const rows = publicOnly(toRows([shared, secret], log));
|
||||
expect(rows.map((r) => r.itemId)).toEqual(["21"]);
|
||||
expect(JSON.stringify(rows)).not.toContain("secret.example");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { pmSave } from "../src/pm-save.js";
|
||||
import type { PmPowers } from "../src/pm-save.js";
|
||||
|
||||
/** a fake console that records what was logged, by level */
|
||||
function makeLog() {
|
||||
const entries: { level: string; args: unknown[] }[] = [];
|
||||
const at = (level: string) => (...args: unknown[]) => entries.push({ level, args });
|
||||
return { log: { info: at("info"), warn: at("warn"), error: at("error") }, entries };
|
||||
}
|
||||
|
||||
class FakeCredential {
|
||||
constructor(readonly init: { id: string; password: string }) {}
|
||||
}
|
||||
|
||||
function setup(store: (c: unknown) => Promise<unknown>, petname = "my cache") {
|
||||
const { log, entries } = makeLog();
|
||||
const stored: unknown[] = [];
|
||||
const powers: PmPowers = {
|
||||
petname: { value: petname },
|
||||
credentials: { store: (c) => (stored.push(c), store(c)) },
|
||||
PasswordCredential: FakeCredential,
|
||||
log,
|
||||
};
|
||||
const btn = { textContent: "save to password manager", disabled: false };
|
||||
return { powers, btn, entries, stored };
|
||||
}
|
||||
|
||||
describe("pmSave", () => {
|
||||
it("stores under the petname, and reports a spec-conformant resolve (undefined) as offered, not failed", async () => {
|
||||
const { powers, btn, entries, stored } = setup(() => Promise.resolve(undefined));
|
||||
await pmSave(btn, "the-key", powers);
|
||||
expect(stored).toEqual([new FakeCredential({ id: "my cache", password: "the-key" })]);
|
||||
expect(btn.textContent).not.toMatch(/not saved|fail/);
|
||||
expect(btn.textContent).toMatch(/password manager/);
|
||||
expect(btn.disabled).toBe(false);
|
||||
expect(entries.map((e) => e.level)).toEqual(["info"]);
|
||||
});
|
||||
|
||||
it("logs, never silently, when the browser has no credential store", async () => {
|
||||
const { powers, btn, entries } = setup(() => Promise.resolve(undefined));
|
||||
delete powers.PasswordCredential;
|
||||
await pmSave(btn, "the-key", powers);
|
||||
expect(btn.textContent).toMatch(/no credential store/);
|
||||
expect(entries.map((e) => e.level)).toEqual(["warn"]);
|
||||
});
|
||||
|
||||
it("logs the error when the store rejects, without the key", async () => {
|
||||
const { powers, btn, entries } = setup(() => Promise.reject(new Error("NotAllowedError")));
|
||||
await pmSave(btn, "the-key", powers);
|
||||
expect(btn.textContent).toMatch(/save failed/);
|
||||
expect(entries.map((e) => e.level)).toEqual(["error"]);
|
||||
expect(JSON.stringify(entries)).not.toContain("the-key");
|
||||
});
|
||||
|
||||
it("asks for a petname rather than saving under an empty username", async () => {
|
||||
const { powers, btn, stored } = setup(() => Promise.resolve(undefined), " ");
|
||||
await pmSave(btn, "the-key", powers);
|
||||
expect(stored).toEqual([]);
|
||||
expect(btn.textContent).toMatch(/petname/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,139 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
yurl32,
|
||||
unyurl32,
|
||||
hexToBytes,
|
||||
bytesToHex,
|
||||
glyphsFor,
|
||||
bytesInGlyphs,
|
||||
toWebkey,
|
||||
idFromWebkey,
|
||||
isWebkey,
|
||||
shashKey,
|
||||
type DurableObjectIdLike,
|
||||
} from "../src/webkeys.js";
|
||||
|
||||
// @ts-expect-error vite raw import
|
||||
import webkeysSource from "../src/webkeys.ts?raw";
|
||||
|
||||
const sampleHex = "ab9d14667a05453602b28218c3dc60995d890e49c1502ee1fffa2d0815b367ee";
|
||||
|
||||
/** mock a Cloudflare DurableObjectId */
|
||||
function mockDurableObjectId(hex = sampleHex): DurableObjectIdLike {
|
||||
return {
|
||||
toString: () => hex,
|
||||
equals: (other: unknown) =>
|
||||
typeof other === "object" &&
|
||||
other !== null &&
|
||||
"toString" in other &&
|
||||
(other as { toString(): string }).toString() === hex,
|
||||
};
|
||||
}
|
||||
|
||||
describe("yurl32 and hex codecs", () => {
|
||||
it("matches the paper's alphabet sample", () => {
|
||||
expect("mhbqcmmva5ja3").toMatch(/^[a-z2-7]{13}$/);
|
||||
});
|
||||
|
||||
it("encodes 5 bits per glyph", () => {
|
||||
expect(glyphsFor(64)).toBe(13);
|
||||
expect(glyphsFor(128)).toBe(26);
|
||||
expect(glyphsFor(160)).toBe(32);
|
||||
expect(glyphsFor(256)).toBe(52);
|
||||
expect(bytesInGlyphs(26)).toBe(16);
|
||||
expect(bytesInGlyphs(52)).toBe(32);
|
||||
});
|
||||
|
||||
it("round-trips hex bytes", () => {
|
||||
const bytes = hexToBytes(sampleHex);
|
||||
expect(bytes.length).toBe(32);
|
||||
expect(bytesToHex(bytes)).toBe(sampleHex);
|
||||
});
|
||||
|
||||
it("round-trips bytes through yurl32", () => {
|
||||
const bytes = hexToBytes(sampleHex);
|
||||
const encoded = yurl32(bytes);
|
||||
expect(encoded).toMatch(/^[a-z2-7]+$/);
|
||||
expect(encoded.length).toBe(52);
|
||||
expect(unyurl32(encoded)).toEqual(bytes);
|
||||
});
|
||||
|
||||
it("handles leftovers: 0xff encodes to '74'", () => {
|
||||
expect(yurl32(new Uint8Array([0xff]))).toBe("74");
|
||||
});
|
||||
|
||||
it("refuses garbage", () => {
|
||||
expect(() => unyurl32("0OIl")).toThrow(/not YURL32/);
|
||||
expect(() => unyurl32("AAAA-")).toThrow();
|
||||
expect(() => hexToBytes("abc")).toThrow(/invalid hex length/);
|
||||
expect(() => hexToBytes("zz")).toThrow(/invalid hex byte/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("toWebkey and idFromWebkey (aligning with Worker unique ID)", () => {
|
||||
it("converts a DurableObjectId to a 52-glyph YURL32 webkey", () => {
|
||||
const doId = mockDurableObjectId();
|
||||
const webkey = toWebkey(doId);
|
||||
expect(webkey.length).toBe(52);
|
||||
expect(webkey).toMatch(/^[a-z2-7]{52}$/);
|
||||
expect(isWebkey(webkey)).toBe(true);
|
||||
});
|
||||
|
||||
it("round-trips DurableObjectId -> webkey -> hex string", () => {
|
||||
const doId = mockDurableObjectId();
|
||||
const webkey = toWebkey(doId);
|
||||
const hex = idFromWebkey(webkey);
|
||||
expect(hex).toBe(sampleHex);
|
||||
});
|
||||
|
||||
it("accepts a raw 64-char hex string as input to toWebkey", () => {
|
||||
const webkey = toWebkey(sampleHex);
|
||||
expect(webkey.length).toBe(52);
|
||||
expect(idFromWebkey(webkey)).toBe(sampleHex);
|
||||
});
|
||||
|
||||
it("accepts an already-hex ID in idFromWebkey for backward compatibility", () => {
|
||||
expect(idFromWebkey(sampleHex)).toBe(sampleHex);
|
||||
expect(idFromWebkey(sampleHex.toUpperCase())).toBe(sampleHex);
|
||||
});
|
||||
|
||||
it("encodes raw 16-byte (128-bit) entropy directly", () => {
|
||||
const bytes = new Uint8Array(16).fill(42);
|
||||
const webkey = toWebkey(bytes);
|
||||
expect(webkey.length).toBe(26);
|
||||
expect(isWebkey(webkey)).toBe(true);
|
||||
expect(isWebkey(webkey, 128)).toBe(true);
|
||||
expect(unyurl32(webkey)).toEqual(bytes);
|
||||
});
|
||||
|
||||
it("refuses malformed or wrong-sized webkeys in idFromWebkey", () => {
|
||||
expect(() => idFromWebkey("short")).toThrow();
|
||||
// 26-char (128-bit) webkey is valid YURL32, but not a 256-bit Worker ID:
|
||||
const short128 = toWebkey(new Uint8Array(16));
|
||||
expect(() => idFromWebkey(short128)).toThrow(/expected 32 bytes/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shashKey (swissHash discipline)", () => {
|
||||
it("is one-way: identity, not authority", async () => {
|
||||
const doId = mockDurableObjectId();
|
||||
const h = await shashKey(doId);
|
||||
expect(h).toMatch(/^[a-z2-7]+$/);
|
||||
expect(h.length).toBe(52);
|
||||
expect(h).not.toBe(toWebkey(doId));
|
||||
});
|
||||
|
||||
it("is deterministic (joinable in logs)", async () => {
|
||||
const doId = mockDurableObjectId();
|
||||
expect(await shashKey(doId)).toBe(await shashKey(doId));
|
||||
});
|
||||
});
|
||||
|
||||
describe("no ambient authority in this module", () => {
|
||||
it("the module's functions take their effects as arguments", () => {
|
||||
// strip comments before inspecting executable code
|
||||
const code = webkeysSource.replace(/\/\*[\s\S]*?\*\/|\/\/.*/g, "");
|
||||
expect(code.includes("getRandomValues")).toBe(false);
|
||||
expect(code.match(/crypto\.subtle/g)?.length ?? 0).toBe(1); // shashKey only
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user