diigo-cache: the public page only reads
/public/<username> reads through the cache's viewer() facet (PageViewer: stored pages, knownTotal, fill progress — the reader alone), so it can't fetch from diigo or write storage. Behavior change, agreed beforehand: an unfetched page no longer fills itself on a miss; it says "not fetched yet" (JSON: stored: false), and the owner fills via the webkey page. The HIT/MISS line becomes "stored <when>". Strangers can no longer make the cache spend the owner's diigo session; the README's deployment concern shrinks to /api. Tests: viewing an unfetched page makes no webapp call; the page appears once stored; fill progress shows after one fill step. Generated-by: Claude Opus 5.5
This commit is contained in:
1 parent
a17d287e1d
commit
d7190abaf8
5 files changed
+54
-33
No files matched your search
+12
-10
@@ -26,7 +26,11 @@ goal 2 preamble).
|
|||||||
Public page n is diigo page n, filtered (may show fewer than
|
Public page n is diigo page n, filtered (may show fewer than
|
||||||
`PAGE_SIZE`, 100: diigo's cap).
|
`PAGE_SIZE`, 100: diigo's cap).
|
||||||
`/public/<username>` resolves through the `idFromName("index")`
|
`/public/<username>` resolves through the `idFromName("index")`
|
||||||
registry.
|
registry, and reads through `viewer()`, a read-only facet: it shows
|
||||||
|
stored pages and fill progress, and can't fetch from diigo or write.
|
||||||
|
- **Storage**: SQL lives in `src/cache-read.sql` (selects only) and
|
||||||
|
`src/cache-write.sql`; `src/store.ts` turns them into a reader and a
|
||||||
|
writer facet, so the cache logic holds no SQL text.
|
||||||
- **Pages**: home's dump form (cache id + page number, POST) shows raw
|
- **Pages**: home's dump form (cache id + page number, POST) shows raw
|
||||||
dump rows; `/public/<username>?page=n` shows an HTML list with a page
|
dump rows; `/public/<username>?page=n` shows an HTML list with a page
|
||||||
form (JSON for `Accept: application/json`). Both say `cache: HIT` or
|
form (JSON for `Accept: application/json`). Both say `cache: HIT` or
|
||||||
@@ -72,20 +76,18 @@ show only a fingerprint (a prefix of the id's one-way hash).
|
|||||||
Stored state is meant to be durable: see the breaking-change rule in
|
Stored state is meant to be durable: see the breaking-change rule in
|
||||||
[capsite AGENTS.md](../AGENTS.md).
|
[capsite AGENTS.md](../AGENTS.md).
|
||||||
|
|
||||||
## Deployment concern: public pages spend your diigo session
|
## Deployment concern: rate-limit `/api`
|
||||||
|
|
||||||
Anyone who can reach `/public/<username>` can make the cache fetch from
|
Public pages only read storage, so strangers can't make the cache spend
|
||||||
diigo through *your* cookie session: each request may spend up to
|
your diigo session. `/api` is keyed (a cache id is needed), but Cap'n
|
||||||
`FETCH_BUDGET` (40) diigo fetches on uncached pages (goal 1d), paced at
|
Web pipelining makes calls cheap to send. Before exposing a deployment:
|
||||||
`MS_PER_CALL`. Cached pages cost nothing. Before exposing a deployment:
|
|
||||||
|
|
||||||
- rate-limit the public routes and `/api` (Cap'n Web pipelining makes
|
- rate-limit `/api` (and `/public/*`, which reads storage) — on
|
||||||
calls cheap to send) — on Cloudflare, the
|
Cloudflare, the
|
||||||
[Workers rate-limit binding](https://developers.cloudflare.com/workers/runtime-apis/bindings/rate-limit/)
|
[Workers rate-limit binding](https://developers.cloudflare.com/workers/runtime-apis/bindings/rate-limit/)
|
||||||
(keyed by client and/or username); on other workerd hosts, a rate
|
(keyed by client and/or username); on other workerd hosts, a rate
|
||||||
limit in the proxy in front;
|
limit in the proxy in front;
|
||||||
- run the fill job (goal 1e) so most pages are cached before strangers
|
- run the fill job (goal 1e): public pages show only what's stored.
|
||||||
arrive.
|
|
||||||
|
|
||||||
## RPC use (from another worker, same account)
|
## RPC use (from another worker, same account)
|
||||||
|
|
||||||
|
|||||||
@@ -417,6 +417,11 @@ export class PageViewer extends RpcTarget {
|
|||||||
return stored && { ...stored, cached: true };
|
return stored && { ...stored, cached: true };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** diigo's total, as last stored, or null if nothing is stored */
|
||||||
|
knownTotal(): number | null {
|
||||||
|
return this.#reader.knownTotal();
|
||||||
|
}
|
||||||
|
|
||||||
/** the cache's fill job, if any */
|
/** the cache's fill job, if any */
|
||||||
fillStatus(): Promise<FillStatus | undefined> {
|
fillStatus(): Promise<FillStatus | undefined> {
|
||||||
return this.#fillStatus();
|
return this.#fillStatus();
|
||||||
|
|||||||
@@ -479,24 +479,28 @@ async function publicPage(request: Request, env: Env, username: string): Promise
|
|||||||
const raw = new URL(request.url).searchParams.get("page");
|
const raw = new URL(request.url).searchParams.get("page");
|
||||||
const pageN = raw === null ? 1 : Number.parseInt(raw, 10);
|
const pageN = raw === null ? 1 : Number.parseInt(raw, 10);
|
||||||
const valid = !Number.isNaN(pageN) && pageN >= 1 && pageN <= 10000;
|
const valid = !Number.isNaN(pageN) && pageN >= 1 && pageN <= 10000;
|
||||||
let page;
|
// read-only: the viewer facet serves stored pages and cannot fetch
|
||||||
|
// from diigo or write storage; filling is the owner's (webkey page)
|
||||||
|
const viewer = await cache.viewer();
|
||||||
|
let page, total, fill;
|
||||||
try {
|
try {
|
||||||
// a hit is storage-only; a miss asks the webapp (private rows are
|
page = await viewer.page(valid ? pageN : 1, PUBLIC_PER_PAGE);
|
||||||
// dropped before etching)
|
total = page ? page.total : await viewer.knownTotal();
|
||||||
page = await cache.page(valid ? pageN : 1, PUBLIC_PER_PAGE);
|
fill = await viewer.fillStatus();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error({ event: "public-page", page: pageN, error: String(err) });
|
console.error({ event: "public-page", page: pageN, error: String(err) });
|
||||||
return new Response(`could not load page ${valid ? pageN : 1}: ${String(err)}`, { status: 502 });
|
return new Response(`could not load page ${valid ? pageN : 1}: ${String(err)}`, { status: 502 });
|
||||||
|
} finally {
|
||||||
|
viewer[Symbol.dispose]();
|
||||||
}
|
}
|
||||||
// total and pages count all of diigo's items: public page n is diigo
|
// total and pages count all of diigo's items: public page n is diigo
|
||||||
// page n with the private rows dropped
|
// page n with the private rows dropped
|
||||||
const pages = Math.max(1, Math.ceil(page.total / PUBLIC_PER_PAGE));
|
const pages = total === null ? 1 : Math.max(1, Math.ceil(total / PUBLIC_PER_PAGE));
|
||||||
const items = page.items.map((j) => JSON.parse(j) as Item);
|
const items = page ? page.items.map((j) => JSON.parse(j) as Item) : [];
|
||||||
const fill = await cache.fillStatus();
|
|
||||||
|
|
||||||
const accept = request.headers.get("accept") ?? "";
|
const accept = request.headers.get("accept") ?? "";
|
||||||
if (accept.includes("application/json") && !accept.includes("text/html")) {
|
if (accept.includes("application/json") && !accept.includes("text/html")) {
|
||||||
return Response.json({ username, page: pageN, pages, total: page.total, items });
|
return Response.json({ username, page: pageN, pages, total, stored: page !== null, etchedAt: page?.etchedAt ?? null, items });
|
||||||
}
|
}
|
||||||
|
|
||||||
let notice = "";
|
let notice = "";
|
||||||
@@ -527,7 +531,7 @@ ${items
|
|||||||
<body>
|
<body>
|
||||||
<h1>${esc(username)}'s shared bookmarks</h1>
|
<h1>${esc(username)}'s shared bookmarks</h1>
|
||||||
<p>${pages} page${pages === 1 ? "" : "s"} (diigo's, ${PUBLIC_PER_PAGE} bookmarks each; private ones are left out, so a page may show fewer).</p>
|
<p>${pages} page${pages === 1 ? "" : "s"} (diigo's, ${PUBLIC_PER_PAGE} bookmarks each; private ones are left out, so a page may show fewer).</p>
|
||||||
<p><strong>cache: ${page.cached ? "HIT (from storage)" : "MISS (fetched from diigo now)"}</strong> — page ${valid ? pageN : 1}, ${items.length} shared</p>
|
<p>${page ? `<strong>stored ${esc(page.etchedAt ?? "")}</strong> — page ${valid ? pageN : 1}, ${items.length} shared` : `<strong>not fetched yet</strong> — page ${valid ? pageN : 1}; the cache's owner fills it`}</p>
|
||||||
${fill ? fillLine(fill) : ""}
|
${fill ? fillLine(fill) : ""}
|
||||||
<form method="get" action="${here}">
|
<form method="get" action="${here}">
|
||||||
<p>${nav(pageN - 1, "← prev")}
|
<p>${nav(pageN - 1, "← prev")}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { env } from "cloudflare:workers";
|
|||||||
import { evictDurableObject, runInDurableObject } from "cloudflare:test";
|
import { evictDurableObject, runInDurableObject } from "cloudflare:test";
|
||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import worker from "../src/index.js";
|
import worker from "../src/index.js";
|
||||||
import { toWebkey } from "../src/webkeys.js";
|
import { idFromWebkey, toWebkey } from "../src/webkeys.js";
|
||||||
import { FETCH_BUDGET } from "../src/query.js";
|
import { FETCH_BUDGET } from "../src/query.js";
|
||||||
import { PAGE_SIZE } from "../src/cache.js";
|
import { PAGE_SIZE } from "../src/cache.js";
|
||||||
|
|
||||||
@@ -225,22 +225,28 @@ describe("worker routes", () => {
|
|||||||
const call = (path: string, init?: RequestInit) =>
|
const call = (path: string, init?: RequestInit) =>
|
||||||
worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
|
worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
|
||||||
|
|
||||||
it("wires a public cache whose page fills itself on a miss", async () => {
|
it("the public page only reads: an unfetched page says so; the owner's fill makes it appear", async () => {
|
||||||
const wired = await call("/_wire", {
|
const wired = await call("/_wire", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "content-type": "application/json" },
|
headers: { "content-type": "application/json" },
|
||||||
body: JSON.stringify({ webappId: WEBAPP_ID, mode: "public" }),
|
body: JSON.stringify({ webappId: WEBAPP_ID, mode: "public" }),
|
||||||
});
|
});
|
||||||
expect(wired.status).toBe(200);
|
expect(wired.status).toBe(200);
|
||||||
const { username } = (await wired.json()) as { username: string };
|
const { username, publicId } = (await wired.json()) as { username: string; publicId: string };
|
||||||
expect(username).toBe("alice_99");
|
expect(username).toBe("alice_99");
|
||||||
|
|
||||||
// page 2 was never fetched by anyone: the public cache asks the webapp
|
// page 2 was never fetched: the public page can't fetch it
|
||||||
|
const before = await testEnv.WEBAPP.calls();
|
||||||
|
const unfetched = await (await call(`/public/${username}?page=2`)).text();
|
||||||
|
expect(unfetched).toContain('<form method="get"');
|
||||||
|
expect(unfetched).toContain("<strong>not fetched yet</strong>");
|
||||||
|
expect(await testEnv.WEBAPP.calls()).toBe(before);
|
||||||
|
|
||||||
|
// the owner's side stores it (here, directly on the public cache)
|
||||||
|
const pub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.idFromString(idFromWebkey(publicId)));
|
||||||
|
await pub.page(2, PAGE_SIZE);
|
||||||
const html = await (await call(`/public/${username}?page=2`)).text();
|
const html = await (await call(`/public/${username}?page=2`)).text();
|
||||||
expect(html).toContain('<form method="get"');
|
expect(html).toMatch(/<strong>stored \d{4}-\d\d-\d\dT[\d:.]+Z<\/strong>/);
|
||||||
expect(html).toContain("<strong>cache: MISS");
|
|
||||||
const again = await (await call(`/public/${username}?page=2`)).text();
|
|
||||||
expect(again).toContain("<strong>cache: HIT");
|
|
||||||
// PAGE_SIZE 100: page 2 is bookmarks 101..200
|
// PAGE_SIZE 100: page 2 is bookmarks 101..200
|
||||||
expect(html).toContain(">bookmark 130<");
|
expect(html).toContain(">bookmark 130<");
|
||||||
// the date comes first, in toISOString() form
|
// the date comes first, in toISOString() form
|
||||||
@@ -249,8 +255,9 @@ describe("worker routes", () => {
|
|||||||
|
|
||||||
const json = (await (
|
const json = (await (
|
||||||
await call(`/public/${username}?page=2`, { headers: { accept: "application/json" } })
|
await call(`/public/${username}?page=2`, { headers: { accept: "application/json" } })
|
||||||
).json()) as { pages: number; items: unknown[] };
|
).json()) as { pages: number; stored: boolean; items: unknown[] };
|
||||||
expect(json.pages).toBe(3);
|
expect(json.pages).toBe(3);
|
||||||
|
expect(json.stored).toBe(true);
|
||||||
expect(json.items).toHaveLength(67); // 101..200 minus its 33 multiples of 3
|
expect(json.items).toHaveLength(67); // 101..200 minus its 33 multiples of 3
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import { describe, expect, it, vi } from "vitest";
|
|||||||
import worker from "../src/index.js";
|
import worker from "../src/index.js";
|
||||||
import { newHttpBatchRpcSession } from "capnweb";
|
import { newHttpBatchRpcSession } from "capnweb";
|
||||||
import type { CacheApi } from "../src/api.js";
|
import type { CacheApi } from "../src/api.js";
|
||||||
import { toWebkey } from "../src/webkeys.js";
|
import { idFromWebkey, toWebkey } from "../src/webkeys.js";
|
||||||
import { FIXED_NOW } from "./test-worker.js";
|
import { FIXED_NOW } from "./test-worker.js";
|
||||||
import { PAGE_SIZE } from "../src/cache.js";
|
import { PAGE_SIZE } from "../src/cache.js";
|
||||||
|
|
||||||
@@ -203,7 +203,7 @@ describe("fill via webkey and Cap'n Web (goal 1e)", () => {
|
|||||||
headers: { "content-type": "application/json" },
|
headers: { "content-type": "application/json" },
|
||||||
body: JSON.stringify({ webappId: "fake-webapp-id", mode }),
|
body: JSON.stringify({ webappId: "fake-webapp-id", mode }),
|
||||||
});
|
});
|
||||||
return (await res.json()) as { cacheId: string; username?: string };
|
return (await res.json()) as { cacheId: string; username?: string; publicId?: string };
|
||||||
};
|
};
|
||||||
|
|
||||||
it("trading the key and starting the fill pipeline into one round trip", async () => {
|
it("trading the key and starting the fill pipeline into one round trip", async () => {
|
||||||
@@ -332,14 +332,17 @@ describe("fill via webkey and Cap'n Web (goal 1e)", () => {
|
|||||||
expect((await call("/_fill", { method: "POST" })).status).toBe(404);
|
expect((await call("/_fill", { method: "POST" })).status).toBe(404);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("the public page shows a running fill's progress", async () => {
|
it("the public page shows a fill's progress", async () => {
|
||||||
const { cacheId, username } = await wireJson("public");
|
const { cacheId, username, publicId } = (await wireJson("public")) as { cacheId: string; username: string; publicId: string };
|
||||||
const { api, done } = session();
|
const { api, done } = session();
|
||||||
try {
|
try {
|
||||||
await api.cache(cacheId).publicCache().startFill();
|
await api.cache(cacheId).publicCache().startFill({ pagesPerStep: 1 });
|
||||||
} finally {
|
} finally {
|
||||||
done();
|
done();
|
||||||
}
|
}
|
||||||
|
// one step of the job: page 1 stored, so the page count is known
|
||||||
|
const pub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.idFromString(idFromWebkey(publicId)));
|
||||||
|
await runDurableObjectAlarm(pub);
|
||||||
const html = await (await call(`/public/${username}`)).text();
|
const html = await (await call(`/public/${username}`)).text();
|
||||||
expect(html).toMatch(/fill(ed|ing)[^<]*\d+ of \d+ pages/);
|
expect(html).toMatch(/fill(ed|ing)[^<]*\d+ of \d+ pages/);
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in new issue
Block a user