diff --git a/diigo-cache/src/cache.ts b/diigo-cache/src/cache.ts index 4d6b5f1..fbb3b52 100644 --- a/diigo-cache/src/cache.ts +++ b/diigo-cache/src/cache.ts @@ -56,8 +56,13 @@ export type CachePage = { const MS_PER_CALL = 500; -/** items per diigo page, for the fill job and the pages it etches */ -const PAGE_SIZE = 24; +/** + * items per diigo page — the one size the worker asks for everywhere + * (fill, public pages, dumps), so a cache never mixes sizes. Pages are + * cached by (page number, size): changing this turns every cached page + * into a miss (items survive; pages refetch). + */ +export const PAGE_SIZE = 128; /** the pause before each fill step's alarm: a step already paces its * pages at MS_PER_CALL; this keeps steps from running back to back */ @@ -284,6 +289,14 @@ abstract class PagedCache extends DurableObject { } finally { for (const stub of [queried, joined, app]) disposeStub(stub); } + // diigo may cap the page size below what we asked: a short page that + // isn't the last would silently skip items, so refuse it loudly + const lastPage = Math.max(1, Math.ceil(data.total / count)); + if (data.items.length < count && pageNum < lastPage) { + const error = `diigo returned ${data.items.length} items for page ${pageNum} of ${lastPage} (asked for ${count}): page size not honored?`; + console.error({ event: "cache", kind: this.kind, page: pageNum, outcome: "short-page", fetched: data.items.length, asked: count }); + throw new Error(error); + } const kept = this.keep(toRows(data.items, console)); // fetched vs kept: the public cache's drop of private rows shows here console.log({ event: "cache", kind: this.kind, page: pageNum, outcome: "miss", fetched: data.items.length, kept: kept.length }); diff --git a/diigo-cache/src/index.ts b/diigo-cache/src/index.ts index 75ecde9..8962e8a 100644 --- a/diigo-cache/src/index.ts +++ b/diigo-cache/src/index.ts @@ -18,6 +18,7 @@ import { CacheApi } from "./api.js"; import { CAPNWEB_BROWSER_JS } from "./capnweb-browser.js"; import { CACHE_PAGE_HTML } from "./cache-page.js"; import { DiigoCacheLogic, PublicCacheLogic } from "./cache.js"; +import { PAGE_SIZE } from "./cache.js"; import type { CacheEnv, Clock, FillStatus } from "./cache.js"; import type { Item } from "./cache.js"; import { updatedISO, usernameOf } from "./items.js"; @@ -69,7 +70,7 @@ const DIIGO_INTERACT = "https://www.diigo.com/interact_api/"; export class DiigoBridge extends WorkerEntrypoint { /** bulk dump from the caller's address space: `pages * count` bookmarks * through the cache (cache-hits are storage reads) */ - async dumpNdjson(pages: number, count = 24): Promise { + async dumpNdjson(pages: number, count = PAGE_SIZE): Promise { const ns = (this.env as unknown as Env).CACHE_DO; const cacheId = (this.env as unknown as Env).DIIGO_CACHE_ID; if (!cacheId) throw new Error("no cache id supplied (caller config error)"); @@ -285,7 +286,7 @@ async function wire(request: Request, env: Env): Promise { const cacheId = toWebkey(env.CACHE_DO.newUniqueId()); const cacheStub = env.CACHE_DO.get(env.CACHE_DO.idFromString(idFromWebkey(cacheId))); await cacheStub.setup({ webappId, publicId }); - const first = await cacheStub.page(1, 24); + const first = await cacheStub.page(1, PAGE_SIZE); if (publicId) { const firstItem = first.items[0] ? (JSON.parse(first.items[0]) as Item) : undefined; @@ -401,7 +402,7 @@ async function publishCache(request: Request, env: Env): Promise { // seed the projection *now*, while the cache is still private-only: // only rows already cached and marked shared can enter the public // side; the username rides the same rows - const firstPage = await cache.page(1, 24); + const firstPage = await cache.page(1, PAGE_SIZE); const firstItem = firstPage.items[0] ? (JSON.parse(firstPage.items[0]) as Item) : undefined; const username = usernameOf(firstItem); if (!username) { @@ -452,7 +453,7 @@ async function dump(request: Request, env: Env): Promise { try { const ns = env.CACHE_DO; const cache = ns.get(ns.idFromString(idFromWebkey(cacheId))); - const page = await cache.page(pageN, 24); + const page = await cache.page(pageN, PAGE_SIZE); const data = `total: ${page.total}\nitems: ${page.items.length}\ncache: ${page.cached ? "HIT (from storage)" : "MISS (etched now)"}` + `\n\n${page.items.join("\n")}`; return home(env, null, data, cacheId); @@ -462,7 +463,7 @@ async function dump(request: Request, env: Env): Promise { } } -const PUBLIC_PER_PAGE = 24; +const PUBLIC_PER_PAGE = PAGE_SIZE; /** the public page for a username: an HTML page with a page-number form * (GET — nothing secret here, so `?page=` in the URL is fine, unlike the diff --git a/diigo-cache/tests/caches.test.ts b/diigo-cache/tests/caches.test.ts index 2c949e7..6978681 100644 --- a/diigo-cache/tests/caches.test.ts +++ b/diigo-cache/tests/caches.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest"; import worker from "../src/index.js"; import { toWebkey } from "../src/webkeys.js"; import { FETCH_BUDGET } from "../src/query.js"; +import { PAGE_SIZE } from "../src/cache.js"; // matches tests/fake-webapp.js: 30 items, every third private const WEBAPP_ID = "fake-webapp-id"; @@ -25,7 +26,7 @@ describe("PublicCache", () => { const before = await testEnv.WEBAPP.calls(); const miss = await stub.page(1, 24); expect(miss.cached).toBe(false); - expect(miss.total).toBe(30); + expect(miss.total).toBe(300); // diigo page 1 = items 1..24, of which 8 are private expect(parse(miss.items).map((i) => i.private)).toEqual(Array(16).fill(false)); const afterMiss = await testEnv.WEBAPP.calls(); @@ -68,6 +69,31 @@ describe("PublicCache", () => { }); }); +describe("page size", () => { + it("refuses, loudly, a short page that isn't the last (diigo capping the size would skip items)", async () => { + const spy = vi.spyOn(console, "error"); + try { + const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId()); + await stub.setup({ webappId: "fake-capped-webapp" }); + // settle the RPC promise ourselves: expect().rejects on a DO stub's + // promise leaves the rejection unhandled + const failure = () => stub.page(1, PAGE_SIZE).then(() => null, (e: unknown) => String(e)); + expect(await failure()).toMatch(/returned 100 items for page 1 of 3 \(asked for 128\)/); + expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.outcome === "short-page")).toHaveLength(1); + // nothing was stored: asking again refuses again + expect(await failure()).toMatch(/page size not honored/); + } finally { + spy.mockRestore(); + } + }); + + it("the last page may be short", async () => { + const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId()); + await stub.setup({ webappId: WEBAPP_ID }); + expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(300 - 2 * PAGE_SIZE); + }); +}); + describe("eviction", () => { it("a cache miss leaves no undisposed RPC stubs to block eviction", async () => { const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId()); @@ -127,16 +153,17 @@ describe("worker routes", () => { expect(html).toContain("cache: MISS"); const again = await (await call(`/public/${username}?page=2`)).text(); expect(again).toContain("cache: HIT"); - expect(html).toContain("bookmark 25"); + // PAGE_SIZE 128: page 2 is bookmarks 129..256 + expect(html).toContain(">bookmark 130<"); // the date comes first, in toISOString() form expect(html).toMatch(/