diigo-cache: fetch 128 items per diigo page; refuse short pages
PAGE_SIZE = 128 (was 24), one constant for the fill job, public pages, dumps, and DiigoBridge, so a cache never mixes sizes: ~131 pages instead of ~698 for 16.7k bookmarks. Cache invalidation, agreed beforehand: pages are cached by (page number, size), so pages cached at 24 become misses and refetch at 128. Items are keyed by link_id and just re-upsert; nothing is lost. Unknown whether diigo honors count=128, so a short page that isn't the last is refused loudly (logged, thrown; a fill fails with the reason) rather than silently skipping items. The fake gains a capped account to test it; the plain account grows to 300 (3 pages), the deep one to 6000. Generated-by: Claude Opus 5.5
This commit is contained in:
dckc committed
1 parent
efaf74023d
commit
826153e35f
5 files changed
+111
-47
No files matched your search
@@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import worker from "../src/index.js";
|
||||
import { toWebkey } from "../src/webkeys.js";
|
||||
import { FETCH_BUDGET } from "../src/query.js";
|
||||
import { PAGE_SIZE } from "../src/cache.js";
|
||||
|
||||
// matches tests/fake-webapp.js: 30 items, every third private
|
||||
const WEBAPP_ID = "fake-webapp-id";
|
||||
@@ -25,7 +26,7 @@ describe("PublicCache", () => {
|
||||
const before = await testEnv.WEBAPP.calls();
|
||||
const miss = await stub.page(1, 24);
|
||||
expect(miss.cached).toBe(false);
|
||||
expect(miss.total).toBe(30);
|
||||
expect(miss.total).toBe(300);
|
||||
// diigo page 1 = items 1..24, of which 8 are private
|
||||
expect(parse(miss.items).map((i) => i.private)).toEqual(Array(16).fill(false));
|
||||
const afterMiss = await testEnv.WEBAPP.calls();
|
||||
@@ -68,6 +69,31 @@ describe("PublicCache", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("page size", () => {
|
||||
it("refuses, loudly, a short page that isn't the last (diigo capping the size would skip items)", async () => {
|
||||
const spy = vi.spyOn(console, "error");
|
||||
try {
|
||||
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
|
||||
await stub.setup({ webappId: "fake-capped-webapp" });
|
||||
// settle the RPC promise ourselves: expect().rejects on a DO stub's
|
||||
// promise leaves the rejection unhandled
|
||||
const failure = () => stub.page(1, PAGE_SIZE).then(() => null, (e: unknown) => String(e));
|
||||
expect(await failure()).toMatch(/returned 100 items for page 1 of 3 \(asked for 128\)/);
|
||||
expect(spy.mock.calls.map((c) => c[0]).filter((e) => e?.outcome === "short-page")).toHaveLength(1);
|
||||
// nothing was stored: asking again refuses again
|
||||
expect(await failure()).toMatch(/page size not honored/);
|
||||
} finally {
|
||||
spy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("the last page may be short", async () => {
|
||||
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
|
||||
await stub.setup({ webappId: WEBAPP_ID });
|
||||
expect((await stub.page(3, PAGE_SIZE)).items).toHaveLength(300 - 2 * PAGE_SIZE);
|
||||
});
|
||||
});
|
||||
|
||||
describe("eviction", () => {
|
||||
it("a cache miss leaves no undisposed RPC stubs to block eviction", async () => {
|
||||
const stub = testEnv.CACHE_DO.get(testEnv.CACHE_DO.newUniqueId());
|
||||
@@ -127,16 +153,17 @@ describe("worker routes", () => {
|
||||
expect(html).toContain("<strong>cache: MISS");
|
||||
const again = await (await call(`/public/${username}?page=2`)).text();
|
||||
expect(again).toContain("<strong>cache: HIT");
|
||||
expect(html).toContain("bookmark 25");
|
||||
// PAGE_SIZE 128: page 2 is bookmarks 129..256
|
||||
expect(html).toContain(">bookmark 130<");
|
||||
// the date comes first, in toISOString() form
|
||||
expect(html).toMatch(/<li><time datetime="\d{4}-\d\d-\d\dT[\d:.]+Z">[^<]+<\/time> <a /);
|
||||
expect(html).not.toContain("bookmark 27"); // private (27 % 3 === 0)
|
||||
expect(html).not.toContain(">bookmark 129<"); // private (129 % 3 === 0)
|
||||
|
||||
const json = (await (
|
||||
await call(`/public/${username}?page=2`, { headers: { accept: "application/json" } })
|
||||
).json()) as { pages: number; items: unknown[] };
|
||||
expect(json.pages).toBe(2);
|
||||
expect(json.items).toHaveLength(4); // 25..30 minus 27, 30
|
||||
expect(json.pages).toBe(3);
|
||||
expect(json.items).toHaveLength(85); // 129..256 minus its 43 multiples of 3
|
||||
});
|
||||
|
||||
it("publishes a private cache later, without the cache id in the URL", async () => {
|
||||
@@ -204,14 +231,23 @@ describe.skip("public page filters (goal 1d)", () => {
|
||||
expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 42, 49, 56, 35));
|
||||
});
|
||||
|
||||
it("?days=30 pages the matches 24 at a time; links keep the filter", async () => {
|
||||
const page = await wireDated();
|
||||
const first = await (await call(`${page}?days=30`)).text();
|
||||
const want = Array.from({ length: 30 }, (_, k) => k + 1).filter((n) => n % 10 !== 0);
|
||||
expect(titlesIn(first)).toEqual(dated(...want.slice(0, 24)));
|
||||
expect(first).toContain(`href="${page}?days=30&page=2"`);
|
||||
const second = await (await call(`${page}?days=30&page=2`)).text();
|
||||
expect(titlesIn(second)).toEqual(dated(...want.slice(24)));
|
||||
it("?days=300 pages the matches PAGE_SIZE at a time; links keep the filter", async () => {
|
||||
const res = await call("/_wire", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
const page = "/public/deep_dee";
|
||||
// deep_dee: one a day; days=300 holds 1..300, minus every 10th (private)
|
||||
const want = Array.from({ length: 300 }, (_, k) => k + 1).filter((n) => n % 10 !== 0);
|
||||
const first = await (await call(`${page}?days=300`)).text();
|
||||
expect(titlesIn(first)).toEqual(dated(...want.slice(0, PAGE_SIZE)));
|
||||
expect(first).toContain(`href="${page}?days=300&page=2"`);
|
||||
const second = await (await call(`${page}?days=300&page=2`)).text();
|
||||
expect(titlesIn(second)).toEqual(dated(...want.slice(PAGE_SIZE, 2 * PAGE_SIZE)));
|
||||
const third = await (await call(`${page}?days=300&page=3`)).text();
|
||||
expect(titlesIn(third)).toEqual(dated(...want.slice(2 * PAGE_SIZE)));
|
||||
});
|
||||
|
||||
it("?after=&before= selects a date range", async () => {
|
||||
@@ -242,7 +278,7 @@ describe.skip("public page filters (goal 1d)", () => {
|
||||
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
// deep_dee: 50 diigo pages; a search matching nothing must walk them all
|
||||
// deep_dee: 47 diigo pages of 128; a search matching nothing must walk them all
|
||||
const before = await testEnv.WEBAPP.calls();
|
||||
const first = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
|
||||
expect(await first.json()).toMatchObject({ complete: false, items: [] });
|
||||
@@ -269,10 +305,10 @@ describe.skip("public page filters (goal 1d)", () => {
|
||||
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
|
||||
});
|
||||
expect(res.status).toBe(200);
|
||||
// deep_dee: 1200 bookmarks, one a day, 50 diigo pages; a week ~2.5
|
||||
// years back sits near page 38 — a walk from page 1 would cost ~38
|
||||
const twoAndAHalfYearsAgo = new Date(Date.now() - 900 * 24 * 60 * 60 * 1000);
|
||||
const week = isoWeek(twoAndAHalfYearsAgo);
|
||||
// deep_dee: 6000 bookmarks, one a day, 47 diigo pages of 128; a week
|
||||
// ~14 years back sits near page 40 — a walk from page 1 would cost ~40
|
||||
const longAgo = new Date(Date.now() - 5100 * 24 * 60 * 60 * 1000);
|
||||
const week = isoWeek(longAgo);
|
||||
const before = await testEnv.WEBAPP.calls();
|
||||
const html = await (await call(`/public/deep_dee?in=${week}`)).text();
|
||||
// one a day; every 10th is private
|
||||
|
||||
Reference in new issue
Block a user