diigo-cache: fetch 128 items per diigo page; refuse short pages
PAGE_SIZE = 128 (was 24), one constant for the fill job, public pages, dumps, and DiigoBridge, so a cache never mixes sizes: ~131 pages instead of ~698 for 16.7k bookmarks. Cache invalidation, agreed beforehand: pages are cached by (page number, size), so pages cached at 24 become misses and refetch at 128. Items are keyed by link_id and just re-upsert; nothing is lost. Unknown whether diigo honors count=128, so a short page that isn't the last is refused loudly (logged, thrown; a fill fails with the reason) rather than silently skipping items. The fake gains a capped account to test it; the plain account grows to 300 (3 pages), the deep one to 6000. Generated-by: Claude Opus 5.5
This commit is contained in:
1 parent
efaf74023d
commit
826153e35f
5 files changed
+111
-47
No files matched your search
@@ -6,10 +6,11 @@ import { newHttpBatchRpcSession } from "capnweb";
|
||||
import type { CacheApi } from "../src/api.js";
|
||||
import { toWebkey } from "../src/webkeys.js";
|
||||
import { FIXED_NOW } from "./test-worker.js";
|
||||
import { PAGE_SIZE } from "../src/cache.js";
|
||||
|
||||
// goal 1e (capsite checklist): a job that grabs every page. Fake
|
||||
// accounts: tests/fake-webapp.js ("fake-webapp-id": 30 items = 2 pages
|
||||
// of 24, every 3rd private).
|
||||
// accounts: tests/fake-webapp.js ("fake-webapp-id": 300 items = 3 pages
|
||||
// of PAGE_SIZE 128, every 3rd private).
|
||||
|
||||
type TestEnv = {
|
||||
CACHE_DO: DurableObjectNamespace<import("../src/index.js").DiigoCache>;
|
||||
@@ -40,12 +41,12 @@ describe("fill job (goal 1e)", () => {
|
||||
expect((await stub.startFill()).state).toBe("running");
|
||||
await drain(stub);
|
||||
const status = await stub.fillStatus();
|
||||
expect(status).toMatchObject({ state: "done", pages: 2, fetched: 2 });
|
||||
expect(status).toMatchObject({ state: "done", pages: 3, fetched: 3 });
|
||||
// one diigo call per page fetched (no extra page-1 fetch per miss)
|
||||
expect((await testEnv.WEBAPP.calls()) - atStart).toBe(2);
|
||||
expect((await testEnv.WEBAPP.calls()) - atStart).toBe(3);
|
||||
|
||||
const before = await testEnv.WEBAPP.calls();
|
||||
for (const n of [1, 2]) expect((await stub.page(n, 24)).cached).toBe(true);
|
||||
for (const n of [1, 2, 3]) expect((await stub.page(n, PAGE_SIZE)).cached).toBe(true);
|
||||
expect(await testEnv.WEBAPP.calls()).toBe(before);
|
||||
});
|
||||
|
||||
@@ -53,7 +54,7 @@ describe("fill job (goal 1e)", () => {
|
||||
const stub = await wired();
|
||||
await stub.startFill({ pagesPerStep: 1 });
|
||||
expect(await runDurableObjectAlarm(stub)).toBe(true);
|
||||
expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2, pages: 2 });
|
||||
expect(await stub.fillStatus()).toMatchObject({ state: "running", nextPage: 2, pages: 3 });
|
||||
await drain(stub);
|
||||
expect((await stub.fillStatus())?.state).toBe("done");
|
||||
});
|
||||
@@ -79,10 +80,10 @@ describe("fill job (goal 1e)", () => {
|
||||
|
||||
it("skips pages already cached", async () => {
|
||||
const stub = await wired();
|
||||
await stub.page(1, 24);
|
||||
await stub.page(1, PAGE_SIZE);
|
||||
await stub.startFill();
|
||||
await drain(stub);
|
||||
expect(await stub.fillStatus()).toMatchObject({ state: "done", fetched: 1 });
|
||||
expect(await stub.fillStatus()).toMatchObject({ state: "done", fetched: 2 });
|
||||
});
|
||||
|
||||
it("stops when asked; cached pages stay", async () => {
|
||||
@@ -91,7 +92,7 @@ describe("fill job (goal 1e)", () => {
|
||||
await runDurableObjectAlarm(stub);
|
||||
expect((await stub.stopFill()).state).toBe("stopped");
|
||||
expect(await runDurableObjectAlarm(stub)).toBe(false);
|
||||
expect((await stub.page(1, 24)).cached).toBe(true);
|
||||
expect((await stub.page(1, PAGE_SIZE)).cached).toBe(true);
|
||||
});
|
||||
|
||||
it("a webapp failure fails the job loudly, once: status says why, the log too", async () => {
|
||||
@@ -121,6 +122,15 @@ describe("fill job (goal 1e)", () => {
|
||||
expect(await stub.fillStatus()).toMatchObject({ state: "done", updatedAt: at });
|
||||
});
|
||||
|
||||
it("a capped page size fails the fill with the reason, instead of skipping items", async () => {
|
||||
const stub = await wired("fake-capped-webapp");
|
||||
await stub.startFill();
|
||||
await drain(stub);
|
||||
const status = await stub.fillStatus();
|
||||
expect(status?.state).toBe("failed");
|
||||
expect(status?.error).toMatch(/page size not honored/);
|
||||
});
|
||||
|
||||
it("a public cache's fill stores no private rows", async () => {
|
||||
const stub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.newUniqueId());
|
||||
await stub.setup({ webappId: "fake-webapp-id" });
|
||||
|
||||
Reference in new issue
Block a user