diigo-cache: page size 100, diigo's cap; fill jobs record their page size

Asked for 128, diigo returned 100 (the short-page guard caught it on the
first real fill), so PAGE_SIZE = 100: ~168 pages for 16.7k bookmarks.

The same run showed a fill job outliving a page-size change: progress
counted in 24s ("page 25 of 698") resumed as if counted in 128s. Stored
state reinterpreted, so: FillStatus records pageSize (additive; jobs
saved without it counted in 24), and a job whose size differs from
PAGE_SIZE fails with "page size changed …; start the fill again"
instead of resuming; starting again replaces it. Tested with the old
shape in storage.

Generated-by: Claude Opus 5.5
This commit is contained in:
dckc committed 2026-10-07 04:59:39 -05:00
1 parent 3baffcab4c
commit 563cb83995
4 files changed
+60 -20

No files matched your search

+26 -3
View File
@@ -60,9 +60,14 @@ const MS_PER_CALL = 500;
* items per diigo page — the one size the worker asks for everywhere
* (fill, public pages, dumps), so a cache never mixes sizes. Pages are
* cached by (page number, size): changing this turns every cached page
* into a miss (items survive; pages refetch).
* into a miss (items survive; pages refetch), and fails any fill job
* counting in the old size. 100 is diigo's cap: asked for 128, it
* returned 100 (observed 2026-10-07).
*/
export const PAGE_SIZE = 128;
export const PAGE_SIZE = 100;
/** the size fill jobs counted in before FillStatus recorded it */
const LEGACY_FILL_PAGE_SIZE = 24;
/** the pause before each fill step's alarm: a step already paces its
* pages at MS_PER_CALL; this keeps steps from running back to back */
@@ -78,6 +83,9 @@ export type FillStatus = {
nextPage: number;
/** diigo's page count, known after page 1 */
pages: number | null;
/** the page size `nextPage` and `pages` count in; absent on jobs
* saved before this field existed (they counted in 24s) */
pageSize?: number;
/** pages fetched from diigo by this job (cache hits not counted) */
fetched: number;
startedAt: string;
@@ -86,6 +94,11 @@ export type FillStatus = {
error?: string;
};
/** the page size a fill job counts in (old jobs didn't record it) */
function fillPageSize(status: FillStatus): number {
return status.pageSize ?? LEGACY_FILL_PAGE_SIZE;
}
/** release an RPC stub, if it is one (plain objects are left alone) */
function disposeStub(stub: unknown): void {
const dispose = (stub as { [Symbol.dispose]?: () => void } | null)?.[Symbol.dispose];
@@ -169,7 +182,7 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
*/
async startFill(opts: { pagesPerStep?: number } = {}): Promise<FillStatus> {
const current = await this.fillStatus();
if (current?.state === "running") return current;
if (current?.state === "running" && fillPageSize(current) === PAGE_SIZE) return current;
const step = Math.trunc(opts.pagesPerStep ?? 20);
if (!(step >= 1 && step <= 50)) throw new Error("pagesPerStep: expected 1–50");
const now = this.#iso();
@@ -177,6 +190,7 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
state: "running",
nextPage: 1,
pages: this.#knownPages(),
pageSize: PAGE_SIZE,
fetched: 0,
startedAt: now,
updatedAt: now,
@@ -211,6 +225,15 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
async alarm(): Promise<void> {
const step = (await this.ctx.storage.get<number>("fillStep")) ?? 20;
let status = await this.fillStatus();
// a job saved under another page size counts its pages differently:
// resuming it would skip or repeat items, so it fails instead
if (status?.state === "running" && fillPageSize(status) !== PAGE_SIZE) {
const error = `page size changed (from ${fillPageSize(status)} to ${PAGE_SIZE}); start the fill again`;
status = { ...status, state: "failed", error, updatedAt: this.#iso() };
console.error({ event: "fill", kind: this.kind, state: "failed", error });
await this.ctx.storage.put("fill", status);
return;
}
for (let i = 0; i < step && status?.state === "running"; i += 1) {
if (status.pages !== null && status.nextPage > status.pages) {
status = { ...status, state: "done" };