diigo-cache: page size 100, diigo's cap; fill jobs record their page size
Asked for 128, diigo returned 100 (the short-page guard caught it on the
first real fill), so PAGE_SIZE = 100: ~168 pages for 16.7k bookmarks.
The same run showed a fill job outliving a page-size change: progress
counted in 24s ("page 25 of 698") resumed as if counted in 128s. Stored
state reinterpreted, so: FillStatus records pageSize (additive; jobs
saved without it counted in 24), and a job whose size differs from
PAGE_SIZE fails with "page size changed …; start the fill again"
instead of resuming; starting again replaces it. Tested with the old
shape in storage.
Generated-by: Claude Opus 5.5
This commit is contained in:
1 parent
3baffcab4c
commit
563cb83995
4 files changed
+60
-20
No files matched your search
@@ -60,9 +60,14 @@ const MS_PER_CALL = 500;
|
||||
* items per diigo page — the one size the worker asks for everywhere
|
||||
* (fill, public pages, dumps), so a cache never mixes sizes. Pages are
|
||||
* cached by (page number, size): changing this turns every cached page
|
||||
* into a miss (items survive; pages refetch).
|
||||
* into a miss (items survive; pages refetch), and fails any fill job
|
||||
* counting in the old size. 100 is diigo's cap: asked for 128, it
|
||||
* returned 100 (observed 2026-10-07).
|
||||
*/
|
||||
export const PAGE_SIZE = 128;
|
||||
export const PAGE_SIZE = 100;
|
||||
|
||||
/** the size fill jobs counted in before FillStatus recorded it */
|
||||
const LEGACY_FILL_PAGE_SIZE = 24;
|
||||
|
||||
/** the pause before each fill step's alarm: a step already paces its
|
||||
* pages at MS_PER_CALL; this keeps steps from running back to back */
|
||||
@@ -78,6 +83,9 @@ export type FillStatus = {
|
||||
nextPage: number;
|
||||
/** diigo's page count, known after page 1 */
|
||||
pages: number | null;
|
||||
/** the page size `nextPage` and `pages` count in; absent on jobs
|
||||
* saved before this field existed (they counted in 24s) */
|
||||
pageSize?: number;
|
||||
/** pages fetched from diigo by this job (cache hits not counted) */
|
||||
fetched: number;
|
||||
startedAt: string;
|
||||
@@ -86,6 +94,11 @@ export type FillStatus = {
|
||||
error?: string;
|
||||
};
|
||||
|
||||
/** the page size a fill job counts in (old jobs didn't record it) */
|
||||
function fillPageSize(status: FillStatus): number {
|
||||
return status.pageSize ?? LEGACY_FILL_PAGE_SIZE;
|
||||
}
|
||||
|
||||
/** release an RPC stub, if it is one (plain objects are left alone) */
|
||||
function disposeStub(stub: unknown): void {
|
||||
const dispose = (stub as { [Symbol.dispose]?: () => void } | null)?.[Symbol.dispose];
|
||||
@@ -169,7 +182,7 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
|
||||
*/
|
||||
async startFill(opts: { pagesPerStep?: number } = {}): Promise<FillStatus> {
|
||||
const current = await this.fillStatus();
|
||||
if (current?.state === "running") return current;
|
||||
if (current?.state === "running" && fillPageSize(current) === PAGE_SIZE) return current;
|
||||
const step = Math.trunc(opts.pagesPerStep ?? 20);
|
||||
if (!(step >= 1 && step <= 50)) throw new Error("pagesPerStep: expected 1–50");
|
||||
const now = this.#iso();
|
||||
@@ -177,6 +190,7 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
|
||||
state: "running",
|
||||
nextPage: 1,
|
||||
pages: this.#knownPages(),
|
||||
pageSize: PAGE_SIZE,
|
||||
fetched: 0,
|
||||
startedAt: now,
|
||||
updatedAt: now,
|
||||
@@ -211,6 +225,15 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
|
||||
async alarm(): Promise<void> {
|
||||
const step = (await this.ctx.storage.get<number>("fillStep")) ?? 20;
|
||||
let status = await this.fillStatus();
|
||||
// a job saved under another page size counts its pages differently:
|
||||
// resuming it would skip or repeat items, so it fails instead
|
||||
if (status?.state === "running" && fillPageSize(status) !== PAGE_SIZE) {
|
||||
const error = `page size changed (from ${fillPageSize(status)} to ${PAGE_SIZE}); start the fill again`;
|
||||
status = { ...status, state: "failed", error, updatedAt: this.#iso() };
|
||||
console.error({ event: "fill", kind: this.kind, state: "failed", error });
|
||||
await this.ctx.storage.put("fill", status);
|
||||
return;
|
||||
}
|
||||
for (let i = 0; i < step && status?.state === "running"; i += 1) {
|
||||
if (status.pages !== null && status.nextPage > status.pages) {
|
||||
status = { ...status, state: "done" };
|
||||
|
||||
Reference in new issue
Block a user