dckc/captoys
1
0 Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

diigo-cache: diigo counts pages from 0; migrate stored pages (schema v2)

Browse Source 
Comparing a cache with diigo's RSS export: the size-100 walk had missed
exactly the 100 newest bookmarks (everything updated since 2026-09-16).
diigo's page_num is 0-based, and we sent our 1-based page number
unchanged, so "page 1" was really page 2 and the newest page was never
fetched. diigo's total (16744) was right; the "total-overstated"
reading was a misdiagnosis.

- page() asks for page_num = n - 1
- BREAKING for stored state, agreed beforehand, migrated in place:
  schema v2 renumbers every stored page +1 (pages and page_members, via
  negation so no number collides), once, recorded as kv "schema" = 2;
  nothing is refetched, page 1 just becomes a miss
- fill jobs record their numbering; a job saved without it fails ("page
  numbering changed …; start the fill again") instead of resuming
- the fake pages from 0 like diigo (7 tests failed until the fix);
  tests feed v1-shaped storage and a v1-numbered job to the new code
- the next-page peek guard stays; its comments no longer claim diigo
  overstates its total
- the fixed test clock moves to 2099: pinned to "today" its alarms came
  due in real time and fired on their own

Generated-by: Claude Opus 5.5
This commit is contained in:
dckc committed 2026-10-07 08:26:13 -05:00
1 parent e6deb9fda0
commit cffb1fa9b2
5 files changed
+116 -17
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

diigo-cache/src/cache.ts
+57 -11
View File
@@ -45,6 +45,22 @@ create table if not exists page_members (
);
`;
/**
* V2: renumber stored pages +1. Pages are 1-based here, but diigo's
* `page_num` counts from 0, and V1 sent our page number unchanged — so a
* V1 "page n" holds what is really page n+1 (and the newest page was never
* fetched). Via negation, so no row's new number collides with an old one.
*/
const SCHEMA_V2_RENUMBER = `
update page_members set page_num = -(page_num + 1);
update page_members set page_num = -page_num;
update pages set page_num = -(page_num + 1);
update pages set page_num = -page_num;
`;
/** the schema version a cache's storage is at (kv key "schema") */
const SCHEMA_VERSION = 2;
/** one cached page, raw: `items` are the dump rows (JSON strings), ready
* to re-join with newlines */
export type CachePage = {
@@ -86,6 +102,9 @@ export type FillStatus = {
/** the page size `nextPage` and `pages` count in; absent on jobs
* saved before this field existed (they counted in 24s) */
pageSize?: number;
/** the page numbering `nextPage` counts in: 2 since diigo's 0-based
* page_num is translated; absent on jobs that sent it unchanged */
numbering?: number;
/** pages fetched from diigo by this job (cache hits not counted) */
fetched: number;
startedAt: string;
@@ -99,6 +118,11 @@ function fillPageSize(status: FillStatus): number {
return status.pageSize ?? LEGACY_FILL_PAGE_SIZE;
}
/** does a stored fill job count pages the way this code does? */
function fillCountsAsNow(status: FillStatus): boolean {
return fillPageSize(status) === PAGE_SIZE && status.numbering === SCHEMA_VERSION;
}
/** release an RPC stub, if it is one (plain objects are left alone) */
function disposeStub(stub: unknown): void {
const dispose = (stub as { [Symbol.dispose]?: () => void } | null)?.[Symbol.dispose];
@@ -182,7 +206,7 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
*/
async startFill(opts: { pagesPerStep?: number } = {}): Promise<FillStatus> {
const current = await this.fillStatus();
if (current?.state === "running" && fillPageSize(current) === PAGE_SIZE) return current;
if (current?.state === "running" && fillCountsAsNow(current)) return current;
const step = Math.trunc(opts.pagesPerStep ?? 20);
if (!(step >= 1 && step <= 50)) throw new Error("pagesPerStep: expected 1–50");
const now = this.#iso();
@@ -191,6 +215,7 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
nextPage: 1,
pages: this.#knownPages(),
pageSize: PAGE_SIZE,
numbering: SCHEMA_VERSION,
fetched: 0,
startedAt: now,
updatedAt: now,
@@ -225,10 +250,13 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
async alarm(): Promise<void> {
const step = (await this.ctx.storage.get<number>("fillStep")) ?? 20;
let status = await this.fillStatus();
// a job saved under another page size counts its pages differently:
// resuming it would skip or repeat items, so it fails instead
if (status?.state === "running" && fillPageSize(status) !== PAGE_SIZE) {
const error = `page size changed (from ${fillPageSize(status)} to ${PAGE_SIZE}); start the fill again`;
// a job saved under another page size or numbering counts its pages
// differently: resuming it would skip or repeat items, so it fails
if (status?.state === "running" && !fillCountsAsNow(status)) {
const error =
fillPageSize(status) !== PAGE_SIZE
? `page size changed (from ${fillPageSize(status)} to ${PAGE_SIZE}); start the fill again`
: "page numbering changed (diigo counts pages from 0); start the fill again";
status = { ...status, state: "failed", error, updatedAt: this.#iso() };
console.error({ event: "fill", kind: this.kind, state: "failed", error });
await this.ctx.storage.put("fill", status);
@@ -274,10 +302,27 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
if (status.state === "running") await this.ctx.storage.setAlarm(this.#clock.now() + FILL_STEP_DELAY_MS);
}
#schemaReady = false;
/** create tables, and migrate stored state once per activation */
#schema(): void {
if (this.#schemaReady) return;
this.ctx.storage.sql.exec(SCHEMA_V1);
const version = this.ctx.storage.kv.get<number>("schema") ?? 1;
if (version < 2) {
this.ctx.storage.transactionSync(() => {
this.ctx.storage.sql.exec(SCHEMA_V2_RENUMBER);
this.ctx.storage.kv.put("schema", SCHEMA_VERSION);
});
console.log({ event: "schema", kind: this.kind, from: version, to: SCHEMA_VERSION });
}
this.#schemaReady = true;
}
/** diigo's page count, if any cached page recorded `total` (pages past
* the end record 0, so they don't count) */
#knownPages(): number | null {
this.ctx.storage.sql.exec(SCHEMA_V1);
this.#schema();
const row = this.ctx.storage.sql
.exec<{ total: number | null }>("select total from pages where total > 0 order by etched_at desc limit 1")
.toArray();
@@ -293,7 +338,7 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
async page(pageNum: number, count: number): Promise<CachePage> {
const init = await this.wiring();
if (!init) throw new Error("not wired: this cache id was never set up");
this.ctx.storage.sql.exec(SCHEMA_V1);
this.#schema();
const hit = await this.#pageFromStorage(pageNum, count);
if (hit) {
@@ -322,9 +367,9 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
// page is in hand: undisposed stubs keep this DO from being evicted
const data = await this.#fetchPage(init.webappId, pageNum, count);
// diigo may cap the page size below what we asked: a short page that
// isn't the last would silently skip items. But diigo's `total` can
// also overstate what it returns, making the true last page look
// short. The next page tells them apart: empty means the data ends
// isn't the last would silently skip items. If `total` ever
// overstated what diigo returns, the true last page would look short
// too. The next page tells them apart: empty means the data ends
// here; anything else means items would be skipped — refuse loudly.
const lastPage = Math.max(1, Math.ceil(data.total / count));
if (data.items.length < count && pageNum < lastPage) {
@@ -350,7 +395,8 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
async #fetchPage(webappId: string, pageNum: number, count: number): Promise<{ total: number; items: Item[] }> {
const app = await this.env.WEBAPP.at(webappId);
const joined = app.pathjoin("load_user_items");
const queried = joined.query({ sort: "updated", count, page_num: pageNum });
// diigo's page_num counts from 0; ours from 1
const queried = joined.query({ sort: "updated", count, page_num: pageNum - 1 });
try {
return JSON.parse(await queried.get()) as { total: number; items: Item[] };
} finally {