diigo-cache: fetch 128 items per diigo page; refuse short pages
PAGE_SIZE = 128 (was 24), one constant for the fill job, public pages, dumps, and DiigoBridge, so a cache never mixes sizes: ~131 pages instead of ~698 for 16.7k bookmarks. Cache invalidation, agreed beforehand: pages are cached by (page number, size), so pages cached at 24 become misses and refetch at 128. Items are keyed by link_id and just re-upsert; nothing is lost. Unknown whether diigo honors count=128, so a short page that isn't the last is refused loudly (logged, thrown; a fill fails with the reason) rather than silently skipping items. The fake gains a capped account to test it; the plain account grows to 300 (3 pages), the deep one to 6000. Generated-by: Claude Opus 5.5
This commit is contained in:
1 parent
efaf74023d
commit
826153e35f
5 files changed
+111
-47
No files matched your search
@@ -56,8 +56,13 @@ export type CachePage = {
|
||||
|
||||
const MS_PER_CALL = 500;
|
||||
|
||||
/** items per diigo page, for the fill job and the pages it etches */
|
||||
const PAGE_SIZE = 24;
|
||||
/**
|
||||
* items per diigo page — the one size the worker asks for everywhere
|
||||
* (fill, public pages, dumps), so a cache never mixes sizes. Pages are
|
||||
* cached by (page number, size): changing this turns every cached page
|
||||
* into a miss (items survive; pages refetch).
|
||||
*/
|
||||
export const PAGE_SIZE = 128;
|
||||
|
||||
/** the pause before each fill step's alarm: a step already paces its
|
||||
* pages at MS_PER_CALL; this keeps steps from running back to back */
|
||||
@@ -284,6 +289,14 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
|
||||
} finally {
|
||||
for (const stub of [queried, joined, app]) disposeStub(stub);
|
||||
}
|
||||
// diigo may cap the page size below what we asked: a short page that
|
||||
// isn't the last would silently skip items, so refuse it loudly
|
||||
const lastPage = Math.max(1, Math.ceil(data.total / count));
|
||||
if (data.items.length < count && pageNum < lastPage) {
|
||||
const error = `diigo returned ${data.items.length} items for page ${pageNum} of ${lastPage} (asked for ${count}): page size not honored?`;
|
||||
console.error({ event: "cache", kind: this.kind, page: pageNum, outcome: "short-page", fetched: data.items.length, asked: count });
|
||||
throw new Error(error);
|
||||
}
|
||||
const kept = this.keep(toRows(data.items, console));
|
||||
// fetched vs kept: the public cache's drop of private rows shows here
|
||||
console.log({ event: "cache", kind: this.kind, page: pageNum, outcome: "miss", fetched: data.items.length, kept: kept.length });
|
||||
|
||||
Reference in new issue
Block a user