From 002f08b456ebf13f51f6cb63a0b1dbe703ae9793 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Fri, 9 Oct 2026 01:17:03 -0500 Subject: [PATCH] diigo-cache: date windows and search on the public page (1d) /public/ takes days, after, before, in (year, month, quarter, ISO week), by=updated|created, and q (words, "phrases", #tags, -negation, gmail/Slack/GitHub date operators). JSON answers echo the parsed filter; a malformed one is a 400. Filters query the stored items in sqlite through the read-only viewer instead of walking diigo pages: the public page keeps no fetch authority, and a filled cache already holds every item. Results say `complete` when every diigo page is stored; freshness is 1f's head refresh. So the fetch budget and the O(log pages) walk test go, and the fake deep account shrinks to 300. Generated-by: Claude Opus 5.5 --- capsite-checklist.md | 16 +-- diigo-cache/src/cache-read.sql | 11 ++ diigo-cache/src/cache.ts | 34 ++++- diigo-cache/src/index.ts | 164 +++++++++++++++++++----- diigo-cache/src/query.ts | 207 ++++++++++++++++++++++++++++--- diigo-cache/src/store.ts | 6 + diigo-cache/tests/caches.test.ts | 86 ++++++------- diigo-cache/tests/fake-webapp.js | 4 +- diigo-cache/tests/query.test.ts | 10 +- 9 files changed, 418 insertions(+), 120 deletions(-) diff --git a/capsite-checklist.md b/capsite-checklist.md index b3acb0c..5cc1c73 100644 --- a/capsite-checklist.md +++ b/capsite-checklist.md @@ -42,20 +42,16 @@ narrative. 2026-10-07, via `scripts/compare-export.ts`. Getting there found diigo's `page_num` is 0-based (schema v2 migration). - - [ ] **1d. Date windows and search on the public page**, toward the + - [x] **1d. Date windows and search on the public page**, toward the weekly PESOS roundup of [#23](https://github.com/dckc/madmode-blog/issues/23) (a week's permalink: `in=2026-W41`). One filter, two spellings: devs pass `{ days: 7, q: "#weekly" }`, URLs the same fields; `q` takes gmail/Slack/GitHub operators. UTC, ISO weeks, half-open - windows; a fetch budget per request. Design: the API in - `diigo-cache/src/query.ts` and the skipped suites in - `tests/query.test.ts` and `tests/caches.test.ts` ("public page - filters"); done when they're unskipped and green. - Reconcile with 1f first: `query.ts` plans to filter while walking - diigo's pages (binary search by `updated`), but after 1c-vi a - filled cache holds every item, so 1d can query `items` in sqlite - and spend diigo fetches only on a head refresh (1f). Choose - before implementing; the skipped suites may need adjusting. + windows (`diigo-cache/src/query.ts`; `tests/query.test.ts`, + `tests/caches.test.ts` "public page filters"). Filters read + stored `items` only (the public page never fetches): results are + as fresh as the cache, and say `complete` when every diigo page + is stored. Keeping them fresh is 1f. 2026-10-09. - [x] **1e. Fill job: a button that grabs every page** into a cache from the webkey page `/cache/#`, which trades the key for a diff --git a/diigo-cache/src/cache-read.sql b/diigo-cache/src/cache-read.sql index f361f55..5088871 100644 --- a/diigo-cache/src/cache-read.sql +++ b/diigo-cache/src/cache-read.sql @@ -19,3 +19,14 @@ where cast(items.updated as integer) >= ?; -- pagesThrough: select count(*) as n from pages where page_num <= ?; + +-- itemsByUpdated: +select json from items +where cast(updated as integer) >= ? and cast(updated as integer) < ? +order by cast(updated as integer) desc, item_id; + +-- itemsByCreated: +select json from items +where cast(json_extract(json, '$.created_at') as integer) >= ? + and cast(json_extract(json, '$.created_at') as integer) < ? +order by cast(json_extract(json, '$.created_at') as integer) desc, item_id; diff --git a/diigo-cache/src/cache.ts b/diigo-cache/src/cache.ts index 5b71f5c..7d5b3c4 100644 --- a/diigo-cache/src/cache.ts +++ b/diigo-cache/src/cache.ts @@ -6,6 +6,8 @@ import { DurableObject, RpcTarget } from "cloudflare:workers"; import type { PagedGet } from "./diigo.js"; import { publicOnly, toRows } from "./items.js"; +import { matches } from "./query.js"; +import type { Query } from "./query.js"; import { makeReader, makeWriter, SCHEMA_VERSION } from "./store.js"; import type { Reader, Writer } from "./store.js"; import type { Item, ItemRow } from "./items.js"; @@ -398,7 +400,18 @@ abstract class PagedCache extends DurableObject { } } -/** what anyone may see of a cache: stored pages and fill progress */ +/** one page of a filter's matches over stored items */ +export type QueryPage = { + /** the matches on this page, raw dump rows (JSON strings) */ + items: string[]; + /** matches in all */ + matched: number; + /** true when every diigo page is stored, so no match can be missing */ + complete: boolean; +}; + +/** what anyone may see of a cache: stored pages, stored items by filter, + * and fill progress */ export class PageViewer extends RpcTarget { #reader: Reader; #fillStatus: () => Promise; @@ -415,6 +428,25 @@ export class PageViewer extends RpcTarget { return stored && { ...stored, cached: true }; } + /** + * page `pageNum` (`count` per page) of the stored items matching + * `query`, newest first by its date. The window narrows in SQL; terms + * are checked per item. Reads storage only: items not yet stored can't + * match, which `complete` reports. + */ + query(query: Query, pageNum: number, count: number): QueryPage { + const from = query.after ? Math.ceil(query.after.getTime() / 1000) : Number.MIN_SAFE_INTEGER; + const to = query.before ? Math.ceil(query.before.getTime() / 1000) : Number.MAX_SAFE_INTEGER; + const hits = this.#reader.itemsIn(query.by, from, to).filter((json) => matches(JSON.parse(json) as Item, query)); + const total = this.#reader.knownTotal(); + const pages = total === null ? null : Math.max(1, Math.ceil(total / PAGE_SIZE)); + return { + items: hits.slice((pageNum - 1) * count, pageNum * count), + matched: hits.length, + complete: pages !== null && this.#reader.pagesThrough(pages) >= pages, + }; + } + /** diigo's total, as last stored, or null if nothing is stored */ knownTotal(): number | null { return this.#reader.knownTotal(); diff --git a/diigo-cache/src/index.ts b/diigo-cache/src/index.ts index c5be9ca..febd4cc 100644 --- a/diigo-cache/src/index.ts +++ b/diigo-cache/src/index.ts @@ -10,9 +10,11 @@ import { CAPNWEB_BROWSER_JS } from "./capnweb-browser.js"; import { CACHE_PAGE_HTML } from "./cache-page.js"; import { DiigoCacheLogic, PublicCacheLogic } from "./cache.js"; import { PAGE_SIZE } from "./cache.js"; -import type { CacheEnv, Clock, FillStatus } from "./cache.js"; +import type { CacheEnv, Clock, FillStatus, QueryPage } from "./cache.js"; import type { Item } from "./cache.js"; -import { updatedISO, usernameOf } from "./items.js"; +import { usernameOf } from "./items.js"; +import { dateOf, isEmpty, parseQuery } from "./query.js"; +import type { Query } from "./query.js"; import { toWebkey, idFromWebkey, isWebkey, shashKey } from "./webkeys.js"; import { pmSave } from "./pm-save.js"; @@ -443,10 +445,16 @@ async function dump(request: Request, env: Env): Promise { const PUBLIC_PER_PAGE = PAGE_SIZE; +/** the public page's day presets */ +const DAY_PRESETS = [7, 30, 90, 365]; + /** * the public page for a username: stored pages only, through the cache's - * read-only `viewer()` (it can't fetch or write). HTML with a page-number - * form, or JSON for `Accept: application/json`. `?page=` in the URL is + * read-only `viewer()` (it can't fetch or write). HTML with a filter form + * and a page-number form, or JSON for `Accept: application/json`. A + * filter (goal 1d: `days`, `after`, `before`, `in`, `by`, `q`; see + * {@link parseQuery}) lists the stored items that match, `page` counting + * matches; with none, `page` is diigo's page as stored. Query params are * fine here: nothing on this page is secret. */ async function publicPage(request: Request, env: Env, username: string): Promise { @@ -456,14 +464,33 @@ async function publicPage(request: Request, env: Env, username: string): Promise const ns = env.PUBLIC_DO; const cache = ns.get(ns.idFromString(idFromWebkey(publicId))); - const raw = new URL(request.url).searchParams.get("page"); + const params = new URL(request.url).searchParams; + const accept = request.headers.get("accept") ?? ""; + const wantsJson = accept.includes("application/json") && !accept.includes("text/html"); + const here = `/public/${encodeURIComponent(username)}`; + const raw = params.get("page"); const pageN = raw === null ? 1 : Number.parseInt(raw, 10); const valid = !Number.isNaN(pageN) && pageN >= 1 && pageN <= 10000; + + const query = parseQuery(params, new Date()); + if ("error" in query) { + console.log({ event: "notice", page: "public", text: "filter refused", error: query.error }); + if (wantsJson) return Response.json({ error: query.error }, { status: 400 }); + const body = publicFrame(username, `${filterForm(here, params)} +

${esc(query.error)}

`); + return new Response(body, { status: 400, headers: { "content-type": "text/html;charset=utf-8" } }); + } + const viewer = await cache.viewer(); - let page, total, fill; + let page, fill, found; + let total: number | null = null; try { - page = await viewer.page(valid ? pageN : 1, PUBLIC_PER_PAGE); - total = page ? page.total : await viewer.knownTotal(); + if (isEmpty(query)) { + page = await viewer.page(valid ? pageN : 1, PUBLIC_PER_PAGE); + total = page ? page.total : await viewer.knownTotal(); + } else { + found = await viewer.query(query, valid ? pageN : 1, PUBLIC_PER_PAGE); + } fill = await viewer.fillStatus(); } catch (err) { console.error({ event: "public-page", page: pageN, error: String(err) }); @@ -471,13 +498,14 @@ async function publicPage(request: Request, env: Env, username: string): Promise } finally { viewer[Symbol.dispose](); } + if (found) return filteredPage(username, here, params, query, found, valid ? pageN : 1, wantsJson, fill); + // total and pages count all of diigo's items: public page n is diigo // page n with the private rows dropped const pages = total === null ? 1 : Math.max(1, Math.ceil(total / PUBLIC_PER_PAGE)); const items = page ? page.items.map((j) => JSON.parse(j) as Item) : []; - const accept = request.headers.get("accept") ?? ""; - if (accept.includes("application/json") && !accept.includes("text/html")) { + if (wantsJson) { return Response.json({ username, page: pageN, pages, total, stored: page !== null, etchedAt: page?.etchedAt ?? null, items }); } @@ -486,42 +514,112 @@ async function publicPage(request: Request, env: Env, username: string): Promise console.log({ event: "notice", page: "public", text: "page out of range", pages }); notice = `

no page ${esc(String(raw))}: pages run 1–${pages}

`; } - const here = `/public/${encodeURIComponent(username)}`; const nav = (n: number, label: string) => n >= 1 && n <= pages ? `${label}` : `${label}`; - const list = notice - ? "" - : `
    -${items - .map((it) => { - const url = typeof it.url === "string" ? it.url : ""; - const title = typeof it.title === "string" && it.title ? it.title : url; - const desc = typeof it.description === "string" && it.description ? `
    ${esc(it.description)}` : ""; - const href = /^https?:\/\//i.test(url) ? url : "#"; - const when = updatedISO(it); - const date = when ? `` : "(no date)"; - return `
  1. ${date} ${esc(title)}${desc}
    2. `; - }) - .join("\n")} -
`; - const body = ` -${esc(username)}'s shared bookmarks - -

${esc(username)}'s shared bookmarks

-

${pages} page${pages === 1 ? "" : "s"} (diigo's, ${PUBLIC_PER_PAGE} bookmarks each; private ones are left out, so a page may show fewer).

+ const list = notice ? "" : itemList(items, "updated"); + const body = publicFrame( + username, + `

${pages} page${pages === 1 ? "" : "s"} (diigo's, ${PUBLIC_PER_PAGE} bookmarks each; private ones are left out, so a page may show fewer).

${page ? `stored ${esc(page.etchedAt ?? "")} — page ${valid ? pageN : 1}, ${items.length} shared` : `not fetched yet — page ${valid ? pageN : 1}; the cache's owner fills it`}

${fill ? fillLine(fill) : ""} +${filterForm(here, params)}

${nav(pageN - 1, "← prev")} ${nav(pageN + 1, "next →")}

-${notice}${list} -`; +${notice}${list}`, + ); return new Response(body, { headers: { "content-type": "text/html;charset=utf-8" } }); } +/** a filter's matches, as the public page's HTML or JSON */ +function filteredPage( + username: string, + here: string, + params: URLSearchParams, + query: Query, + found: QueryPage, + pageN: number, + wantsJson: boolean, + fill: FillStatus | undefined, +): Response { + const pages = Math.max(1, Math.ceil(found.matched / PUBLIC_PER_PAGE)); + const items = found.items.map((j) => JSON.parse(j) as Item); + if (wantsJson) { + const { matched, complete } = found; + return Response.json({ username, filter: query, page: pageN, pages, matched, complete, items }); + } + // links keep the filter: the request's own non-empty params, in order + const kept = [...params].filter(([k, v]) => k !== "page" && v !== ""); + const href = (n: number) => `${here}?${new URLSearchParams([...kept, ["page", String(n)]])}`; + const nav = (n: number, label: string) => + n >= 1 && n <= pages ? `${label}` : `${label}`; + const status = found.matched + ? `${found.matched} match${found.matched === 1 ? "" : "es"}, page ${pageN} of ${pages}, newest ${query.by} first` + : "no matches"; + const caveat = found.complete ? "" : " — not every diigo page is stored yet, so some matches may be missing"; + const body = publicFrame( + username, + `

${esc(status)}${caveat}

+${fill ? fillLine(fill) : ""} +${filterForm(here, params)} +

${nav(pageN - 1, "← prev")} ${nav(pageN + 1, "next →")} all, by diigo page

+${itemList(items, query.by)}`, + ); + return new Response(body, { headers: { "content-type": "text/html;charset=utf-8" } }); +} + +/** the public page's frame: title and heading around `content` */ +function publicFrame(username: string, content: string): string { + return ` +${esc(username)}'s shared bookmarks + +

${esc(username)}'s shared bookmarks

+${content} +`; +} + +/** the filter form, echoing the request's filter */ +function filterForm(here: string, params: URLSearchParams): string { + const field = (name: string) => esc(params.get(name) ?? ""); + const days = params.get("days") ?? ""; + const presets = DAY_PRESETS.map(String); + if (days && !presets.includes(days)) presets.push(days); + const option = (value: string, label: string, current: string) => + ``; + return `
+

+ + + +

+

+

+

Dates and periods (2026, 2026-03, 2026-Q1, 2026-W41) are UTC. Search takes +words, "phrases", #tags, -negation, and after:, before:, newer_than:7d, during:2026-Q1, created:2026.

+
`; +} + +/** bookmarks as a list, each led by its `by` date */ +function itemList(items: Item[], by: Query["by"]): string { + return `
    +${items + .map((it) => { + const url = typeof it.url === "string" ? it.url : ""; + const title = typeof it.title === "string" && it.title ? it.title : url; + const desc = typeof it.description === "string" && it.description ? `
    ${esc(it.description)}` : ""; + const href = /^https?:\/\//i.test(url) ? url : "#"; + const secs = dateOf(it, by); + const when = secs === undefined ? "" : new Date(secs * 1000).toISOString(); + const date = when ? `` : "(no date)"; + return `
  1. ${date} ${esc(title)}${desc}
    2. `; + }) + .join("\n")} +
`; +} + /** home page: the connect link (webapp's cookie form, diigo-prefilled) + * the wire step for an already-minted id + the dump form. `home` renders * the error path too, so it takes env for the connect link. Notices (the diff --git a/diigo-cache/src/query.ts b/diigo-cache/src/query.ts index 67767d7..d75f71a 100644 --- a/diigo-cache/src/query.ts +++ b/diigo-cache/src/query.ts @@ -1,18 +1,10 @@ /** - * @file the public page's filters: date windows and search (goal 1d, **not - * yet implemented**; the design is the skipped specs in tests/query.test.ts). + * @file the public page's filters: date windows and search over a cache's + * stored items (goal 1d; tests/query.test.ts). */ import type { Item } from "./items.js"; -/** - * diigo fetches one request may spend (cache hits are free); under the - * Workers free plan's 50 external subrequests per invocation - * - * @see https://developers.cloudflare.com/workers/platform/limits/ - */ -export const FETCH_BUDGET = 40; - /** * a filter as devs write it (`{ days: 7, q: "#weekly" }`); URL params map * onto these fields one-to-one (strings, with "" meaning absent). Dates @@ -56,6 +48,76 @@ export type Query = { /** a refused filter: the message is safe to show and log */ export type QueryError = { error: string }; +const DAY_MS = 24 * 60 * 60 * 1000; + +/** thrown inside {@link toQuery}, returned as a {@link QueryError} */ +class Refused extends Error {} + +/** `yyyy-mm-dd` or `yyyy/mm/dd` as 00:00Z of that UTC day */ +function parseDay(text: string, what: string): Date { + const m = /^(\d{4})([-/])(\d{2})\2(\d{2})$/.exec(text); + const d = m && new Date(Date.UTC(Number(m[1]), Number(m[3]) - 1, Number(m[4]))); + if (!m || !d || d.getUTCMonth() !== Number(m[3]) - 1 || d.getUTCDate() !== Number(m[4])) { + throw new Refused(`${what}: expected a date as yyyy-mm-dd`); + } + return d; +} + +/** Monday 00:00Z of ISO week 1 of `year` (the week holding January 4) */ +function isoWeekOne(year: number): number { + const jan4 = Date.UTC(year, 0, 4); + const weekday = new Date(jan4).getUTCDay() || 7; + return jan4 - (weekday - 1) * DAY_MS; +} + +/** a named period as its half-open [after, before) window */ +function parsePeriod(text: string, what: string): [Date, Date] { + const refuse = () => new Refused(`${what}: expected a period as 2025, 2025-03, 2025-Q1, or 2025-W01`); + let m; + if ((m = /^(\d{4})$/.exec(text))) { + const y = Number(m[1]); + return [new Date(Date.UTC(y, 0, 1)), new Date(Date.UTC(y + 1, 0, 1))]; + } + if ((m = /^(\d{4})-(\d{2})$/.exec(text))) { + const [y, mo] = [Number(m[1]), Number(m[2])]; + if (mo < 1 || mo > 12) throw refuse(); + return [new Date(Date.UTC(y, mo - 1, 1)), new Date(Date.UTC(y, mo, 1))]; + } + if ((m = /^(\d{4})-Q([1-4])$/.exec(text))) { + const [y, q] = [Number(m[1]), Number(m[2])]; + return [new Date(Date.UTC(y, 3 * (q - 1), 1)), new Date(Date.UTC(y, 3 * q, 1))]; + } + if ((m = /^(\d{4})-W(\d{2})$/.exec(text))) { + const [y, w] = [Number(m[1]), Number(m[2])]; + const weeks = (isoWeekOne(y + 1) - isoWeekOne(y)) / (7 * DAY_MS); + if (w < 1 || w > weeks) throw refuse(); + const monday = isoWeekOne(y) + (w - 1) * 7 * DAY_MS; + return [new Date(monday), new Date(monday + 7 * DAY_MS)]; + } + throw refuse(); +} + +/** gmail's `newer_than:` amount (`7d`, `2w`, `1m`, `1y`) counted back from + * `now`; months and years are calendar steps */ +function parseNewerThan(text: string, now: Date): Date { + const m = /^(\d{1,4})([dwmy])$/.exec(text); + if (!m || Number(m[1]) < 1) throw new Refused("newer_than: expected an amount like 7d, 2w, 1m, or 1y"); + const n = Number(m[1]); + const d = new Date(now.getTime()); + if (m[2] === "d") d.setTime(now.getTime() - n * DAY_MS); + if (m[2] === "w") d.setTime(now.getTime() - 7 * n * DAY_MS); + if (m[2] === "m") d.setUTCMonth(d.getUTCMonth() - n); + if (m[2] === "y") d.setUTCFullYear(d.getUTCFullYear() - n); + return d; +} + +/** `days` as a whole number from 1 to 3650 */ +function parseDays(days: number | string): number { + const n = typeof days === "number" ? days : /^\d+$/.test(days) ? Number(days) : NaN; + if (!Number.isInteger(n) || n < 1 || n > 3650) throw new Refused("days: expected a whole number from 1 to 3650"); + return n; +} + /** * resolve a filter: fields and `q` operators combine; a bound given twice * (by fields, operators, or both) is refused, as is an empty window. @@ -75,21 +137,128 @@ export type QueryError = { error: string }; * not read ambiently, so tests pin it) */ export function toQuery(spec: FilterSpec, now: Date): Query | QueryError { - void spec; - void now; - throw new Error("toQuery: not implemented (goal 1d)"); + try { + return resolve(spec, now); + } catch (err) { + if (err instanceof Refused) return { error: err.message }; + throw err; + } +} + +function resolve(spec: FilterSpec, now: Date): Query { + const words: Term[] = []; + const tags: Term[] = []; + let by: { value: Query["by"]; from: string } | undefined; + let after: { value: Date; from: string } | undefined; + let before: { value: Date; from: string } | undefined; + + const setBy = (value: Query["by"], from: string) => { + if (by && by.value !== value) throw new Refused(`${by.from} and ${from} name different dates`); + by = { value, from }; + }; + const setAfter = (value: Date, from: string) => { + if (after) throw new Refused(`${after.from} and ${from} both set where the window starts`); + after = { value, from }; + }; + const setBefore = (value: Date, from: string) => { + if (before) throw new Refused(`${before.from} and ${from} both set where the window ends`); + before = { value, from }; + }; + const setPeriod = ([lo, hi]: [Date, Date], from: string) => { + setAfter(lo, from); + setBefore(hi, from); + }; + + const present = (v: string | number | undefined) => v !== undefined && v !== ""; + if (present(spec.by)) { + if (spec.by !== "updated" && spec.by !== "created") throw new Refused("by: expected updated or created"); + setBy(spec.by, `by=${spec.by}`); + } + if (present(spec.days)) setAfter(new Date(now.getTime() - parseDays(spec.days!) * DAY_MS), "days"); + if (present(spec.after)) setAfter(parseDay(spec.after!, "after"), "after"); + if (present(spec.before)) setBefore(parseDay(spec.before!, "before"), "before"); + if (present(spec.in)) setPeriod(parsePeriod(spec.in!, "in"), "in"); + + for (const m of (spec.q ?? "").matchAll(/(-?)"([^"]*)"|(\S+)/g)) { + if (m[2] !== undefined) { + if (m[2].trim()) words.push({ text: m[2].trim().toLowerCase(), negated: m[1] === "-" }); + continue; + } + const token = m[3]; + const negated = token.length > 1 && token.startsWith("-"); + const body = negated ? token.slice(1) : token; + const op = /^([a-z_]+):(.+)$/i.exec(body); + const name = op?.[1].toLowerCase(); + const arg = op?.[2] ?? ""; + const windowOp = ["after", "before", "newer_than", "during", "in", "created", "updated"]; + if (name && windowOp.includes(name) && negated) throw new Refused(`${name}: can't be negated`); + if (name === "after") setAfter(parseDay(arg, "after:"), "after:"); + else if (name === "before") setBefore(parseDay(arg, "before:"), "before:"); + else if (name === "newer_than") setAfter(parseNewerThan(arg, now), "newer_than:"); + else if (name === "during" || name === "in") setPeriod(parsePeriod(arg, `${name}:`), `${name}:`); + else if (name === "created" || name === "updated") { + setBy(name, `${name}:`); + setPeriod(parsePeriod(arg, `${name}:`), `${name}:`); + } else if (name === "tag") tags.push({ text: arg.toLowerCase(), negated }); + else if (body.length > 1 && body.startsWith("#")) tags.push({ text: body.slice(1).toLowerCase(), negated }); + else words.push({ text: body.toLowerCase(), negated }); + } + + if (after && before && after.value.getTime() >= before.value.getTime()) { + throw new Refused(`the window is empty: ${after.from} is not earlier than ${before.from}`); + } + const query: Query = { by: by?.value ?? "updated", words, tags }; + if (after) query.after = after.value; + if (before) query.before = before.value; + return query; } /** {@link toQuery} over the public page's URL params */ export function parseQuery(params: URLSearchParams, now: Date): Query | QueryError { - void params; - void now; - throw new Error("parseQuery: not implemented (goal 1d)"); + const field = (name: string) => params.get(name) ?? ""; + const by = field("by"); + return toQuery( + { + days: field("days"), + after: field("after"), + before: field("before"), + in: field("in"), + by: by as FilterSpec["by"], + q: field("q"), + }, + now, + ); +} + +/** does the query filter anything? (an empty one lists diigo's pages as stored) */ +export function isEmpty(query: Query): boolean { + return !query.after && !query.before && !query.words.length && !query.tags.length && query.by === "updated"; +} + +/** the item's date that `by` names, in epoch seconds (diigo's unit), or + * undefined when it carries none */ +export function dateOf(item: Item, by: Query["by"]): number | undefined { + const secs = by === "created" ? item.created_at : item.updated_at; + return typeof secs === "number" && Number.isFinite(secs) ? secs : undefined; } /** does `item` fall in the window and satisfy every term? */ export function matches(item: Item, query: Query): boolean { - void item; - void query; - throw new Error("matches: not implemented (goal 1d)"); + if (query.after || query.before) { + const secs = dateOf(item, query.by); + if (secs === undefined) return false; + if (query.after && secs * 1000 < query.after.getTime()) return false; + if (query.before && secs * 1000 >= query.before.getTime()) return false; + } + const text = (v: unknown) => (typeof v === "string" ? v : ""); + // fields joined by newline, so a phrase never spans two of them + const haystack = [item.title, item.description, item.url].map(text).join("\n").toLowerCase(); + if (!query.words.every((w) => haystack.includes(w.text) !== w.negated)) return false; + const itemTags = new Set( + text(item.tags) + .split(",") + .map((t) => t.trim().toLowerCase()) + .filter(Boolean), + ); + return query.tags.every((t) => itemTags.has(t.text) !== t.negated); } diff --git a/diigo-cache/src/store.ts b/diigo-cache/src/store.ts index c091607..e81bc8f 100644 --- a/diigo-cache/src/store.ts +++ b/diigo-cache/src/store.ts @@ -41,6 +41,12 @@ export function makeReader(sql: Sql) { deepestPageSince(cutoff: number): number | null { return run<{ last: number | null }>("deepestPageSince", cutoff)[0]?.last ?? null; }, + /** stored items whose `by` date (epoch seconds) is in [`from`, `to`), + * newest first; the raw dump rows (JSON strings) */ + itemsIn(by: "updated" | "created", from: number, to: number): string[] { + const name = by === "created" ? "itemsByCreated" : "itemsByUpdated"; + return run<{ json: string }>(name, from, to).map((r) => r.json); + }, /** how many cached pages are numbered `pageNum` or lower */ pagesThrough(pageNum: number): number { return Number(run<{ n: number }>("pagesThrough", pageNum)[0]?.n ?? 0); diff --git a/diigo-cache/tests/caches.test.ts b/diigo-cache/tests/caches.test.ts index 41c34fd..79d0184 100644 --- a/diigo-cache/tests/caches.test.ts +++ b/diigo-cache/tests/caches.test.ts @@ -3,7 +3,6 @@ import { evictDurableObject, runInDurableObject } from "cloudflare:test"; import { describe, expect, it, vi } from "vitest"; import worker from "../src/index.js"; import { idFromWebkey, toWebkey } from "../src/webkeys.js"; -import { FETCH_BUDGET } from "../src/query.js"; import { PAGE_SIZE } from "../src/cache.js"; // matches tests/fake-webapp.js: 30 items, every third private @@ -283,30 +282,34 @@ function isoWeek(d: Date): string { return `${year}-W${String(week).padStart(2, "0")}`; } -describe.skip("public page filters (goal 1d)", () => { +describe("public page filters (goal 1d)", () => { // the fake's dated account: "dated n" updated n-1 days + 1h ago; // #weekly on every 7th; every 10th private; every 5th created 100 - // days before it was updated + // days before it was updated. The public page only reads, so each + // test stores the pages it needs on the owner's side first. const call = (path: string, init?: RequestInit) => worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never); - const wireDated = async () => { + /** wire a public cache to `webappId` and store diigo pages 1..`pages` */ + const wireStored = async (webappId: string, pages: number) => { const res = await call("/_wire", { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ webappId: "fake-dated-webapp", mode: "public" }), + body: JSON.stringify({ webappId, mode: "public" }), }); expect(res.status).toBe(200); - return "/public/dated_dan"; + const { username, publicId } = (await res.json()) as { username: string; publicId: string }; + const pub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.idFromString(idFromWebkey(publicId))); + for (let n = 1; n <= pages; n += 1) await pub.page(n, PAGE_SIZE); + return { page: `/public/${username}`, pub }; }; + const wireDated = async () => (await wireStored("fake-dated-webapp", 1)).page; const titlesIn = (html: string) => [...html.matchAll(/>(dated \d+)<\/a>/g)].map((m) => m[1]); const dated = (...ns: number[]) => ns.map((n) => `dated ${n}`); - it("?days=7 lists the last week, walking only the diigo pages it needs", async () => { + it("?days=7 lists the last week", async () => { const page = await wireDated(); - const before = await testEnv.WEBAPP.calls(); const html = await (await call(`${page}?days=7`)).text(); expect(titlesIn(html)).toEqual(dated(1, 2, 3, 4, 5, 6, 7)); - expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(1); }); it("?by=created windows, lists, and dates by when it was bookmarked", async () => { @@ -327,13 +330,7 @@ describe.skip("public page filters (goal 1d)", () => { }); it("?days=300 pages the matches PAGE_SIZE at a time; links keep the filter", async () => { - const res = await call("/_wire", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }), - }); - expect(res.status).toBe(200); - const page = "/public/deep_dee"; + const { page } = await wireStored("fake-deep-webapp", 3); // deep_dee: one a day; days=300 holds 1..300, minus every 10th (private) const want = Array.from({ length: 300 }, (_, k) => k + 1).filter((n) => n % 10 !== 0); const first = await (await call(`${page}?days=300`)).text(); @@ -354,6 +351,15 @@ describe.skip("public page filters (goal 1d)", () => { expect(titlesIn(html)).toEqual(dated(11, 12)); }); + it("?in= names an ISO week", async () => { + const { page } = await wireStored("fake-deep-webapp", 3); + const week = isoWeek(new Date(Date.now() - 100 * 24 * 60 * 60 * 1000)); + const html = await (await call(`${page}?in=${week}`)).text(); + // one a day; every 10th is private + expect(titlesIn(html).length).toBeGreaterThanOrEqual(6); + expect(titlesIn(html).length).toBeLessThanOrEqual(7); + }); + it("?q=#weekly&days=60 filters by tag; private items stay out", async () => { const page = await wireDated(); const html = await (await call(`${page}?days=60&q=${encodeURIComponent("#weekly")}`)).text(); @@ -366,22 +372,21 @@ describe.skip("public page filters (goal 1d)", () => { expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 35, 42, 49, 56)); }); - it("a request spends at most FETCH_BUDGET diigo fetches; incomplete results say so, and a reload continues", async () => { - const res = await call("/_wire", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }), - }); - expect(res.status).toBe(200); - // deep_dee: 60 diigo pages of 100; a search matching nothing must walk them all + it("a filter reads only stored items, never diigo; results say whether storage is complete", async () => { + const { page, pub } = await wireStored("fake-deep-webapp", 1); + const json = async () => + (await call(`${page}?days=300`, { headers: { accept: "application/json" } })).json() as Promise<{ + matched: number; + complete: boolean; + }>; const before = await testEnv.WEBAPP.calls(); - const first = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } }); - expect(await first.json()).toMatchObject({ complete: false, items: [] }); - expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(FETCH_BUDGET + 1); - const html = await (await call("/public/deep_dee?q=nomatch")).text(); - expect(html).toMatch(/complete|no matches/); - const again = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } }); - expect(await again.json()).toMatchObject({ complete: true, items: [] }); + // page 1 holds 1..100, minus 10 private + expect(await json()).toMatchObject({ matched: 90, complete: false }); + expect(await (await call(`${page}?days=300`)).text()).toMatch(/some matches may be missing/); + expect(await testEnv.WEBAPP.calls()).toBe(before); + await pub.page(2, PAGE_SIZE); + await pub.page(3, PAGE_SIZE); + expect(await json()).toMatchObject({ matched: 270, complete: true }); }); it("JSON answers carry the canonical filter, so devs see what was understood", async () => { @@ -393,25 +398,6 @@ describe.skip("public page filters (goal 1d)", () => { }); }); - it("?in= an old week costs O(log pages) diigo fetches with by=updated, not a walk from page 1", async () => { - const res = await call("/_wire", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }), - }); - expect(res.status).toBe(200); - // deep_dee: 6000 bookmarks, one a day, 60 diigo pages of 100; a week - // ~14 years back sits near page 51 — a walk from page 1 would cost ~51 - const longAgo = new Date(Date.now() - 5100 * 24 * 60 * 60 * 1000); - const week = isoWeek(longAgo); - const before = await testEnv.WEBAPP.calls(); - const html = await (await call(`/public/deep_dee?in=${week}`)).text(); - // one a day; every 10th is private - expect(titlesIn(html).length).toBeGreaterThanOrEqual(6); - expect(titlesIn(html).length).toBeLessThanOrEqual(7); - expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(12); - }); - it("a malformed filter is a 400 with a message, not an empty list", async () => { const page = await wireDated(); const res = await call(`${page}?days=7x`); diff --git a/diigo-cache/tests/fake-webapp.js b/diigo-cache/tests/fake-webapp.js index 7badf0b..483a0aa 100644 --- a/diigo-cache/tests/fake-webapp.js +++ b/diigo-cache/tests/fake-webapp.js @@ -18,9 +18,9 @@ const OVERSTATEMENT = 60; * 5th an old bookmark edited recently (created 100 days before updated) */ const DATED_WEBAPP = "fake-dated-webapp"; const DATED_TOTAL = 60; -/** the same shape, 6000 deep (~16 years, 60 pages of 100): for walk cost */ +/** the same shape, 300 deep (3 pages of 100): for paging matches */ const DEEP_WEBAPP = "fake-deep-webapp"; -const DEEP_TOTAL = 6000; +const DEEP_TOTAL = 300; const HOUR = 60 * 60 * 1000; function datedItem(n, now, user = "dated_dan") { diff --git a/diigo-cache/tests/query.test.ts b/diigo-cache/tests/query.test.ts index 886627e..0fbebe8 100644 --- a/diigo-cache/tests/query.test.ts +++ b/diigo-cache/tests/query.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { matches, parseQuery, toQuery } from "../src/query.js"; import type { FilterSpec, Query, QueryError } from "../src/query.js"; -// goal 1d (capsite checklist): the design as tests; unskip as it lands +// goal 1d (capsite checklist) const NOW = new Date("2026-10-07T12:00:00.000Z"); const DAY = 24 * 60 * 60 * 1000; @@ -21,7 +21,7 @@ const refused = (q: Query | QueryError) => { const day = (iso: string) => new Date(`${iso}T00:00:00.000Z`); const range = (q: Query) => [q.after?.toISOString().slice(0, 10), q.before?.toISOString().slice(0, 10)]; -describe.skip("filters: one spec, two spellings (goal 1d)", () => { +describe("filters: one spec, two spellings (goal 1d)", () => { it("devs write { days: 7 }; URLs write ?days=7; same query", () => { expect(spec({ days: 7 })).toEqual(url("days=7")); expect(spec({ days: 7 }).after).toEqual(new Date(NOW.getTime() - 7 * DAY)); @@ -42,7 +42,7 @@ describe.skip("filters: one spec, two spellings (goal 1d)", () => { }); }); -describe.skip("windows: UTC, half-open (goal 1d)", () => { +describe("windows: UTC, half-open (goal 1d)", () => { it("after/before are UTC days: after inclusive from 00:00Z, before exclusive", () => { expect(range(url("after=2026-01-01&before=2026-02-01"))).toEqual(["2026-01-01", "2026-02-01"]); expect(url("after=2026/01/01").after).toEqual(day("2026-01-01")); @@ -88,7 +88,7 @@ describe.skip("windows: UTC, half-open (goal 1d)", () => { }); }); -describe.skip("q: terms and operators (goal 1d)", () => { +describe("q: terms and operators (goal 1d)", () => { const q = (text: string) => url(`q=${encodeURIComponent(text)}`); it("words, phrases, #tags, tag:, and - negation; case-insensitive", () => { @@ -160,7 +160,7 @@ const item = (iso: string, extra: Record = {}) => ({ }); const query = (q: Partial): Query => ({ by: "updated", words: [], tags: [], ...q }); -describe.skip("matches (goal 1d)", () => { +describe("matches (goal 1d)", () => { it("the window is half-open: after inclusive, before exclusive", () => { const q = query({ after: day("2026-01-01"), before: day("2026-02-01") }); expect(matches(item("2026-01-01T00:00:00.000Z"), q)).toBe(true);