diff --git a/diigo-cache/src/query.ts b/diigo-cache/src/query.ts new file mode 100644 index 0000000..596cb1c --- /dev/null +++ b/diigo-cache/src/query.ts @@ -0,0 +1,107 @@ +/** + * @file the public page's filters — goal 1d, **not yet implemented**: the + * design is pinned by the skipped suites in tests/query.test.ts and + * tests/caches.test.ts ("public page filters"). + * + * One filter, two spellings: devs pass a {@link FilterSpec} object + * (`{ days: 7, q: "#weekly" }`); URLs carry the same fields as params + * (`?days=7&q=%23weekly`, empty fields = absent). `q` also takes the + * search operators large audiences already know — gmail's `after:`, + * `before:`, `newer_than:7d`; Slack's `during:` (alias `in:`); GitHub's + * `created:`/`updated:` — plus words, `"phrases"`, `#tag`/`tag:tag`, and + * `-` negation. + * + * Time is UTC (Z) throughout; weeks are ISO 8601 (Monday first, so a + * weekend stays together). Every window is half-open: `after` inclusive + * from 00:00Z, `before` exclusive at 00:00Z — so GitHub's inclusive + * `a..b` ranges are deliberately not offered. + * + * Applied while walking diigo's pages (updated-desc). `by=updated` finds + * an old window by binary search over the sorted pages; `by=created` + * walks from page 1 (an old bookmark edited lately sits near page 1). + * Either way a request spends at most {@link FETCH_BUDGET} diigo fetches + * (cached pages are free) and says so when results are incomplete. + * + * @see https://mailmeteor.com/blog/search-gmail-by-date + * @see https://slack.com/help/articles/202528808 + * @see https://docs.github.com/en/search-github/searching-on-github/searching-for-repositories + */ + +import type { Item } from "./items.js"; + +/** + * diigo fetches one request may spend (cache hits are free); under the + * Workers free plan's 50 external subrequests per invocation + * + * @see https://developers.cloudflare.com/workers/platform/limits/ + */ +export const FETCH_BUDGET = 40; + +/** + * a filter as devs write it; URL params map onto these fields one-to-one + * (strings, with "" meaning absent). Dates are `yyyy-mm-dd` (gmail's + * `yyyy/mm/dd` too); periods are `2025`, `2025-03`, `2025-Q1`, or ISO week + * `2025-W01`. + */ +export type FilterSpec = { + /** the last N days, counted back from now (1–3650) */ + days?: number | string; + /** inclusive lower bound, from 00:00Z */ + after?: string; + /** exclusive upper bound, at 00:00Z */ + before?: string; + /** a named period: shorthand for its after/before pair */ + in?: string; + /** which date the window, the listed date, and the sort use */ + by?: "updated" | "created" | ""; + /** words, phrases, tags, and operators */ + q?: string; +}; + +/** one search term; `negated` for a leading `-` */ +export type Term = { text: string; negated: boolean }; + +/** a parsed filter; an empty one (no window, no terms) filters nothing */ +export type Query = { + /** which date the window, the listed date, and the sort use (default + * `updated`, diigo's own order) */ + by: "updated" | "created"; + /** inclusive lower bound */ + after?: Date; + /** exclusive upper bound */ + before?: Date; + /** words and phrases: each must appear in title, description, or url */ + words: Term[]; + /** tags: exact, case-insensitive match against the item's tags */ + tags: Term[]; +}; + +/** a refused filter: the message is safe to show and log */ +export type QueryError = { error: string }; + +/** + * resolve a filter: fields and `q` operators combine; a bound given twice + * (by fields, operators, or both) is refused, as is an empty window. + * + * @param now the clock `days`/`newer_than` count back from (passed in, + * not read ambiently, so tests pin it) + */ +export function toQuery(spec: FilterSpec, now: Date): Query | QueryError { + void spec; + void now; + throw new Error("toQuery: not implemented (goal 1d)"); +} + +/** {@link toQuery} over the public page's URL params */ +export function parseQuery(params: URLSearchParams, now: Date): Query | QueryError { + void params; + void now; + throw new Error("parseQuery: not implemented (goal 1d)"); +} + +/** does `item` fall in the window and satisfy every term? */ +export function matches(item: Item, query: Query): boolean { + void item; + void query; + throw new Error("matches: not implemented (goal 1d)"); +} diff --git a/diigo-cache/tests/caches.test.ts b/diigo-cache/tests/caches.test.ts index b406de6..fe01bf7 100644 --- a/diigo-cache/tests/caches.test.ts +++ b/diigo-cache/tests/caches.test.ts @@ -3,6 +3,7 @@ import { runInDurableObject } from "cloudflare:test"; import { describe, expect, it, vi } from "vitest"; import worker from "../src/index.js"; import { toWebkey } from "../src/webkeys.js"; +import { FETCH_BUDGET } from "../src/query.js"; // matches tests/fake-webapp.js: 30 items, every third private const WEBAPP_ID = "fake-webapp-id"; @@ -139,3 +140,155 @@ describe("worker routes", () => { expect(res.status).toBe(404); }); }); + +/** `yyyy-Www` for the ISO 8601 week holding `d` (UTC) */ +function isoWeek(d: Date): string { + const t = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())); + const day = t.getUTCDay() || 7; + t.setUTCDate(t.getUTCDate() + 4 - day); // the week's Thursday names its year + const year = t.getUTCFullYear(); + const week = Math.ceil(((t.getTime() - Date.UTC(year, 0, 1)) / 86400000 + 1) / 7); + return `${year}-W${String(week).padStart(2, "0")}`; +} + +describe.skip("public page filters (goal 1d)", () => { + // the fake's dated account: "dated n" updated n-1 days + 1h ago; + // #weekly on every 7th; every 10th private; every 5th created 100 + // days before it was updated + const call = (path: string, init?: RequestInit) => + worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never); + const wireDated = async () => { + const res = await call("/_wire", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ webappId: "fake-dated-webapp", mode: "public" }), + }); + expect(res.status).toBe(200); + return "/public/dated_dan"; + }; + const titlesIn = (html: string) => [...html.matchAll(/>(dated \d+)<\/a>/g)].map((m) => m[1]); + const dated = (...ns: number[]) => ns.map((n) => `dated ${n}`); + + it("?days=7 lists the last week, walking only the diigo pages it needs", async () => { + const page = await wireDated(); + const before = await testEnv.WEBAPP.calls(); + const html = await (await call(`${page}?days=7`)).text(); + expect(titlesIn(html)).toEqual(dated(1, 2, 3, 4, 5, 6, 7)); + expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(1); + }); + + it("?by=created windows, lists, and dates by when it was bookmarked", async () => { + const page = await wireDated(); + const html = await (await call(`${page}?days=7&by=created`)).text(); + // 5 was updated this week but bookmarked 100 days earlier + expect(titlesIn(html)).toEqual(dated(1, 2, 3, 4, 6, 7)); + const updated = await (await call(`${page}?days=7`)).text(); + expect(titlesIn(updated)).toContain("dated 5"); + }); + + it("?by=created over a longer window sorts by created, newest first", async () => { + const page = await wireDated(); + const html = await (await call(`${page}?days=150&by=created&q=${encodeURIComponent("#weekly")}`)).text(); + // weekly: 7,14,21,28,35,42,49,56; 35 was created ~134 days ago + // (100 days before its update), so it sorts after 56 (~55 days) + expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 42, 49, 56, 35)); + }); + + it("?days=30 pages the matches 24 at a time; links keep the filter", async () => { + const page = await wireDated(); + const first = await (await call(`${page}?days=30`)).text(); + const want = Array.from({ length: 30 }, (_, k) => k + 1).filter((n) => n % 10 !== 0); + expect(titlesIn(first)).toEqual(dated(...want.slice(0, 24))); + expect(first).toContain(`href="${page}?days=30&page=2"`); + const second = await (await call(`${page}?days=30&page=2`)).text(); + expect(titlesIn(second)).toEqual(dated(...want.slice(24))); + }); + + it("?after=&before= selects a date range", async () => { + const page = await wireDated(); + const day = (n: number) => new Date(Date.now() - (n - 1) * 24 * 60 * 60 * 1000 - 60 * 60 * 1000).toISOString().slice(0, 10); + // items are 24h apart, so each has its own UTC day: from dated 12's + // day (inclusive) up to dated 9's day (exclusive); 10 is private + const html = await (await call(`${page}?after=${day(12)}&before=${day(9)}`)).text(); + expect(titlesIn(html)).toEqual(dated(11, 12)); + }); + + it("?q=#weekly&days=60 filters by tag; private items stay out", async () => { + const page = await wireDated(); + const html = await (await call(`${page}?days=60&q=${encodeURIComponent("#weekly")}`)).text(); + expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 35, 42, 49, 56)); + }); + + it("?q= alone searches all time", async () => { + const page = await wireDated(); + const html = await (await call(`${page}?q=fragments`)).text(); + expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 35, 42, 49, 56)); + }); + + it("a request spends at most FETCH_BUDGET diigo fetches; incomplete results say so, and a reload continues", async () => { + const res = await call("/_wire", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }), + }); + expect(res.status).toBe(200); + // deep_dee: 50 diigo pages; a search matching nothing must walk them all + const before = await testEnv.WEBAPP.calls(); + const first = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } }); + expect(await first.json()).toMatchObject({ complete: false, items: [] }); + expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(FETCH_BUDGET + 1); + const html = await (await call("/public/deep_dee?q=nomatch")).text(); + expect(html).toMatch(/complete|no matches/); + const again = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } }); + expect(await again.json()).toMatchObject({ complete: true, items: [] }); + }); + + it("JSON answers carry the canonical filter, so devs see what was understood", async () => { + const page = await wireDated(); + const res = await call(`${page}?days=7&q=${encodeURIComponent("#Weekly")}`, { headers: { accept: "application/json" } }); + expect(await res.json()).toMatchObject({ + filter: { by: "updated", tags: [{ text: "weekly", negated: false }] }, + complete: true, + }); + }); + + it("?in= an old week costs O(log pages) diigo fetches with by=updated, not a walk from page 1", async () => { + const res = await call("/_wire", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }), + }); + expect(res.status).toBe(200); + // deep_dee: 1200 bookmarks, one a day, 50 diigo pages; a week ~2.5 + // years back sits near page 38 — a walk from page 1 would cost ~38 + const twoAndAHalfYearsAgo = new Date(Date.now() - 900 * 24 * 60 * 60 * 1000); + const week = isoWeek(twoAndAHalfYearsAgo); + const before = await testEnv.WEBAPP.calls(); + const html = await (await call(`/public/deep_dee?in=${week}`)).text(); + // one a day; every 10th is private + expect(titlesIn(html).length).toBeGreaterThanOrEqual(6); + expect(titlesIn(html).length).toBeLessThanOrEqual(7); + expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(12); + }); + + it("a malformed filter is a 400 with a message, not an empty list", async () => { + const page = await wireDated(); + const res = await call(`${page}?days=7x`); + expect(res.status).toBe(400); + expect(await res.text()).toMatch(/days/); + }); + + it("the form offers the day presets, date inputs, and search, echoing the current filter", async () => { + const page = await wireDated(); + const html = await (await call(`${page}?days=30&q=ocap`)).text(); + expect(html).toMatch(//); + for (const n of [7, 30, 90, 365]) expect(html).toContain(`