diigo-cache: date windows and search on the public page (1d)
/public/<username> takes days, after, before, in (year, month, quarter, ISO week), by=updated|created, and q (words, "phrases", #tags, -negation, gmail/Slack/GitHub date operators). JSON answers echo the parsed filter; a malformed one is a 400. Filters query the stored items in sqlite through the read-only viewer instead of walking diigo pages: the public page keeps no fetch authority, and a filled cache already holds every item. Results say `complete` when every diigo page is stored; freshness is 1f's head refresh. So the fetch budget and the O(log pages) walk test go, and the fake deep account shrinks to 300. Generated-by: Claude Opus 5.5
This commit is contained in:
1 parent
efb54ad4cc
commit
002f08b456
9 files changed
+418
-120
No files matched your search
+6
-10
@@ -42,20 +42,16 @@ narrative.
|
|||||||
2026-10-07, via `scripts/compare-export.ts`. Getting there
|
2026-10-07, via `scripts/compare-export.ts`. Getting there
|
||||||
found diigo's `page_num` is 0-based (schema v2 migration).
|
found diigo's `page_num` is 0-based (schema v2 migration).
|
||||||
|
|
||||||
- [ ] **1d. Date windows and search on the public page**, toward the
|
- [x] **1d. Date windows and search on the public page**, toward the
|
||||||
weekly PESOS roundup of [#23](https://github.com/dckc/madmode-blog/issues/23)
|
weekly PESOS roundup of [#23](https://github.com/dckc/madmode-blog/issues/23)
|
||||||
(a week's permalink: `in=2026-W41`). One filter, two spellings:
|
(a week's permalink: `in=2026-W41`). One filter, two spellings:
|
||||||
devs pass `{ days: 7, q: "#weekly" }`, URLs the same fields;
|
devs pass `{ days: 7, q: "#weekly" }`, URLs the same fields;
|
||||||
`q` takes gmail/Slack/GitHub operators. UTC, ISO weeks, half-open
|
`q` takes gmail/Slack/GitHub operators. UTC, ISO weeks, half-open
|
||||||
windows; a fetch budget per request. Design: the API in
|
windows (`diigo-cache/src/query.ts`; `tests/query.test.ts`,
|
||||||
`diigo-cache/src/query.ts` and the skipped suites in
|
`tests/caches.test.ts` "public page filters"). Filters read
|
||||||
`tests/query.test.ts` and `tests/caches.test.ts` ("public page
|
stored `items` only (the public page never fetches): results are
|
||||||
filters"); done when they're unskipped and green.
|
as fresh as the cache, and say `complete` when every diigo page
|
||||||
Reconcile with 1f first: `query.ts` plans to filter while walking
|
is stored. Keeping them fresh is 1f. 2026-10-09.
|
||||||
diigo's pages (binary search by `updated`), but after 1c-vi a
|
|
||||||
filled cache holds every item, so 1d can query `items` in sqlite
|
|
||||||
and spend diigo fetches only on a head refresh (1f). Choose
|
|
||||||
before implementing; the skipped suites may need adjusting.
|
|
||||||
|
|
||||||
- [x] **1e. Fill job: a button that grabs every page** into a cache
|
- [x] **1e. Fill job: a button that grabs every page** into a cache
|
||||||
from the webkey page `/cache/#<key>`, which trades the key for a
|
from the webkey page `/cache/#<key>`, which trades the key for a
|
||||||
|
|||||||
@@ -19,3 +19,14 @@ where cast(items.updated as integer) >= ?;
|
|||||||
|
|
||||||
-- pagesThrough:
|
-- pagesThrough:
|
||||||
select count(*) as n from pages where page_num <= ?;
|
select count(*) as n from pages where page_num <= ?;
|
||||||
|
|
||||||
|
-- itemsByUpdated:
|
||||||
|
select json from items
|
||||||
|
where cast(updated as integer) >= ? and cast(updated as integer) < ?
|
||||||
|
order by cast(updated as integer) desc, item_id;
|
||||||
|
|
||||||
|
-- itemsByCreated:
|
||||||
|
select json from items
|
||||||
|
where cast(json_extract(json, '$.created_at') as integer) >= ?
|
||||||
|
and cast(json_extract(json, '$.created_at') as integer) < ?
|
||||||
|
order by cast(json_extract(json, '$.created_at') as integer) desc, item_id;
|
||||||
@@ -6,6 +6,8 @@
|
|||||||
import { DurableObject, RpcTarget } from "cloudflare:workers";
|
import { DurableObject, RpcTarget } from "cloudflare:workers";
|
||||||
import type { PagedGet } from "./diigo.js";
|
import type { PagedGet } from "./diigo.js";
|
||||||
import { publicOnly, toRows } from "./items.js";
|
import { publicOnly, toRows } from "./items.js";
|
||||||
|
import { matches } from "./query.js";
|
||||||
|
import type { Query } from "./query.js";
|
||||||
import { makeReader, makeWriter, SCHEMA_VERSION } from "./store.js";
|
import { makeReader, makeWriter, SCHEMA_VERSION } from "./store.js";
|
||||||
import type { Reader, Writer } from "./store.js";
|
import type { Reader, Writer } from "./store.js";
|
||||||
import type { Item, ItemRow } from "./items.js";
|
import type { Item, ItemRow } from "./items.js";
|
||||||
@@ -398,7 +400,18 @@ abstract class PagedCache extends DurableObject<CacheEnv> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** what anyone may see of a cache: stored pages and fill progress */
|
/** one page of a filter's matches over stored items */
|
||||||
|
export type QueryPage = {
|
||||||
|
/** the matches on this page, raw dump rows (JSON strings) */
|
||||||
|
items: string[];
|
||||||
|
/** matches in all */
|
||||||
|
matched: number;
|
||||||
|
/** true when every diigo page is stored, so no match can be missing */
|
||||||
|
complete: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
/** what anyone may see of a cache: stored pages, stored items by filter,
|
||||||
|
* and fill progress */
|
||||||
export class PageViewer extends RpcTarget {
|
export class PageViewer extends RpcTarget {
|
||||||
#reader: Reader;
|
#reader: Reader;
|
||||||
#fillStatus: () => Promise<FillStatus | undefined>;
|
#fillStatus: () => Promise<FillStatus | undefined>;
|
||||||
@@ -415,6 +428,25 @@ export class PageViewer extends RpcTarget {
|
|||||||
return stored && { ...stored, cached: true };
|
return stored && { ...stored, cached: true };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* page `pageNum` (`count` per page) of the stored items matching
|
||||||
|
* `query`, newest first by its date. The window narrows in SQL; terms
|
||||||
|
* are checked per item. Reads storage only: items not yet stored can't
|
||||||
|
* match, which `complete` reports.
|
||||||
|
*/
|
||||||
|
query(query: Query, pageNum: number, count: number): QueryPage {
|
||||||
|
const from = query.after ? Math.ceil(query.after.getTime() / 1000) : Number.MIN_SAFE_INTEGER;
|
||||||
|
const to = query.before ? Math.ceil(query.before.getTime() / 1000) : Number.MAX_SAFE_INTEGER;
|
||||||
|
const hits = this.#reader.itemsIn(query.by, from, to).filter((json) => matches(JSON.parse(json) as Item, query));
|
||||||
|
const total = this.#reader.knownTotal();
|
||||||
|
const pages = total === null ? null : Math.max(1, Math.ceil(total / PAGE_SIZE));
|
||||||
|
return {
|
||||||
|
items: hits.slice((pageNum - 1) * count, pageNum * count),
|
||||||
|
matched: hits.length,
|
||||||
|
complete: pages !== null && this.#reader.pagesThrough(pages) >= pages,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/** diigo's total, as last stored, or null if nothing is stored */
|
/** diigo's total, as last stored, or null if nothing is stored */
|
||||||
knownTotal(): number | null {
|
knownTotal(): number | null {
|
||||||
return this.#reader.knownTotal();
|
return this.#reader.knownTotal();
|
||||||
|
|||||||
+131
-33
@@ -10,9 +10,11 @@ import { CAPNWEB_BROWSER_JS } from "./capnweb-browser.js";
|
|||||||
import { CACHE_PAGE_HTML } from "./cache-page.js";
|
import { CACHE_PAGE_HTML } from "./cache-page.js";
|
||||||
import { DiigoCacheLogic, PublicCacheLogic } from "./cache.js";
|
import { DiigoCacheLogic, PublicCacheLogic } from "./cache.js";
|
||||||
import { PAGE_SIZE } from "./cache.js";
|
import { PAGE_SIZE } from "./cache.js";
|
||||||
import type { CacheEnv, Clock, FillStatus } from "./cache.js";
|
import type { CacheEnv, Clock, FillStatus, QueryPage } from "./cache.js";
|
||||||
import type { Item } from "./cache.js";
|
import type { Item } from "./cache.js";
|
||||||
import { updatedISO, usernameOf } from "./items.js";
|
import { usernameOf } from "./items.js";
|
||||||
|
import { dateOf, isEmpty, parseQuery } from "./query.js";
|
||||||
|
import type { Query } from "./query.js";
|
||||||
import { toWebkey, idFromWebkey, isWebkey, shashKey } from "./webkeys.js";
|
import { toWebkey, idFromWebkey, isWebkey, shashKey } from "./webkeys.js";
|
||||||
import { pmSave } from "./pm-save.js";
|
import { pmSave } from "./pm-save.js";
|
||||||
|
|
||||||
@@ -443,10 +445,16 @@ async function dump(request: Request, env: Env): Promise<Response> {
|
|||||||
|
|
||||||
const PUBLIC_PER_PAGE = PAGE_SIZE;
|
const PUBLIC_PER_PAGE = PAGE_SIZE;
|
||||||
|
|
||||||
|
/** the public page's day presets */
|
||||||
|
const DAY_PRESETS = [7, 30, 90, 365];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* the public page for a username: stored pages only, through the cache's
|
* the public page for a username: stored pages only, through the cache's
|
||||||
* read-only `viewer()` (it can't fetch or write). HTML with a page-number
|
* read-only `viewer()` (it can't fetch or write). HTML with a filter form
|
||||||
* form, or JSON for `Accept: application/json`. `?page=` in the URL is
|
* and a page-number form, or JSON for `Accept: application/json`. A
|
||||||
|
* filter (goal 1d: `days`, `after`, `before`, `in`, `by`, `q`; see
|
||||||
|
* {@link parseQuery}) lists the stored items that match, `page` counting
|
||||||
|
* matches; with none, `page` is diigo's page as stored. Query params are
|
||||||
* fine here: nothing on this page is secret.
|
* fine here: nothing on this page is secret.
|
||||||
*/
|
*/
|
||||||
async function publicPage(request: Request, env: Env, username: string): Promise<Response> {
|
async function publicPage(request: Request, env: Env, username: string): Promise<Response> {
|
||||||
@@ -456,14 +464,33 @@ async function publicPage(request: Request, env: Env, username: string): Promise
|
|||||||
const ns = env.PUBLIC_DO;
|
const ns = env.PUBLIC_DO;
|
||||||
const cache = ns.get(ns.idFromString(idFromWebkey(publicId)));
|
const cache = ns.get(ns.idFromString(idFromWebkey(publicId)));
|
||||||
|
|
||||||
const raw = new URL(request.url).searchParams.get("page");
|
const params = new URL(request.url).searchParams;
|
||||||
|
const accept = request.headers.get("accept") ?? "";
|
||||||
|
const wantsJson = accept.includes("application/json") && !accept.includes("text/html");
|
||||||
|
const here = `/public/${encodeURIComponent(username)}`;
|
||||||
|
const raw = params.get("page");
|
||||||
const pageN = raw === null ? 1 : Number.parseInt(raw, 10);
|
const pageN = raw === null ? 1 : Number.parseInt(raw, 10);
|
||||||
const valid = !Number.isNaN(pageN) && pageN >= 1 && pageN <= 10000;
|
const valid = !Number.isNaN(pageN) && pageN >= 1 && pageN <= 10000;
|
||||||
|
|
||||||
|
const query = parseQuery(params, new Date());
|
||||||
|
if ("error" in query) {
|
||||||
|
console.log({ event: "notice", page: "public", text: "filter refused", error: query.error });
|
||||||
|
if (wantsJson) return Response.json({ error: query.error }, { status: 400 });
|
||||||
|
const body = publicFrame(username, `${filterForm(here, params)}
|
||||||
|
<p><em>${esc(query.error)}</em></p>`);
|
||||||
|
return new Response(body, { status: 400, headers: { "content-type": "text/html;charset=utf-8" } });
|
||||||
|
}
|
||||||
|
|
||||||
const viewer = await cache.viewer();
|
const viewer = await cache.viewer();
|
||||||
let page, total, fill;
|
let page, fill, found;
|
||||||
|
let total: number | null = null;
|
||||||
try {
|
try {
|
||||||
page = await viewer.page(valid ? pageN : 1, PUBLIC_PER_PAGE);
|
if (isEmpty(query)) {
|
||||||
total = page ? page.total : await viewer.knownTotal();
|
page = await viewer.page(valid ? pageN : 1, PUBLIC_PER_PAGE);
|
||||||
|
total = page ? page.total : await viewer.knownTotal();
|
||||||
|
} else {
|
||||||
|
found = await viewer.query(query, valid ? pageN : 1, PUBLIC_PER_PAGE);
|
||||||
|
}
|
||||||
fill = await viewer.fillStatus();
|
fill = await viewer.fillStatus();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error({ event: "public-page", page: pageN, error: String(err) });
|
console.error({ event: "public-page", page: pageN, error: String(err) });
|
||||||
@@ -471,13 +498,14 @@ async function publicPage(request: Request, env: Env, username: string): Promise
|
|||||||
} finally {
|
} finally {
|
||||||
viewer[Symbol.dispose]();
|
viewer[Symbol.dispose]();
|
||||||
}
|
}
|
||||||
|
if (found) return filteredPage(username, here, params, query, found, valid ? pageN : 1, wantsJson, fill);
|
||||||
|
|
||||||
// total and pages count all of diigo's items: public page n is diigo
|
// total and pages count all of diigo's items: public page n is diigo
|
||||||
// page n with the private rows dropped
|
// page n with the private rows dropped
|
||||||
const pages = total === null ? 1 : Math.max(1, Math.ceil(total / PUBLIC_PER_PAGE));
|
const pages = total === null ? 1 : Math.max(1, Math.ceil(total / PUBLIC_PER_PAGE));
|
||||||
const items = page ? page.items.map((j) => JSON.parse(j) as Item) : [];
|
const items = page ? page.items.map((j) => JSON.parse(j) as Item) : [];
|
||||||
|
|
||||||
const accept = request.headers.get("accept") ?? "";
|
if (wantsJson) {
|
||||||
if (accept.includes("application/json") && !accept.includes("text/html")) {
|
|
||||||
return Response.json({ username, page: pageN, pages, total, stored: page !== null, etchedAt: page?.etchedAt ?? null, items });
|
return Response.json({ username, page: pageN, pages, total, stored: page !== null, etchedAt: page?.etchedAt ?? null, items });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -486,42 +514,112 @@ async function publicPage(request: Request, env: Env, username: string): Promise
|
|||||||
console.log({ event: "notice", page: "public", text: "page out of range", pages });
|
console.log({ event: "notice", page: "public", text: "page out of range", pages });
|
||||||
notice = `<p><em>no page ${esc(String(raw))}: pages run 1–${pages}</em></p>`;
|
notice = `<p><em>no page ${esc(String(raw))}: pages run 1–${pages}</em></p>`;
|
||||||
}
|
}
|
||||||
const here = `/public/${encodeURIComponent(username)}`;
|
|
||||||
const nav = (n: number, label: string) =>
|
const nav = (n: number, label: string) =>
|
||||||
n >= 1 && n <= pages ? `<a href="${here}?page=${n}">${label}</a>` : `<span>${label}</span>`;
|
n >= 1 && n <= pages ? `<a href="${here}?page=${n}">${label}</a>` : `<span>${label}</span>`;
|
||||||
const list = notice
|
const list = notice ? "" : itemList(items, "updated");
|
||||||
? ""
|
const body = publicFrame(
|
||||||
: `<ol>
|
username,
|
||||||
${items
|
`<p>${pages} page${pages === 1 ? "" : "s"} (diigo's, ${PUBLIC_PER_PAGE} bookmarks each; private ones are left out, so a page may show fewer).</p>
|
||||||
.map((it) => {
|
|
||||||
const url = typeof it.url === "string" ? it.url : "";
|
|
||||||
const title = typeof it.title === "string" && it.title ? it.title : url;
|
|
||||||
const desc = typeof it.description === "string" && it.description ? `<br><small>${esc(it.description)}</small>` : "";
|
|
||||||
const href = /^https?:\/\//i.test(url) ? url : "#";
|
|
||||||
const when = updatedISO(it);
|
|
||||||
const date = when ? `<time datetime="${when}">${when}</time>` : "(no date)";
|
|
||||||
return `<li>${date} <a href="${esc(href)}">${esc(title)}</a>${desc}</li>`;
|
|
||||||
})
|
|
||||||
.join("\n")}
|
|
||||||
</ol>`;
|
|
||||||
const body = `<!doctype html>
|
|
||||||
<html><head><meta charset="utf-8"><title>${esc(username)}'s shared bookmarks</title></head>
|
|
||||||
<body>
|
|
||||||
<h1>${esc(username)}'s shared bookmarks</h1>
|
|
||||||
<p>${pages} page${pages === 1 ? "" : "s"} (diigo's, ${PUBLIC_PER_PAGE} bookmarks each; private ones are left out, so a page may show fewer).</p>
|
|
||||||
<p>${page ? `<strong>stored ${esc(page.etchedAt ?? "")}</strong> — page ${valid ? pageN : 1}, ${items.length} shared` : `<strong>not fetched yet</strong> — page ${valid ? pageN : 1}; the cache's owner fills it`}</p>
|
<p>${page ? `<strong>stored ${esc(page.etchedAt ?? "")}</strong> — page ${valid ? pageN : 1}, ${items.length} shared` : `<strong>not fetched yet</strong> — page ${valid ? pageN : 1}; the cache's owner fills it`}</p>
|
||||||
${fill ? fillLine(fill) : ""}
|
${fill ? fillLine(fill) : ""}
|
||||||
|
${filterForm(here, params)}
|
||||||
<form method="get" action="${here}">
|
<form method="get" action="${here}">
|
||||||
<p>${nav(pageN - 1, "← prev")}
|
<p>${nav(pageN - 1, "← prev")}
|
||||||
<label>page <input type="number" name="page" min="1" value="${Number.isNaN(pageN) ? 1 : pageN}"></label>
|
<label>page <input type="number" name="page" min="1" value="${Number.isNaN(pageN) ? 1 : pageN}"></label>
|
||||||
<button type="submit">go</button>
|
<button type="submit">go</button>
|
||||||
${nav(pageN + 1, "next →")}</p>
|
${nav(pageN + 1, "next →")}</p>
|
||||||
</form>
|
</form>
|
||||||
${notice}${list}
|
${notice}${list}`,
|
||||||
</body></html>`;
|
);
|
||||||
return new Response(body, { headers: { "content-type": "text/html;charset=utf-8" } });
|
return new Response(body, { headers: { "content-type": "text/html;charset=utf-8" } });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** a filter's matches, as the public page's HTML or JSON */
|
||||||
|
function filteredPage(
|
||||||
|
username: string,
|
||||||
|
here: string,
|
||||||
|
params: URLSearchParams,
|
||||||
|
query: Query,
|
||||||
|
found: QueryPage,
|
||||||
|
pageN: number,
|
||||||
|
wantsJson: boolean,
|
||||||
|
fill: FillStatus | undefined,
|
||||||
|
): Response {
|
||||||
|
const pages = Math.max(1, Math.ceil(found.matched / PUBLIC_PER_PAGE));
|
||||||
|
const items = found.items.map((j) => JSON.parse(j) as Item);
|
||||||
|
if (wantsJson) {
|
||||||
|
const { matched, complete } = found;
|
||||||
|
return Response.json({ username, filter: query, page: pageN, pages, matched, complete, items });
|
||||||
|
}
|
||||||
|
// links keep the filter: the request's own non-empty params, in order
|
||||||
|
const kept = [...params].filter(([k, v]) => k !== "page" && v !== "");
|
||||||
|
const href = (n: number) => `${here}?${new URLSearchParams([...kept, ["page", String(n)]])}`;
|
||||||
|
const nav = (n: number, label: string) =>
|
||||||
|
n >= 1 && n <= pages ? `<a href="${esc(href(n))}">${label}</a>` : `<span>${label}</span>`;
|
||||||
|
const status = found.matched
|
||||||
|
? `${found.matched} match${found.matched === 1 ? "" : "es"}, page ${pageN} of ${pages}, newest ${query.by} first`
|
||||||
|
: "no matches";
|
||||||
|
const caveat = found.complete ? "" : " — not every diigo page is stored yet, so some matches may be missing";
|
||||||
|
const body = publicFrame(
|
||||||
|
username,
|
||||||
|
`<p><strong>${esc(status)}</strong>${caveat}</p>
|
||||||
|
${fill ? fillLine(fill) : ""}
|
||||||
|
${filterForm(here, params)}
|
||||||
|
<p>${nav(pageN - 1, "← prev")} ${nav(pageN + 1, "next →")} <a href="${here}">all, by diigo page</a></p>
|
||||||
|
${itemList(items, query.by)}`,
|
||||||
|
);
|
||||||
|
return new Response(body, { headers: { "content-type": "text/html;charset=utf-8" } });
|
||||||
|
}
|
||||||
|
|
||||||
|
/** the public page's frame: title and heading around `content` */
|
||||||
|
function publicFrame(username: string, content: string): string {
|
||||||
|
return `<!doctype html>
|
||||||
|
<html><head><meta charset="utf-8"><title>${esc(username)}'s shared bookmarks</title></head>
|
||||||
|
<body>
|
||||||
|
<h1>${esc(username)}'s shared bookmarks</h1>
|
||||||
|
${content}
|
||||||
|
</body></html>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** the filter form, echoing the request's filter */
|
||||||
|
function filterForm(here: string, params: URLSearchParams): string {
|
||||||
|
const field = (name: string) => esc(params.get(name) ?? "");
|
||||||
|
const days = params.get("days") ?? "";
|
||||||
|
const presets = DAY_PRESETS.map(String);
|
||||||
|
if (days && !presets.includes(days)) presets.push(days);
|
||||||
|
const option = (value: string, label: string, current: string) =>
|
||||||
|
`<option value="${esc(value)}"${value === current ? " selected" : ""}>${esc(label)}</option>`;
|
||||||
|
return `<form method="get" action="${here}">
|
||||||
|
<p><label>last <select name="days">${option("", "any", days)}${presets.map((n) => option(n, `${n} days`, days)).join("")}</select></label>
|
||||||
|
<label>or after <input type="date" name="after" value="${field("after")}"></label>
|
||||||
|
<label>before <input type="date" name="before" value="${field("before")}"></label>
|
||||||
|
<label>or in <input name="in" size="9" placeholder="2026-W41" value="${field("in")}"></label>
|
||||||
|
<label>by <select name="by">${option("updated", "updated", params.get("by") ?? "")}${option("created", "created", params.get("by") ?? "")}</select></label></p>
|
||||||
|
<p><label>search <input type="search" name="q" size="40" value="${field("q")}"></label>
|
||||||
|
<button type="submit">filter</button></p>
|
||||||
|
<p><small>Dates and periods (2026, 2026-03, 2026-Q1, 2026-W41) are UTC. Search takes
|
||||||
|
words, "phrases", #tags, -negation, and after:, before:, newer_than:7d, during:2026-Q1, created:2026.</small></p>
|
||||||
|
</form>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** bookmarks as a list, each led by its `by` date */
|
||||||
|
function itemList(items: Item[], by: Query["by"]): string {
|
||||||
|
return `<ol>
|
||||||
|
${items
|
||||||
|
.map((it) => {
|
||||||
|
const url = typeof it.url === "string" ? it.url : "";
|
||||||
|
const title = typeof it.title === "string" && it.title ? it.title : url;
|
||||||
|
const desc = typeof it.description === "string" && it.description ? `<br><small>${esc(it.description)}</small>` : "";
|
||||||
|
const href = /^https?:\/\//i.test(url) ? url : "#";
|
||||||
|
const secs = dateOf(it, by);
|
||||||
|
const when = secs === undefined ? "" : new Date(secs * 1000).toISOString();
|
||||||
|
const date = when ? `<time datetime="${when}">${when}</time>` : "(no date)";
|
||||||
|
return `<li>${date} <a href="${esc(href)}">${esc(title)}</a>${desc}</li>`;
|
||||||
|
})
|
||||||
|
.join("\n")}
|
||||||
|
</ol>`;
|
||||||
|
}
|
||||||
|
|
||||||
/** home page: the connect link (webapp's cookie form, diigo-prefilled) +
|
/** home page: the connect link (webapp's cookie form, diigo-prefilled) +
|
||||||
* the wire step for an already-minted id + the dump form. `home` renders
|
* the wire step for an already-minted id + the dump form. `home` renders
|
||||||
* the error path too, so it takes env for the connect link. Notices (the
|
* the error path too, so it takes env for the connect link. Notices (the
|
||||||
|
|||||||
+188
-19
@@ -1,18 +1,10 @@
|
|||||||
/**
|
/**
|
||||||
* @file the public page's filters: date windows and search (goal 1d, **not
|
* @file the public page's filters: date windows and search over a cache's
|
||||||
* yet implemented**; the design is the skipped specs in tests/query.test.ts).
|
* stored items (goal 1d; tests/query.test.ts).
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { Item } from "./items.js";
|
import type { Item } from "./items.js";
|
||||||
|
|
||||||
/**
|
|
||||||
* diigo fetches one request may spend (cache hits are free); under the
|
|
||||||
* Workers free plan's 50 external subrequests per invocation
|
|
||||||
*
|
|
||||||
* @see https://developers.cloudflare.com/workers/platform/limits/
|
|
||||||
*/
|
|
||||||
export const FETCH_BUDGET = 40;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* a filter as devs write it (`{ days: 7, q: "#weekly" }`); URL params map
|
* a filter as devs write it (`{ days: 7, q: "#weekly" }`); URL params map
|
||||||
* onto these fields one-to-one (strings, with "" meaning absent). Dates
|
* onto these fields one-to-one (strings, with "" meaning absent). Dates
|
||||||
@@ -56,6 +48,76 @@ export type Query = {
|
|||||||
/** a refused filter: the message is safe to show and log */
|
/** a refused filter: the message is safe to show and log */
|
||||||
export type QueryError = { error: string };
|
export type QueryError = { error: string };
|
||||||
|
|
||||||
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
/** thrown inside {@link toQuery}, returned as a {@link QueryError} */
|
||||||
|
class Refused extends Error {}
|
||||||
|
|
||||||
|
/** `yyyy-mm-dd` or `yyyy/mm/dd` as 00:00Z of that UTC day */
|
||||||
|
function parseDay(text: string, what: string): Date {
|
||||||
|
const m = /^(\d{4})([-/])(\d{2})\2(\d{2})$/.exec(text);
|
||||||
|
const d = m && new Date(Date.UTC(Number(m[1]), Number(m[3]) - 1, Number(m[4])));
|
||||||
|
if (!m || !d || d.getUTCMonth() !== Number(m[3]) - 1 || d.getUTCDate() !== Number(m[4])) {
|
||||||
|
throw new Refused(`${what}: expected a date as yyyy-mm-dd`);
|
||||||
|
}
|
||||||
|
return d;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Monday 00:00Z of ISO week 1 of `year` (the week holding January 4) */
|
||||||
|
function isoWeekOne(year: number): number {
|
||||||
|
const jan4 = Date.UTC(year, 0, 4);
|
||||||
|
const weekday = new Date(jan4).getUTCDay() || 7;
|
||||||
|
return jan4 - (weekday - 1) * DAY_MS;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** a named period as its half-open [after, before) window */
|
||||||
|
function parsePeriod(text: string, what: string): [Date, Date] {
|
||||||
|
const refuse = () => new Refused(`${what}: expected a period as 2025, 2025-03, 2025-Q1, or 2025-W01`);
|
||||||
|
let m;
|
||||||
|
if ((m = /^(\d{4})$/.exec(text))) {
|
||||||
|
const y = Number(m[1]);
|
||||||
|
return [new Date(Date.UTC(y, 0, 1)), new Date(Date.UTC(y + 1, 0, 1))];
|
||||||
|
}
|
||||||
|
if ((m = /^(\d{4})-(\d{2})$/.exec(text))) {
|
||||||
|
const [y, mo] = [Number(m[1]), Number(m[2])];
|
||||||
|
if (mo < 1 || mo > 12) throw refuse();
|
||||||
|
return [new Date(Date.UTC(y, mo - 1, 1)), new Date(Date.UTC(y, mo, 1))];
|
||||||
|
}
|
||||||
|
if ((m = /^(\d{4})-Q([1-4])$/.exec(text))) {
|
||||||
|
const [y, q] = [Number(m[1]), Number(m[2])];
|
||||||
|
return [new Date(Date.UTC(y, 3 * (q - 1), 1)), new Date(Date.UTC(y, 3 * q, 1))];
|
||||||
|
}
|
||||||
|
if ((m = /^(\d{4})-W(\d{2})$/.exec(text))) {
|
||||||
|
const [y, w] = [Number(m[1]), Number(m[2])];
|
||||||
|
const weeks = (isoWeekOne(y + 1) - isoWeekOne(y)) / (7 * DAY_MS);
|
||||||
|
if (w < 1 || w > weeks) throw refuse();
|
||||||
|
const monday = isoWeekOne(y) + (w - 1) * 7 * DAY_MS;
|
||||||
|
return [new Date(monday), new Date(monday + 7 * DAY_MS)];
|
||||||
|
}
|
||||||
|
throw refuse();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** gmail's `newer_than:` amount (`7d`, `2w`, `1m`, `1y`) counted back from
|
||||||
|
* `now`; months and years are calendar steps */
|
||||||
|
function parseNewerThan(text: string, now: Date): Date {
|
||||||
|
const m = /^(\d{1,4})([dwmy])$/.exec(text);
|
||||||
|
if (!m || Number(m[1]) < 1) throw new Refused("newer_than: expected an amount like 7d, 2w, 1m, or 1y");
|
||||||
|
const n = Number(m[1]);
|
||||||
|
const d = new Date(now.getTime());
|
||||||
|
if (m[2] === "d") d.setTime(now.getTime() - n * DAY_MS);
|
||||||
|
if (m[2] === "w") d.setTime(now.getTime() - 7 * n * DAY_MS);
|
||||||
|
if (m[2] === "m") d.setUTCMonth(d.getUTCMonth() - n);
|
||||||
|
if (m[2] === "y") d.setUTCFullYear(d.getUTCFullYear() - n);
|
||||||
|
return d;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `days` as a whole number from 1 to 3650 */
|
||||||
|
function parseDays(days: number | string): number {
|
||||||
|
const n = typeof days === "number" ? days : /^\d+$/.test(days) ? Number(days) : NaN;
|
||||||
|
if (!Number.isInteger(n) || n < 1 || n > 3650) throw new Refused("days: expected a whole number from 1 to 3650");
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* resolve a filter: fields and `q` operators combine; a bound given twice
|
* resolve a filter: fields and `q` operators combine; a bound given twice
|
||||||
* (by fields, operators, or both) is refused, as is an empty window.
|
* (by fields, operators, or both) is refused, as is an empty window.
|
||||||
@@ -75,21 +137,128 @@ export type QueryError = { error: string };
|
|||||||
* not read ambiently, so tests pin it)
|
* not read ambiently, so tests pin it)
|
||||||
*/
|
*/
|
||||||
export function toQuery(spec: FilterSpec, now: Date): Query | QueryError {
|
export function toQuery(spec: FilterSpec, now: Date): Query | QueryError {
|
||||||
void spec;
|
try {
|
||||||
void now;
|
return resolve(spec, now);
|
||||||
throw new Error("toQuery: not implemented (goal 1d)");
|
} catch (err) {
|
||||||
|
if (err instanceof Refused) return { error: err.message };
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolve(spec: FilterSpec, now: Date): Query {
|
||||||
|
const words: Term[] = [];
|
||||||
|
const tags: Term[] = [];
|
||||||
|
let by: { value: Query["by"]; from: string } | undefined;
|
||||||
|
let after: { value: Date; from: string } | undefined;
|
||||||
|
let before: { value: Date; from: string } | undefined;
|
||||||
|
|
||||||
|
const setBy = (value: Query["by"], from: string) => {
|
||||||
|
if (by && by.value !== value) throw new Refused(`${by.from} and ${from} name different dates`);
|
||||||
|
by = { value, from };
|
||||||
|
};
|
||||||
|
const setAfter = (value: Date, from: string) => {
|
||||||
|
if (after) throw new Refused(`${after.from} and ${from} both set where the window starts`);
|
||||||
|
after = { value, from };
|
||||||
|
};
|
||||||
|
const setBefore = (value: Date, from: string) => {
|
||||||
|
if (before) throw new Refused(`${before.from} and ${from} both set where the window ends`);
|
||||||
|
before = { value, from };
|
||||||
|
};
|
||||||
|
const setPeriod = ([lo, hi]: [Date, Date], from: string) => {
|
||||||
|
setAfter(lo, from);
|
||||||
|
setBefore(hi, from);
|
||||||
|
};
|
||||||
|
|
||||||
|
const present = (v: string | number | undefined) => v !== undefined && v !== "";
|
||||||
|
if (present(spec.by)) {
|
||||||
|
if (spec.by !== "updated" && spec.by !== "created") throw new Refused("by: expected updated or created");
|
||||||
|
setBy(spec.by, `by=${spec.by}`);
|
||||||
|
}
|
||||||
|
if (present(spec.days)) setAfter(new Date(now.getTime() - parseDays(spec.days!) * DAY_MS), "days");
|
||||||
|
if (present(spec.after)) setAfter(parseDay(spec.after!, "after"), "after");
|
||||||
|
if (present(spec.before)) setBefore(parseDay(spec.before!, "before"), "before");
|
||||||
|
if (present(spec.in)) setPeriod(parsePeriod(spec.in!, "in"), "in");
|
||||||
|
|
||||||
|
for (const m of (spec.q ?? "").matchAll(/(-?)"([^"]*)"|(\S+)/g)) {
|
||||||
|
if (m[2] !== undefined) {
|
||||||
|
if (m[2].trim()) words.push({ text: m[2].trim().toLowerCase(), negated: m[1] === "-" });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const token = m[3];
|
||||||
|
const negated = token.length > 1 && token.startsWith("-");
|
||||||
|
const body = negated ? token.slice(1) : token;
|
||||||
|
const op = /^([a-z_]+):(.+)$/i.exec(body);
|
||||||
|
const name = op?.[1].toLowerCase();
|
||||||
|
const arg = op?.[2] ?? "";
|
||||||
|
const windowOp = ["after", "before", "newer_than", "during", "in", "created", "updated"];
|
||||||
|
if (name && windowOp.includes(name) && negated) throw new Refused(`${name}: can't be negated`);
|
||||||
|
if (name === "after") setAfter(parseDay(arg, "after:"), "after:");
|
||||||
|
else if (name === "before") setBefore(parseDay(arg, "before:"), "before:");
|
||||||
|
else if (name === "newer_than") setAfter(parseNewerThan(arg, now), "newer_than:");
|
||||||
|
else if (name === "during" || name === "in") setPeriod(parsePeriod(arg, `${name}:`), `${name}:`);
|
||||||
|
else if (name === "created" || name === "updated") {
|
||||||
|
setBy(name, `${name}:`);
|
||||||
|
setPeriod(parsePeriod(arg, `${name}:`), `${name}:`);
|
||||||
|
} else if (name === "tag") tags.push({ text: arg.toLowerCase(), negated });
|
||||||
|
else if (body.length > 1 && body.startsWith("#")) tags.push({ text: body.slice(1).toLowerCase(), negated });
|
||||||
|
else words.push({ text: body.toLowerCase(), negated });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (after && before && after.value.getTime() >= before.value.getTime()) {
|
||||||
|
throw new Refused(`the window is empty: ${after.from} is not earlier than ${before.from}`);
|
||||||
|
}
|
||||||
|
const query: Query = { by: by?.value ?? "updated", words, tags };
|
||||||
|
if (after) query.after = after.value;
|
||||||
|
if (before) query.before = before.value;
|
||||||
|
return query;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** {@link toQuery} over the public page's URL params */
|
/** {@link toQuery} over the public page's URL params */
|
||||||
export function parseQuery(params: URLSearchParams, now: Date): Query | QueryError {
|
export function parseQuery(params: URLSearchParams, now: Date): Query | QueryError {
|
||||||
void params;
|
const field = (name: string) => params.get(name) ?? "";
|
||||||
void now;
|
const by = field("by");
|
||||||
throw new Error("parseQuery: not implemented (goal 1d)");
|
return toQuery(
|
||||||
|
{
|
||||||
|
days: field("days"),
|
||||||
|
after: field("after"),
|
||||||
|
before: field("before"),
|
||||||
|
in: field("in"),
|
||||||
|
by: by as FilterSpec["by"],
|
||||||
|
q: field("q"),
|
||||||
|
},
|
||||||
|
now,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** does the query filter anything? (an empty one lists diigo's pages as stored) */
|
||||||
|
export function isEmpty(query: Query): boolean {
|
||||||
|
return !query.after && !query.before && !query.words.length && !query.tags.length && query.by === "updated";
|
||||||
|
}
|
||||||
|
|
||||||
|
/** the item's date that `by` names, in epoch seconds (diigo's unit), or
|
||||||
|
* undefined when it carries none */
|
||||||
|
export function dateOf(item: Item, by: Query["by"]): number | undefined {
|
||||||
|
const secs = by === "created" ? item.created_at : item.updated_at;
|
||||||
|
return typeof secs === "number" && Number.isFinite(secs) ? secs : undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** does `item` fall in the window and satisfy every term? */
|
/** does `item` fall in the window and satisfy every term? */
|
||||||
export function matches(item: Item, query: Query): boolean {
|
export function matches(item: Item, query: Query): boolean {
|
||||||
void item;
|
if (query.after || query.before) {
|
||||||
void query;
|
const secs = dateOf(item, query.by);
|
||||||
throw new Error("matches: not implemented (goal 1d)");
|
if (secs === undefined) return false;
|
||||||
|
if (query.after && secs * 1000 < query.after.getTime()) return false;
|
||||||
|
if (query.before && secs * 1000 >= query.before.getTime()) return false;
|
||||||
|
}
|
||||||
|
const text = (v: unknown) => (typeof v === "string" ? v : "");
|
||||||
|
// fields joined by newline, so a phrase never spans two of them
|
||||||
|
const haystack = [item.title, item.description, item.url].map(text).join("\n").toLowerCase();
|
||||||
|
if (!query.words.every((w) => haystack.includes(w.text) !== w.negated)) return false;
|
||||||
|
const itemTags = new Set(
|
||||||
|
text(item.tags)
|
||||||
|
.split(",")
|
||||||
|
.map((t) => t.trim().toLowerCase())
|
||||||
|
.filter(Boolean),
|
||||||
|
);
|
||||||
|
return query.tags.every((t) => itemTags.has(t.text) !== t.negated);
|
||||||
}
|
}
|
||||||
@@ -41,6 +41,12 @@ export function makeReader(sql: Sql) {
|
|||||||
deepestPageSince(cutoff: number): number | null {
|
deepestPageSince(cutoff: number): number | null {
|
||||||
return run<{ last: number | null }>("deepestPageSince", cutoff)[0]?.last ?? null;
|
return run<{ last: number | null }>("deepestPageSince", cutoff)[0]?.last ?? null;
|
||||||
},
|
},
|
||||||
|
/** stored items whose `by` date (epoch seconds) is in [`from`, `to`),
|
||||||
|
* newest first; the raw dump rows (JSON strings) */
|
||||||
|
itemsIn(by: "updated" | "created", from: number, to: number): string[] {
|
||||||
|
const name = by === "created" ? "itemsByCreated" : "itemsByUpdated";
|
||||||
|
return run<{ json: string }>(name, from, to).map((r) => r.json);
|
||||||
|
},
|
||||||
/** how many cached pages are numbered `pageNum` or lower */
|
/** how many cached pages are numbered `pageNum` or lower */
|
||||||
pagesThrough(pageNum: number): number {
|
pagesThrough(pageNum: number): number {
|
||||||
return Number(run<{ n: number }>("pagesThrough", pageNum)[0]?.n ?? 0);
|
return Number(run<{ n: number }>("pagesThrough", pageNum)[0]?.n ?? 0);
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ import { evictDurableObject, runInDurableObject } from "cloudflare:test";
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import worker from "../src/index.js";
|
import worker from "../src/index.js";
|
||||||
import { idFromWebkey, toWebkey } from "../src/webkeys.js";
|
import { idFromWebkey, toWebkey } from "../src/webkeys.js";
|
||||||
import { FETCH_BUDGET } from "../src/query.js";
|
|
||||||
import { PAGE_SIZE } from "../src/cache.js";
|
import { PAGE_SIZE } from "../src/cache.js";
|
||||||
|
|
||||||
// matches tests/fake-webapp.js: 30 items, every third private
|
// matches tests/fake-webapp.js: 30 items, every third private
|
||||||
@@ -283,30 +282,34 @@ function isoWeek(d: Date): string {
|
|||||||
return `${year}-W${String(week).padStart(2, "0")}`;
|
return `${year}-W${String(week).padStart(2, "0")}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
describe.skip("public page filters (goal 1d)", () => {
|
describe("public page filters (goal 1d)", () => {
|
||||||
// the fake's dated account: "dated n" updated n-1 days + 1h ago;
|
// the fake's dated account: "dated n" updated n-1 days + 1h ago;
|
||||||
// #weekly on every 7th; every 10th private; every 5th created 100
|
// #weekly on every 7th; every 10th private; every 5th created 100
|
||||||
// days before it was updated
|
// days before it was updated. The public page only reads, so each
|
||||||
|
// test stores the pages it needs on the owner's side first.
|
||||||
const call = (path: string, init?: RequestInit) =>
|
const call = (path: string, init?: RequestInit) =>
|
||||||
worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
|
worker.fetch(new Request(`https://diigo-cache.test${path}`, init), env as never);
|
||||||
const wireDated = async () => {
|
/** wire a public cache to `webappId` and store diigo pages 1..`pages` */
|
||||||
|
const wireStored = async (webappId: string, pages: number) => {
|
||||||
const res = await call("/_wire", {
|
const res = await call("/_wire", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "content-type": "application/json" },
|
headers: { "content-type": "application/json" },
|
||||||
body: JSON.stringify({ webappId: "fake-dated-webapp", mode: "public" }),
|
body: JSON.stringify({ webappId, mode: "public" }),
|
||||||
});
|
});
|
||||||
expect(res.status).toBe(200);
|
expect(res.status).toBe(200);
|
||||||
return "/public/dated_dan";
|
const { username, publicId } = (await res.json()) as { username: string; publicId: string };
|
||||||
|
const pub = testEnv.PUBLIC_DO.get(testEnv.PUBLIC_DO.idFromString(idFromWebkey(publicId)));
|
||||||
|
for (let n = 1; n <= pages; n += 1) await pub.page(n, PAGE_SIZE);
|
||||||
|
return { page: `/public/${username}`, pub };
|
||||||
};
|
};
|
||||||
|
const wireDated = async () => (await wireStored("fake-dated-webapp", 1)).page;
|
||||||
const titlesIn = (html: string) => [...html.matchAll(/>(dated \d+)<\/a>/g)].map((m) => m[1]);
|
const titlesIn = (html: string) => [...html.matchAll(/>(dated \d+)<\/a>/g)].map((m) => m[1]);
|
||||||
const dated = (...ns: number[]) => ns.map((n) => `dated ${n}`);
|
const dated = (...ns: number[]) => ns.map((n) => `dated ${n}`);
|
||||||
|
|
||||||
it("?days=7 lists the last week, walking only the diigo pages it needs", async () => {
|
it("?days=7 lists the last week", async () => {
|
||||||
const page = await wireDated();
|
const page = await wireDated();
|
||||||
const before = await testEnv.WEBAPP.calls();
|
|
||||||
const html = await (await call(`${page}?days=7`)).text();
|
const html = await (await call(`${page}?days=7`)).text();
|
||||||
expect(titlesIn(html)).toEqual(dated(1, 2, 3, 4, 5, 6, 7));
|
expect(titlesIn(html)).toEqual(dated(1, 2, 3, 4, 5, 6, 7));
|
||||||
expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(1);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("?by=created windows, lists, and dates by when it was bookmarked", async () => {
|
it("?by=created windows, lists, and dates by when it was bookmarked", async () => {
|
||||||
@@ -327,13 +330,7 @@ describe.skip("public page filters (goal 1d)", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("?days=300 pages the matches PAGE_SIZE at a time; links keep the filter", async () => {
|
it("?days=300 pages the matches PAGE_SIZE at a time; links keep the filter", async () => {
|
||||||
const res = await call("/_wire", {
|
const { page } = await wireStored("fake-deep-webapp", 3);
|
||||||
method: "POST",
|
|
||||||
headers: { "content-type": "application/json" },
|
|
||||||
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
|
|
||||||
});
|
|
||||||
expect(res.status).toBe(200);
|
|
||||||
const page = "/public/deep_dee";
|
|
||||||
// deep_dee: one a day; days=300 holds 1..300, minus every 10th (private)
|
// deep_dee: one a day; days=300 holds 1..300, minus every 10th (private)
|
||||||
const want = Array.from({ length: 300 }, (_, k) => k + 1).filter((n) => n % 10 !== 0);
|
const want = Array.from({ length: 300 }, (_, k) => k + 1).filter((n) => n % 10 !== 0);
|
||||||
const first = await (await call(`${page}?days=300`)).text();
|
const first = await (await call(`${page}?days=300`)).text();
|
||||||
@@ -354,6 +351,15 @@ describe.skip("public page filters (goal 1d)", () => {
|
|||||||
expect(titlesIn(html)).toEqual(dated(11, 12));
|
expect(titlesIn(html)).toEqual(dated(11, 12));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("?in= names an ISO week", async () => {
|
||||||
|
const { page } = await wireStored("fake-deep-webapp", 3);
|
||||||
|
const week = isoWeek(new Date(Date.now() - 100 * 24 * 60 * 60 * 1000));
|
||||||
|
const html = await (await call(`${page}?in=${week}`)).text();
|
||||||
|
// one a day; every 10th is private
|
||||||
|
expect(titlesIn(html).length).toBeGreaterThanOrEqual(6);
|
||||||
|
expect(titlesIn(html).length).toBeLessThanOrEqual(7);
|
||||||
|
});
|
||||||
|
|
||||||
it("?q=#weekly&days=60 filters by tag; private items stay out", async () => {
|
it("?q=#weekly&days=60 filters by tag; private items stay out", async () => {
|
||||||
const page = await wireDated();
|
const page = await wireDated();
|
||||||
const html = await (await call(`${page}?days=60&q=${encodeURIComponent("#weekly")}`)).text();
|
const html = await (await call(`${page}?days=60&q=${encodeURIComponent("#weekly")}`)).text();
|
||||||
@@ -366,22 +372,21 @@ describe.skip("public page filters (goal 1d)", () => {
|
|||||||
expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 35, 42, 49, 56));
|
expect(titlesIn(html)).toEqual(dated(7, 14, 21, 28, 35, 42, 49, 56));
|
||||||
});
|
});
|
||||||
|
|
||||||
it("a request spends at most FETCH_BUDGET diigo fetches; incomplete results say so, and a reload continues", async () => {
|
it("a filter reads only stored items, never diigo; results say whether storage is complete", async () => {
|
||||||
const res = await call("/_wire", {
|
const { page, pub } = await wireStored("fake-deep-webapp", 1);
|
||||||
method: "POST",
|
const json = async () =>
|
||||||
headers: { "content-type": "application/json" },
|
(await call(`${page}?days=300`, { headers: { accept: "application/json" } })).json() as Promise<{
|
||||||
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
|
matched: number;
|
||||||
});
|
complete: boolean;
|
||||||
expect(res.status).toBe(200);
|
}>;
|
||||||
// deep_dee: 60 diigo pages of 100; a search matching nothing must walk them all
|
|
||||||
const before = await testEnv.WEBAPP.calls();
|
const before = await testEnv.WEBAPP.calls();
|
||||||
const first = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
|
// page 1 holds 1..100, minus 10 private
|
||||||
expect(await first.json()).toMatchObject({ complete: false, items: [] });
|
expect(await json()).toMatchObject({ matched: 90, complete: false });
|
||||||
expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(FETCH_BUDGET + 1);
|
expect(await (await call(`${page}?days=300`)).text()).toMatch(/some matches may be missing/);
|
||||||
const html = await (await call("/public/deep_dee?q=nomatch")).text();
|
expect(await testEnv.WEBAPP.calls()).toBe(before);
|
||||||
expect(html).toMatch(/complete|no matches/);
|
await pub.page(2, PAGE_SIZE);
|
||||||
const again = await call("/public/deep_dee?q=nomatch", { headers: { accept: "application/json" } });
|
await pub.page(3, PAGE_SIZE);
|
||||||
expect(await again.json()).toMatchObject({ complete: true, items: [] });
|
expect(await json()).toMatchObject({ matched: 270, complete: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("JSON answers carry the canonical filter, so devs see what was understood", async () => {
|
it("JSON answers carry the canonical filter, so devs see what was understood", async () => {
|
||||||
@@ -393,25 +398,6 @@ describe.skip("public page filters (goal 1d)", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("?in= an old week costs O(log pages) diigo fetches with by=updated, not a walk from page 1", async () => {
|
|
||||||
const res = await call("/_wire", {
|
|
||||||
method: "POST",
|
|
||||||
headers: { "content-type": "application/json" },
|
|
||||||
body: JSON.stringify({ webappId: "fake-deep-webapp", mode: "public" }),
|
|
||||||
});
|
|
||||||
expect(res.status).toBe(200);
|
|
||||||
// deep_dee: 6000 bookmarks, one a day, 60 diigo pages of 100; a week
|
|
||||||
// ~14 years back sits near page 51 — a walk from page 1 would cost ~51
|
|
||||||
const longAgo = new Date(Date.now() - 5100 * 24 * 60 * 60 * 1000);
|
|
||||||
const week = isoWeek(longAgo);
|
|
||||||
const before = await testEnv.WEBAPP.calls();
|
|
||||||
const html = await (await call(`/public/deep_dee?in=${week}`)).text();
|
|
||||||
// one a day; every 10th is private
|
|
||||||
expect(titlesIn(html).length).toBeGreaterThanOrEqual(6);
|
|
||||||
expect(titlesIn(html).length).toBeLessThanOrEqual(7);
|
|
||||||
expect((await testEnv.WEBAPP.calls()) - before).toBeLessThanOrEqual(12);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("a malformed filter is a 400 with a message, not an empty list", async () => {
|
it("a malformed filter is a 400 with a message, not an empty list", async () => {
|
||||||
const page = await wireDated();
|
const page = await wireDated();
|
||||||
const res = await call(`${page}?days=7x`);
|
const res = await call(`${page}?days=7x`);
|
||||||
|
|||||||
@@ -18,9 +18,9 @@ const OVERSTATEMENT = 60;
|
|||||||
* 5th an old bookmark edited recently (created 100 days before updated) */
|
* 5th an old bookmark edited recently (created 100 days before updated) */
|
||||||
const DATED_WEBAPP = "fake-dated-webapp";
|
const DATED_WEBAPP = "fake-dated-webapp";
|
||||||
const DATED_TOTAL = 60;
|
const DATED_TOTAL = 60;
|
||||||
/** the same shape, 6000 deep (~16 years, 60 pages of 100): for walk cost */
|
/** the same shape, 300 deep (3 pages of 100): for paging matches */
|
||||||
const DEEP_WEBAPP = "fake-deep-webapp";
|
const DEEP_WEBAPP = "fake-deep-webapp";
|
||||||
const DEEP_TOTAL = 6000;
|
const DEEP_TOTAL = 300;
|
||||||
const HOUR = 60 * 60 * 1000;
|
const HOUR = 60 * 60 * 1000;
|
||||||
|
|
||||||
function datedItem(n, now, user = "dated_dan") {
|
function datedItem(n, now, user = "dated_dan") {
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest";
|
|||||||
import { matches, parseQuery, toQuery } from "../src/query.js";
|
import { matches, parseQuery, toQuery } from "../src/query.js";
|
||||||
import type { FilterSpec, Query, QueryError } from "../src/query.js";
|
import type { FilterSpec, Query, QueryError } from "../src/query.js";
|
||||||
|
|
||||||
// goal 1d (capsite checklist): the design as tests; unskip as it lands
|
// goal 1d (capsite checklist)
|
||||||
|
|
||||||
const NOW = new Date("2026-10-07T12:00:00.000Z");
|
const NOW = new Date("2026-10-07T12:00:00.000Z");
|
||||||
const DAY = 24 * 60 * 60 * 1000;
|
const DAY = 24 * 60 * 60 * 1000;
|
||||||
@@ -21,7 +21,7 @@ const refused = (q: Query | QueryError) => {
|
|||||||
const day = (iso: string) => new Date(`${iso}T00:00:00.000Z`);
|
const day = (iso: string) => new Date(`${iso}T00:00:00.000Z`);
|
||||||
const range = (q: Query) => [q.after?.toISOString().slice(0, 10), q.before?.toISOString().slice(0, 10)];
|
const range = (q: Query) => [q.after?.toISOString().slice(0, 10), q.before?.toISOString().slice(0, 10)];
|
||||||
|
|
||||||
describe.skip("filters: one spec, two spellings (goal 1d)", () => {
|
describe("filters: one spec, two spellings (goal 1d)", () => {
|
||||||
it("devs write { days: 7 }; URLs write ?days=7; same query", () => {
|
it("devs write { days: 7 }; URLs write ?days=7; same query", () => {
|
||||||
expect(spec({ days: 7 })).toEqual(url("days=7"));
|
expect(spec({ days: 7 })).toEqual(url("days=7"));
|
||||||
expect(spec({ days: 7 }).after).toEqual(new Date(NOW.getTime() - 7 * DAY));
|
expect(spec({ days: 7 }).after).toEqual(new Date(NOW.getTime() - 7 * DAY));
|
||||||
@@ -42,7 +42,7 @@ describe.skip("filters: one spec, two spellings (goal 1d)", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe.skip("windows: UTC, half-open (goal 1d)", () => {
|
describe("windows: UTC, half-open (goal 1d)", () => {
|
||||||
it("after/before are UTC days: after inclusive from 00:00Z, before exclusive", () => {
|
it("after/before are UTC days: after inclusive from 00:00Z, before exclusive", () => {
|
||||||
expect(range(url("after=2026-01-01&before=2026-02-01"))).toEqual(["2026-01-01", "2026-02-01"]);
|
expect(range(url("after=2026-01-01&before=2026-02-01"))).toEqual(["2026-01-01", "2026-02-01"]);
|
||||||
expect(url("after=2026/01/01").after).toEqual(day("2026-01-01"));
|
expect(url("after=2026/01/01").after).toEqual(day("2026-01-01"));
|
||||||
@@ -88,7 +88,7 @@ describe.skip("windows: UTC, half-open (goal 1d)", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe.skip("q: terms and operators (goal 1d)", () => {
|
describe("q: terms and operators (goal 1d)", () => {
|
||||||
const q = (text: string) => url(`q=${encodeURIComponent(text)}`);
|
const q = (text: string) => url(`q=${encodeURIComponent(text)}`);
|
||||||
|
|
||||||
it("words, phrases, #tags, tag:, and - negation; case-insensitive", () => {
|
it("words, phrases, #tags, tag:, and - negation; case-insensitive", () => {
|
||||||
@@ -160,7 +160,7 @@ const item = (iso: string, extra: Record<string, unknown> = {}) => ({
|
|||||||
});
|
});
|
||||||
const query = (q: Partial<Query>): Query => ({ by: "updated", words: [], tags: [], ...q });
|
const query = (q: Partial<Query>): Query => ({ by: "updated", words: [], tags: [], ...q });
|
||||||
|
|
||||||
describe.skip("matches (goal 1d)", () => {
|
describe("matches (goal 1d)", () => {
|
||||||
it("the window is half-open: after inclusive, before exclusive", () => {
|
it("the window is half-open: after inclusive, before exclusive", () => {
|
||||||
const q = query({ after: day("2026-01-01"), before: day("2026-02-01") });
|
const q = query({ after: day("2026-01-01"), before: day("2026-02-01") });
|
||||||
expect(matches(item("2026-01-01T00:00:00.000Z"), q)).toBe(true);
|
expect(matches(item("2026-01-01T00:00:00.000Z"), q)).toBe(true);
|
||||||
|
|||||||
Reference in new issue
Block a user