diigo-cache: date windows and search on the public page (1d)
/public/<username> takes days, after, before, in (year, month, quarter, ISO week), by=updated|created, and q (words, "phrases", #tags, -negation, gmail/Slack/GitHub date operators). JSON answers echo the parsed filter; a malformed one is a 400. Filters query the stored items in sqlite through the read-only viewer instead of walking diigo pages: the public page keeps no fetch authority, and a filled cache already holds every item. Results say `complete` when every diigo page is stored; freshness is 1f's head refresh. So the fetch budget and the O(log pages) walk test go, and the fake deep account shrinks to 300. Generated-by: Claude Opus 5.5
This commit is contained in:
dckc committed
1 parent
efb54ad4cc
commit
002f08b456
9 files changed
+418
-120
No files matched your search
+188
-19
@@ -1,18 +1,10 @@
|
||||
/**
|
||||
* @file the public page's filters: date windows and search (goal 1d, **not
|
||||
* yet implemented**; the design is the skipped specs in tests/query.test.ts).
|
||||
* @file the public page's filters: date windows and search over a cache's
|
||||
* stored items (goal 1d; tests/query.test.ts).
|
||||
*/
|
||||
|
||||
import type { Item } from "./items.js";
|
||||
|
||||
/**
|
||||
* diigo fetches one request may spend (cache hits are free); under the
|
||||
* Workers free plan's 50 external subrequests per invocation
|
||||
*
|
||||
* @see https://developers.cloudflare.com/workers/platform/limits/
|
||||
*/
|
||||
export const FETCH_BUDGET = 40;
|
||||
|
||||
/**
|
||||
* a filter as devs write it (`{ days: 7, q: "#weekly" }`); URL params map
|
||||
* onto these fields one-to-one (strings, with "" meaning absent). Dates
|
||||
@@ -56,6 +48,76 @@ export type Query = {
|
||||
/** a refused filter: the message is safe to show and log */
|
||||
export type QueryError = { error: string };
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/** thrown inside {@link toQuery}, returned as a {@link QueryError} */
|
||||
class Refused extends Error {}
|
||||
|
||||
/** `yyyy-mm-dd` or `yyyy/mm/dd` as 00:00Z of that UTC day */
|
||||
function parseDay(text: string, what: string): Date {
|
||||
const m = /^(\d{4})([-/])(\d{2})\2(\d{2})$/.exec(text);
|
||||
const d = m && new Date(Date.UTC(Number(m[1]), Number(m[3]) - 1, Number(m[4])));
|
||||
if (!m || !d || d.getUTCMonth() !== Number(m[3]) - 1 || d.getUTCDate() !== Number(m[4])) {
|
||||
throw new Refused(`${what}: expected a date as yyyy-mm-dd`);
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
/** Monday 00:00Z of ISO week 1 of `year` (the week holding January 4) */
|
||||
function isoWeekOne(year: number): number {
|
||||
const jan4 = Date.UTC(year, 0, 4);
|
||||
const weekday = new Date(jan4).getUTCDay() || 7;
|
||||
return jan4 - (weekday - 1) * DAY_MS;
|
||||
}
|
||||
|
||||
/** a named period as its half-open [after, before) window */
|
||||
function parsePeriod(text: string, what: string): [Date, Date] {
|
||||
const refuse = () => new Refused(`${what}: expected a period as 2025, 2025-03, 2025-Q1, or 2025-W01`);
|
||||
let m;
|
||||
if ((m = /^(\d{4})$/.exec(text))) {
|
||||
const y = Number(m[1]);
|
||||
return [new Date(Date.UTC(y, 0, 1)), new Date(Date.UTC(y + 1, 0, 1))];
|
||||
}
|
||||
if ((m = /^(\d{4})-(\d{2})$/.exec(text))) {
|
||||
const [y, mo] = [Number(m[1]), Number(m[2])];
|
||||
if (mo < 1 || mo > 12) throw refuse();
|
||||
return [new Date(Date.UTC(y, mo - 1, 1)), new Date(Date.UTC(y, mo, 1))];
|
||||
}
|
||||
if ((m = /^(\d{4})-Q([1-4])$/.exec(text))) {
|
||||
const [y, q] = [Number(m[1]), Number(m[2])];
|
||||
return [new Date(Date.UTC(y, 3 * (q - 1), 1)), new Date(Date.UTC(y, 3 * q, 1))];
|
||||
}
|
||||
if ((m = /^(\d{4})-W(\d{2})$/.exec(text))) {
|
||||
const [y, w] = [Number(m[1]), Number(m[2])];
|
||||
const weeks = (isoWeekOne(y + 1) - isoWeekOne(y)) / (7 * DAY_MS);
|
||||
if (w < 1 || w > weeks) throw refuse();
|
||||
const monday = isoWeekOne(y) + (w - 1) * 7 * DAY_MS;
|
||||
return [new Date(monday), new Date(monday + 7 * DAY_MS)];
|
||||
}
|
||||
throw refuse();
|
||||
}
|
||||
|
||||
/** gmail's `newer_than:` amount (`7d`, `2w`, `1m`, `1y`) counted back from
|
||||
* `now`; months and years are calendar steps */
|
||||
function parseNewerThan(text: string, now: Date): Date {
|
||||
const m = /^(\d{1,4})([dwmy])$/.exec(text);
|
||||
if (!m || Number(m[1]) < 1) throw new Refused("newer_than: expected an amount like 7d, 2w, 1m, or 1y");
|
||||
const n = Number(m[1]);
|
||||
const d = new Date(now.getTime());
|
||||
if (m[2] === "d") d.setTime(now.getTime() - n * DAY_MS);
|
||||
if (m[2] === "w") d.setTime(now.getTime() - 7 * n * DAY_MS);
|
||||
if (m[2] === "m") d.setUTCMonth(d.getUTCMonth() - n);
|
||||
if (m[2] === "y") d.setUTCFullYear(d.getUTCFullYear() - n);
|
||||
return d;
|
||||
}
|
||||
|
||||
/** `days` as a whole number from 1 to 3650 */
|
||||
function parseDays(days: number | string): number {
|
||||
const n = typeof days === "number" ? days : /^\d+$/.test(days) ? Number(days) : NaN;
|
||||
if (!Number.isInteger(n) || n < 1 || n > 3650) throw new Refused("days: expected a whole number from 1 to 3650");
|
||||
return n;
|
||||
}
|
||||
|
||||
/**
|
||||
* resolve a filter: fields and `q` operators combine; a bound given twice
|
||||
* (by fields, operators, or both) is refused, as is an empty window.
|
||||
@@ -75,21 +137,128 @@ export type QueryError = { error: string };
|
||||
* not read ambiently, so tests pin it)
|
||||
*/
|
||||
export function toQuery(spec: FilterSpec, now: Date): Query | QueryError {
|
||||
void spec;
|
||||
void now;
|
||||
throw new Error("toQuery: not implemented (goal 1d)");
|
||||
try {
|
||||
return resolve(spec, now);
|
||||
} catch (err) {
|
||||
if (err instanceof Refused) return { error: err.message };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function resolve(spec: FilterSpec, now: Date): Query {
|
||||
const words: Term[] = [];
|
||||
const tags: Term[] = [];
|
||||
let by: { value: Query["by"]; from: string } | undefined;
|
||||
let after: { value: Date; from: string } | undefined;
|
||||
let before: { value: Date; from: string } | undefined;
|
||||
|
||||
const setBy = (value: Query["by"], from: string) => {
|
||||
if (by && by.value !== value) throw new Refused(`${by.from} and ${from} name different dates`);
|
||||
by = { value, from };
|
||||
};
|
||||
const setAfter = (value: Date, from: string) => {
|
||||
if (after) throw new Refused(`${after.from} and ${from} both set where the window starts`);
|
||||
after = { value, from };
|
||||
};
|
||||
const setBefore = (value: Date, from: string) => {
|
||||
if (before) throw new Refused(`${before.from} and ${from} both set where the window ends`);
|
||||
before = { value, from };
|
||||
};
|
||||
const setPeriod = ([lo, hi]: [Date, Date], from: string) => {
|
||||
setAfter(lo, from);
|
||||
setBefore(hi, from);
|
||||
};
|
||||
|
||||
const present = (v: string | number | undefined) => v !== undefined && v !== "";
|
||||
if (present(spec.by)) {
|
||||
if (spec.by !== "updated" && spec.by !== "created") throw new Refused("by: expected updated or created");
|
||||
setBy(spec.by, `by=${spec.by}`);
|
||||
}
|
||||
if (present(spec.days)) setAfter(new Date(now.getTime() - parseDays(spec.days!) * DAY_MS), "days");
|
||||
if (present(spec.after)) setAfter(parseDay(spec.after!, "after"), "after");
|
||||
if (present(spec.before)) setBefore(parseDay(spec.before!, "before"), "before");
|
||||
if (present(spec.in)) setPeriod(parsePeriod(spec.in!, "in"), "in");
|
||||
|
||||
for (const m of (spec.q ?? "").matchAll(/(-?)"([^"]*)"|(\S+)/g)) {
|
||||
if (m[2] !== undefined) {
|
||||
if (m[2].trim()) words.push({ text: m[2].trim().toLowerCase(), negated: m[1] === "-" });
|
||||
continue;
|
||||
}
|
||||
const token = m[3];
|
||||
const negated = token.length > 1 && token.startsWith("-");
|
||||
const body = negated ? token.slice(1) : token;
|
||||
const op = /^([a-z_]+):(.+)$/i.exec(body);
|
||||
const name = op?.[1].toLowerCase();
|
||||
const arg = op?.[2] ?? "";
|
||||
const windowOp = ["after", "before", "newer_than", "during", "in", "created", "updated"];
|
||||
if (name && windowOp.includes(name) && negated) throw new Refused(`${name}: can't be negated`);
|
||||
if (name === "after") setAfter(parseDay(arg, "after:"), "after:");
|
||||
else if (name === "before") setBefore(parseDay(arg, "before:"), "before:");
|
||||
else if (name === "newer_than") setAfter(parseNewerThan(arg, now), "newer_than:");
|
||||
else if (name === "during" || name === "in") setPeriod(parsePeriod(arg, `${name}:`), `${name}:`);
|
||||
else if (name === "created" || name === "updated") {
|
||||
setBy(name, `${name}:`);
|
||||
setPeriod(parsePeriod(arg, `${name}:`), `${name}:`);
|
||||
} else if (name === "tag") tags.push({ text: arg.toLowerCase(), negated });
|
||||
else if (body.length > 1 && body.startsWith("#")) tags.push({ text: body.slice(1).toLowerCase(), negated });
|
||||
else words.push({ text: body.toLowerCase(), negated });
|
||||
}
|
||||
|
||||
if (after && before && after.value.getTime() >= before.value.getTime()) {
|
||||
throw new Refused(`the window is empty: ${after.from} is not earlier than ${before.from}`);
|
||||
}
|
||||
const query: Query = { by: by?.value ?? "updated", words, tags };
|
||||
if (after) query.after = after.value;
|
||||
if (before) query.before = before.value;
|
||||
return query;
|
||||
}
|
||||
|
||||
/** {@link toQuery} over the public page's URL params */
|
||||
export function parseQuery(params: URLSearchParams, now: Date): Query | QueryError {
|
||||
void params;
|
||||
void now;
|
||||
throw new Error("parseQuery: not implemented (goal 1d)");
|
||||
const field = (name: string) => params.get(name) ?? "";
|
||||
const by = field("by");
|
||||
return toQuery(
|
||||
{
|
||||
days: field("days"),
|
||||
after: field("after"),
|
||||
before: field("before"),
|
||||
in: field("in"),
|
||||
by: by as FilterSpec["by"],
|
||||
q: field("q"),
|
||||
},
|
||||
now,
|
||||
);
|
||||
}
|
||||
|
||||
/** does the query filter anything? (an empty one lists diigo's pages as stored) */
|
||||
export function isEmpty(query: Query): boolean {
|
||||
return !query.after && !query.before && !query.words.length && !query.tags.length && query.by === "updated";
|
||||
}
|
||||
|
||||
/** the item's date that `by` names, in epoch seconds (diigo's unit), or
|
||||
* undefined when it carries none */
|
||||
export function dateOf(item: Item, by: Query["by"]): number | undefined {
|
||||
const secs = by === "created" ? item.created_at : item.updated_at;
|
||||
return typeof secs === "number" && Number.isFinite(secs) ? secs : undefined;
|
||||
}
|
||||
|
||||
/** does `item` fall in the window and satisfy every term? */
|
||||
export function matches(item: Item, query: Query): boolean {
|
||||
void item;
|
||||
void query;
|
||||
throw new Error("matches: not implemented (goal 1d)");
|
||||
if (query.after || query.before) {
|
||||
const secs = dateOf(item, query.by);
|
||||
if (secs === undefined) return false;
|
||||
if (query.after && secs * 1000 < query.after.getTime()) return false;
|
||||
if (query.before && secs * 1000 >= query.before.getTime()) return false;
|
||||
}
|
||||
const text = (v: unknown) => (typeof v === "string" ? v : "");
|
||||
// fields joined by newline, so a phrase never spans two of them
|
||||
const haystack = [item.title, item.description, item.url].map(text).join("\n").toLowerCase();
|
||||
if (!query.words.every((w) => haystack.includes(w.text) !== w.negated)) return false;
|
||||
const itemTags = new Set(
|
||||
text(item.tags)
|
||||
.split(",")
|
||||
.map((t) => t.trim().toLowerCase())
|
||||
.filter(Boolean),
|
||||
);
|
||||
return query.tags.every((t) => itemTags.has(t.text) !== t.negated);
|
||||
}
|
||||
Reference in new issue
Block a user