From d145841a2e1bc3a086a93b0ac9303e091ae9442f Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 7 Oct 2026 08:34:02 -0500 Subject: [PATCH] fixup! diigo-cache: RSS in diigo's export shape, from cached items compare-export.ts under ocap discipline: parseExport and compare are pure; main is the program's edge and takes every power it uses (argv, file reads and writes, the sqlite read, the clock, console, the exit code) as default arguments. Nothing ambient at module level; the last line just calls main(). Generated-by: Claude Opus 5.5 --- diigo-cache/scripts/compare-export.ts | 172 +++++++++++++++----------- 1 file changed, 103 insertions(+), 69 deletions(-) diff --git a/diigo-cache/scripts/compare-export.ts b/diigo-cache/scripts/compare-export.ts index c486da4..ad50748 100644 --- a/diigo-cache/scripts/compare-export.ts +++ b/diigo-cache/scripts/compare-export.ts @@ -1,16 +1,9 @@ #!/usr/bin/env node /** * @file program script: does a cache hold everything in a diigo RSS - * export? Generates RSS from the cache's items (src/rss.ts) and compares - * it, item by item keyed on link, with the export: title, tags, - * description, and highlights + sticky notes. A program script, so it - * reaches the filesystem and the clock itself. + * export? See {@link main}. * * node scripts/compare-export.ts - * - * Prints counts only; writes /from-cache.rss and - * /report.json (mismatched links, by category). diigo's export - * is not well-formed XML, so it is read leniently, item by item. */ import { readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -19,75 +12,116 @@ import { itemDescription, toRss } from "../src/rss.ts"; type Item = Record; -const [dbPath, exportPath, outDir] = process.argv.slice(2); -if (!dbPath || !exportPath || !outDir) { - console.error("usage: node scripts/compare-export.ts "); - process.exit(2); -} +/** one export item, as read */ +type Exported = { title: string; link: string; description: string }; const FORBIDDEN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g; const clean = (s: string) => s.replace(FORBIDDEN, ""); -// the cache: every item it holds, newest update first -const db = new DatabaseSync(dbPath, { readOnly: true }); -const ours = (db.prepare("select json from items order by updated desc").all() as { json: string }[]).map( - (r) => JSON.parse(r.json) as Item, -); -writeFileSync(join(outDir, "from-cache.rss"), toRss(ours, { title: "from the cache", pubDate: new Date().toUTCString() })); - -// the export, leniently -const field = (it: string, name: string) => - it.match(new RegExp(`<${name}>`))?.[1] ?? ""; -const theirs = [...readFileSync(exportPath, "utf8").matchAll(/([\s\S]*?)<\/item>/g)].map((m) => ({ - title: clean(field(m[1], "title")), - link: clean(field(m[1], "link")), - description: clean(field(m[1], "description")), -})); - -const byUrl = new Map(); -for (const it of ours) { - const url = String(it.url ?? ""); - byUrl.set(url, [...(byUrl.get(url) ?? []), it]); +/** diigo's RSS export, read leniently item by item (it isn't well-formed + * XML: it carries characters XML forbids) */ +export function parseExport(xml: string): Exported[] { + const field = (it: string, name: string) => + it.match(new RegExp(`<${name}>`))?.[1] ?? ""; + return [...xml.matchAll(/([\s\S]*?)<\/item>/g)].map((m) => ({ + title: clean(field(m[1], "title")), + link: clean(field(m[1], "link")), + description: clean(field(m[1], "description")), + })); } const PARTS = /^Tags:([\s\S]*?);Description:([\s\S]*?);Annotations and Sticky notes:([\s\S]*)$/; -const report: Record = {}; -const note = (category: string, link: string) => (report[category] ??= []).push(link); -let complete = 0; -for (const t of theirs) { - const candidates = byUrl.get(t.link); - if (!candidates) { - note("missing from cache", t.link); - continue; - } - // a link bookmarked twice: any candidate that matches will do - const verdicts = candidates.map((o) => { - const problems: string[] = []; - const title = clean(String(o.title ?? "")); - if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) problems.push("title differs"); - const mine = clean(itemDescription(o)).match(PARTS); - const yours = t.description.match(PARTS); - if (!mine || !yours) problems.push("description unparsed"); - else { - if (mine[1] !== yours[1]) problems.push("tags differ"); - if (mine[2] !== yours[2]) problems.push("description differs"); - if (mine[3] !== yours[3]) problems.push("annotations differ"); - } - return problems; - }); - const best = verdicts.reduce((a, b) => (b.length < a.length ? b : a)); - if (best.length === 0) complete += 1; - for (const p of best) note(p, t.link); +/** what's wrong with `ours` as a rendering of `theirs` (empty: nothing) */ +function problems(o: Item, t: Exported): string[] { + const found: string[] = []; + const title = clean(String(o.title ?? "")); + // the export cuts titles at 255 + if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) found.push("title differs"); + const mine = clean(itemDescription(o)).match(PARTS); + const yours = t.description.match(PARTS); + if (!mine || !yours) return [...found, "description unparsed"]; + if (mine[1] !== yours[1]) found.push("tags differ"); + if (mine[2] !== yours[2]) found.push("description differs"); + if (mine[3] !== yours[3]) found.push("annotations differ"); + return found; } -const exportLinks = new Set(theirs.map((t) => t.link)); -for (const url of byUrl.keys()) if (!exportLinks.has(url)) note("in cache, not in export", url); +/** + * compare cached items with an export, keyed on link: each export item + * is complete, missing, or has problems by category; cached links absent + * from the export are reported too. Links only, never contents. + */ +export function compare(ours: Item[], theirs: Exported[]): { complete: number; report: Record } { + const byUrl = new Map(); + for (const it of ours) { + const url = String(it.url ?? ""); + byUrl.set(url, [...(byUrl.get(url) ?? []), it]); + } + const report: Record = {}; + const note = (category: string, link: string) => (report[category] ??= []).push(link); + let complete = 0; + for (const t of theirs) { + const candidates = byUrl.get(t.link); + if (!candidates) { + note("missing from cache", t.link); + continue; + } + // a link bookmarked twice: the best-matching candidate counts + const best = candidates.map((o) => problems(o, t)).reduce((a, b) => (b.length < a.length ? b : a)); + if (best.length === 0) complete += 1; + for (const p of best) note(p, t.link); + } + const exportLinks = new Set(theirs.map((t) => t.link)); + for (const url of byUrl.keys()) if (!exportLinks.has(url)) note("in cache, not in export", url); + return { complete, report }; +} -writeFileSync(join(outDir, "report.json"), JSON.stringify(report, null, 1)); -console.log({ - export: theirs.length, - cache: ours.length, - complete, - ...Object.fromEntries(Object.entries(report).map(([k, v]) => [k, v.length])), -}); +/** + * Generates RSS from a cache's items (src/rss.ts) and compares it with a + * diigo RSS export: title, tags, description, highlights + sticky notes. + * Prints counts only; writes /from-cache.rss and + * /report.json (mismatched links, by category). + * + * The program's edge: every power it uses arrives as a default argument + * (ocap discipline), so a test can pass others. + * + * Ends by reporting an exit code (2: usage) through `exit`. + */ +export function main( + argv = process.argv.slice(2), + { + readText = (path: string) => readFileSync(path, "utf8"), + writeText = (path: string, text: string) => writeFileSync(path, text), + readItems = (dbPath: string): Item[] => + (new DatabaseSync(dbPath, { readOnly: true }).prepare("select json from items order by updated desc").all() as { + json: string; + }[]).map((r) => JSON.parse(r.json) as Item), + now = () => new Date(), + log = (x: unknown) => console.log(x), + warn = (x: unknown) => console.error(x), + exit = (code: number) => { + process.exitCode = code; + }, + } = {}, +): void { + const [dbPath, exportPath, outDir] = argv; + if (!dbPath || !exportPath || !outDir) { + warn("usage: node scripts/compare-export.ts "); + return exit(2); + } + const ours = readItems(dbPath); + writeText(join(outDir, "from-cache.rss"), toRss(ours, { title: "from the cache", pubDate: now().toUTCString() })); + const theirs = parseExport(readText(exportPath)); + const { complete, report } = compare(ours, theirs); + writeText(join(outDir, "report.json"), JSON.stringify(report, null, 1)); + log({ + export: theirs.length, + cache: ours.length, + complete, + ...Object.fromEntries(Object.entries(report).map(([k, v]) => [k, v.length])), + }); + exit(0); +} + +main();