diff --git a/diigo-cache/scripts/compare-export.ts b/diigo-cache/scripts/compare-export.ts index c486da4..ad50748 100644 --- a/diigo-cache/scripts/compare-export.ts +++ b/diigo-cache/scripts/compare-export.ts @@ -1,16 +1,9 @@ #!/usr/bin/env node /** * @file program script: does a cache hold everything in a diigo RSS - * export? Generates RSS from the cache's items (src/rss.ts) and compares - * it, item by item keyed on link, with the export: title, tags, - * description, and highlights + sticky notes. A program script, so it - * reaches the filesystem and the clock itself. + * export? See {@link main}. * * node scripts/compare-export.ts - * - * Prints counts only; writes /from-cache.rss and - * /report.json (mismatched links, by category). diigo's export - * is not well-formed XML, so it is read leniently, item by item. */ import { readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -19,75 +12,116 @@ import { itemDescription, toRss } from "../src/rss.ts"; type Item = Record; -const [dbPath, exportPath, outDir] = process.argv.slice(2); -if (!dbPath || !exportPath || !outDir) { - console.error("usage: node scripts/compare-export.ts "); - process.exit(2); -} +/** one export item, as read */ +type Exported = { title: string; link: string; description: string }; const FORBIDDEN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g; const clean = (s: string) => s.replace(FORBIDDEN, ""); -// the cache: every item it holds, newest update first -const db = new DatabaseSync(dbPath, { readOnly: true }); -const ours = (db.prepare("select json from items order by updated desc").all() as { json: string }[]).map( - (r) => JSON.parse(r.json) as Item, -); -writeFileSync(join(outDir, "from-cache.rss"), toRss(ours, { title: "from the cache", pubDate: new Date().toUTCString() })); - -// the export, leniently -const field = (it: string, name: string) => - it.match(new RegExp(`<${name}>`))?.[1] ?? ""; -const theirs = [...readFileSync(exportPath, "utf8").matchAll(/([\s\S]*?)<\/item>/g)].map((m) => ({ - title: clean(field(m[1], "title")), - link: clean(field(m[1], "link")), - description: clean(field(m[1], "description")), -})); - -const byUrl = new Map(); -for (const it of ours) { - const url = String(it.url ?? ""); - byUrl.set(url, [...(byUrl.get(url) ?? []), it]); +/** diigo's RSS export, read leniently item by item (it isn't well-formed + * XML: it carries characters XML forbids) */ +export function parseExport(xml: string): Exported[] { + const field = (it: string, name: string) => + it.match(new RegExp(`<${name}>`))?.[1] ?? ""; + return [...xml.matchAll(/([\s\S]*?)<\/item>/g)].map((m) => ({ + title: clean(field(m[1], "title")), + link: clean(field(m[1], "link")), + description: clean(field(m[1], "description")), + })); } const PARTS = /^Tags:([\s\S]*?);Description:([\s\S]*?);Annotations and Sticky notes:([\s\S]*)$/; -const report: Record = {}; -const note = (category: string, link: string) => (report[category] ??= []).push(link); -let complete = 0; -for (const t of theirs) { - const candidates = byUrl.get(t.link); - if (!candidates) { - note("missing from cache", t.link); - continue; - } - // a link bookmarked twice: any candidate that matches will do - const verdicts = candidates.map((o) => { - const problems: string[] = []; - const title = clean(String(o.title ?? "")); - if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) problems.push("title differs"); - const mine = clean(itemDescription(o)).match(PARTS); - const yours = t.description.match(PARTS); - if (!mine || !yours) problems.push("description unparsed"); - else { - if (mine[1] !== yours[1]) problems.push("tags differ"); - if (mine[2] !== yours[2]) problems.push("description differs"); - if (mine[3] !== yours[3]) problems.push("annotations differ"); - } - return problems; - }); - const best = verdicts.reduce((a, b) => (b.length < a.length ? b : a)); - if (best.length === 0) complete += 1; - for (const p of best) note(p, t.link); +/** what's wrong with `ours` as a rendering of `theirs` (empty: nothing) */ +function problems(o: Item, t: Exported): string[] { + const found: string[] = []; + const title = clean(String(o.title ?? "")); + // the export cuts titles at 255 + if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) found.push("title differs"); + const mine = clean(itemDescription(o)).match(PARTS); + const yours = t.description.match(PARTS); + if (!mine || !yours) return [...found, "description unparsed"]; + if (mine[1] !== yours[1]) found.push("tags differ"); + if (mine[2] !== yours[2]) found.push("description differs"); + if (mine[3] !== yours[3]) found.push("annotations differ"); + return found; } -const exportLinks = new Set(theirs.map((t) => t.link)); -for (const url of byUrl.keys()) if (!exportLinks.has(url)) note("in cache, not in export", url); +/** + * compare cached items with an export, keyed on link: each export item + * is complete, missing, or has problems by category; cached links absent + * from the export are reported too. Links only, never contents. + */ +export function compare(ours: Item[], theirs: Exported[]): { complete: number; report: Record } { + const byUrl = new Map(); + for (const it of ours) { + const url = String(it.url ?? ""); + byUrl.set(url, [...(byUrl.get(url) ?? []), it]); + } + const report: Record = {}; + const note = (category: string, link: string) => (report[category] ??= []).push(link); + let complete = 0; + for (const t of theirs) { + const candidates = byUrl.get(t.link); + if (!candidates) { + note("missing from cache", t.link); + continue; + } + // a link bookmarked twice: the best-matching candidate counts + const best = candidates.map((o) => problems(o, t)).reduce((a, b) => (b.length < a.length ? b : a)); + if (best.length === 0) complete += 1; + for (const p of best) note(p, t.link); + } + const exportLinks = new Set(theirs.map((t) => t.link)); + for (const url of byUrl.keys()) if (!exportLinks.has(url)) note("in cache, not in export", url); + return { complete, report }; +} -writeFileSync(join(outDir, "report.json"), JSON.stringify(report, null, 1)); -console.log({ - export: theirs.length, - cache: ours.length, - complete, - ...Object.fromEntries(Object.entries(report).map(([k, v]) => [k, v.length])), -}); +/** + * Generates RSS from a cache's items (src/rss.ts) and compares it with a + * diigo RSS export: title, tags, description, highlights + sticky notes. + * Prints counts only; writes /from-cache.rss and + * /report.json (mismatched links, by category). + * + * The program's edge: every power it uses arrives as a default argument + * (ocap discipline), so a test can pass others. + * + * Ends by reporting an exit code (2: usage) through `exit`. + */ +export function main( + argv = process.argv.slice(2), + { + readText = (path: string) => readFileSync(path, "utf8"), + writeText = (path: string, text: string) => writeFileSync(path, text), + readItems = (dbPath: string): Item[] => + (new DatabaseSync(dbPath, { readOnly: true }).prepare("select json from items order by updated desc").all() as { + json: string; + }[]).map((r) => JSON.parse(r.json) as Item), + now = () => new Date(), + log = (x: unknown) => console.log(x), + warn = (x: unknown) => console.error(x), + exit = (code: number) => { + process.exitCode = code; + }, + } = {}, +): void { + const [dbPath, exportPath, outDir] = argv; + if (!dbPath || !exportPath || !outDir) { + warn("usage: node scripts/compare-export.ts "); + return exit(2); + } + const ours = readItems(dbPath); + writeText(join(outDir, "from-cache.rss"), toRss(ours, { title: "from the cache", pubDate: now().toUTCString() })); + const theirs = parseExport(readText(exportPath)); + const { complete, report } = compare(ours, theirs); + writeText(join(outDir, "report.json"), JSON.stringify(report, null, 1)); + log({ + export: theirs.length, + cache: ours.length, + complete, + ...Object.fromEntries(Object.entries(report).map(([k, v]) => [k, v.length])), + }); + exit(0); +} + +main();