dckc/captoys
1
0 Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

fixup! diigo-cache: RSS in diigo's export shape, from cached items

Browse Source 
compare-export.ts under ocap discipline: parseExport and compare are
pure; main is the program's edge and takes every power it uses (argv,
file reads and writes, the sqlite read, the clock, console, the exit
code) as default arguments. Nothing ambient at module level; the last
line just calls main().

Generated-by: Claude Opus 5.5
This commit is contained in:
dckc committed 2026-10-07 08:34:02 -05:00
1 parent 7be87d37dc
commit d145841a2e
1 file changed
+103 -69
Download Patch File Download Diff File Expand all files Collapse all files
diigo-cache/scripts/compare-export.ts
+103 -69
View File
@@ -1,16 +1,9 @@
#!/usr/bin/env node #!/usr/bin/env node
/** /**
* @file program script: does a cache hold everything in a diigo RSS * @file program script: does a cache hold everything in a diigo RSS
* export? Generates RSS from the cache's items (src/rss.ts) and compares * export? See {@link main}.
* it, item by item keyed on link, with the export: title, tags,
* description, and highlights + sticky notes. A program script, so it
* reaches the filesystem and the clock itself.
* *
* node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir> * node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir>
*
* Prints counts only; writes <out-dir>/from-cache.rss and
* <out-dir>/report.json (mismatched links, by category). diigo's export
* is not well-formed XML, so it is read leniently, item by item.
*/ */
import { readFileSync, writeFileSync } from "node:fs"; import { readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path"; import { join } from "node:path";
@@ -19,75 +12,116 @@ import { itemDescription, toRss } from "../src/rss.ts";
type Item = Record<string, unknown>; type Item = Record<string, unknown>;
const [dbPath, exportPath, outDir] = process.argv.slice(2); /** one export item, as read */
if (!dbPath || !exportPath || !outDir) { type Exported = { title: string; link: string; description: string };
console.error("usage: node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir>");
process.exit(2);
}
const FORBIDDEN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g; const FORBIDDEN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g;
const clean = (s: string) => s.replace(FORBIDDEN, ""); const clean = (s: string) => s.replace(FORBIDDEN, "");
// the cache: every item it holds, newest update first /** diigo's RSS export, read leniently item by item (it isn't well-formed
const db = new DatabaseSync(dbPath, { readOnly: true }); * XML: it carries characters XML forbids) */
const ours = (db.prepare("select json from items order by updated desc").all() as { json: string }[]).map( export function parseExport(xml: string): Exported[] {
(r) => JSON.parse(r.json) as Item, const field = (it: string, name: string) =>
); it.match(new RegExp(`<${name}><!\\[CDATA\\[([\\s\\S]*?)\\]\\]></${name}>`))?.[1] ?? "";
writeFileSync(join(outDir, "from-cache.rss"), toRss(ours, { title: "from the cache", pubDate: new Date().toUTCString() })); return [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => ({
title: clean(field(m[1], "title")),
// the export, leniently link: clean(field(m[1], "link")),
const field = (it: string, name: string) => description: clean(field(m[1], "description")),
it.match(new RegExp(`<${name}><!\\[CDATA\\[([\\s\\S]*?)\\]\\]></${name}>`))?.[1] ?? ""; }));
const theirs = [...readFileSync(exportPath, "utf8").matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => ({
title: clean(field(m[1], "title")),
link: clean(field(m[1], "link")),
description: clean(field(m[1], "description")),
}));
const byUrl = new Map<string, Item[]>();
for (const it of ours) {
const url = String(it.url ?? "");
byUrl.set(url, [...(byUrl.get(url) ?? []), it]);
} }
const PARTS = /^Tags:([\s\S]*?);Description:([\s\S]*?);Annotations and Sticky notes:([\s\S]*)$/; const PARTS = /^Tags:([\s\S]*?);Description:([\s\S]*?);Annotations and Sticky notes:([\s\S]*)$/;
const report: Record<string, string[]> = {};
const note = (category: string, link: string) => (report[category] ??= []).push(link);
let complete = 0;
for (const t of theirs) { /** what's wrong with `ours` as a rendering of `theirs` (empty: nothing) */
const candidates = byUrl.get(t.link); function problems(o: Item, t: Exported): string[] {
if (!candidates) { const found: string[] = [];
note("missing from cache", t.link); const title = clean(String(o.title ?? ""));
continue; // the export cuts titles at 255
} if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) found.push("title differs");
// a link bookmarked twice: any candidate that matches will do const mine = clean(itemDescription(o)).match(PARTS);
const verdicts = candidates.map((o) => { const yours = t.description.match(PARTS);
const problems: string[] = []; if (!mine || !yours) return [...found, "description unparsed"];
const title = clean(String(o.title ?? "")); if (mine[1] !== yours[1]) found.push("tags differ");
if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) problems.push("title differs"); if (mine[2] !== yours[2]) found.push("description differs");
const mine = clean(itemDescription(o)).match(PARTS); if (mine[3] !== yours[3]) found.push("annotations differ");
const yours = t.description.match(PARTS); return found;
if (!mine || !yours) problems.push("description unparsed");
else {
if (mine[1] !== yours[1]) problems.push("tags differ");
if (mine[2] !== yours[2]) problems.push("description differs");
if (mine[3] !== yours[3]) problems.push("annotations differ");
}
return problems;
});
const best = verdicts.reduce((a, b) => (b.length < a.length ? b : a));
if (best.length === 0) complete += 1;
for (const p of best) note(p, t.link);
} }
const exportLinks = new Set(theirs.map((t) => t.link)); /**
for (const url of byUrl.keys()) if (!exportLinks.has(url)) note("in cache, not in export", url); * compare cached items with an export, keyed on link: each export item
* is complete, missing, or has problems by category; cached links absent
* from the export are reported too. Links only, never contents.
*/
export function compare(ours: Item[], theirs: Exported[]): { complete: number; report: Record<string, string[]> } {
const byUrl = new Map<string, Item[]>();
for (const it of ours) {
const url = String(it.url ?? "");
byUrl.set(url, [...(byUrl.get(url) ?? []), it]);
}
const report: Record<string, string[]> = {};
const note = (category: string, link: string) => (report[category] ??= []).push(link);
let complete = 0;
for (const t of theirs) {
const candidates = byUrl.get(t.link);
if (!candidates) {
note("missing from cache", t.link);
continue;
}
// a link bookmarked twice: the best-matching candidate counts
const best = candidates.map((o) => problems(o, t)).reduce((a, b) => (b.length < a.length ? b : a));
if (best.length === 0) complete += 1;
for (const p of best) note(p, t.link);
}
const exportLinks = new Set(theirs.map((t) => t.link));
for (const url of byUrl.keys()) if (!exportLinks.has(url)) note("in cache, not in export", url);
return { complete, report };
}
writeFileSync(join(outDir, "report.json"), JSON.stringify(report, null, 1)); /**
console.log({ * Generates RSS from a cache's items (src/rss.ts) and compares it with a
export: theirs.length, * diigo RSS export: title, tags, description, highlights + sticky notes.
cache: ours.length, * Prints counts only; writes <out-dir>/from-cache.rss and
complete, * <out-dir>/report.json (mismatched links, by category).
...Object.fromEntries(Object.entries(report).map(([k, v]) => [k, v.length])), *
}); * The program's edge: every power it uses arrives as a default argument
* (ocap discipline), so a test can pass others.
*
* Ends by reporting an exit code (2: usage) through `exit`.
*/
export function main(
argv = process.argv.slice(2),
{
readText = (path: string) => readFileSync(path, "utf8"),
writeText = (path: string, text: string) => writeFileSync(path, text),
readItems = (dbPath: string): Item[] =>
(new DatabaseSync(dbPath, { readOnly: true }).prepare("select json from items order by updated desc").all() as {
json: string;
}[]).map((r) => JSON.parse(r.json) as Item),
now = () => new Date(),
log = (x: unknown) => console.log(x),
warn = (x: unknown) => console.error(x),
exit = (code: number) => {
process.exitCode = code;
},
} = {},
): void {
const [dbPath, exportPath, outDir] = argv;
if (!dbPath || !exportPath || !outDir) {
warn("usage: node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir>");
return exit(2);
}
const ours = readItems(dbPath);
writeText(join(outDir, "from-cache.rss"), toRss(ours, { title: "from the cache", pubDate: now().toUTCString() }));
const theirs = parseExport(readText(exportPath));
const { complete, report } = compare(ours, theirs);
writeText(join(outDir, "report.json"), JSON.stringify(report, null, 1));
log({
export: theirs.length,
cache: ours.length,
complete,
...Object.fromEntries(Object.entries(report).map(([k, v]) => [k, v.length])),
});
exit(0);
}
main();