fixup! diigo-cache: RSS in diigo's export shape, from cached items
compare-export.ts under ocap discipline: parseExport and compare are pure; main is the program's edge and takes every power it uses (argv, file reads and writes, the sqlite read, the clock, console, the exit code) as default arguments. Nothing ambient at module level; the last line just calls main(). Generated-by: Claude Opus 5.5
This commit is contained in:
dckc committed
1 parent
7be87d37dc
commit
d145841a2e
1 file changed
+78
-44
@@ -1,16 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file program script: does a cache hold everything in a diigo RSS
|
||||
* export? Generates RSS from the cache's items (src/rss.ts) and compares
|
||||
* it, item by item keyed on link, with the export: title, tags,
|
||||
* description, and highlights + sticky notes. A program script, so it
|
||||
* reaches the filesystem and the clock itself.
|
||||
* export? See {@link main}.
|
||||
*
|
||||
* node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir>
|
||||
*
|
||||
* Prints counts only; writes <out-dir>/from-cache.rss and
|
||||
* <out-dir>/report.json (mismatched links, by category). diigo's export
|
||||
* is not well-formed XML, so it is read leniently, item by item.
|
||||
*/
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
@@ -19,75 +12,116 @@ import { itemDescription, toRss } from "../src/rss.ts";
|
||||
|
||||
type Item = Record<string, unknown>;
|
||||
|
||||
const [dbPath, exportPath, outDir] = process.argv.slice(2);
|
||||
if (!dbPath || !exportPath || !outDir) {
|
||||
console.error("usage: node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir>");
|
||||
process.exit(2);
|
||||
}
|
||||
/** one export item, as read */
|
||||
type Exported = { title: string; link: string; description: string };
|
||||
|
||||
const FORBIDDEN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g;
|
||||
const clean = (s: string) => s.replace(FORBIDDEN, "");
|
||||
|
||||
// the cache: every item it holds, newest update first
|
||||
const db = new DatabaseSync(dbPath, { readOnly: true });
|
||||
const ours = (db.prepare("select json from items order by updated desc").all() as { json: string }[]).map(
|
||||
(r) => JSON.parse(r.json) as Item,
|
||||
);
|
||||
writeFileSync(join(outDir, "from-cache.rss"), toRss(ours, { title: "from the cache", pubDate: new Date().toUTCString() }));
|
||||
|
||||
// the export, leniently
|
||||
/** diigo's RSS export, read leniently item by item (it isn't well-formed
|
||||
* XML: it carries characters XML forbids) */
|
||||
export function parseExport(xml: string): Exported[] {
|
||||
const field = (it: string, name: string) =>
|
||||
it.match(new RegExp(`<${name}><!\\[CDATA\\[([\\s\\S]*?)\\]\\]></${name}>`))?.[1] ?? "";
|
||||
const theirs = [...readFileSync(exportPath, "utf8").matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => ({
|
||||
return [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => ({
|
||||
title: clean(field(m[1], "title")),
|
||||
link: clean(field(m[1], "link")),
|
||||
description: clean(field(m[1], "description")),
|
||||
}));
|
||||
}
|
||||
|
||||
const PARTS = /^Tags:([\s\S]*?);Description:([\s\S]*?);Annotations and Sticky notes:([\s\S]*)$/;
|
||||
|
||||
/** what's wrong with `ours` as a rendering of `theirs` (empty: nothing) */
|
||||
function problems(o: Item, t: Exported): string[] {
|
||||
const found: string[] = [];
|
||||
const title = clean(String(o.title ?? ""));
|
||||
// the export cuts titles at 255
|
||||
if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) found.push("title differs");
|
||||
const mine = clean(itemDescription(o)).match(PARTS);
|
||||
const yours = t.description.match(PARTS);
|
||||
if (!mine || !yours) return [...found, "description unparsed"];
|
||||
if (mine[1] !== yours[1]) found.push("tags differ");
|
||||
if (mine[2] !== yours[2]) found.push("description differs");
|
||||
if (mine[3] !== yours[3]) found.push("annotations differ");
|
||||
return found;
|
||||
}
|
||||
|
||||
/**
|
||||
* compare cached items with an export, keyed on link: each export item
|
||||
* is complete, missing, or has problems by category; cached links absent
|
||||
* from the export are reported too. Links only, never contents.
|
||||
*/
|
||||
export function compare(ours: Item[], theirs: Exported[]): { complete: number; report: Record<string, string[]> } {
|
||||
const byUrl = new Map<string, Item[]>();
|
||||
for (const it of ours) {
|
||||
const url = String(it.url ?? "");
|
||||
byUrl.set(url, [...(byUrl.get(url) ?? []), it]);
|
||||
}
|
||||
|
||||
const PARTS = /^Tags:([\s\S]*?);Description:([\s\S]*?);Annotations and Sticky notes:([\s\S]*)$/;
|
||||
const report: Record<string, string[]> = {};
|
||||
const note = (category: string, link: string) => (report[category] ??= []).push(link);
|
||||
let complete = 0;
|
||||
|
||||
for (const t of theirs) {
|
||||
const candidates = byUrl.get(t.link);
|
||||
if (!candidates) {
|
||||
note("missing from cache", t.link);
|
||||
continue;
|
||||
}
|
||||
// a link bookmarked twice: any candidate that matches will do
|
||||
const verdicts = candidates.map((o) => {
|
||||
const problems: string[] = [];
|
||||
const title = clean(String(o.title ?? ""));
|
||||
if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) problems.push("title differs");
|
||||
const mine = clean(itemDescription(o)).match(PARTS);
|
||||
const yours = t.description.match(PARTS);
|
||||
if (!mine || !yours) problems.push("description unparsed");
|
||||
else {
|
||||
if (mine[1] !== yours[1]) problems.push("tags differ");
|
||||
if (mine[2] !== yours[2]) problems.push("description differs");
|
||||
if (mine[3] !== yours[3]) problems.push("annotations differ");
|
||||
}
|
||||
return problems;
|
||||
});
|
||||
const best = verdicts.reduce((a, b) => (b.length < a.length ? b : a));
|
||||
// a link bookmarked twice: the best-matching candidate counts
|
||||
const best = candidates.map((o) => problems(o, t)).reduce((a, b) => (b.length < a.length ? b : a));
|
||||
if (best.length === 0) complete += 1;
|
||||
for (const p of best) note(p, t.link);
|
||||
}
|
||||
|
||||
const exportLinks = new Set(theirs.map((t) => t.link));
|
||||
for (const url of byUrl.keys()) if (!exportLinks.has(url)) note("in cache, not in export", url);
|
||||
return { complete, report };
|
||||
}
|
||||
|
||||
writeFileSync(join(outDir, "report.json"), JSON.stringify(report, null, 1));
|
||||
console.log({
|
||||
/**
|
||||
* Generates RSS from a cache's items (src/rss.ts) and compares it with a
|
||||
* diigo RSS export: title, tags, description, highlights + sticky notes.
|
||||
* Prints counts only; writes <out-dir>/from-cache.rss and
|
||||
* <out-dir>/report.json (mismatched links, by category).
|
||||
*
|
||||
* The program's edge: every power it uses arrives as a default argument
|
||||
* (ocap discipline), so a test can pass others.
|
||||
*
|
||||
* Ends by reporting an exit code (2: usage) through `exit`.
|
||||
*/
|
||||
export function main(
|
||||
argv = process.argv.slice(2),
|
||||
{
|
||||
readText = (path: string) => readFileSync(path, "utf8"),
|
||||
writeText = (path: string, text: string) => writeFileSync(path, text),
|
||||
readItems = (dbPath: string): Item[] =>
|
||||
(new DatabaseSync(dbPath, { readOnly: true }).prepare("select json from items order by updated desc").all() as {
|
||||
json: string;
|
||||
}[]).map((r) => JSON.parse(r.json) as Item),
|
||||
now = () => new Date(),
|
||||
log = (x: unknown) => console.log(x),
|
||||
warn = (x: unknown) => console.error(x),
|
||||
exit = (code: number) => {
|
||||
process.exitCode = code;
|
||||
},
|
||||
} = {},
|
||||
): void {
|
||||
const [dbPath, exportPath, outDir] = argv;
|
||||
if (!dbPath || !exportPath || !outDir) {
|
||||
warn("usage: node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir>");
|
||||
return exit(2);
|
||||
}
|
||||
const ours = readItems(dbPath);
|
||||
writeText(join(outDir, "from-cache.rss"), toRss(ours, { title: "from the cache", pubDate: now().toUTCString() }));
|
||||
const theirs = parseExport(readText(exportPath));
|
||||
const { complete, report } = compare(ours, theirs);
|
||||
writeText(join(outDir, "report.json"), JSON.stringify(report, null, 1));
|
||||
log({
|
||||
export: theirs.length,
|
||||
cache: ours.length,
|
||||
complete,
|
||||
...Object.fromEntries(Object.entries(report).map(([k, v]) => [k, v.length])),
|
||||
});
|
||||
exit(0);
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in new issue
Block a user