dckc
e6deb9fda0
src/rss.ts renders items as RSS 2.0 in the shape of diigo's RSS export: title, link, and a description packing Tags:…;Description:…; Annotations and Sticky notes:… (Highlight:/Sticky notes: lines). Unlike the export it is well-formed (characters XML forbids are dropped; the export carries some raw) and keeps whole titles (the export cuts at 255). Pure: dates come from the caller. scripts/compare-export.ts (a program script: node, node:sqlite) loads a cache's items, writes the RSS, and compares it with an export, keyed on link, reporting counts by category. Generated-by: Claude Opus 5.5
94 lines
3.7 KiB
JavaScript
94 lines
3.7 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* @file program script: does a cache hold everything in a diigo RSS
|
|
* export? Generates RSS from the cache's items (src/rss.ts) and compares
|
|
* it, item by item keyed on link, with the export: title, tags,
|
|
* description, and highlights + sticky notes. A program script, so it
|
|
* reaches the filesystem and the clock itself.
|
|
*
|
|
* node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir>
|
|
*
|
|
* Prints counts only; writes <out-dir>/from-cache.rss and
|
|
* <out-dir>/report.json (mismatched links, by category). diigo's export
|
|
* is not well-formed XML, so it is read leniently, item by item.
|
|
*/
|
|
import { readFileSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { DatabaseSync } from "node:sqlite";
|
|
import { itemDescription, toRss } from "../src/rss.ts";
|
|
|
|
type Item = Record<string, unknown>;
|
|
|
|
const [dbPath, exportPath, outDir] = process.argv.slice(2);
|
|
if (!dbPath || !exportPath || !outDir) {
|
|
console.error("usage: node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir>");
|
|
process.exit(2);
|
|
}
|
|
|
|
const FORBIDDEN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g;
|
|
const clean = (s: string) => s.replace(FORBIDDEN, "");
|
|
|
|
// the cache: every item it holds, newest update first
|
|
const db = new DatabaseSync(dbPath, { readOnly: true });
|
|
const ours = (db.prepare("select json from items order by updated desc").all() as { json: string }[]).map(
|
|
(r) => JSON.parse(r.json) as Item,
|
|
);
|
|
writeFileSync(join(outDir, "from-cache.rss"), toRss(ours, { title: "from the cache", pubDate: new Date().toUTCString() }));
|
|
|
|
// the export, leniently
|
|
const field = (it: string, name: string) =>
|
|
it.match(new RegExp(`<${name}><!\\[CDATA\\[([\\s\\S]*?)\\]\\]></${name}>`))?.[1] ?? "";
|
|
const theirs = [...readFileSync(exportPath, "utf8").matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => ({
|
|
title: clean(field(m[1], "title")),
|
|
link: clean(field(m[1], "link")),
|
|
description: clean(field(m[1], "description")),
|
|
}));
|
|
|
|
const byUrl = new Map<string, Item[]>();
|
|
for (const it of ours) {
|
|
const url = String(it.url ?? "");
|
|
byUrl.set(url, [...(byUrl.get(url) ?? []), it]);
|
|
}
|
|
|
|
const PARTS = /^Tags:([\s\S]*?);Description:([\s\S]*?);Annotations and Sticky notes:([\s\S]*)$/;
|
|
const report: Record<string, string[]> = {};
|
|
const note = (category: string, link: string) => (report[category] ??= []).push(link);
|
|
let complete = 0;
|
|
|
|
for (const t of theirs) {
|
|
const candidates = byUrl.get(t.link);
|
|
if (!candidates) {
|
|
note("missing from cache", t.link);
|
|
continue;
|
|
}
|
|
// a link bookmarked twice: any candidate that matches will do
|
|
const verdicts = candidates.map((o) => {
|
|
const problems: string[] = [];
|
|
const title = clean(String(o.title ?? ""));
|
|
if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) problems.push("title differs");
|
|
const mine = clean(itemDescription(o)).match(PARTS);
|
|
const yours = t.description.match(PARTS);
|
|
if (!mine || !yours) problems.push("description unparsed");
|
|
else {
|
|
if (mine[1] !== yours[1]) problems.push("tags differ");
|
|
if (mine[2] !== yours[2]) problems.push("description differs");
|
|
if (mine[3] !== yours[3]) problems.push("annotations differ");
|
|
}
|
|
return problems;
|
|
});
|
|
const best = verdicts.reduce((a, b) => (b.length < a.length ? b : a));
|
|
if (best.length === 0) complete += 1;
|
|
for (const p of best) note(p, t.link);
|
|
}
|
|
|
|
const exportLinks = new Set(theirs.map((t) => t.link));
|
|
for (const url of byUrl.keys()) if (!exportLinks.has(url)) note("in cache, not in export", url);
|
|
|
|
writeFileSync(join(outDir, "report.json"), JSON.stringify(report, null, 1));
|
|
console.log({
|
|
export: theirs.length,
|
|
cache: ours.length,
|
|
complete,
|
|
...Object.fromEntries(Object.entries(report).map(([k, v]) => [k, v.length])),
|
|
});
|