captoys/diigo-cache/scripts/compare-export.ts

#!/usr/bin/env node
/** @file Does a cache hold everything in a diigo RSS export? See {@link HELP}. */

/** printed on bad usage */
const HELP = `usage: node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir>

Generates RSS from the cache's items (src/rss.ts) and compares it with the
export, keyed on link: title, tags, description, highlights and sticky
notes. Prints counts to stdout; writes <out-dir>/from-cache.rss and
<out-dir>/report.json (mismatched links, by category).
`;

import * as nodeFs from "node:fs/promises";
import { basename, join } from "node:path";
import { DatabaseSync } from "node:sqlite";
import { itemDescription, toRss } from "../src/rss.ts";

type Item = Record<string, unknown>;

/**
 * the exit status for incorrect usage (invalid options, missing
 * arguments): the shell's convention
 *
 * @see https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
 */
const BAD_USAGE = 2;

/** one export item, as read */
type Exported = { title: string; link: string; description: string };

/** a new object with each value mapped; keys kept (as `objectMap` in
 * @endo/common) */
export function objectMap<V, W>(original: Record<string, V>, mapFn: (value: V, key: string) => W): Record<string, W> {
	return Object.fromEntries(Object.entries(original).map(([k, v]) => [k, mapFn(v, k)]));
}

const FORBIDDEN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g;
const clean = (s: string) => s.replace(FORBIDDEN, "");

/** diigo's RSS export, read leniently item by item (it isn't well-formed
 * XML: it carries characters XML forbids) */
export function parseExport(xml: string): Exported[] {
	const field = (it: string, name: string) =>
		it.match(new RegExp(`<${name}><!\\[CDATA\\[([\\s\\S]*?)\\]\\]></${name}>`))?.[1] ?? "";
	return [...xml.matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => ({
		title: clean(field(m[1], "title")),
		link: clean(field(m[1], "link")),
		description: clean(field(m[1], "description")),
	}));
}

const PARTS = /^Tags:([\s\S]*?);Description:([\s\S]*?);Annotations and Sticky notes:([\s\S]*)$/;

/** what's wrong with `ours` as a rendering of `theirs` (empty: nothing) */
function problems(o: Item, t: Exported): string[] {
	const found: string[] = [];
	const title = clean(String(o.title ?? ""));
	// the export cuts titles at 255
	if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) found.push("title differs");
	const mine = clean(itemDescription(o)).match(PARTS);
	const yours = t.description.match(PARTS);
	if (!mine || !yours) return [...found, "description unparsed"];
	if (mine[1] !== yours[1]) found.push("tags differ");
	if (mine[2] !== yours[2]) found.push("description differs");
	if (mine[3] !== yours[3]) found.push("annotations differ");
	return found;
}

/**
 * compare cached items with an export, keyed on link: each export item
 * is complete, missing, or has problems by category; cached links absent
 * from the export are reported too. Links only, never contents.
 */
export function compare(ours: Item[], theirs: Exported[]): { complete: number; report: Record<string, string[]> } {
	const byUrl = new Map<string, Item[]>();
	for (const it of ours) {
		const url = String(it.url ?? "");
		byUrl.set(url, [...(byUrl.get(url) ?? []), it]);
	}
	const report: Record<string, string[]> = {};
	const note = (category: string, link: string) => (report[category] ??= []).push(link);
	let complete = 0;
	for (const t of theirs) {
		const candidates = byUrl.get(t.link);
		if (!candidates) {
			note("missing from cache", t.link);
			continue;
		}
		// a link bookmarked twice: the best-matching candidate counts
		const best = candidates.map((o) => problems(o, t)).reduce((a, b) => (b.length < a.length ? b : a));
		if (best.length === 0) complete += 1;
		for (const p of best) note(p, t.link);
	}
	const exportLinks = new Set(theirs.map((t) => t.link));
	for (const url of byUrl.keys()) if (!exportLinks.has(url)) note("in cache, not in export", url);
	return { complete, report };
}

/** read-only access to one cache's items: attenuates a database */
export function cacheItems(db: { prepare(sql: string): { all(): unknown[] } }) {
	return {
		/** every item, newest update first */
		all: (): Item[] =>
			(db.prepare("select json from items order by updated desc").all() as { json: string }[]).map(
				(r) => JSON.parse(r.json) as Item,
			),
	};
}

/** write access to files directly inside one directory: attenuates fs */
export function outDir(fs: Pick<typeof nodeFs, "writeFile">, dir: string) {
	return {
		async write(name: string, text: string): Promise<void> {
			if (basename(name) !== name) throw new Error(`not a plain file name: ${name}`);
			await fs.writeFile(join(dir, name), text);
		},
	};
}

/**
 * Run the comparison. Every power it uses comes in as a default argument
 * and is narrowed here — the database to {@link cacheItems}, the output
 * directory to {@link outDir} — before anything else uses it.
 */
export async function main(
	argv = process.argv.slice(2),
	{
		fs = nodeFs,
		// node:sqlite has only a synchronous API; one read of a local file
		connect = (path: string) => new DatabaseSync(path, { readOnly: true }),
		now = () => new Date(),
		stdout = process.stdout,
		stderr = process.stderr,
		exit = (code: number) => {
			process.exitCode = code;
		},
	}: {
		fs?: Pick<typeof nodeFs, "readFile" | "writeFile">;
		connect?: (path: string) => { prepare(sql: string): { all(): unknown[] } };
		now?: () => Date;
		stdout?: { write(text: string): unknown };
		stderr?: { write(text: string): unknown };
		exit?: (code: number) => void;
	} = {},
): Promise<void> {
	const [dbPath, exportPath, outPath] = argv;
	if (!dbPath || !exportPath || !outPath) {
		stderr.write(HELP);
		return exit(BAD_USAGE);
	}
	const cache = cacheItems(connect(dbPath));
	const out = outDir(fs, outPath);
	const exportXml = await fs.readFile(exportPath, "utf8");

	const ours = cache.all();
	await out.write("from-cache.rss", toRss(ours, { title: "from the cache", pubDate: now().toUTCString() }));
	const theirs = parseExport(exportXml);
	const { complete, report } = compare(ours, theirs);
	await out.write("report.json", JSON.stringify(report, null, 1));
	const counts = {
		export: theirs.length,
		cache: ours.length,
		complete,
		...objectMap(report, (links) => links.length),
	};
	stdout.write(`${JSON.stringify(counts, null, 1)}\n`);
	exit(0);
}

// run only as a program, not when imported (e.g. by a test)
if (import.meta.main) await main();