diigo-cache: RSS in diigo's export shape, from cached items

src/rss.ts renders items as RSS 2.0 in the shape of diigo's RSS export:
title, link, and a description packing Tags:…;Description:…;
Annotations and Sticky notes:… (Highlight:/Sticky notes: lines). Unlike
the export it is well-formed (characters XML forbids are dropped; the
export carries some raw) and keeps whole titles (the export cuts at
255). Pure: dates come from the caller.

scripts/compare-export.ts (a program script: node, node:sqlite) loads a
cache's items, writes the RSS, and compares it with an export, keyed on
link, reporting counts by category.

Generated-by: Claude Opus 5.5

diigo-cache/scripts/compare-export.ts

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+/**
+ * @file program script: does a cache hold everything in a diigo RSS
+ * export? Generates RSS from the cache's items (src/rss.ts) and compares
+ * it, item by item keyed on link, with the export: title, tags,
+ * description, and highlights + sticky notes. A program script, so it
+ * reaches the filesystem and the clock itself.
+ *
+ *     node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir>
+ *
+ * Prints counts only; writes <out-dir>/from-cache.rss and
+ * <out-dir>/report.json (mismatched links, by category). diigo's export
+ * is not well-formed XML, so it is read leniently, item by item.
+ */
+import { readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { itemDescription, toRss } from "../src/rss.ts";
+
+type Item = Record<string, unknown>;
+
+const [dbPath, exportPath, outDir] = process.argv.slice(2);
+if (!dbPath || !exportPath || !outDir) {
+	console.error("usage: node scripts/compare-export.ts <cache.sqlite> <export.xml> <out-dir>");
+	process.exit(2);
+}
+
+const FORBIDDEN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g;
+const clean = (s: string) => s.replace(FORBIDDEN, "");
+
+// the cache: every item it holds, newest update first
+const db = new DatabaseSync(dbPath, { readOnly: true });
+const ours = (db.prepare("select json from items order by updated desc").all() as { json: string }[]).map(
+	(r) => JSON.parse(r.json) as Item,
+);
+writeFileSync(join(outDir, "from-cache.rss"), toRss(ours, { title: "from the cache", pubDate: new Date().toUTCString() }));
+
+// the export, leniently
+const field = (it: string, name: string) =>
+	it.match(new RegExp(`<${name}><!\\[CDATA\\[([\\s\\S]*?)\\]\\]></${name}>`))?.[1] ?? "";
+const theirs = [...readFileSync(exportPath, "utf8").matchAll(/<item>([\s\S]*?)<\/item>/g)].map((m) => ({
+	title: clean(field(m[1], "title")),
+	link: clean(field(m[1], "link")),
+	description: clean(field(m[1], "description")),
+}));
+
+const byUrl = new Map<string, Item[]>();
+for (const it of ours) {
+	const url = String(it.url ?? "");
+	byUrl.set(url, [...(byUrl.get(url) ?? []), it]);
+}
+
+const PARTS = /^Tags:([\s\S]*?);Description:([\s\S]*?);Annotations and Sticky notes:([\s\S]*)$/;
+const report: Record<string, string[]> = {};
+const note = (category: string, link: string) => (report[category] ??= []).push(link);
+let complete = 0;
+
+for (const t of theirs) {
+	const candidates = byUrl.get(t.link);
+	if (!candidates) {
+		note("missing from cache", t.link);
+		continue;
+	}
+	// a link bookmarked twice: any candidate that matches will do
+	const verdicts = candidates.map((o) => {
+		const problems: string[] = [];
+		const title = clean(String(o.title ?? ""));
+		if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) problems.push("title differs");
+		const mine = clean(itemDescription(o)).match(PARTS);
+		const yours = t.description.match(PARTS);
+		if (!mine || !yours) problems.push("description unparsed");
+		else {
+			if (mine[1] !== yours[1]) problems.push("tags differ");
+			if (mine[2] !== yours[2]) problems.push("description differs");
+			if (mine[3] !== yours[3]) problems.push("annotations differ");
+		}
+		return problems;
+	});
+	const best = verdicts.reduce((a, b) => (b.length < a.length ? b : a));
+	if (best.length === 0) complete += 1;
+	for (const p of best) note(p, t.link);
+}
+
+const exportLinks = new Set(theirs.map((t) => t.link));
+for (const url of byUrl.keys()) if (!exportLinks.has(url)) note("in cache, not in export", url);
+
+writeFileSync(join(outDir, "report.json"), JSON.stringify(report, null, 1));
+console.log({
+	export: theirs.length,
+	cache: ours.length,
+	complete,
+	...Object.fromEntries(Object.entries(report).map(([k, v]) => [k, v.length])),
+});

diigo-cache/src/rss.ts

@@ -0,0 +1,61 @@
+/**
+ * @file cached diigo items as RSS 2.0, in the shape of diigo's RSS export
+ * (`Tags:…;Description:…;Annotations and Sticky notes:…`), but well-formed
+ * and with titles uncut. Pure: the caller supplies the dates.
+ */
+
+import type { Item } from "./items.js";
+
+/** characters XML 1.0 forbids (the C0 controls but tab, LF, CR);
+ * diigo's export carries some raw, so strict parsers reject it */
+const XML_FORBIDDEN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g;
+
+/** text as one or more CDATA sections: forbidden characters dropped,
+ * a literal `]]>` split across two sections */
+function cdata(text: string): string {
+	return `<![CDATA[${text.replace(XML_FORBIDDEN, "").replaceAll("]]>", "]]]]><![CDATA[>")}]]>`;
+}
+
+const str = (v: unknown): string => (typeof v === "string" ? v : "");
+const list = (v: unknown): Record<string, unknown>[] => (Array.isArray(v) ? (v as Record<string, unknown>[]) : []);
+
+/**
+ * the packed description diigo's export uses:
+ * `Tags:a,b;Description:…;Annotations and Sticky notes:…` — tags
+ * comma-separated (`no_tag` when none), then each highlight as
+ * `Highlight:<text>\n` followed by its sticky notes, one
+ * `Sticky notes:<text>\n` each (notes may span lines)
+ */
+export function itemDescription(item: Item): string {
+	const tags = str(item.tags) || "no_tag";
+	const notes = list(item.annotations)
+		.map((a) => `Highlight:${str(a.content)}\n${list(a.stickyNotes).map((n) => `Sticky notes:${str(n.content)}\n`).join("")}`)
+		.join("");
+	return `Tags:${tags};Description:${str(item.description)};Annotations and Sticky notes:${notes}`;
+}
+
+/** one `<item>`: title, link, packed description. The title is whole
+ * (diigo's export cuts titles at 255 characters). */
+export function rssItem(item: Item): string {
+	return `<item><title>${cdata(str(item.title))}</title><link>${cdata(str(item.url))}</link><description>${cdata(itemDescription(item))}</description></item>`;
+}
+
+/**
+ * a whole feed
+ *
+ * @param channel `pubDate` in RFC 822 form is the caller's to supply
+ * (no clock here)
+ */
+export function toRss(items: Item[], channel: { title: string; pubDate: string }): string {
+	return `<?xml version="1.0"?>
+<rss version="2.0">
+<channel>
+<title>${cdata(channel.title)}</title>
+<link></link>
+<pubDate>${channel.pubDate}</pubDate>
+<lastBuildDate>${channel.pubDate}</lastBuildDate>
+${items.map(rssItem).join("\n")}
+</channel>
+</rss>
+`;
+}

diigo-cache/tests/rss.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, it } from "vitest";
+import { itemDescription, rssItem, toRss } from "../src/rss.js";
+
+// shaped like diigo's export (field layout observed 2026-10-07), made-up values
+
+describe("itemDescription", () => {
+	it("packs tags, description, and annotations the way diigo's export does", () => {
+		expect(
+			itemDescription({
+				tags: "ocap,js",
+				description: "a note",
+				annotations: [
+					{ content: "first highlight", stickyNotes: [] },
+					{ content: "second", stickyNotes: [{ content: "a sticky\n\nwith lines" }, { content: "another" }] },
+				],
+			}),
+		).toBe(
+			"Tags:ocap,js;Description:a note;Annotations and Sticky notes:" +
+				"Highlight:first highlight\n" +
+				"Highlight:second\nSticky notes:a sticky\n\nwith lines\nSticky notes:another\n",
+		);
+	});
+
+	it("says no_tag when there are no tags, and leaves empty parts empty", () => {
+		expect(itemDescription({ tags: "", description: "", annotations: [] })).toBe(
+			"Tags:no_tag;Description:;Annotations and Sticky notes:",
+		);
+	});
+});
+
+describe("rssItem / toRss", () => {
+	it("is well-formed even when diigo's text isn't: forbidden characters dropped, ]]> split", () => {
+		const xml = rssItem({ title: "bad\u0003char", url: "https://a.example/", tags: "", description: "x]]>y" });
+		expect(xml).not.toContain("\u0003");
+		expect(xml).toContain("<title><![CDATA[badchar]]></title>");
+		expect(xml).toContain("Description:x]]]]><![CDATA[>y;");
+	});
+
+	it("keeps the whole title (diigo's export cuts at 255)", () => {
+		const title = "t".repeat(300);
+		expect(rssItem({ title, url: "u" })).toContain(title);
+	});
+
+	it("wraps items in an RSS 2.0 channel, dated by the caller", () => {
+		const feed = toRss([{ title: "a", url: "u" }], { title: "my items", pubDate: "Wed, 07 Oct 2026 12:00:00 +0000" });
+		expect(feed).toMatch(/^<\?xml version="1.0"\?>\n<rss version="2.0">\n<channel>/);
+		expect(feed).toContain("<pubDate>Wed, 07 Oct 2026 12:00:00 +0000</pubDate>");
+		expect(feed).toContain("<item><title><![CDATA[a]]></title>");
+	});
+});