From e6deb9fda0e7733855c18a51edc4a656205bfb90 Mon Sep 17 00:00:00 2001 From: Dan Connolly Date: Wed, 7 Oct 2026 05:22:12 -0500 Subject: [PATCH] diigo-cache: RSS in diigo's export shape, from cached items MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/rss.ts renders items as RSS 2.0 in the shape of diigo's RSS export: title, link, and a description packing Tags:…;Description:…; Annotations and Sticky notes:… (Highlight:/Sticky notes: lines). Unlike the export it is well-formed (characters XML forbids are dropped; the export carries some raw) and keeps whole titles (the export cuts at 255). Pure: dates come from the caller. scripts/compare-export.ts (a program script: node, node:sqlite) loads a cache's items, writes the RSS, and compares it with an export, keyed on link, reporting counts by category. Generated-by: Claude Opus 5.5 --- diigo-cache/scripts/compare-export.ts | 93 +++++++++++++++++++++++++++ diigo-cache/src/rss.ts | 61 ++++++++++++++++++ diigo-cache/tests/rss.test.ts | 50 ++++++++++++++ 3 files changed, 204 insertions(+) create mode 100644 diigo-cache/scripts/compare-export.ts create mode 100644 diigo-cache/src/rss.ts create mode 100644 diigo-cache/tests/rss.test.ts diff --git a/diigo-cache/scripts/compare-export.ts b/diigo-cache/scripts/compare-export.ts new file mode 100644 index 0000000..c486da4 --- /dev/null +++ b/diigo-cache/scripts/compare-export.ts @@ -0,0 +1,93 @@ +#!/usr/bin/env node +/** + * @file program script: does a cache hold everything in a diigo RSS + * export? Generates RSS from the cache's items (src/rss.ts) and compares + * it, item by item keyed on link, with the export: title, tags, + * description, and highlights + sticky notes. A program script, so it + * reaches the filesystem and the clock itself. + * + * node scripts/compare-export.ts + * + * Prints counts only; writes /from-cache.rss and + * /report.json (mismatched links, by category). diigo's export + * is not well-formed XML, so it is read leniently, item by item. + */ +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { itemDescription, toRss } from "../src/rss.ts"; + +type Item = Record; + +const [dbPath, exportPath, outDir] = process.argv.slice(2); +if (!dbPath || !exportPath || !outDir) { + console.error("usage: node scripts/compare-export.ts "); + process.exit(2); +} + +const FORBIDDEN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g; +const clean = (s: string) => s.replace(FORBIDDEN, ""); + +// the cache: every item it holds, newest update first +const db = new DatabaseSync(dbPath, { readOnly: true }); +const ours = (db.prepare("select json from items order by updated desc").all() as { json: string }[]).map( + (r) => JSON.parse(r.json) as Item, +); +writeFileSync(join(outDir, "from-cache.rss"), toRss(ours, { title: "from the cache", pubDate: new Date().toUTCString() })); + +// the export, leniently +const field = (it: string, name: string) => + it.match(new RegExp(`<${name}>`))?.[1] ?? ""; +const theirs = [...readFileSync(exportPath, "utf8").matchAll(/([\s\S]*?)<\/item>/g)].map((m) => ({ + title: clean(field(m[1], "title")), + link: clean(field(m[1], "link")), + description: clean(field(m[1], "description")), +})); + +const byUrl = new Map(); +for (const it of ours) { + const url = String(it.url ?? ""); + byUrl.set(url, [...(byUrl.get(url) ?? []), it]); +} + +const PARTS = /^Tags:([\s\S]*?);Description:([\s\S]*?);Annotations and Sticky notes:([\s\S]*)$/; +const report: Record = {}; +const note = (category: string, link: string) => (report[category] ??= []).push(link); +let complete = 0; + +for (const t of theirs) { + const candidates = byUrl.get(t.link); + if (!candidates) { + note("missing from cache", t.link); + continue; + } + // a link bookmarked twice: any candidate that matches will do + const verdicts = candidates.map((o) => { + const problems: string[] = []; + const title = clean(String(o.title ?? "")); + if (title !== t.title && !(t.title.length === 255 && title.startsWith(t.title))) problems.push("title differs"); + const mine = clean(itemDescription(o)).match(PARTS); + const yours = t.description.match(PARTS); + if (!mine || !yours) problems.push("description unparsed"); + else { + if (mine[1] !== yours[1]) problems.push("tags differ"); + if (mine[2] !== yours[2]) problems.push("description differs"); + if (mine[3] !== yours[3]) problems.push("annotations differ"); + } + return problems; + }); + const best = verdicts.reduce((a, b) => (b.length < a.length ? b : a)); + if (best.length === 0) complete += 1; + for (const p of best) note(p, t.link); +} + +const exportLinks = new Set(theirs.map((t) => t.link)); +for (const url of byUrl.keys()) if (!exportLinks.has(url)) note("in cache, not in export", url); + +writeFileSync(join(outDir, "report.json"), JSON.stringify(report, null, 1)); +console.log({ + export: theirs.length, + cache: ours.length, + complete, + ...Object.fromEntries(Object.entries(report).map(([k, v]) => [k, v.length])), +}); diff --git a/diigo-cache/src/rss.ts b/diigo-cache/src/rss.ts new file mode 100644 index 0000000..303225b --- /dev/null +++ b/diigo-cache/src/rss.ts @@ -0,0 +1,61 @@ +/** + * @file cached diigo items as RSS 2.0, in the shape of diigo's RSS export + * (`Tags:…;Description:…;Annotations and Sticky notes:…`), but well-formed + * and with titles uncut. Pure: the caller supplies the dates. + */ + +import type { Item } from "./items.js"; + +/** characters XML 1.0 forbids (the C0 controls but tab, LF, CR); + * diigo's export carries some raw, so strict parsers reject it */ +const XML_FORBIDDEN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g; + +/** text as one or more CDATA sections: forbidden characters dropped, + * a literal `]]>` split across two sections */ +function cdata(text: string): string { + return `", "]]]]>")}]]>`; +} + +const str = (v: unknown): string => (typeof v === "string" ? v : ""); +const list = (v: unknown): Record[] => (Array.isArray(v) ? (v as Record[]) : []); + +/** + * the packed description diigo's export uses: + * `Tags:a,b;Description:…;Annotations and Sticky notes:…` — tags + * comma-separated (`no_tag` when none), then each highlight as + * `Highlight:\n` followed by its sticky notes, one + * `Sticky notes:\n` each (notes may span lines) + */ +export function itemDescription(item: Item): string { + const tags = str(item.tags) || "no_tag"; + const notes = list(item.annotations) + .map((a) => `Highlight:${str(a.content)}\n${list(a.stickyNotes).map((n) => `Sticky notes:${str(n.content)}\n`).join("")}`) + .join(""); + return `Tags:${tags};Description:${str(item.description)};Annotations and Sticky notes:${notes}`; +} + +/** one ``: title, link, packed description. The title is whole + * (diigo's export cuts titles at 255 characters). */ +export function rssItem(item: Item): string { + return `${cdata(str(item.title))}${cdata(str(item.url))}${cdata(itemDescription(item))}`; +} + +/** + * a whole feed + * + * @param channel `pubDate` in RFC 822 form is the caller's to supply + * (no clock here) + */ +export function toRss(items: Item[], channel: { title: string; pubDate: string }): string { + return ` + + +${cdata(channel.title)} + +${channel.pubDate} +${channel.pubDate} +${items.map(rssItem).join("\n")} + + +`; +} diff --git a/diigo-cache/tests/rss.test.ts b/diigo-cache/tests/rss.test.ts new file mode 100644 index 0000000..d1986ab --- /dev/null +++ b/diigo-cache/tests/rss.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { itemDescription, rssItem, toRss } from "../src/rss.js"; + +// shaped like diigo's export (field layout observed 2026-10-07), made-up values + +describe("itemDescription", () => { + it("packs tags, description, and annotations the way diigo's export does", () => { + expect( + itemDescription({ + tags: "ocap,js", + description: "a note", + annotations: [ + { content: "first highlight", stickyNotes: [] }, + { content: "second", stickyNotes: [{ content: "a sticky\n\nwith lines" }, { content: "another" }] }, + ], + }), + ).toBe( + "Tags:ocap,js;Description:a note;Annotations and Sticky notes:" + + "Highlight:first highlight\n" + + "Highlight:second\nSticky notes:a sticky\n\nwith lines\nSticky notes:another\n", + ); + }); + + it("says no_tag when there are no tags, and leaves empty parts empty", () => { + expect(itemDescription({ tags: "", description: "", annotations: [] })).toBe( + "Tags:no_tag;Description:;Annotations and Sticky notes:", + ); + }); +}); + +describe("rssItem / toRss", () => { + it("is well-formed even when diigo's text isn't: forbidden characters dropped, ]]> split", () => { + const xml = rssItem({ title: "bad\u0003char", url: "https://a.example/", tags: "", description: "x]]>y" }); + expect(xml).not.toContain("\u0003"); + expect(xml).toContain("<![CDATA[badchar]]>"); + expect(xml).toContain("Description:x]]]]>y;"); + }); + + it("keeps the whole title (diigo's export cuts at 255)", () => { + const title = "t".repeat(300); + expect(rssItem({ title, url: "u" })).toContain(title); + }); + + it("wraps items in an RSS 2.0 channel, dated by the caller", () => { + const feed = toRss([{ title: "a", url: "u" }], { title: "my items", pubDate: "Wed, 07 Oct 2026 12:00:00 +0000" }); + expect(feed).toMatch(/^<\?xml version="1.0"\?>\n\n/); + expect(feed).toContain("Wed, 07 Oct 2026 12:00:00 +0000"); + expect(feed).toContain("<![CDATA[a]]>"); + }); +});