refactor: rename diigo-dump to diigo-cache

Pure rename ahead of the dump worker's evolution into a bookmark cache (goal 1c), so history follows the files. The diigo-login helper script is dropped: the cookie now enters through webapp's mint form, never a local script.
2026-10-07 03:25:56 -05:00
1 parent 2c7d8c9b4d
commit 58be2ccaf8
@@ -0,0 +1,48 @@
+/**
+ * @file Diigo dump logic: the bookmark generator over any fluent handle.
+ * Site-flavored configuration (URL, headers, cookie) lives where the webapp
+ * instance is minted (src/index.ts: mint); only the traversal is here.
+ */
+
+export type PagedGet = {
+	pathjoin(ref: string): PagedGet;
+	query(params: Record<string, string | number>): PagedGet;
+	get(): Promise<string>;
+};
+
+export type DumpOptions = {
+	/** fluent handle for the diigo account, cookie attached upstream */
+	diigoAPI: PagedGet;
+	pages?: number;
+	count?: number;
+	log?: (event: Record<string, unknown>) => void;
+};
+
+/** yield diigo bookmarks, most-recently-updated first, `pages * count` at
+ * most. `diigoAPI` is attenuated authority; over RPC the generator runs in
+ * the caller's address space against a stub. */
+export async function* dumpBookmarks({
+	diigoAPI,
+	pages = 2,
+	count = 24,
+	log = () => {},
+}: DumpOptions): AsyncGenerator<Record<string, unknown>> {
+	const getUserItems = (page: number) =>
+		diigoAPI
+			.pathjoin("load_user_items")
+			.query({ sort: "updated", count, page_num: page })
+			.get()
+			.then(JSON.parse) as Promise<{ total: number; items: Record<string, unknown>[] }>;
+
+	const first = await getUserItems(1);
+	const totalPages = Math.ceil(first.total / count);
+	log({ event: "start", total: first.total, pages, totalPages });
+
+	let got = 0;
+	for (let page = 1; page <= Math.min(pages, totalPages); page += 1) {
+		const data = page === 1 ? first : await getUserItems(page);
+		yield* data.items;
+		got += data.items.length;
+		log({ event: "page", page, of: pages, got });
+	}
+}
@@ -0,0 +1,244 @@
+/**
+ * @file diigo-dump worker: cookie access to diigo's
+ * interact_api/load_user_items via a WebApp Durable Object instance from
+ * the webapp worker. The id is a bearer capability held in DIIGO_WEB_ID;
+ * the cookie lives only inside that DO's storage.
+ *
+ * Connect flow (no askpass, no cookie-on-disk): the home page links to
+ * /_connect, which proxies webapp's mint form with the diigo prefills
+ * (cookie field blank) — paste the cookie, mint lands via the service
+ * binding, and the page shows the DIIGO_WEB_ID to put in .dev.vars.
+ * (scripts/diigo-login automates the same flow for terminals.)
+ * @see {@link DiigoBridge}
+ */
+
+import { WorkerEntrypoint } from "cloudflare:workers";
+import { dumpBookmarks, type PagedGet } from "./diigo.js";
+
+/** the WEBAPP service binding to webapp/src/index.ts:WebAppFactory; typed
+ * structurally here pending a shared types package */
+type WebAppFactoryShape = {
+	newWebApp(init: {
+		url: string;
+		headers?: Record<string, string>;
+		msPerCall?: number;
+	}): Promise<{ id: string; app: PagedGet }>;
+	at(id: string): Promise<PagedGet>;
+};
+
+type Env = {
+	DIIGO_WEB_ID?: string;
+	WEBAPP: WebAppFactoryShape;
+};
+
+const UA = "Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/141.0";
+const DIIGO_INTERACT = "https://www.diigo.com/interact_api/";
+/** prefill for webapp's mint form: everything but the cookie */
+const DIIGO_PREFILL = JSON.stringify({
+	"user-agent": UA,
+	"x-requested-with": "XMLHttpRequest",
+});
+
+/** connect to the configured instance, or give the setup pointer */
+async function instance(env: Env): Promise<PagedGet> {
+	const id = env.DIIGO_WEB_ID;
+	if (!id) {
+		throw new Error("no instance: use the connect flow on the home page, then set DIIGO_WEB_ID");
+	}
+	return env.WEBAPP.at(id);
+}
+
+export class DiigoBridge extends WorkerEntrypoint {
+	/** the diigo-attenuated handle: fluent chain over the instance */
+	open(): Promise<PagedGet> {
+		return instance(this.env as unknown as Env);
+	}
+
+	/** bulk dump from the caller's address space */
+	async dumpNdjson(pages: number, count = 24): Promise<string> {
+		const diigoAPI = await instance(this.env as unknown as Env);
+		const lines: string[] = [];
+		for await (const item of dumpBookmarks({ diigoAPI, pages, count })) {
+			lines.push(JSON.stringify(item));
+		}
+		return lines.join("\n");
+	}
+}
+
+export default {
+	async fetch(request: Request, env: Env): Promise<Response> {
+		const url = new URL(request.url);
+
+		// the connect flow: proxy webapp's mint form (prefilled, cookie blank)
+		if (url.pathname === "/_connect") {
+			return env.DIIGO_WEB_ID ? connectedPage(env) : connectPage();
+		}
+		// webapp's form POSTs back through us: forward to the binding
+		if (url.pathname === "/_mint" && request.method === "POST") {
+			return proxyMint(request, env);
+		}
+		if (url.pathname !== "/") return new Response("not found", { status: 404 });
+
+		if (request.method !== "POST") return home(null, undefined, Boolean(env.DIIGO_WEB_ID));
+
+		const form = await request.formData();
+		const raw = form.get("page");
+		const page = typeof raw === "string" ? Number.parseInt(raw, 10) : Number.NaN;
+		if (Number.isNaN(page) || page < 1 || page > 10000) {
+			return home("enter a page number (1 = most recently updated)", undefined, Boolean(env.DIIGO_WEB_ID));
+		}
+
+		try {
+			const app = await instance(env);
+			const text = await app
+				.pathjoin("load_user_items")
+				.query({ sort: "updated", count: 24, page_num: page })
+				.get();
+			const data = JSON.parse(text) as { total?: number; items?: Record<string, unknown>[] };
+			return home(
+				`total: ${data.total}\nitems: ${data.items?.length ?? 0}\n\n${JSON.stringify(data.items ?? data, null, 2)}`,
+				undefined,
+				true,
+			);
+		} catch (err) {
+			console.error({ event: "dump", page, error: String(err) });
+			return home(null, String(err), Boolean(env.DIIGO_WEB_ID));
+		}
+	},
+};
+
+/** the connect page: webapp's form with diigo defaults. The mint POST is
+ * proxied (/_mint) so one port serves the whole flow; the response
+ * page shows the id and the .dev.vars line to add. */
+async function connectPage(): Promise<Response> {
+	// proxy: render our own form with action="/_mint" so one port serves
+	// the whole flow. TODO(webapp): one canonical mint form; consider
+	// serving it from webapp directly.
+	const body = `<!doctype html>
+<html><head><meta charset="utf-8"><title>connect diigo session</title></head>
+<body>
+<h1>connect diigo session</h1>
+<ol>
+<li>copy the <code>Cookie</code> header from diigo.com devtools</li>
+<li>paste below, mint, then add the <code>DIIGO_WEB_ID</code> line it
+shows to <code>.dev.vars</code> (deployed: <code>wrangler secret put DIIGO_WEB_ID</code>)</li>
+<li>restart wrangler, dump from the <a href="/">home page</a></li>
+</ol>
+<form method="post" action="/_mint">
+<input type="hidden" name="url" value="${esc(DIIGO_INTERACT)}">
+<input type="hidden" name="headers" value="${esc(JSON.stringify({ "user-agent": UA, "x-requested-with": "XMLHttpRequest", cookie: "" }))}">
+<p><label>cookie <input type="password" name="cookie" size="80" autocomplete="off"></label></p>
+<p><button type="submit">mint</button></p>
+</form>
+</body></html>`;
+	return new Response(body, { headers: { "content-type": "text/html;charset=utf-8" } });
+}
+async function connectedPage(env: Env): Promise<Response> {
+	let health = "";
+	try {
+		const app = await instance(env);
+		// a cheap call: one item, exercises the cookie + rate gate
+		const text = await app
+			.pathjoin("load_user_items")
+			.query({ sort: "updated", count: 1, page_num: 1 })
+			.get();
+		const data = JSON.parse(text) as { total?: number };
+		health = `<p>instance is <b>live</b>: diigo reports <b>${esc(String(data.total ?? "?"))}</b> bookmarks.</p>`;
+	} catch (err) {
+		console.error({ event: "connect-check", error: String(err) });
+		health = `<p>instance is configured but the last check <b>failed</b>:</p><pre>${esc(String(err))}</pre><p>If diigo expired the cookie, re-mint below.</p>`;
+	}
+	const body = `<!doctype html>
+<html><head><meta charset="utf-8"><title>diigo session</title></head>
+<body>
+<h1>diigo session</h1>
+<p>connected: id <code>${esc(`${env.DIIGO_WEB_ID!.slice(0, 6)}…${env.DIIGO_WEB_ID!.slice(-4)}`)}</code> (from .dev.vars).</p>
+${health}
+<h2>replace the session</h2>
+<p>Only needed if the cookie is stale (diigo will say 401/403). Minting a
+new instance invalidates nothing, but the id rotates and .dev.vars needs
+the new <code>DIIGO_WEB_ID</code>.</p>
+<form method="post" action="/_mint">
+<input type="hidden" name="url" value="${esc(DIIGO_INTERACT)}">
+<input type="hidden" name="headers" value="${esc(JSON.stringify({ "user-agent": UA, "x-requested-with": "XMLHttpRequest", cookie: "" }))}">
+<p><label>cookie <input type="password" name="cookie" size="80" autocomplete="off"></label></p>
+<p><button type="submit">mint</button></p>
+</form>
+</body></html>`;
+	return new Response(body, { headers: { "content-type": "text/html;charset=utf-8" } });
+}
+
+/** proxy the mint: merge the pasted cookie into the headers JSON, mint via
+ * the service binding, show the id (plus the .dev.vars line to copy) */
+async function proxyMint(request: Request, env: Env): Promise<Response> {
+	const form = await request.formData();
+	const cookie = String(form.get("cookie") || "");
+	if (!cookie) {
+		return new Response("no cookie supplied", { status: 400 });
+	}
+	const headers = {
+		"user-agent": UA,
+		"x-requested-with": "XMLHttpRequest",
+		cookie,
+	};
+	const { id } = await env.WEBAPP.newWebApp({
+		url: DIIGO_INTERACT,
+		headers,
+		msPerCall: 500,
+	});
+	return idPage(id);
+}
+
+/** the id shown on the success page: a narrow text field (scrollable but
+ * too small to show most of the value) with a copy button for the full
+ * version. navigator.clipboard needs the page served from localhost/https
+ * — true in dev and at the deployed URL */
+function idPage(id: string): Response {
+	const body = `<!doctype html>
+<html><head><meta charset="utf-8"><title>minted</title></head>
+<body>
+<h1>minted</h1>
+<p>id: <input id="wid" readonly size="4" value="${esc(id)}" onfocus="this.select()">
+<button type="button" onclick='navigator.clipboard.writeText(${JSON.stringify(id)}).then(() => this.textContent = "copied ✓")'>copy</button></p>
+<p>add to <code>.dev.vars</code>:</p>
+<pre>DIIGO_WEB_ID=</pre>
+<p>…click copy, then paste the full value after it. <a href="/">back to diigo dump</a> (restart wrangler first).</p>
+</body></html>`;
+	return new Response(body, {
+		headers: {
+			"content-type": "text/html;charset=utf-8",
+			// for scripts: diigo-login reads this instead of scraping HTML
+			"x-diigo-web-id": id,
+		},
+	});
+}
+
+function home(detail: string | null, error?: string, connected?: boolean): Response {
+	const form = `<form method="post" action="/"><label>page <input type="number" name="page" min="1"></label> <button type="submit">dump</button></form>`;
+	const result = error
+		? `<p>status: <b>error</b></p><pre readonly style="white-space:pre-wrap;border:1px solid #c00;padding:1rem">${escapeHtml(error)}</pre>`
+		: detail
+			? `<pre readonly style="white-space:pre-wrap;border:1px solid #ccc;padding:1rem;max-height:60vh;overflow:auto">${escapeHtml(detail)}</pre>`
+			: "";
+	return new Response(
+		`<!doctype html>
+<html><head><meta charset="utf-8"><title>diigo dump</title></head>
+<body>
+<h1>diigo dump</h1>
+<p><a href="/_connect">${connected ? "diigo session status/replace…" : "connect diigo session…"}</a></p>
+${form}
+${result}
+</body></html>`,
+		{ headers: { "content-type": "text/html;charset=utf-8" } },
+	);
+}
+
+function esc(s: string): string {
+	return s
+		.replaceAll("&", "&amp;")
+		.replaceAll("<", "&lt;")
+		.replaceAll(">", "&gt;")
+		.replaceAll('"', "&quot;");
+}
+
+const escapeHtml = esc;