From 309f003fa378744fd2cd5335782ce025e2ad1adb Mon Sep 17 00:00:00 2001 From: Connolly Date: Tue, 6 Oct 2026 22:13:58 -0500 Subject: [PATCH] feat(diigo-dump): fluent WebApp over RPC + bound fetch + logged errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restore the fluent client from mm-diigo's WebApp.js as RpcTarget classes (src/webapp.ts): WebApp with pathjoin/query/withHeaders/cookies/get/post, flower-style unions (NuxiNL/flower v0.11, Jan 2019 — merge only if no shared key contradicts), pathjoin containment guard; RateLimitedWebApp shares one gate across all derived handles, enforced at get/post only. src/diigo.ts ports makeDiigoAPI and the dumpBookmarks generator. RPC surface: DiigoService (WorkerEntrypoint) with open() handing a rate-limited stub whose cookie is attached server-side only, and dumpNdjson. Promise pipelining makes a whole chain a single round trip. Divergences from the original for the RPC setting: - url is a private field: a public property on an RpcTarget is readable by stub holders, and query strings can carry credentials - Error messages carry no header/param values; URLs have query stripped (structured details via @endo/errors deferred until node_modules) - fetch default is fetch.bind(globalThis): a detached fetch throws 'Illegal invocation' in workerd (found live, fixed, verified 200) - dumpPage/open log failures via console.error for observability README documents config, RPC use, and each divergence. Generated-by: opencode (GLM) --- diigo-dump/README.md | 34 ++++- diigo-dump/src/cloudflare-env.d.ts | 10 ++ diigo-dump/src/diigo.ts | 73 ++++++++++ diigo-dump/src/index.ts | 96 ++++++++----- diigo-dump/src/webapp.ts | 213 +++++++++++++++++++++++++++++ diigo-dump/tsconfig.json | 13 ++ 6 files changed, 404 insertions(+), 35 deletions(-) create mode 100644 diigo-dump/src/cloudflare-env.d.ts create mode 100644 diigo-dump/src/diigo.ts create mode 100644 diigo-dump/src/webapp.ts create mode 100644 diigo-dump/tsconfig.json diff --git a/diigo-dump/README.md b/diigo-dump/README.md index ed721ea..a7dbe04 100644 --- a/diigo-dump/README.md +++ b/diigo-dump/README.md @@ -16,9 +16,37 @@ with the session cookie and shows status, `total`, and the JSON items in a read-only `
`. GET `/` is inert — dumps happen only on explicit submits.
 
 Provenance notes: the `user-agent` / `x-requested-with` headers and the
-`interact_api` endpoint mirror the original script; the ocap-style
-attenuation there (pathjoin/query/rateLimit in WebApp.js) is reduced here to
-a plain fetch with the cookie never exposed beyond this one call.
+`interact_api` endpoint mirror the original script; the fluent client from
+`WebApp.js` is restored in `src/webapp.ts` with `WebApp` and the rate-limit
+wrapper as `RpcTarget` subclasses. `src/diigo.ts` holds `makeDiigoAPI` and
+the `dumpBookmarks` generator; `DiigoService.open()` hands an RPC caller a
+rate-limited stub with the cookie attached only on the server side, and
+promise pipelining makes a whole `pathjoin().query().get()` chain a single
+round trip. One deliberate divergence from the original: `url` is a private
+field, not a public property — on an `RpcTarget`, a public property is
+readable over RPC (a property read is an argument-less method call), and a
+query string can carry credentials, so a public `url` would leak them to any
+stub holder.
+
+## RPC use (from another worker, same account)
+
+```jsonc
+// wrangler.jsonc of the caller
+"services": [{ "binding": "DIIGO", "service": "diigo-dump", "entrypoint": "DiigoService" }]
+```
+
+```ts
+import { dumpBookmarks } from "./diigo.js";
+
+const stub = env.DIIGO.open(); // rate-limited, cookie stays server-side
+// pipeline: one round trip for the whole chain
+const first = JSON.parse(await stub.pathjoin("load_user_items")
+	.query({ sort: "updated", count: 24, page_num: 1 }).get());
+// or run the whole dump in this worker's address space:
+for await (const item of dumpBookmarks({ diigoAPI: stub, pages: 2 })) {
+	console.log(item);
+}
+```
 
 ## Configuration
 
diff --git a/diigo-dump/src/cloudflare-env.d.ts b/diigo-dump/src/cloudflare-env.d.ts
new file mode 100644
index 0000000..3a08acc
--- /dev/null
+++ b/diigo-dump/src/cloudflare-env.d.ts
@@ -0,0 +1,10 @@
+declare module "cloudflare:workers" {
+	export abstract class RpcTarget {
+		constructor(...args: never[]);
+	}
+	export abstract class WorkerEntrypoint> {
+		protected ctx: { waitUntil(p: Promise): void };
+		protected env: Env;
+		constructor(ctx: unknown, env: Env);
+	}
+}
diff --git a/diigo-dump/src/diigo.ts b/diigo-dump/src/diigo.ts
new file mode 100644
index 0000000..63d18a9
--- /dev/null
+++ b/diigo-dump/src/diigo.ts
@@ -0,0 +1,73 @@
+/**
+ * @file Diigo via browser session cookie, like their web UI does.
+ * Local mode uses the objects directly; over RPC, a caller receives a
+ * rate-limited stub rooted at interact_api/ (never the cookie) and runs
+ * {@link dumpBookmarks} in its own address space.
+ * @see {@link makeDiigoAPI}
+ */
+
+import { WebApp, rateLimit, type PagedGet } from "./webapp.js";
+
+export type { PagedGet } from "./webapp.js";
+
+export const USER_AGENT =
+	"Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/141.0";
+
+/**
+ * Attenuated authority: a client rooted at
+ * https://www.diigo.com/interact_api/, with no cookie, fetch, or host
+ * authority of its own. Attach the cookie last:
+ * `makeDiigoAPI().cookies(env.DIIGO_COOKIE)` — or leave it off and keep the
+ * handle cookie-free (e.g. for passing to less-trusted code).
+ *
+ * The UA / x-requested-with headers and the endpoint path mirror
+ * mm-diigo's diigo-interact.mjs (see
+ * https://github.com/dckc/mm-diigo — file projects/diigo-bak/diigo-interact.mjs,
+ * commit 7dbb367, Oct 6, 2026), which is the canonical source of this shape.
+ */
+export const makeDiigoAPI = (fetchImpl: typeof fetch = fetch.bind(globalThis)): WebApp =>
+	new WebApp("https://www.diigo.com", fetchImpl)
+		.withHeaders({
+			"user-agent": USER_AGENT,
+			"x-requested-with": "XMLHttpRequest",
+		})
+		.pathjoin("interact_api/");
+
+export type DumpOptions = {
+	/** rate-limited client, cookie attached (see makeDiigoAPI) */
+	diigoAPI: PagedGet;
+	pages?: number;
+	count?: number;
+	log?: (event: Record) => void;
+};
+
+/** yield diigo bookmarks, most-recently-updated first, `pages * count` at
+ * most. `diigoAPI` is attenuated authority (see {@link makeDiigoAPI}); over
+ * RPC this generator runs in the *caller's* address space against a stub. */
+export async function* dumpBookmarks({
+	diigoAPI,
+	pages = 2,
+	count = 24,
+	log = () => {},
+}: DumpOptions): AsyncGenerator> {
+	const getUserItems = (page: number) =>
+		diigoAPI
+			.pathjoin("load_user_items")
+			.query({ sort: "updated", count, page_num: page })
+			.get()
+			.then(JSON.parse) as Promise<{ total: number; items: Record[] }>;
+
+	const first = await getUserItems(1);
+	const totalPages = Math.ceil(first.total / count);
+	log({ event: "start", total: first.total, pages, totalPages });
+
+	let got = 0;
+	for (let page = 1; page <= Math.min(pages, totalPages); page += 1) {
+		const data = page === 1 ? first : await getUserItems(page);
+		yield* data.items;
+		got += data.items.length;
+		log({ event: "page", page, of: pages, got });
+	}
+}
+
+export { rateLimit };
\ No newline at end of file
diff --git a/diigo-dump/src/index.ts b/diigo-dump/src/index.ts
index 7aadc65..5333523 100644
--- a/diigo-dump/src/index.ts
+++ b/diigo-dump/src/index.ts
@@ -1,14 +1,54 @@
-// diigo-dump as a worker: one page-number field + button.
-// Port of mm-diigo/projects/diigo-bak/diigo-dump.mjs —
-// browser-session-cookie access to diigo's interact_api/load_user_items
-// (the API-key path is bkmkeep.py / the EmDash plugin; this one is the cookie path).
-// Secret: DIIGO_COOKIE (copy the "Cookie" header from diigo.com devtools).
+/**
+ * @file diigo-dump worker: one page-number field + button, plus the
+ * {@link DiigoService} RPC entrypoint. Browser-session-cookie access to
+ * diigo's interact_api/load_user_items — the API-key path is bkmkeep.py /
+ * the EmDash plugin; this one is the cookie path.
+ *
+ * Secret: DIIGO_COOKIE (copy the "Cookie" header from diigo.com devtools).
+ * The fluent WebApp client lives in ./webapp.ts and ./diigo.ts; the cookie
+ * is attached only here, inside this worker's address space.
+ * Failures reach both surfaces: console.error for observability (wrapped
+ * workers never see exceptions the fetch handler renders itself), plus the
+ * rendered status/detail on the page.
+ * @see {@link DiigoService}
+ */
 
-const DIIGO_BASE = "https://www.diigo.com/interact_api/";
-const COUNT = 24;
+import { WorkerEntrypoint } from "cloudflare:workers";
+import { makeDiigoAPI, dumpBookmarks, rateLimit } from "./diigo.js";
 
 type Env = { DIIGO_COOKIE?: string };
 
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * RPC entrypoint for other workers on the account. Public methods shape the
+ * ABI: `open()` hands out a rate-limited stub with the cookie attached
+ * server-side (the caller can pathjoin/query/get but never sees the cookie
+ * or fetch authority); `dumpNdjson` runs a whole dump here and returns ndjson.
+ *
+ * @see https://developers.cloudflare.com/workers/runtime-apis/rpc/ — stub
+ * lifecycle; use `using` / `Symbol.dispose` on the caller side
+ */
+export class DiigoService extends WorkerEntrypoint {
+	open(): ReturnType {
+		const { DIIGO_COOKIE: cookie } = this.env as Env;
+		if (!cookie) {
+			console.error({ event: "open", error: "Missing DIIGO_COOKIE in the diigo-dump worker." });
+			throw new Error("Missing DIIGO_COOKIE in the diigo-dump worker.");
+		}
+		return rateLimit(makeDiigoAPI().cookies(cookie), () => sleep(500));
+	}
+
+	async dumpNdjson(pages: number, count = 24): Promise {
+		const diigoAPI = this.open();
+		const lines: string[] = [];
+		for await (const item of dumpBookmarks({ diigoAPI, pages, count })) {
+			lines.push(JSON.stringify(item));
+		}
+		return lines.join("\n");
+	}
+}
+
 export default {
 	async fetch(request: Request, env: Env): Promise {
 		const url = new URL(request.url);
@@ -27,38 +67,30 @@ export default {
 	},
 };
 
+/** the HTTP form handler's dump path: one page via the same fluent chain an
+ * RPC caller would drive (see {@link DiigoService.open}); shows raw JSON. */
 async function dumpPage(env: Env, page: number): Promise<{ status: string; detail: string }> {
 	const cookie = env.DIIGO_COOKIE;
 	if (!cookie) {
-		return { status: "no-creds", detail: "Missing DIIGO_COOKIE.\nSet it: npx wrangler secret put DIIGO_COOKIE (or .dev.vars locally)." };
+		return {
+			status: "no-creds",
+			detail: "Missing DIIGO_COOKIE.\nSet it: npx wrangler secret put DIIGO_COOKIE (or .dev.vars locally).",
+		};
 	}
 
-	const target = new URL(DIIGO_BASE + "load_user_items");
-	target.searchParams.set("sort", "updated");
-	target.searchParams.set("count", String(COUNT));
-	target.searchParams.set("page_num", String(page));
-
+	const diigoAPI = rateLimit(makeDiigoAPI().cookies(cookie), () => sleep(500));
 	try {
-		const response = await fetch(target.toString(), {
-			headers: {
-				cookie,
-				// same attenuated-request shape diigo-dump.mjs / diigo-interact.mjs use
-				"user-agent": "Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/141.0",
-				"x-requested-with": "XMLHttpRequest",
-			},
-		});
-		const text = await response.text();
-		let detail = text;
-		if (response.ok) {
-			try {
-				const data = JSON.parse(text) as { total?: number; items?: unknown[] };
-				detail = `total: ${data.total}\nitems: ${data.items?.length ?? 0}\n\n${JSON.stringify(data.items ?? data, null, 2)}`;
-			} catch {
-				// non-JSON; show raw
-			}
-		}
-		return { status: String(response.status), detail };
+		const text = await diigoAPI
+			.pathjoin("load_user_items")
+			.query({ sort: "updated", count: 24, page_num: page })
+			.get();
+		const data = JSON.parse(text) as { total?: number; items?: Record[] };
+		return {
+			status: "200",
+			detail: `total: ${data.total}\nitems: ${data.items?.length ?? 0}\n\n${JSON.stringify(data.items ?? data, null, 2)}`,
+		};
 	} catch (err) {
+		console.error({ event: "dump", page, error: String(err) });
 		return { status: "error", detail: String(err) };
 	}
 }
diff --git a/diigo-dump/src/webapp.ts b/diigo-dump/src/webapp.ts
new file mode 100644
index 0000000..9ce6472
--- /dev/null
+++ b/diigo-dump/src/webapp.ts
@@ -0,0 +1,213 @@
+/**
+ * @file Fluent web client, ported from mm-diigo's WebApp.js, with handles
+ * that can travel over service-binding RPC.
+ * @see {@link WebApp}
+ */
+
+import { RpcTarget } from "cloudflare:workers";
+
+export type Query = Record;
+type Headers = Record;
+
+/** structural type both WebApp and RateLimitedWebApp satisfy */
+export interface PagedGet {
+	pathjoin(ref: string): PagedGet;
+	query(params: Query): PagedGet;
+	get(): Promise;
+}
+
+/**
+ * Attenuated authority for one URL: derived handles can only narrow, never
+ * widen. `pathjoin` must stay within the old path. `query`/`withHeaders`
+ * unions are flower-style, after [Flower, the label-based network backplane
+ * from the Nuxi/CloudABI world](https://github.com/NuxiNL/flower) (v0.11,
+ * Jan 2019): label sets may merge only if no shared key carries a
+ * contradicting value, and both sides end up holding the union — here,
+ * merging query params or headers throws on contradiction instead of
+ * letting a later grant quietly contradict an earlier one.
+ *
+ * The fetch function and any attached cookie headers never leave the creating
+ * worker — a remote holder of a stub only gets the method calls. The URL is
+ * private for the same reason: on an RpcTarget, a public property is readable
+ * by remote holders (a property read is an argument-less method call), and a
+ * query string can carry credentials (api_key=… and friends), so the URL
+ * must not be part of the exposed surface. Derived handles address more
+ * specific endpoints via {@link pathjoin}/{@link query}; nobody needs to read
+ * the URL back out.
+ *
+ * Extends RpcTarget, so remote holders can promise-pipeline: a whole
+ * `pathjoin().query().get()` chain is one round trip, with only the terminal
+ * `get`/`post` doing I/O.
+ *
+ * @see https://developers.cloudflare.com/workers/runtime-apis/rpc/ — promise
+ * pipelining, the stub security model, and property access
+ *
+ * Error hygiene: Error messages must not contain any header or param values —
+ * they can cross RPC as exceptions and end up in logs. URLs appear only with
+ * their query string stripped, unions report key names alone (safe here: the
+ * diigo cookie travels as a header, but the rule holds for keyed APIs too).
+ * In due course: when this package gains node_modules, reach for
+ * @endo/errors so error details can travel off-band from the message.
+ */
+export class WebApp extends RpcTarget {
+	#url: string;
+	#fetch: typeof fetch;
+	#headers: Headers;
+
+	// bind: a detached `fetch` (e.g. a bare method reference) misbehaves in
+	// workerd — "Illegal invocation" — so the default receiver is globalThis.
+	constructor(url: string, fetchImpl: typeof fetch = fetch.bind(globalThis), headers: Headers = {}) {
+		super();
+		this.#url = url;
+		this.#fetch = fetchImpl;
+		this.#headers = headers;
+	}
+
+	/** Join a relative path, merging the base's query.
+	 * The new path must stay within the old path (query excluded from the check)
+	 * — the attenuation guard that keeps a remote holder of this stub inside
+	 * the subtree it was granted. */
+	pathjoin(ref: string): WebApp {
+		const here = new URL(this.#url);
+		const there = new URL(ref, here);
+		const hereNoQuery = new URL(here.href);
+		hereNoQuery.search = "";
+		const thereNoQuery = new URL(there.href);
+		thereNoQuery.search = "";
+		if (!thereNoQuery.href.startsWith(hereNoQuery.href)) {
+			throw new Error(`pathjoin: ${ref} not within ${hereNoQuery.href}`);
+		}
+		const baseParams: Query = Object.fromEntries(here.searchParams);
+		return new WebApp(unionQuery(there.href, baseParams), this.#fetch, this.#headers);
+	}
+
+	/** Union headers, flower-style (see {@link WebApp}): a shared key with a
+	 * different value throws; the same value is fine. */
+	withHeaders(h: Headers): WebApp {
+		return new WebApp(this.#url, this.#fetch, unionHeaders(this.#headers, h));
+	}
+
+	cookies(cookie: string): WebApp {
+		return this.withHeaders({ cookie });
+	}
+
+	/** Union params into the existing query, flower-style (see {@link WebApp}). */
+	query(params: Query): WebApp {
+		return new WebApp(unionQuery(this.#url, params), this.#fetch, this.#headers);
+	}
+
+	async get(): Promise {
+		const res = await this.#fetch(this.#url, { headers: this.#headers });
+		return checkOK(this.#url, res).text();
+	}
+
+	async post(body: string): Promise {
+		const res = await this.#fetch(this.#url, { method: "POST", headers: this.#headers, body });
+		return checkOK(this.#url, res).text();
+	}
+}
+
+/** `rateLimit(web, delay)` from the original, as an RpcTarget wrapper: the
+ * gate is shared by every handle derived from this one, so derived handles
+ * stay under the same rate limit even when passed to other workers. Only
+ * `get`/`post` wait on the gate; constructor-style methods (`pathjoin`,
+ * `query`, …) pipeline through freely. */
+export class RateLimitedWebApp extends RpcTarget implements PagedGet {
+	#web: WebApp;
+	#gate: Gate;
+
+	constructor(web: WebApp, gate: Gate) {
+		super();
+		this.#web = web;
+		this.#gate = gate;
+	}
+
+	pathjoin(ref: string): RateLimitedWebApp {
+		return new RateLimitedWebApp(this.#web.pathjoin(ref), this.#gate);
+	}
+
+	query(params: Query): RateLimitedWebApp {
+		return new RateLimitedWebApp(this.#web.query(params), this.#gate);
+	}
+
+	withHeaders(h: Headers): RateLimitedWebApp {
+		return new RateLimitedWebApp(this.#web.withHeaders(h), this.#gate);
+	}
+
+	cookies(cookie: string): RateLimitedWebApp {
+		return new RateLimitedWebApp(this.#web.cookies(cookie), this.#gate);
+	}
+
+	async get(): Promise {
+		await this.#gate.waitMyTurn();
+		return this.#web.get();
+	}
+
+	async post(body: string): Promise {
+		await this.#gate.waitMyTurn();
+		return this.#web.post(body);
+	}
+}
+
+export function rateLimit(web: WebApp, delay: () => Promise): RateLimitedWebApp {
+	return new RateLimitedWebApp(web, new Gate(delay));
+}
+
+/** @internal — shared gate behind {@link RateLimitedWebApp}:
+ * of all those waiting on pause, only 1 gets to go next */
+class Gate {
+	#step = 0;
+	#pause?: Promise;
+	#delay: () => Promise;
+
+	constructor(delay: () => Promise) {
+		this.#delay = delay;
+	}
+
+	async waitMyTurn(): Promise {
+		let before;
+		do {
+			before = this.#step;
+			if (this.#pause) {
+				await this.#pause;
+			}
+		} while (before !== this.#step);
+		this.#step += 1;
+		this.#pause = this.#delay();
+	}
+}
+
+function unionHeaders(base: Headers, more: Headers): Headers {
+	const merged = { ...base };
+	for (const [key, value] of Object.entries(more)) {
+		if (key in merged && merged[key] !== value) {
+			// key name only: header values must not appear in Error messages
+			throw new Error(`headers: contradicts on ${key}`);
+		}
+		merged[key] = value;
+	}
+	return merged;
+}
+
+function unionQuery(base: string, params: Query): string {
+	const there = new URL(base);
+	for (const [key, value] of Object.entries(params)) {
+		const v = `${value}`;
+		if (there.searchParams.has(key) && there.searchParams.get(key) !== v) {
+			// key name only: param values must not appear in Error messages
+			throw new Error(`query: contradicts on ${key}`);
+		}
+		there.searchParams.set(key, v);
+	}
+	return there.href;
+}
+
+function checkOK(url: string, res: Response): Response {
+	if (!res.ok) {
+		// query stripped: param values must not appear in Error messages
+		const noQuery = new URL(url);
+		noQuery.search = "";
+		throw new Error(`${res.status} ${res.statusText} on ${noQuery.href}`);
+	}
+	return res;
+}
\ No newline at end of file
diff --git a/diigo-dump/tsconfig.json b/diigo-dump/tsconfig.json
new file mode 100644
index 0000000..db0fc4a
--- /dev/null
+++ b/diigo-dump/tsconfig.json
@@ -0,0 +1,13 @@
+{
+	"compilerOptions": {
+		"target": "ES2022",
+		"module": "preserve",
+		"moduleResolution": "bundler",
+		"strict": true,
+		"esModuleInterop": true,
+		"verbatimModuleSyntax": true,
+		"skipLibCheck": true,
+		"types": []
+	},
+	"include": ["src/**/*"]
+}