diff --git a/diigo-dump/README.md b/diigo-dump/README.md index ed721ea..a7dbe04 100644 --- a/diigo-dump/README.md +++ b/diigo-dump/README.md @@ -16,9 +16,37 @@ with the session cookie and shows status, `total`, and the JSON items in a read-only `
`. GET `/` is inert — dumps happen only on explicit submits.
Provenance notes: the `user-agent` / `x-requested-with` headers and the
-`interact_api` endpoint mirror the original script; the ocap-style
-attenuation there (pathjoin/query/rateLimit in WebApp.js) is reduced here to
-a plain fetch with the cookie never exposed beyond this one call.
+`interact_api` endpoint mirror the original script; the fluent client from
+`WebApp.js` is restored in `src/webapp.ts` with `WebApp` and the rate-limit
+wrapper as `RpcTarget` subclasses. `src/diigo.ts` holds `makeDiigoAPI` and
+the `dumpBookmarks` generator; `DiigoService.open()` hands an RPC caller a
+rate-limited stub with the cookie attached only on the server side, and
+promise pipelining makes a whole `pathjoin().query().get()` chain a single
+round trip. One deliberate divergence from the original: `url` is a private
+field, not a public property — on an `RpcTarget`, a public property is
+readable over RPC (a property read is an argument-less method call), and a
+query string can carry credentials, so a public `url` would leak them to any
+stub holder.
+
+## RPC use (from another worker, same account)
+
+```jsonc
+// wrangler.jsonc of the caller
+"services": [{ "binding": "DIIGO", "service": "diigo-dump", "entrypoint": "DiigoService" }]
+```
+
+```ts
+import { dumpBookmarks } from "./diigo.js";
+
+const stub = env.DIIGO.open(); // rate-limited, cookie stays server-side
+// pipeline: one round trip for the whole chain
+const first = JSON.parse(await stub.pathjoin("load_user_items")
+ .query({ sort: "updated", count: 24, page_num: 1 }).get());
+// or run the whole dump in this worker's address space:
+for await (const item of dumpBookmarks({ diigoAPI: stub, pages: 2 })) {
+ console.log(item);
+}
+```
## Configuration
diff --git a/diigo-dump/src/cloudflare-env.d.ts b/diigo-dump/src/cloudflare-env.d.ts
new file mode 100644
index 0000000..3a08acc
--- /dev/null
+++ b/diigo-dump/src/cloudflare-env.d.ts
@@ -0,0 +1,10 @@
+declare module "cloudflare:workers" {
+ export abstract class RpcTarget {
+ constructor(...args: never[]);
+ }
+ export abstract class WorkerEntrypoint> {
+ protected ctx: { waitUntil(p: Promise): void };
+ protected env: Env;
+ constructor(ctx: unknown, env: Env);
+ }
+}
diff --git a/diigo-dump/src/diigo.ts b/diigo-dump/src/diigo.ts
new file mode 100644
index 0000000..63d18a9
--- /dev/null
+++ b/diigo-dump/src/diigo.ts
@@ -0,0 +1,73 @@
+/**
+ * @file Diigo via browser session cookie, like their web UI does.
+ * Local mode uses the objects directly; over RPC, a caller receives a
+ * rate-limited stub rooted at interact_api/ (never the cookie) and runs
+ * {@link dumpBookmarks} in its own address space.
+ * @see {@link makeDiigoAPI}
+ */
+
+import { WebApp, rateLimit, type PagedGet } from "./webapp.js";
+
+export type { PagedGet } from "./webapp.js";
+
+export const USER_AGENT =
+ "Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/141.0";
+
+/**
+ * Attenuated authority: a client rooted at
+ * https://www.diigo.com/interact_api/, with no cookie, fetch, or host
+ * authority of its own. Attach the cookie last:
+ * `makeDiigoAPI().cookies(env.DIIGO_COOKIE)` — or leave it off and keep the
+ * handle cookie-free (e.g. for passing to less-trusted code).
+ *
+ * The UA / x-requested-with headers and the endpoint path mirror
+ * mm-diigo's diigo-interact.mjs (see
+ * https://github.com/dckc/mm-diigo — file projects/diigo-bak/diigo-interact.mjs,
+ * commit 7dbb367, Oct 6, 2026), which is the canonical source of this shape.
+ */
+export const makeDiigoAPI = (fetchImpl: typeof fetch = fetch.bind(globalThis)): WebApp =>
+ new WebApp("https://www.diigo.com", fetchImpl)
+ .withHeaders({
+ "user-agent": USER_AGENT,
+ "x-requested-with": "XMLHttpRequest",
+ })
+ .pathjoin("interact_api/");
+
+export type DumpOptions = {
+ /** rate-limited client, cookie attached (see makeDiigoAPI) */
+ diigoAPI: PagedGet;
+ pages?: number;
+ count?: number;
+ log?: (event: Record) => void;
+};
+
+/** yield diigo bookmarks, most-recently-updated first, `pages * count` at
+ * most. `diigoAPI` is attenuated authority (see {@link makeDiigoAPI}); over
+ * RPC this generator runs in the *caller's* address space against a stub. */
+export async function* dumpBookmarks({
+ diigoAPI,
+ pages = 2,
+ count = 24,
+ log = () => {},
+}: DumpOptions): AsyncGenerator> {
+ const getUserItems = (page: number) =>
+ diigoAPI
+ .pathjoin("load_user_items")
+ .query({ sort: "updated", count, page_num: page })
+ .get()
+ .then(JSON.parse) as Promise<{ total: number; items: Record[] }>;
+
+ const first = await getUserItems(1);
+ const totalPages = Math.ceil(first.total / count);
+ log({ event: "start", total: first.total, pages, totalPages });
+
+ let got = 0;
+ for (let page = 1; page <= Math.min(pages, totalPages); page += 1) {
+ const data = page === 1 ? first : await getUserItems(page);
+ yield* data.items;
+ got += data.items.length;
+ log({ event: "page", page, of: pages, got });
+ }
+}
+
+export { rateLimit };
\ No newline at end of file
diff --git a/diigo-dump/src/index.ts b/diigo-dump/src/index.ts
index 7aadc65..5333523 100644
--- a/diigo-dump/src/index.ts
+++ b/diigo-dump/src/index.ts
@@ -1,14 +1,54 @@
-// diigo-dump as a worker: one page-number field + button.
-// Port of mm-diigo/projects/diigo-bak/diigo-dump.mjs —
-// browser-session-cookie access to diigo's interact_api/load_user_items
-// (the API-key path is bkmkeep.py / the EmDash plugin; this one is the cookie path).
-// Secret: DIIGO_COOKIE (copy the "Cookie" header from diigo.com devtools).
+/**
+ * @file diigo-dump worker: one page-number field + button, plus the
+ * {@link DiigoService} RPC entrypoint. Browser-session-cookie access to
+ * diigo's interact_api/load_user_items — the API-key path is bkmkeep.py /
+ * the EmDash plugin; this one is the cookie path.
+ *
+ * Secret: DIIGO_COOKIE (copy the "Cookie" header from diigo.com devtools).
+ * The fluent WebApp client lives in ./webapp.ts and ./diigo.ts; the cookie
+ * is attached only here, inside this worker's address space.
+ * Failures reach both surfaces: console.error for observability (wrapped
+ * workers never see exceptions the fetch handler renders itself), plus the
+ * rendered status/detail on the page.
+ * @see {@link DiigoService}
+ */
-const DIIGO_BASE = "https://www.diigo.com/interact_api/";
-const COUNT = 24;
+import { WorkerEntrypoint } from "cloudflare:workers";
+import { makeDiigoAPI, dumpBookmarks, rateLimit } from "./diigo.js";
type Env = { DIIGO_COOKIE?: string };
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * RPC entrypoint for other workers on the account. Public methods shape the
+ * ABI: `open()` hands out a rate-limited stub with the cookie attached
+ * server-side (the caller can pathjoin/query/get but never sees the cookie
+ * or fetch authority); `dumpNdjson` runs a whole dump here and returns ndjson.
+ *
+ * @see https://developers.cloudflare.com/workers/runtime-apis/rpc/ — stub
+ * lifecycle; use `using` / `Symbol.dispose` on the caller side
+ */
+export class DiigoService extends WorkerEntrypoint {
+ open(): ReturnType {
+ const { DIIGO_COOKIE: cookie } = this.env as Env;
+ if (!cookie) {
+ console.error({ event: "open", error: "Missing DIIGO_COOKIE in the diigo-dump worker." });
+ throw new Error("Missing DIIGO_COOKIE in the diigo-dump worker.");
+ }
+ return rateLimit(makeDiigoAPI().cookies(cookie), () => sleep(500));
+ }
+
+ async dumpNdjson(pages: number, count = 24): Promise {
+ const diigoAPI = this.open();
+ const lines: string[] = [];
+ for await (const item of dumpBookmarks({ diigoAPI, pages, count })) {
+ lines.push(JSON.stringify(item));
+ }
+ return lines.join("\n");
+ }
+}
+
export default {
async fetch(request: Request, env: Env): Promise {
const url = new URL(request.url);
@@ -27,38 +67,30 @@ export default {
},
};
+/** the HTTP form handler's dump path: one page via the same fluent chain an
+ * RPC caller would drive (see {@link DiigoService.open}); shows raw JSON. */
async function dumpPage(env: Env, page: number): Promise<{ status: string; detail: string }> {
const cookie = env.DIIGO_COOKIE;
if (!cookie) {
- return { status: "no-creds", detail: "Missing DIIGO_COOKIE.\nSet it: npx wrangler secret put DIIGO_COOKIE (or .dev.vars locally)." };
+ return {
+ status: "no-creds",
+ detail: "Missing DIIGO_COOKIE.\nSet it: npx wrangler secret put DIIGO_COOKIE (or .dev.vars locally).",
+ };
}
- const target = new URL(DIIGO_BASE + "load_user_items");
- target.searchParams.set("sort", "updated");
- target.searchParams.set("count", String(COUNT));
- target.searchParams.set("page_num", String(page));
-
+ const diigoAPI = rateLimit(makeDiigoAPI().cookies(cookie), () => sleep(500));
try {
- const response = await fetch(target.toString(), {
- headers: {
- cookie,
- // same attenuated-request shape diigo-dump.mjs / diigo-interact.mjs use
- "user-agent": "Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/141.0",
- "x-requested-with": "XMLHttpRequest",
- },
- });
- const text = await response.text();
- let detail = text;
- if (response.ok) {
- try {
- const data = JSON.parse(text) as { total?: number; items?: unknown[] };
- detail = `total: ${data.total}\nitems: ${data.items?.length ?? 0}\n\n${JSON.stringify(data.items ?? data, null, 2)}`;
- } catch {
- // non-JSON; show raw
- }
- }
- return { status: String(response.status), detail };
+ const text = await diigoAPI
+ .pathjoin("load_user_items")
+ .query({ sort: "updated", count: 24, page_num: page })
+ .get();
+ const data = JSON.parse(text) as { total?: number; items?: Record[] };
+ return {
+ status: "200",
+ detail: `total: ${data.total}\nitems: ${data.items?.length ?? 0}\n\n${JSON.stringify(data.items ?? data, null, 2)}`,
+ };
} catch (err) {
+ console.error({ event: "dump", page, error: String(err) });
return { status: "error", detail: String(err) };
}
}
diff --git a/diigo-dump/src/webapp.ts b/diigo-dump/src/webapp.ts
new file mode 100644
index 0000000..9ce6472
--- /dev/null
+++ b/diigo-dump/src/webapp.ts
@@ -0,0 +1,213 @@
+/**
+ * @file Fluent web client, ported from mm-diigo's WebApp.js, with handles
+ * that can travel over service-binding RPC.
+ * @see {@link WebApp}
+ */
+
+import { RpcTarget } from "cloudflare:workers";
+
+export type Query = Record;
+type Headers = Record;
+
+/** structural type both WebApp and RateLimitedWebApp satisfy */
+export interface PagedGet {
+ pathjoin(ref: string): PagedGet;
+ query(params: Query): PagedGet;
+ get(): Promise;
+}
+
+/**
+ * Attenuated authority for one URL: derived handles can only narrow, never
+ * widen. `pathjoin` must stay within the old path. `query`/`withHeaders`
+ * unions are flower-style, after [Flower, the label-based network backplane
+ * from the Nuxi/CloudABI world](https://github.com/NuxiNL/flower) (v0.11,
+ * Jan 2019): label sets may merge only if no shared key carries a
+ * contradicting value, and both sides end up holding the union — here,
+ * merging query params or headers throws on contradiction instead of
+ * letting a later grant quietly contradict an earlier one.
+ *
+ * The fetch function and any attached cookie headers never leave the creating
+ * worker — a remote holder of a stub only gets the method calls. The URL is
+ * private for the same reason: on an RpcTarget, a public property is readable
+ * by remote holders (a property read is an argument-less method call), and a
+ * query string can carry credentials (api_key=… and friends), so the URL
+ * must not be part of the exposed surface. Derived handles address more
+ * specific endpoints via {@link pathjoin}/{@link query}; nobody needs to read
+ * the URL back out.
+ *
+ * Extends RpcTarget, so remote holders can promise-pipeline: a whole
+ * `pathjoin().query().get()` chain is one round trip, with only the terminal
+ * `get`/`post` doing I/O.
+ *
+ * @see https://developers.cloudflare.com/workers/runtime-apis/rpc/ — promise
+ * pipelining, the stub security model, and property access
+ *
+ * Error hygiene: Error messages must not contain any header or param values —
+ * they can cross RPC as exceptions and end up in logs. URLs appear only with
+ * their query string stripped, unions report key names alone (safe here: the
+ * diigo cookie travels as a header, but the rule holds for keyed APIs too).
+ * In due course: when this package gains node_modules, reach for
+ * @endo/errors so error details can travel off-band from the message.
+ */
+export class WebApp extends RpcTarget {
+ #url: string;
+ #fetch: typeof fetch;
+ #headers: Headers;
+
+ // bind: a detached `fetch` (e.g. a bare method reference) misbehaves in
+ // workerd — "Illegal invocation" — so the default receiver is globalThis.
+ constructor(url: string, fetchImpl: typeof fetch = fetch.bind(globalThis), headers: Headers = {}) {
+ super();
+ this.#url = url;
+ this.#fetch = fetchImpl;
+ this.#headers = headers;
+ }
+
+ /** Join a relative path, merging the base's query.
+ * The new path must stay within the old path (query excluded from the check)
+ * — the attenuation guard that keeps a remote holder of this stub inside
+ * the subtree it was granted. */
+ pathjoin(ref: string): WebApp {
+ const here = new URL(this.#url);
+ const there = new URL(ref, here);
+ const hereNoQuery = new URL(here.href);
+ hereNoQuery.search = "";
+ const thereNoQuery = new URL(there.href);
+ thereNoQuery.search = "";
+ if (!thereNoQuery.href.startsWith(hereNoQuery.href)) {
+ throw new Error(`pathjoin: ${ref} not within ${hereNoQuery.href}`);
+ }
+ const baseParams: Query = Object.fromEntries(here.searchParams);
+ return new WebApp(unionQuery(there.href, baseParams), this.#fetch, this.#headers);
+ }
+
+ /** Union headers, flower-style (see {@link WebApp}): a shared key with a
+ * different value throws; the same value is fine. */
+ withHeaders(h: Headers): WebApp {
+ return new WebApp(this.#url, this.#fetch, unionHeaders(this.#headers, h));
+ }
+
+ cookies(cookie: string): WebApp {
+ return this.withHeaders({ cookie });
+ }
+
+ /** Union params into the existing query, flower-style (see {@link WebApp}). */
+ query(params: Query): WebApp {
+ return new WebApp(unionQuery(this.#url, params), this.#fetch, this.#headers);
+ }
+
+ async get(): Promise {
+ const res = await this.#fetch(this.#url, { headers: this.#headers });
+ return checkOK(this.#url, res).text();
+ }
+
+ async post(body: string): Promise {
+ const res = await this.#fetch(this.#url, { method: "POST", headers: this.#headers, body });
+ return checkOK(this.#url, res).text();
+ }
+}
+
+/** `rateLimit(web, delay)` from the original, as an RpcTarget wrapper: the
+ * gate is shared by every handle derived from this one, so derived handles
+ * stay under the same rate limit even when passed to other workers. Only
+ * `get`/`post` wait on the gate; constructor-style methods (`pathjoin`,
+ * `query`, …) pipeline through freely. */
+export class RateLimitedWebApp extends RpcTarget implements PagedGet {
+ #web: WebApp;
+ #gate: Gate;
+
+ constructor(web: WebApp, gate: Gate) {
+ super();
+ this.#web = web;
+ this.#gate = gate;
+ }
+
+ pathjoin(ref: string): RateLimitedWebApp {
+ return new RateLimitedWebApp(this.#web.pathjoin(ref), this.#gate);
+ }
+
+ query(params: Query): RateLimitedWebApp {
+ return new RateLimitedWebApp(this.#web.query(params), this.#gate);
+ }
+
+ withHeaders(h: Headers): RateLimitedWebApp {
+ return new RateLimitedWebApp(this.#web.withHeaders(h), this.#gate);
+ }
+
+ cookies(cookie: string): RateLimitedWebApp {
+ return new RateLimitedWebApp(this.#web.cookies(cookie), this.#gate);
+ }
+
+ async get(): Promise {
+ await this.#gate.waitMyTurn();
+ return this.#web.get();
+ }
+
+ async post(body: string): Promise {
+ await this.#gate.waitMyTurn();
+ return this.#web.post(body);
+ }
+}
+
+export function rateLimit(web: WebApp, delay: () => Promise): RateLimitedWebApp {
+ return new RateLimitedWebApp(web, new Gate(delay));
+}
+
+/** @internal — shared gate behind {@link RateLimitedWebApp}:
+ * of all those waiting on pause, only 1 gets to go next */
+class Gate {
+ #step = 0;
+ #pause?: Promise;
+ #delay: () => Promise;
+
+ constructor(delay: () => Promise) {
+ this.#delay = delay;
+ }
+
+ async waitMyTurn(): Promise {
+ let before;
+ do {
+ before = this.#step;
+ if (this.#pause) {
+ await this.#pause;
+ }
+ } while (before !== this.#step);
+ this.#step += 1;
+ this.#pause = this.#delay();
+ }
+}
+
+function unionHeaders(base: Headers, more: Headers): Headers {
+ const merged = { ...base };
+ for (const [key, value] of Object.entries(more)) {
+ if (key in merged && merged[key] !== value) {
+ // key name only: header values must not appear in Error messages
+ throw new Error(`headers: contradicts on ${key}`);
+ }
+ merged[key] = value;
+ }
+ return merged;
+}
+
+function unionQuery(base: string, params: Query): string {
+ const there = new URL(base);
+ for (const [key, value] of Object.entries(params)) {
+ const v = `${value}`;
+ if (there.searchParams.has(key) && there.searchParams.get(key) !== v) {
+ // key name only: param values must not appear in Error messages
+ throw new Error(`query: contradicts on ${key}`);
+ }
+ there.searchParams.set(key, v);
+ }
+ return there.href;
+}
+
+function checkOK(url: string, res: Response): Response {
+ if (!res.ok) {
+ // query stripped: param values must not appear in Error messages
+ const noQuery = new URL(url);
+ noQuery.search = "";
+ throw new Error(`${res.status} ${res.statusText} on ${noQuery.href}`);
+ }
+ return res;
+}
\ No newline at end of file
diff --git a/diigo-dump/tsconfig.json b/diigo-dump/tsconfig.json
new file mode 100644
index 0000000..db0fc4a
--- /dev/null
+++ b/diigo-dump/tsconfig.json
@@ -0,0 +1,13 @@
+{
+ "compilerOptions": {
+ "target": "ES2022",
+ "module": "preserve",
+ "moduleResolution": "bundler",
+ "strict": true,
+ "esModuleInterop": true,
+ "verbatimModuleSyntax": true,
+ "skipLibCheck": true,
+ "types": []
+ },
+ "include": ["src/**/*"]
+}