From b9561a49d258ad412c9e7a9b4c4c7f6cb61a180e Mon Sep 17 00:00:00 2001 From: Inferencebot Date: Thu, 5 Nov 2026 16:06:52 +0000 Subject: [PATCH] correctness: paginate Cloudron user fetches; atomic held-credit grant (audit M5+M6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M6: a single-page ?per_page=100 fetch silently truncates past 100 users — the box is at 99. New shared walkers (async + sync) page through /api/v1/users like the Matrix paths, capped at 50 pages. All six single- page call sites now walk: duplicate-email check (provisioning 409s), reinvite loop, Loomio activation filter, admin overview, member-email resolution, local MXID resolution. Failure behavior per site is preserved (page-1 failure fails loudly; later-page failure keeps what was read). M5: provisioning-time held-credit grant moved into grant_held_credits(): per hold, the status flip, balance move and ledger row commit in ONE transaction (the old code committed the flip before add_credits, so an exception between them left hold=granted and the member uncredited). Mirrors admin_held_grant, keyed on held: so re-runs are no-ops; mixed-case emails normalized. Tests in tests/test_portal_hardening.py. --- app/main.py | 206 ++++++++++++++++++++++---------- tests/test_portal_hardening.py | 209 +++++++++++++++++++++++++++++++++ 2 files changed, 352 insertions(+), 63 deletions(-) create mode 100644 tests/test_portal_hardening.py diff --git a/app/main.py b/app/main.py index f78b0a6..a68e8bb 100644 --- a/app/main.py +++ b/app/main.py @@ -1819,12 +1819,10 @@ async def filter_loomio_activated(emails: list[str]) -> list[str]: activated: set[str] = set() try: async with httpx.AsyncClient(timeout=15.0) as client: - r = await client.get( - f"{CLOUDRON_API}/api/v1/users?per_page=100", - headers=cloudron_headers(), - ) - r.raise_for_status() - for u in r.json().get("users", []): + users = await cloudron_fetch_all_users(client) + if users is None: + raise RuntimeError("cloudron users fetch failed (page 1)") + for u in users: if u.get("inviteAccepted") and u.get("email"): activated.add(u["email"]) except Exception as e: @@ -1923,6 +1921,74 @@ def cloudron_headers() -> dict: return {"Authorization": f"Bearer {CLOUDRON_TOKEN}"} +# M6 (audit 2026-11-05): a single-page `?per_page=100` fetch silently truncates +# once the box passes 100 users (the membership sits at the threshold). Both +# walkers below page through /api/v1/users like the Matrix paths already do, +# capped sanely for Cloudron (50 pages x 100 = 5000 users). +_CLOUDRON_USERS_PAGE_CAP = 50 + + +async def cloudron_fetch_all_users(client: httpx.AsyncClient) -> list[dict] | None: + """Every Cloudron user, walking /api/v1/users page by page (async callers). + + Returns None only when page 1 is unresolvable (caller keeps its existing + failure behavior); a failure on a LATER page keeps what was read, which is + strictly more than the truncated single-page fetch returned before. + """ + users: list[dict] = [] + for page in range(1, _CLOUDRON_USERS_PAGE_CAP + 1): + r = await client.get( + f"{CLOUDRON_API}/api/v1/users", + params={"page": page, "per_page": 100}, + headers=cloudron_headers(), + ) + if r.status_code != 200: + if page == 1: + logger.warning("Cloudron users fetch page 1 failed: %s", r.status_code) + return None + logger.warning("Cloudron users fetch page %s failed: %s", page, r.status_code) + break + batch = r.json().get("users", []) + users.extend(batch) + if len(batch) < 100: + break + else: + logger.warning("Cloudron users fetch hit the %s-page cap; list truncated", + _CLOUDRON_USERS_PAGE_CAP) + return users + + +def cloudron_fetch_all_users_sync() -> list[dict]: + """Every Cloudron user for the sync (urllib) call sites, same pagination. + + Raises on a page-1 failure (the call sites error the request); later-page + failures keep what was read. + """ + import urllib.request + + users: list[dict] = [] + for page in range(1, _CLOUDRON_USERS_PAGE_CAP + 1): + req = urllib.request.Request( + f"{CLOUDRON_API}/api/v1/users?page={page}&per_page=100", + headers=cloudron_headers(), + ) + try: + with urllib.request.urlopen(req, timeout=15) as r: + batch = json.loads(r.read().decode()).get("users", []) + except Exception: + if page == 1: + raise + logger.warning("Cloudron users fetch page %s failed", page, exc_info=True) + break + users.extend(batch) + if len(batch) < 100: + break + else: + logger.warning("Cloudron users fetch hit the %s-page cap; list truncated", + _CLOUDRON_USERS_PAGE_CAP) + return users + + async def cloudron_create_user(email: str, name: str) -> str: """Create (or return existing) Cloudron user, assigned to the members group. @@ -1933,13 +1999,12 @@ async def cloudron_create_user(email: str, name: str) -> str: with body {"groupIds": [...]}. """ async with httpx.AsyncClient() as client: - # Check if user exists - r = await client.get( - f"{CLOUDRON_API}/api/v1/users?per_page=100", - headers=cloudron_headers(), - ) - r.raise_for_status() - for u in r.json().get("users", []): + # Check if user exists (walks every page — a single-page fetch + # truncated past 100 users and let duplicate users get created). + users = await cloudron_fetch_all_users(client) + if users is None: + raise RuntimeError("cloudron users fetch failed; refusing to create a duplicate") + for u in users: if u.get("email") == email: # If the existing user has no display name (e.g. created with a # placeholder), fill it in. Otherwise leave it alone — the user @@ -2086,12 +2151,10 @@ async def cloudron_is_activated(user_id: str) -> bool: log in — the state we re-invite for. """ async with httpx.AsyncClient() as client: - r = await client.get( - f"{CLOUDRON_API}/api/v1/users?per_page=100", - headers=cloudron_headers(), - ) - r.raise_for_status() - for u in r.json().get("users", []): + users = await cloudron_fetch_all_users(client) + if users is None: + raise RuntimeError("cloudron users fetch failed") + for u in users: if u.get("id") == user_id: return bool(u.get("inviteAccepted")) return False @@ -2148,30 +2211,12 @@ async def provision_member(email: str, name: str, slug: str = "") -> str: granted = 0.0 try: conn = get_db() - rows = conn.execute( - "SELECT id, amount FROM held_credits " - "WHERE status = 'held' AND (LOWER(email) = ? OR LOWER(slug) = ?)", - (email.lower(), slug.lower() if slug else ""), - ).fetchall() - if rows: - hold_ids = [r[0] for r in rows] - granted = sum(float(r[1] or 0.0) for r in rows) - conn.execute( - "UPDATE held_credits SET status = 'granted', resolved_at = datetime('now') " - "WHERE id IN (%s)" % ",".join("?" * len(hold_ids)), - hold_ids, - ) - conn.commit() + granted = grant_held_credits(conn, email, slug) conn.close() if granted > 0: - new_balance = add_credits( - email, granted, - reason="held credit pack(s) granted on membership", - reference=f"held:{','.join(str(r[0]) for r in rows)}", - ) await sync_team_budget_from_balance(email) logger.info("Granted %.2f held credits to new member %s (balance %s)", - granted, email, new_balance) + granted, email, get_credit_balance(email)) except Exception as e: logger.warning("Held-credit grant for %s failed: %s", email, e) @@ -3452,12 +3497,10 @@ async def admin_overview(request: Request): invite_accepted = {} try: async with httpx.AsyncClient(timeout=15.0) as client: - r = await client.get( - f"{CLOUDRON_API}/api/v1/users?per_page=100", - headers={"Authorization": f"Bearer {CLOUDRON_TOKEN}"}, - ) - r.raise_for_status() - for u in r.json().get("users", []): + users = await cloudron_fetch_all_users(client) + if users is None: + raise RuntimeError("cloudron users fetch failed (page 1)") + for u in users: uid = u.get("id") if u.get("username"): usernames[uid] = u["username"] @@ -3855,16 +3898,9 @@ def resolve_member_email(identity: str) -> str: identity = identity.strip() if "@" in identity: return identity.lower() - # uid / username → email via Cloudron user list (one fetch covers both). - import urllib.request - - req = urllib.request.Request( - f"{CLOUDRON_API}/api/v1/users?per_page=100", - headers={"Authorization": f"Bearer {CLOUDRON_TOKEN}"}, - ) - with urllib.request.urlopen(req, timeout=15) as r: - data = json.loads(r.read().decode()) - users = data.get("users", []) + # uid / username → email via the full (paginated) user list; one walk + # covers both matchers. + users = cloudron_fetch_all_users_sync() for u in users: if u.get("id") == identity: return (u.get("email") or "").lower() @@ -3915,15 +3951,8 @@ async def broker_get_matrix_mxid(request: Request): def local_mxid_for_email(email: str) -> str: """Resolve a member's local Matrix ID (@username:inference.coop), or '' if unknown.""" - import urllib.request try: - req = urllib.request.Request( - f"{CLOUDRON_API}/api/v1/users?per_page=100", - headers={"Authorization": f"Bearer {CLOUDRON_TOKEN}"}, - ) - with urllib.request.urlopen(req, timeout=15) as r: - users = json.loads(r.read().decode()).get("users", []) - for u in users: + for u in cloudron_fetch_all_users_sync(): if (u.get("email") or "").lower() == email and u.get("username"): return f"@{u['username']}:inference.coop" except Exception as e: @@ -5057,6 +5086,57 @@ def credit_reference_exists(reference: str) -> bool: return bool(row) +def grant_held_credits(conn: sqlite3.Connection, email: str, slug: str = "") -> float: + """Apply every HELD credit pack matched to this member, atomically (M5). + + Per hold, in ONE transaction: flip the hold `held` → `granted`, move the + member's balance, and write the ledger row — mirroring admin_held_grant. + Committing the three together is the point: an exception can no longer + leave hold=granted with the member uncredited. The ledger reference is + `held:` (the same shape admin_held_grant uses), so re-running a + hold that was already applied is a no-op and the resolve-then-credit can + never double-apply. Returns the total amount granted. + """ + email = (email or "").strip().lower() + rows = conn.execute( + "SELECT id, amount FROM held_credits " + "WHERE status = 'held' AND (LOWER(email) = ? OR LOWER(slug) = ?)", + (email.lower(), slug.lower() if slug else ""), + ).fetchall() + granted = 0.0 + for r in rows: + hold_id, amount = r[0], float(r[1] or 0.0) + reference = f"held:{hold_id}" + if credit_reference_exists(reference): + # Already applied by an earlier run (e.g. an admin grant on this + # very hold): reconcile the status, move no money. + conn.execute( + "UPDATE held_credits SET status = 'granted', resolved_at = " + "COALESCE(resolved_at, datetime('now')) WHERE id = ? AND status = 'held'", + (hold_id,), + ) + conn.commit() + continue + conn.execute( + "UPDATE held_credits SET status = 'granted', resolved_at = datetime('now') " + "WHERE id = ? AND status = 'held'", + (hold_id,), + ) + conn.execute( + "UPDATE members SET credit_balance = " + "MAX(0, COALESCE(credit_balance, 0) + ?) WHERE LOWER(email) = ?", + (amount, email), + ) + conn.execute( + "INSERT INTO credit_ledger (email, delta, reason, reference) " + "VALUES (?, ?, ?, ?)", + (email, amount, "held credit pack(s) granted on membership", reference), + ) + conn.commit() + granted += amount + return granted + + def shortfall_notice_sent(reference: str) -> bool: conn = get_db() row = conn.execute( diff --git a/tests/test_portal_hardening.py b/tests/test_portal_hardening.py new file mode 100644 index 0000000..5a638b7 --- /dev/null +++ b/tests/test_portal_hardening.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Regression tests for M5 (atomic held-credit grant) + M6 (Cloudron user +pagination) — audit 2026-11-05. + +Runnable: + * from the repo root: python3 tests/test_portal_hardening.py + * inside the portal app: PORTAL_CODE_DIR=/app/code python3 /tmp/test_portal_hardening.py + +Covered: + M5 — grant_held_credits + - matched holds (by email AND by slug) flip + credit in one transaction + - a re-run moves no money (held: reference exists-check) + - a hold whose ledger row already exists is reconciled, never re-credited + M6 — cloudron_fetch_all_users (async) / _sync + - pages past the first: >100 users returned in full (the old code silently + truncated at 100) + - page-1 failure -> None / raise (callers keep failing loudly) + - later-page failure -> keep what was read + - the hard page cap holds even if the API never runs short +""" +import asyncio +import os +import sqlite3 +import sys +import tempfile + +_TEST_DB = os.path.join(tempfile.mkdtemp(prefix="hardening-test-"), "members.db") +os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE app.main is imported + +for _cand in (os.environ.get("PORTAL_CODE_DIR"), "/app/code", + os.path.dirname(os.path.dirname(os.path.abspath(__file__)))): + if _cand and os.path.isfile(os.path.join(_cand, "app", "main.py")): + sys.path.insert(0, _cand) + break + +from app.main import ( # noqa: E402 + cloudron_fetch_all_users, cloudron_fetch_all_users_sync, get_credit_balance, + get_db, grant_held_credits, +) + +failures = 0 + + +def check(name, got, want): + global failures + ok = got == want + if not ok: + failures += 1 + print(f"[{'PASS' if ok else 'FAIL'}] {name}") + if not ok: + print(f" got {got!r}\n want {want!r}") + + +# ---------------------------------------------------------------- M5 --------- +conn = get_db() +conn.execute("INSERT INTO members (email, key_token, slug, active) " + "VALUES ('newmember@example.com', '', 'newmember', 1)") +conn.execute("INSERT INTO held_credits (slug, email, name, amount, oc_reference, status) " + "VALUES ('newmember', 'someone-else@example.com', 'W', 10.0, 'oc:newmember:or_1', 'held')") +conn.execute("INSERT INTO held_credits (slug, email, name, amount, oc_reference, status) " + "VALUES ('newmember', NULL, 'X', 5.0, 'oc:other:or_2', 'held')") +conn.commit() + +# Mixed-case email on purpose: the grant must normalize it (the balance +# UPDATE/SELECT join on the member row and silently miss otherwise). +granted = grant_held_credits(conn, "NewMember@Example.com", "newmember") +check("matched holds sum (including the non-member slug hold)", granted, 15.0) +check("balance credited", get_credit_balance("newmember@example.com"), 15.0) +statuses = dict(conn.execute("SELECT id, status FROM held_credits").fetchall()) +check("both holds flipped in the same pass", set(statuses.values()), {"granted"}) +refs = [r[0] for r in conn.execute("SELECT reference FROM credit_ledger ORDER BY id").fetchall()] +check("one ledger row per hold, held: shape", refs, ["held:1", "held:2"]) + +granted2 = grant_held_credits(conn, "newmember@example.com", "newmember") +check("re-run grants nothing (reference exists-check)", granted2, 0.0) +refs = [r[0] for r in conn.execute("SELECT reference FROM credit_ledger ORDER BY id").fetchall()] +check("re-run wrote no extra ledger rows", refs, ["held:1", "held:2"]) +check("re-run left the balance alone", get_credit_balance("newmember@example.com"), 15.0) + +# Crash-recovery shape: a hold flipped but its ledger row lost… inverse — a +# ledger row that exists while the hold reads 'held' (a crash between the two +# commits of the OLD code) must reconcile the status and move no money. +conn.execute("UPDATE held_credits SET status='held', resolved_at=NULL WHERE id = 2") +conn.commit() +granted3 = grant_held_credits(conn, "newmember@example.com", "newmember") +check("already-credited hold reconciles without re-crediting", granted3, 0.0) +check("its status is reconciled to granted", + dict(conn.execute("SELECT id, status FROM held_credits").fetchall()), {1: "granted", 2: "granted"}) +check("balance still 15 after the reconciliation pass", + get_credit_balance("newmember@example.com"), 15.0) + +_conn2 = get_db() # separate handle to prove committed state, not in-transaction state +n = _conn2.execute("SELECT COUNT(*) FROM credit_ledger").fetchone()[0] +check("committed state equals in-call state", n, 2) +_conn2.close() +conn.close() + +# ---------------------------------------------------------------- M6 --------- +class FakeResponse: + def __init__(self, status_code, payload): + self.status_code = status_code + self._payload = payload + + def json(self): + return self._payload + + +class FakeAsyncClient: + def __init__(self, pages): + self.pages = list(pages) # one entry per expected page + self.calls = [] + + async def get(self, url, params=None, headers=None): + self.calls.append(dict(params or {})) + if not self.pages: + return FakeResponse(200, {"users": []}) + item = self.pages.pop(0) + if isinstance(item, int): + return FakeResponse(item, {}) + return FakeResponse(200, {"users": item}) + + +users_120 = [{"id": f"u{i}", "email": f"m{i}@example.com"} for i in range(120)] +page1, page2 = users_120[:100], users_120[100:] + +fc = FakeAsyncClient([page1, page2]) +got = asyncio.run(cloudron_fetch_all_users(fc)) +check("110+ users: walked past the first page", len(got), 120) +check("per_page=100 on every page", {c.get("per_page") for c in fc.calls}, {100}) +check("pages requested in order", [c.get("page") for c in fc.calls], [1, 2]) + +fc = FakeAsyncClient([500]) +check("page-1 failure -> None (caller fails loudly)", + asyncio.run(cloudron_fetch_all_users(fc)), None) + +fc = FakeAsyncClient([page1, 500, page2]) +got = asyncio.run(cloudron_fetch_all_users(fc)) +check("later-page failure keeps what was read", len(got), 100) + + +class EndlessFullPagesClient: + """A fake API that never runs short — the hard cap must stop the walk.""" + + def __init__(self): + self.calls = 0 + + async def get(self, url, params=None, headers=None): + self.calls += 1 + return FakeResponse(200, {"users": users_120[:100]}) + + +fc = EndlessFullPagesClient() +got = asyncio.run(cloudron_fetch_all_users(fc)) +check("hard cap: stops at 50 pages (5000 users)", (len(got), fc.calls), (5000, 50)) + +# --- sync walker (monkeypatch urllib at the call site's own module) --------- +import urllib.request + +_real_urlopen = urllib.request.urlopen +_seen = {"n": 0} + + +def fake_urlopen(req, timeout=None): + _seen["n"] += 1 + if _seen["n"] == 1: + raise RuntimeError("page 1 simulated failure") + class R: + def __enter__(self): + return self + def __exit__(self, *a): + return False + def read(self): + return b'{"users": []}' + return R() + + +urllib.request.urlopen = fake_urlopen +try: + try: + cloudron_fetch_all_users_sync() + check("sync walker: page-1 failure raises", False, True) + except RuntimeError: + check("sync walker: page-1 failure raises", True, True) +finally: + urllib.request.urlopen = _real_urlopen + +_real_urlopen = urllib.request.urlopen +_state = {"pages": [users_120[:100], users_120[100:]], "urls": []} + + +def fake_urlopen2(req, timeout=None): + _state["urls"].append(req.full_url) + page = _state["pages"].pop(0) if _state["pages"] else [] + import io + body = ('{"users": %s}' % __import__("json").dumps(page)).encode() + return io.BytesIO(body) + + +urllib.request.urlopen = fake_urlopen2 +try: + got = cloudron_fetch_all_users_sync() + check("sync walker pages past the first page too", len(got), 120) + check("sync walker issued a page=2 request", + any("page=2" in u for u in _state["urls"]), True) +finally: + urllib.request.urlopen = _real_urlopen + +print(f"\n{'FAILED' if failures else 'OK'} — {failures} failure(s)") +sys.exit(1 if failures else 0)