correctness: paginate Cloudron user fetches; atomic held-credit grant (audit M5+M6)

M6: a single-page ?per_page=100 fetch silently truncates past 100 users —
the box is at 99. New shared walkers (async + sync) page through
/api/v1/users like the Matrix paths, capped at 50 pages. All six single-
page call sites now walk: duplicate-email check (provisioning 409s),
reinvite loop, Loomio activation filter, admin overview, member-email
resolution, local MXID resolution. Failure behavior per site is preserved
(page-1 failure fails loudly; later-page failure keeps what was read).

M5: provisioning-time held-credit grant moved into grant_held_credits():
per hold, the status flip, balance move and ledger row commit in ONE
transaction (the old code committed the flip before add_credits, so an
exception between them left hold=granted and the member uncredited).
Mirrors admin_held_grant, keyed on held:<hold_id> so re-runs are no-ops;
mixed-case emails normalized. Tests in tests/test_portal_hardening.py.
This commit is contained in:
inference-bot committed 2026-11-05 16:06:52 +00:00
1 parent 0f32c3a868
commit b9561a49d2
2 files changed
+352 -63

No files matched your search

+143 -63
View File
@@ -1819,12 +1819,10 @@ async def filter_loomio_activated(emails: list[str]) -> list[str]:
activated: set[str] = set()
try:
async with httpx.AsyncClient(timeout=15.0) as client:
r = await client.get(
f"{CLOUDRON_API}/api/v1/users?per_page=100",
headers=cloudron_headers(),
)
r.raise_for_status()
for u in r.json().get("users", []):
users = await cloudron_fetch_all_users(client)
if users is None:
raise RuntimeError("cloudron users fetch failed (page 1)")
for u in users:
if u.get("inviteAccepted") and u.get("email"):
activated.add(u["email"])
except Exception as e:
@@ -1923,6 +1921,74 @@ def cloudron_headers() -> dict:
return {"Authorization": f"Bearer {CLOUDRON_TOKEN}"}
# M6 (audit 2026-11-05): a single-page `?per_page=100` fetch silently truncates
# once the box passes 100 users (the membership sits at the threshold). Both
# walkers below page through /api/v1/users like the Matrix paths already do,
# capped sanely for Cloudron (50 pages x 100 = 5000 users).
_CLOUDRON_USERS_PAGE_CAP = 50
async def cloudron_fetch_all_users(client: httpx.AsyncClient) -> list[dict] | None:
"""Every Cloudron user, walking /api/v1/users page by page (async callers).
Returns None only when page 1 is unresolvable (caller keeps its existing
failure behavior); a failure on a LATER page keeps what was read, which is
strictly more than the truncated single-page fetch returned before.
"""
users: list[dict] = []
for page in range(1, _CLOUDRON_USERS_PAGE_CAP + 1):
r = await client.get(
f"{CLOUDRON_API}/api/v1/users",
params={"page": page, "per_page": 100},
headers=cloudron_headers(),
)
if r.status_code != 200:
if page == 1:
logger.warning("Cloudron users fetch page 1 failed: %s", r.status_code)
return None
logger.warning("Cloudron users fetch page %s failed: %s", page, r.status_code)
break
batch = r.json().get("users", [])
users.extend(batch)
if len(batch) < 100:
break
else:
logger.warning("Cloudron users fetch hit the %s-page cap; list truncated",
_CLOUDRON_USERS_PAGE_CAP)
return users
def cloudron_fetch_all_users_sync() -> list[dict]:
"""Every Cloudron user for the sync (urllib) call sites, same pagination.
Raises on a page-1 failure (the call sites error the request); later-page
failures keep what was read.
"""
import urllib.request
users: list[dict] = []
for page in range(1, _CLOUDRON_USERS_PAGE_CAP + 1):
req = urllib.request.Request(
f"{CLOUDRON_API}/api/v1/users?page={page}&per_page=100",
headers=cloudron_headers(),
)
try:
with urllib.request.urlopen(req, timeout=15) as r:
batch = json.loads(r.read().decode()).get("users", [])
except Exception:
if page == 1:
raise
logger.warning("Cloudron users fetch page %s failed", page, exc_info=True)
break
users.extend(batch)
if len(batch) < 100:
break
else:
logger.warning("Cloudron users fetch hit the %s-page cap; list truncated",
_CLOUDRON_USERS_PAGE_CAP)
return users
async def cloudron_create_user(email: str, name: str) -> str:
"""Create (or return existing) Cloudron user, assigned to the members group.
@@ -1933,13 +1999,12 @@ async def cloudron_create_user(email: str, name: str) -> str:
with body {"groupIds": [...]}.
"""
async with httpx.AsyncClient() as client:
# Check if user exists
r = await client.get(
f"{CLOUDRON_API}/api/v1/users?per_page=100",
headers=cloudron_headers(),
)
r.raise_for_status()
for u in r.json().get("users", []):
# Check if user exists (walks every page — a single-page fetch
# truncated past 100 users and let duplicate users get created).
users = await cloudron_fetch_all_users(client)
if users is None:
raise RuntimeError("cloudron users fetch failed; refusing to create a duplicate")
for u in users:
if u.get("email") == email:
# If the existing user has no display name (e.g. created with a
# placeholder), fill it in. Otherwise leave it alone — the user
@@ -2086,12 +2151,10 @@ async def cloudron_is_activated(user_id: str) -> bool:
log in — the state we re-invite for.
"""
async with httpx.AsyncClient() as client:
r = await client.get(
f"{CLOUDRON_API}/api/v1/users?per_page=100",
headers=cloudron_headers(),
)
r.raise_for_status()
for u in r.json().get("users", []):
users = await cloudron_fetch_all_users(client)
if users is None:
raise RuntimeError("cloudron users fetch failed")
for u in users:
if u.get("id") == user_id:
return bool(u.get("inviteAccepted"))
return False
@@ -2148,30 +2211,12 @@ async def provision_member(email: str, name: str, slug: str = "") -> str:
granted = 0.0
try:
conn = get_db()
rows = conn.execute(
"SELECT id, amount FROM held_credits "
"WHERE status = 'held' AND (LOWER(email) = ? OR LOWER(slug) = ?)",
(email.lower(), slug.lower() if slug else ""),
).fetchall()
if rows:
hold_ids = [r[0] for r in rows]
granted = sum(float(r[1] or 0.0) for r in rows)
conn.execute(
"UPDATE held_credits SET status = 'granted', resolved_at = datetime('now') "
"WHERE id IN (%s)" % ",".join("?" * len(hold_ids)),
hold_ids,
)
conn.commit()
granted = grant_held_credits(conn, email, slug)
conn.close()
if granted > 0:
new_balance = add_credits(
email, granted,
reason="held credit pack(s) granted on membership",
reference=f"held:{','.join(str(r[0]) for r in rows)}",
)
await sync_team_budget_from_balance(email)
logger.info("Granted %.2f held credits to new member %s (balance %s)",
granted, email, new_balance)
granted, email, get_credit_balance(email))
except Exception as e:
logger.warning("Held-credit grant for %s failed: %s", email, e)
@@ -3452,12 +3497,10 @@ async def admin_overview(request: Request):
invite_accepted = {}
try:
async with httpx.AsyncClient(timeout=15.0) as client:
r = await client.get(
f"{CLOUDRON_API}/api/v1/users?per_page=100",
headers={"Authorization": f"Bearer {CLOUDRON_TOKEN}"},
)
r.raise_for_status()
for u in r.json().get("users", []):
users = await cloudron_fetch_all_users(client)
if users is None:
raise RuntimeError("cloudron users fetch failed (page 1)")
for u in users:
uid = u.get("id")
if u.get("username"):
usernames[uid] = u["username"]
@@ -3855,16 +3898,9 @@ def resolve_member_email(identity: str) -> str:
identity = identity.strip()
if "@" in identity:
return identity.lower()
# uid / username → email via Cloudron user list (one fetch covers both).
import urllib.request
req = urllib.request.Request(
f"{CLOUDRON_API}/api/v1/users?per_page=100",
headers={"Authorization": f"Bearer {CLOUDRON_TOKEN}"},
)
with urllib.request.urlopen(req, timeout=15) as r:
data = json.loads(r.read().decode())
users = data.get("users", [])
# uid / username → email via the full (paginated) user list; one walk
# covers both matchers.
users = cloudron_fetch_all_users_sync()
for u in users:
if u.get("id") == identity:
return (u.get("email") or "").lower()
@@ -3915,15 +3951,8 @@ async def broker_get_matrix_mxid(request: Request):
def local_mxid_for_email(email: str) -> str:
"""Resolve a member's local Matrix ID (@username:inference.coop), or '' if unknown."""
import urllib.request
try:
req = urllib.request.Request(
f"{CLOUDRON_API}/api/v1/users?per_page=100",
headers={"Authorization": f"Bearer {CLOUDRON_TOKEN}"},
)
with urllib.request.urlopen(req, timeout=15) as r:
users = json.loads(r.read().decode()).get("users", [])
for u in users:
for u in cloudron_fetch_all_users_sync():
if (u.get("email") or "").lower() == email and u.get("username"):
return f"@{u['username']}:inference.coop"
except Exception as e:
@@ -5057,6 +5086,57 @@ def credit_reference_exists(reference: str) -> bool:
return bool(row)
def grant_held_credits(conn: sqlite3.Connection, email: str, slug: str = "") -> float:
"""Apply every HELD credit pack matched to this member, atomically (M5).
Per hold, in ONE transaction: flip the hold `held` → `granted`, move the
member's balance, and write the ledger row — mirroring admin_held_grant.
Committing the three together is the point: an exception can no longer
leave hold=granted with the member uncredited. The ledger reference is
`held:<hold_id>` (the same shape admin_held_grant uses), so re-running a
hold that was already applied is a no-op and the resolve-then-credit can
never double-apply. Returns the total amount granted.
"""
email = (email or "").strip().lower()
rows = conn.execute(
"SELECT id, amount FROM held_credits "
"WHERE status = 'held' AND (LOWER(email) = ? OR LOWER(slug) = ?)",
(email.lower(), slug.lower() if slug else ""),
).fetchall()
granted = 0.0
for r in rows:
hold_id, amount = r[0], float(r[1] or 0.0)
reference = f"held:{hold_id}"
if credit_reference_exists(reference):
# Already applied by an earlier run (e.g. an admin grant on this
# very hold): reconcile the status, move no money.
conn.execute(
"UPDATE held_credits SET status = 'granted', resolved_at = "
"COALESCE(resolved_at, datetime('now')) WHERE id = ? AND status = 'held'",
(hold_id,),
)
conn.commit()
continue
conn.execute(
"UPDATE held_credits SET status = 'granted', resolved_at = datetime('now') "
"WHERE id = ? AND status = 'held'",
(hold_id,),
)
conn.execute(
"UPDATE members SET credit_balance = "
"MAX(0, COALESCE(credit_balance, 0) + ?) WHERE LOWER(email) = ?",
(amount, email),
)
conn.execute(
"INSERT INTO credit_ledger (email, delta, reason, reference) "
"VALUES (?, ?, ?, ?)",
(email, amount, "held credit pack(s) granted on membership", reference),
)
conn.commit()
granted += amount
return granted
def shortfall_notice_sent(reference: str) -> bool:
conn = get_db()
row = conn.execute(