correctness: paginate Cloudron user fetches; atomic held-credit grant (audit M5+M6)
M6: a single-page ?per_page=100 fetch silently truncates past 100 users — the box is at 99. New shared walkers (async + sync) page through /api/v1/users like the Matrix paths, capped at 50 pages. All six single- page call sites now walk: duplicate-email check (provisioning 409s), reinvite loop, Loomio activation filter, admin overview, member-email resolution, local MXID resolution. Failure behavior per site is preserved (page-1 failure fails loudly; later-page failure keeps what was read). M5: provisioning-time held-credit grant moved into grant_held_credits(): per hold, the status flip, balance move and ledger row commit in ONE transaction (the old code committed the flip before add_credits, so an exception between them left hold=granted and the member uncredited). Mirrors admin_held_grant, keyed on held:<hold_id> so re-runs are no-ops; mixed-case emails normalized. Tests in tests/test_portal_hardening.py.
This commit is contained in:
1 parent
0f32c3a868
commit
b9561a49d2
2 files changed
+352
-63
No files matched your search
+143
-63
@@ -1819,12 +1819,10 @@ async def filter_loomio_activated(emails: list[str]) -> list[str]:
|
|||||||
activated: set[str] = set()
|
activated: set[str] = set()
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient(timeout=15.0) as client:
|
async with httpx.AsyncClient(timeout=15.0) as client:
|
||||||
r = await client.get(
|
users = await cloudron_fetch_all_users(client)
|
||||||
f"{CLOUDRON_API}/api/v1/users?per_page=100",
|
if users is None:
|
||||||
headers=cloudron_headers(),
|
raise RuntimeError("cloudron users fetch failed (page 1)")
|
||||||
)
|
for u in users:
|
||||||
r.raise_for_status()
|
|
||||||
for u in r.json().get("users", []):
|
|
||||||
if u.get("inviteAccepted") and u.get("email"):
|
if u.get("inviteAccepted") and u.get("email"):
|
||||||
activated.add(u["email"])
|
activated.add(u["email"])
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -1923,6 +1921,74 @@ def cloudron_headers() -> dict:
|
|||||||
return {"Authorization": f"Bearer {CLOUDRON_TOKEN}"}
|
return {"Authorization": f"Bearer {CLOUDRON_TOKEN}"}
|
||||||
|
|
||||||
|
|
||||||
|
# M6 (audit 2026-11-05): a single-page `?per_page=100` fetch silently truncates
|
||||||
|
# once the box passes 100 users (the membership sits at the threshold). Both
|
||||||
|
# walkers below page through /api/v1/users like the Matrix paths already do,
|
||||||
|
# capped sanely for Cloudron (50 pages x 100 = 5000 users).
|
||||||
|
_CLOUDRON_USERS_PAGE_CAP = 50
|
||||||
|
|
||||||
|
|
||||||
|
async def cloudron_fetch_all_users(client: httpx.AsyncClient) -> list[dict] | None:
|
||||||
|
"""Every Cloudron user, walking /api/v1/users page by page (async callers).
|
||||||
|
|
||||||
|
Returns None only when page 1 is unresolvable (caller keeps its existing
|
||||||
|
failure behavior); a failure on a LATER page keeps what was read, which is
|
||||||
|
strictly more than the truncated single-page fetch returned before.
|
||||||
|
"""
|
||||||
|
users: list[dict] = []
|
||||||
|
for page in range(1, _CLOUDRON_USERS_PAGE_CAP + 1):
|
||||||
|
r = await client.get(
|
||||||
|
f"{CLOUDRON_API}/api/v1/users",
|
||||||
|
params={"page": page, "per_page": 100},
|
||||||
|
headers=cloudron_headers(),
|
||||||
|
)
|
||||||
|
if r.status_code != 200:
|
||||||
|
if page == 1:
|
||||||
|
logger.warning("Cloudron users fetch page 1 failed: %s", r.status_code)
|
||||||
|
return None
|
||||||
|
logger.warning("Cloudron users fetch page %s failed: %s", page, r.status_code)
|
||||||
|
break
|
||||||
|
batch = r.json().get("users", [])
|
||||||
|
users.extend(batch)
|
||||||
|
if len(batch) < 100:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
logger.warning("Cloudron users fetch hit the %s-page cap; list truncated",
|
||||||
|
_CLOUDRON_USERS_PAGE_CAP)
|
||||||
|
return users
|
||||||
|
|
||||||
|
|
||||||
|
def cloudron_fetch_all_users_sync() -> list[dict]:
|
||||||
|
"""Every Cloudron user for the sync (urllib) call sites, same pagination.
|
||||||
|
|
||||||
|
Raises on a page-1 failure (the call sites error the request); later-page
|
||||||
|
failures keep what was read.
|
||||||
|
"""
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
users: list[dict] = []
|
||||||
|
for page in range(1, _CLOUDRON_USERS_PAGE_CAP + 1):
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{CLOUDRON_API}/api/v1/users?page={page}&per_page=100",
|
||||||
|
headers=cloudron_headers(),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=15) as r:
|
||||||
|
batch = json.loads(r.read().decode()).get("users", [])
|
||||||
|
except Exception:
|
||||||
|
if page == 1:
|
||||||
|
raise
|
||||||
|
logger.warning("Cloudron users fetch page %s failed", page, exc_info=True)
|
||||||
|
break
|
||||||
|
users.extend(batch)
|
||||||
|
if len(batch) < 100:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
logger.warning("Cloudron users fetch hit the %s-page cap; list truncated",
|
||||||
|
_CLOUDRON_USERS_PAGE_CAP)
|
||||||
|
return users
|
||||||
|
|
||||||
|
|
||||||
async def cloudron_create_user(email: str, name: str) -> str:
|
async def cloudron_create_user(email: str, name: str) -> str:
|
||||||
"""Create (or return existing) Cloudron user, assigned to the members group.
|
"""Create (or return existing) Cloudron user, assigned to the members group.
|
||||||
|
|
||||||
@@ -1933,13 +1999,12 @@ async def cloudron_create_user(email: str, name: str) -> str:
|
|||||||
with body {"groupIds": [...]}.
|
with body {"groupIds": [...]}.
|
||||||
"""
|
"""
|
||||||
async with httpx.AsyncClient() as client:
|
async with httpx.AsyncClient() as client:
|
||||||
# Check if user exists
|
# Check if user exists (walks every page — a single-page fetch
|
||||||
r = await client.get(
|
# truncated past 100 users and let duplicate users get created).
|
||||||
f"{CLOUDRON_API}/api/v1/users?per_page=100",
|
users = await cloudron_fetch_all_users(client)
|
||||||
headers=cloudron_headers(),
|
if users is None:
|
||||||
)
|
raise RuntimeError("cloudron users fetch failed; refusing to create a duplicate")
|
||||||
r.raise_for_status()
|
for u in users:
|
||||||
for u in r.json().get("users", []):
|
|
||||||
if u.get("email") == email:
|
if u.get("email") == email:
|
||||||
# If the existing user has no display name (e.g. created with a
|
# If the existing user has no display name (e.g. created with a
|
||||||
# placeholder), fill it in. Otherwise leave it alone — the user
|
# placeholder), fill it in. Otherwise leave it alone — the user
|
||||||
@@ -2086,12 +2151,10 @@ async def cloudron_is_activated(user_id: str) -> bool:
|
|||||||
log in — the state we re-invite for.
|
log in — the state we re-invite for.
|
||||||
"""
|
"""
|
||||||
async with httpx.AsyncClient() as client:
|
async with httpx.AsyncClient() as client:
|
||||||
r = await client.get(
|
users = await cloudron_fetch_all_users(client)
|
||||||
f"{CLOUDRON_API}/api/v1/users?per_page=100",
|
if users is None:
|
||||||
headers=cloudron_headers(),
|
raise RuntimeError("cloudron users fetch failed")
|
||||||
)
|
for u in users:
|
||||||
r.raise_for_status()
|
|
||||||
for u in r.json().get("users", []):
|
|
||||||
if u.get("id") == user_id:
|
if u.get("id") == user_id:
|
||||||
return bool(u.get("inviteAccepted"))
|
return bool(u.get("inviteAccepted"))
|
||||||
return False
|
return False
|
||||||
@@ -2148,30 +2211,12 @@ async def provision_member(email: str, name: str, slug: str = "") -> str:
|
|||||||
granted = 0.0
|
granted = 0.0
|
||||||
try:
|
try:
|
||||||
conn = get_db()
|
conn = get_db()
|
||||||
rows = conn.execute(
|
granted = grant_held_credits(conn, email, slug)
|
||||||
"SELECT id, amount FROM held_credits "
|
|
||||||
"WHERE status = 'held' AND (LOWER(email) = ? OR LOWER(slug) = ?)",
|
|
||||||
(email.lower(), slug.lower() if slug else ""),
|
|
||||||
).fetchall()
|
|
||||||
if rows:
|
|
||||||
hold_ids = [r[0] for r in rows]
|
|
||||||
granted = sum(float(r[1] or 0.0) for r in rows)
|
|
||||||
conn.execute(
|
|
||||||
"UPDATE held_credits SET status = 'granted', resolved_at = datetime('now') "
|
|
||||||
"WHERE id IN (%s)" % ",".join("?" * len(hold_ids)),
|
|
||||||
hold_ids,
|
|
||||||
)
|
|
||||||
conn.commit()
|
|
||||||
conn.close()
|
conn.close()
|
||||||
if granted > 0:
|
if granted > 0:
|
||||||
new_balance = add_credits(
|
|
||||||
email, granted,
|
|
||||||
reason="held credit pack(s) granted on membership",
|
|
||||||
reference=f"held:{','.join(str(r[0]) for r in rows)}",
|
|
||||||
)
|
|
||||||
await sync_team_budget_from_balance(email)
|
await sync_team_budget_from_balance(email)
|
||||||
logger.info("Granted %.2f held credits to new member %s (balance %s)",
|
logger.info("Granted %.2f held credits to new member %s (balance %s)",
|
||||||
granted, email, new_balance)
|
granted, email, get_credit_balance(email))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Held-credit grant for %s failed: %s", email, e)
|
logger.warning("Held-credit grant for %s failed: %s", email, e)
|
||||||
|
|
||||||
@@ -3452,12 +3497,10 @@ async def admin_overview(request: Request):
|
|||||||
invite_accepted = {}
|
invite_accepted = {}
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient(timeout=15.0) as client:
|
async with httpx.AsyncClient(timeout=15.0) as client:
|
||||||
r = await client.get(
|
users = await cloudron_fetch_all_users(client)
|
||||||
f"{CLOUDRON_API}/api/v1/users?per_page=100",
|
if users is None:
|
||||||
headers={"Authorization": f"Bearer {CLOUDRON_TOKEN}"},
|
raise RuntimeError("cloudron users fetch failed (page 1)")
|
||||||
)
|
for u in users:
|
||||||
r.raise_for_status()
|
|
||||||
for u in r.json().get("users", []):
|
|
||||||
uid = u.get("id")
|
uid = u.get("id")
|
||||||
if u.get("username"):
|
if u.get("username"):
|
||||||
usernames[uid] = u["username"]
|
usernames[uid] = u["username"]
|
||||||
@@ -3855,16 +3898,9 @@ def resolve_member_email(identity: str) -> str:
|
|||||||
identity = identity.strip()
|
identity = identity.strip()
|
||||||
if "@" in identity:
|
if "@" in identity:
|
||||||
return identity.lower()
|
return identity.lower()
|
||||||
# uid / username → email via Cloudron user list (one fetch covers both).
|
# uid / username → email via the full (paginated) user list; one walk
|
||||||
import urllib.request
|
# covers both matchers.
|
||||||
|
users = cloudron_fetch_all_users_sync()
|
||||||
req = urllib.request.Request(
|
|
||||||
f"{CLOUDRON_API}/api/v1/users?per_page=100",
|
|
||||||
headers={"Authorization": f"Bearer {CLOUDRON_TOKEN}"},
|
|
||||||
)
|
|
||||||
with urllib.request.urlopen(req, timeout=15) as r:
|
|
||||||
data = json.loads(r.read().decode())
|
|
||||||
users = data.get("users", [])
|
|
||||||
for u in users:
|
for u in users:
|
||||||
if u.get("id") == identity:
|
if u.get("id") == identity:
|
||||||
return (u.get("email") or "").lower()
|
return (u.get("email") or "").lower()
|
||||||
@@ -3915,15 +3951,8 @@ async def broker_get_matrix_mxid(request: Request):
|
|||||||
|
|
||||||
def local_mxid_for_email(email: str) -> str:
|
def local_mxid_for_email(email: str) -> str:
|
||||||
"""Resolve a member's local Matrix ID (@username:inference.coop), or '' if unknown."""
|
"""Resolve a member's local Matrix ID (@username:inference.coop), or '' if unknown."""
|
||||||
import urllib.request
|
|
||||||
try:
|
try:
|
||||||
req = urllib.request.Request(
|
for u in cloudron_fetch_all_users_sync():
|
||||||
f"{CLOUDRON_API}/api/v1/users?per_page=100",
|
|
||||||
headers={"Authorization": f"Bearer {CLOUDRON_TOKEN}"},
|
|
||||||
)
|
|
||||||
with urllib.request.urlopen(req, timeout=15) as r:
|
|
||||||
users = json.loads(r.read().decode()).get("users", [])
|
|
||||||
for u in users:
|
|
||||||
if (u.get("email") or "").lower() == email and u.get("username"):
|
if (u.get("email") or "").lower() == email and u.get("username"):
|
||||||
return f"@{u['username']}:inference.coop"
|
return f"@{u['username']}:inference.coop"
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -5057,6 +5086,57 @@ def credit_reference_exists(reference: str) -> bool:
|
|||||||
return bool(row)
|
return bool(row)
|
||||||
|
|
||||||
|
|
||||||
|
def grant_held_credits(conn: sqlite3.Connection, email: str, slug: str = "") -> float:
|
||||||
|
"""Apply every HELD credit pack matched to this member, atomically (M5).
|
||||||
|
|
||||||
|
Per hold, in ONE transaction: flip the hold `held` → `granted`, move the
|
||||||
|
member's balance, and write the ledger row — mirroring admin_held_grant.
|
||||||
|
Committing the three together is the point: an exception can no longer
|
||||||
|
leave hold=granted with the member uncredited. The ledger reference is
|
||||||
|
`held:<hold_id>` (the same shape admin_held_grant uses), so re-running a
|
||||||
|
hold that was already applied is a no-op and the resolve-then-credit can
|
||||||
|
never double-apply. Returns the total amount granted.
|
||||||
|
"""
|
||||||
|
email = (email or "").strip().lower()
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT id, amount FROM held_credits "
|
||||||
|
"WHERE status = 'held' AND (LOWER(email) = ? OR LOWER(slug) = ?)",
|
||||||
|
(email.lower(), slug.lower() if slug else ""),
|
||||||
|
).fetchall()
|
||||||
|
granted = 0.0
|
||||||
|
for r in rows:
|
||||||
|
hold_id, amount = r[0], float(r[1] or 0.0)
|
||||||
|
reference = f"held:{hold_id}"
|
||||||
|
if credit_reference_exists(reference):
|
||||||
|
# Already applied by an earlier run (e.g. an admin grant on this
|
||||||
|
# very hold): reconcile the status, move no money.
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE held_credits SET status = 'granted', resolved_at = "
|
||||||
|
"COALESCE(resolved_at, datetime('now')) WHERE id = ? AND status = 'held'",
|
||||||
|
(hold_id,),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
continue
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE held_credits SET status = 'granted', resolved_at = datetime('now') "
|
||||||
|
"WHERE id = ? AND status = 'held'",
|
||||||
|
(hold_id,),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE members SET credit_balance = "
|
||||||
|
"MAX(0, COALESCE(credit_balance, 0) + ?) WHERE LOWER(email) = ?",
|
||||||
|
(amount, email),
|
||||||
|
)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO credit_ledger (email, delta, reason, reference) "
|
||||||
|
"VALUES (?, ?, ?, ?)",
|
||||||
|
(email, amount, "held credit pack(s) granted on membership", reference),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
granted += amount
|
||||||
|
return granted
|
||||||
|
|
||||||
|
|
||||||
def shortfall_notice_sent(reference: str) -> bool:
|
def shortfall_notice_sent(reference: str) -> bool:
|
||||||
conn = get_db()
|
conn = get_db()
|
||||||
row = conn.execute(
|
row = conn.execute(
|
||||||
|
|||||||
@@ -0,0 +1,209 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Regression tests for M5 (atomic held-credit grant) + M6 (Cloudron user
|
||||||
|
pagination) — audit 2026-11-05.
|
||||||
|
|
||||||
|
Runnable:
|
||||||
|
* from the repo root: python3 tests/test_portal_hardening.py
|
||||||
|
* inside the portal app: PORTAL_CODE_DIR=/app/code python3 /tmp/test_portal_hardening.py
|
||||||
|
|
||||||
|
Covered:
|
||||||
|
M5 — grant_held_credits
|
||||||
|
- matched holds (by email AND by slug) flip + credit in one transaction
|
||||||
|
- a re-run moves no money (held:<id> reference exists-check)
|
||||||
|
- a hold whose ledger row already exists is reconciled, never re-credited
|
||||||
|
M6 — cloudron_fetch_all_users (async) / _sync
|
||||||
|
- pages past the first: >100 users returned in full (the old code silently
|
||||||
|
truncated at 100)
|
||||||
|
- page-1 failure -> None / raise (callers keep failing loudly)
|
||||||
|
- later-page failure -> keep what was read
|
||||||
|
- the hard page cap holds even if the API never runs short
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
_TEST_DB = os.path.join(tempfile.mkdtemp(prefix="hardening-test-"), "members.db")
|
||||||
|
os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE app.main is imported
|
||||||
|
|
||||||
|
for _cand in (os.environ.get("PORTAL_CODE_DIR"), "/app/code",
|
||||||
|
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))):
|
||||||
|
if _cand and os.path.isfile(os.path.join(_cand, "app", "main.py")):
|
||||||
|
sys.path.insert(0, _cand)
|
||||||
|
break
|
||||||
|
|
||||||
|
from app.main import ( # noqa: E402
|
||||||
|
cloudron_fetch_all_users, cloudron_fetch_all_users_sync, get_credit_balance,
|
||||||
|
get_db, grant_held_credits,
|
||||||
|
)
|
||||||
|
|
||||||
|
failures = 0
|
||||||
|
|
||||||
|
|
||||||
|
def check(name, got, want):
|
||||||
|
global failures
|
||||||
|
ok = got == want
|
||||||
|
if not ok:
|
||||||
|
failures += 1
|
||||||
|
print(f"[{'PASS' if ok else 'FAIL'}] {name}")
|
||||||
|
if not ok:
|
||||||
|
print(f" got {got!r}\n want {want!r}")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- M5 ---------
|
||||||
|
conn = get_db()
|
||||||
|
conn.execute("INSERT INTO members (email, key_token, slug, active) "
|
||||||
|
"VALUES ('newmember@example.com', '', 'newmember', 1)")
|
||||||
|
conn.execute("INSERT INTO held_credits (slug, email, name, amount, oc_reference, status) "
|
||||||
|
"VALUES ('newmember', 'someone-else@example.com', 'W', 10.0, 'oc:newmember:or_1', 'held')")
|
||||||
|
conn.execute("INSERT INTO held_credits (slug, email, name, amount, oc_reference, status) "
|
||||||
|
"VALUES ('newmember', NULL, 'X', 5.0, 'oc:other:or_2', 'held')")
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
# Mixed-case email on purpose: the grant must normalize it (the balance
|
||||||
|
# UPDATE/SELECT join on the member row and silently miss otherwise).
|
||||||
|
granted = grant_held_credits(conn, "NewMember@Example.com", "newmember")
|
||||||
|
check("matched holds sum (including the non-member slug hold)", granted, 15.0)
|
||||||
|
check("balance credited", get_credit_balance("newmember@example.com"), 15.0)
|
||||||
|
statuses = dict(conn.execute("SELECT id, status FROM held_credits").fetchall())
|
||||||
|
check("both holds flipped in the same pass", set(statuses.values()), {"granted"})
|
||||||
|
refs = [r[0] for r in conn.execute("SELECT reference FROM credit_ledger ORDER BY id").fetchall()]
|
||||||
|
check("one ledger row per hold, held:<id> shape", refs, ["held:1", "held:2"])
|
||||||
|
|
||||||
|
granted2 = grant_held_credits(conn, "newmember@example.com", "newmember")
|
||||||
|
check("re-run grants nothing (reference exists-check)", granted2, 0.0)
|
||||||
|
refs = [r[0] for r in conn.execute("SELECT reference FROM credit_ledger ORDER BY id").fetchall()]
|
||||||
|
check("re-run wrote no extra ledger rows", refs, ["held:1", "held:2"])
|
||||||
|
check("re-run left the balance alone", get_credit_balance("newmember@example.com"), 15.0)
|
||||||
|
|
||||||
|
# Crash-recovery shape: a hold flipped but its ledger row lost… inverse — a
|
||||||
|
# ledger row that exists while the hold reads 'held' (a crash between the two
|
||||||
|
# commits of the OLD code) must reconcile the status and move no money.
|
||||||
|
conn.execute("UPDATE held_credits SET status='held', resolved_at=NULL WHERE id = 2")
|
||||||
|
conn.commit()
|
||||||
|
granted3 = grant_held_credits(conn, "newmember@example.com", "newmember")
|
||||||
|
check("already-credited hold reconciles without re-crediting", granted3, 0.0)
|
||||||
|
check("its status is reconciled to granted",
|
||||||
|
dict(conn.execute("SELECT id, status FROM held_credits").fetchall()), {1: "granted", 2: "granted"})
|
||||||
|
check("balance still 15 after the reconciliation pass",
|
||||||
|
get_credit_balance("newmember@example.com"), 15.0)
|
||||||
|
|
||||||
|
_conn2 = get_db() # separate handle to prove committed state, not in-transaction state
|
||||||
|
n = _conn2.execute("SELECT COUNT(*) FROM credit_ledger").fetchone()[0]
|
||||||
|
check("committed state equals in-call state", n, 2)
|
||||||
|
_conn2.close()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- M6 ---------
|
||||||
|
class FakeResponse:
|
||||||
|
def __init__(self, status_code, payload):
|
||||||
|
self.status_code = status_code
|
||||||
|
self._payload = payload
|
||||||
|
|
||||||
|
def json(self):
|
||||||
|
return self._payload
|
||||||
|
|
||||||
|
|
||||||
|
class FakeAsyncClient:
|
||||||
|
def __init__(self, pages):
|
||||||
|
self.pages = list(pages) # one entry per expected page
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
async def get(self, url, params=None, headers=None):
|
||||||
|
self.calls.append(dict(params or {}))
|
||||||
|
if not self.pages:
|
||||||
|
return FakeResponse(200, {"users": []})
|
||||||
|
item = self.pages.pop(0)
|
||||||
|
if isinstance(item, int):
|
||||||
|
return FakeResponse(item, {})
|
||||||
|
return FakeResponse(200, {"users": item})
|
||||||
|
|
||||||
|
|
||||||
|
users_120 = [{"id": f"u{i}", "email": f"m{i}@example.com"} for i in range(120)]
|
||||||
|
page1, page2 = users_120[:100], users_120[100:]
|
||||||
|
|
||||||
|
fc = FakeAsyncClient([page1, page2])
|
||||||
|
got = asyncio.run(cloudron_fetch_all_users(fc))
|
||||||
|
check("110+ users: walked past the first page", len(got), 120)
|
||||||
|
check("per_page=100 on every page", {c.get("per_page") for c in fc.calls}, {100})
|
||||||
|
check("pages requested in order", [c.get("page") for c in fc.calls], [1, 2])
|
||||||
|
|
||||||
|
fc = FakeAsyncClient([500])
|
||||||
|
check("page-1 failure -> None (caller fails loudly)",
|
||||||
|
asyncio.run(cloudron_fetch_all_users(fc)), None)
|
||||||
|
|
||||||
|
fc = FakeAsyncClient([page1, 500, page2])
|
||||||
|
got = asyncio.run(cloudron_fetch_all_users(fc))
|
||||||
|
check("later-page failure keeps what was read", len(got), 100)
|
||||||
|
|
||||||
|
|
||||||
|
class EndlessFullPagesClient:
|
||||||
|
"""A fake API that never runs short — the hard cap must stop the walk."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.calls = 0
|
||||||
|
|
||||||
|
async def get(self, url, params=None, headers=None):
|
||||||
|
self.calls += 1
|
||||||
|
return FakeResponse(200, {"users": users_120[:100]})
|
||||||
|
|
||||||
|
|
||||||
|
fc = EndlessFullPagesClient()
|
||||||
|
got = asyncio.run(cloudron_fetch_all_users(fc))
|
||||||
|
check("hard cap: stops at 50 pages (5000 users)", (len(got), fc.calls), (5000, 50))
|
||||||
|
|
||||||
|
# --- sync walker (monkeypatch urllib at the call site's own module) ---------
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
_real_urlopen = urllib.request.urlopen
|
||||||
|
_seen = {"n": 0}
|
||||||
|
|
||||||
|
|
||||||
|
def fake_urlopen(req, timeout=None):
|
||||||
|
_seen["n"] += 1
|
||||||
|
if _seen["n"] == 1:
|
||||||
|
raise RuntimeError("page 1 simulated failure")
|
||||||
|
class R:
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
def __exit__(self, *a):
|
||||||
|
return False
|
||||||
|
def read(self):
|
||||||
|
return b'{"users": []}'
|
||||||
|
return R()
|
||||||
|
|
||||||
|
|
||||||
|
urllib.request.urlopen = fake_urlopen
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
cloudron_fetch_all_users_sync()
|
||||||
|
check("sync walker: page-1 failure raises", False, True)
|
||||||
|
except RuntimeError:
|
||||||
|
check("sync walker: page-1 failure raises", True, True)
|
||||||
|
finally:
|
||||||
|
urllib.request.urlopen = _real_urlopen
|
||||||
|
|
||||||
|
_real_urlopen = urllib.request.urlopen
|
||||||
|
_state = {"pages": [users_120[:100], users_120[100:]], "urls": []}
|
||||||
|
|
||||||
|
|
||||||
|
def fake_urlopen2(req, timeout=None):
|
||||||
|
_state["urls"].append(req.full_url)
|
||||||
|
page = _state["pages"].pop(0) if _state["pages"] else []
|
||||||
|
import io
|
||||||
|
body = ('{"users": %s}' % __import__("json").dumps(page)).encode()
|
||||||
|
return io.BytesIO(body)
|
||||||
|
|
||||||
|
|
||||||
|
urllib.request.urlopen = fake_urlopen2
|
||||||
|
try:
|
||||||
|
got = cloudron_fetch_all_users_sync()
|
||||||
|
check("sync walker pages past the first page too", len(got), 120)
|
||||||
|
check("sync walker issued a page=2 request",
|
||||||
|
any("page=2" in u for u in _state["urls"]), True)
|
||||||
|
finally:
|
||||||
|
urllib.request.urlopen = _real_urlopen
|
||||||
|
|
||||||
|
print(f"\n{'FAILED' if failures else 'OK'} — {failures} failure(s)")
|
||||||
|
sys.exit(1 if failures else 0)
|
||||||
Reference in new issue
Block a user