Guest invite: serialize member invites — parallel direct calls can't slip two guests
The one-per-30-days rule is a READ of the guests ledger, provisioning takes seconds of external calls, and the ledger row that closes the window is written LAST — so two PARALLEL direct calls (curl, bypassing the dashboard JS) could both pass guest_request_guard before either row exists and slip two guests past the rule. The member invite path now runs guard+provision under _GUEST_INVITE_LOCK (single-process uvicorn), making check-and-send atomic: the second caller re-runs the guard after the first caller's ledger row exists and is refused 429. tests/test_guest_concurrency.py proves both halves on the fixture DB: unguarded, two parallel invites both provision (the test has teeth); with the lock, exactly one provisions, one is refused 429, one ledger row, one account, one email; sequential semantics unchanged.
This commit is contained in:
1 parent
1d102a4754
commit
2954fe609f
2 files changed
+281
-6
No files matched your search
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parallel guest invitations cannot slip two guests past the 30-day rule.
|
||||
|
||||
WHY THIS EXISTS
|
||||
The one-guest-per-member-per-30-days rule is a READ of the guests ledger,
|
||||
provisioning takes seconds of external calls, and the ledger row that closes
|
||||
the window is written LAST. Two PARALLEL direct calls (curl, not the
|
||||
dashboard JS) could therefore both pass the guard before either row exists.
|
||||
The member invite path now runs under _GUEST_INVITE_LOCK, which makes
|
||||
check-and-send atomic for the single-process uvicorn app.
|
||||
|
||||
Proven here, against a FIXTURE database with the network layer replaced by
|
||||
raising guards (any live call fails the test loudly):
|
||||
1. With the lock DISABLED, two parallel invites from the same member BOTH
|
||||
provision — the race is real, and the test has teeth.
|
||||
2. With the real lock, the same parallel pair yields exactly ONE
|
||||
provisioned guest; the second is refused 429 inside the 30-day window.
|
||||
3. Exactly one ledger row and one members row exist per accepted invite;
|
||||
the refused recipient is unborn (no row, no credit, no email).
|
||||
|
||||
Runnable: python3 tests/test_guest_concurrency.py (from the repo root)
|
||||
"""
|
||||
import asyncio
|
||||
import base64
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
|
||||
os.environ["PORTAL_KEK"] = base64.b64encode(bytes(32)).decode()
|
||||
os.environ.setdefault("GUESTS_GROUP_ID", "gid-fixture-guests")
|
||||
os.environ.setdefault("MEMBERS_GROUP_ID", "gid-fixture-members")
|
||||
os.environ.setdefault("INACTIVE_GROUP_ID", "gid-fixture-inactive")
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
import app.main as m # noqa: E402
|
||||
|
||||
m.DB_PATH = os.environ["DB_PATH"]
|
||||
|
||||
results = []
|
||||
|
||||
|
||||
def check(name, ok, detail=""):
|
||||
results.append((name, bool(ok), detail))
|
||||
print(("PASS " if ok else "FAIL ") + name + (f" [{detail}]" if detail and not ok else ""))
|
||||
|
||||
|
||||
class _NoNetwork:
|
||||
def __init__(self, *a, **k):
|
||||
raise AssertionError("live network attempted during fixture test")
|
||||
|
||||
def __call__(self, *a, **k): # pragma: no cover
|
||||
raise AssertionError("live network attempted during fixture test")
|
||||
|
||||
|
||||
class _FakeClient(_NoNetwork):
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *a):
|
||||
return False
|
||||
|
||||
async def get(self, *a, **k):
|
||||
raise AssertionError("live network attempted during fixture test")
|
||||
|
||||
async def post(self, *a, **k):
|
||||
raise AssertionError("live network attempted during fixture test")
|
||||
|
||||
|
||||
m.httpx.AsyncClient = _FakeClient
|
||||
|
||||
# --- provisioning stubs: same contract as test_guest_requests, but
|
||||
# cloudron_create_user SUSPENDS briefly, forcing real task interleaving so the
|
||||
# parallel window is actually exercised (without a suspension point asyncio
|
||||
# would run the two requests back-to-back and prove nothing). ----------------#
|
||||
calls = {"group": [], "email": [], "listmonk": [], "budget": [], "uids": 0}
|
||||
|
||||
|
||||
async def fake_user_create(email, name):
|
||||
calls["uids"] += 1
|
||||
await asyncio.sleep(0.05) # the suspension that opens the parallel window
|
||||
return f"uid-fixture-{calls['uids']}"
|
||||
|
||||
|
||||
async def rec_group(user_id, group_id=None):
|
||||
calls["group"].append((user_id, group_id or m.MEMBERS_GROUP_ID))
|
||||
|
||||
|
||||
def rec_email(to, subject, text_body=None, html_body=None):
|
||||
calls["email"].append((to, subject or ""))
|
||||
return True
|
||||
|
||||
|
||||
async def rec_listmonk(email, subscribe):
|
||||
calls["listmonk"].append((email, bool(subscribe)))
|
||||
|
||||
|
||||
async def rec_budget(team_id, budget):
|
||||
calls["budget"].append((team_id, round(float(budget), 2)))
|
||||
|
||||
|
||||
async def fake_team(email, budget, budget_duration="30d"):
|
||||
return "team-fixture"
|
||||
|
||||
|
||||
async def fake_key(email, budget, team_id):
|
||||
return "sk-fixture"
|
||||
|
||||
|
||||
async def fake_invite_link(user_id):
|
||||
return "https://my.example/setupaccount.html?inviteToken=x"
|
||||
|
||||
|
||||
def fake_team_db(team_id):
|
||||
return {"max_budget": None, "spend": 0.0, "budget_reset_at": None}
|
||||
|
||||
|
||||
m.cloudron_create_user = fake_user_create
|
||||
m.cloudron_set_group = rec_group
|
||||
m.cloudron_set_active = rec_group
|
||||
m.cloudron_get_invite_link = fake_invite_link
|
||||
m.litellm_get_or_create_team = fake_team
|
||||
m.litellm_create_key = fake_key
|
||||
m.litellm_update_team_budget = rec_budget
|
||||
m.get_team_budget_from_db = fake_team_db
|
||||
m.send_email = rec_email
|
||||
m.listmonk_sync = rec_listmonk
|
||||
|
||||
# --- real Request objects (the slowapi limiter decorator requires them) ----- #
|
||||
from starlette.requests import Request as StarletteRequest # noqa: E402
|
||||
import json as _json # noqa: E402
|
||||
|
||||
|
||||
def make_request(body=None):
|
||||
raw = _json.dumps(body or {}).encode()
|
||||
headers = {"content-type": "application/json", "content-length": str(len(raw))}
|
||||
async def receive():
|
||||
return {"type": "http.request", "body": raw, "more_body": False}
|
||||
scope = {
|
||||
"type": "http", "method": "POST", "path": "/", "query_string": b"",
|
||||
"headers": [(k.lower().encode(), v.encode()) for k, v in headers.items()],
|
||||
"server": ("testserver", 80), "client": ("testclient", 123),
|
||||
}
|
||||
return StarletteRequest(scope, receive)
|
||||
|
||||
|
||||
def seed_member(conn, email):
|
||||
conn.execute(
|
||||
"INSERT INTO members (email, key_token, cloudron_user_id, slug, active, "
|
||||
"balance, credit_balance, account_type) "
|
||||
"VALUES (?, 'enc', 'uid-' || ?, '', 1, ?, 0, 'member')",
|
||||
(email, email, m.MEMBER_BUDGET),
|
||||
)
|
||||
|
||||
|
||||
class _NoLock:
|
||||
"""Stands in for the real lock to demonstrate the unguarded race."""
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *a):
|
||||
return False
|
||||
|
||||
|
||||
async def parallel_pair(inviter, recipients):
|
||||
"""Fire TWO member invites at once (the direct-call attack shape)."""
|
||||
m._verify_broker_secret = lambda request: inviter
|
||||
return await asyncio.gather(
|
||||
*(m.broker_guest_invite(make_request({"email": r})) for r in recipients),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
|
||||
def ledger_count(inviter):
|
||||
conn = m.get_db()
|
||||
n = conn.execute("SELECT COUNT(*) FROM guests WHERE inviter_email=?",
|
||||
(inviter,)).fetchone()[0]
|
||||
conn.close()
|
||||
return n
|
||||
|
||||
|
||||
def row_count(email):
|
||||
conn = m.get_db()
|
||||
n = conn.execute("SELECT COUNT(*) FROM members WHERE email=?", (email,)).fetchone()[0]
|
||||
conn.close()
|
||||
return n
|
||||
|
||||
|
||||
def main():
|
||||
m.get_db().close() # create + migrate the fixture schema
|
||||
conn = m.get_db()
|
||||
m._ensure_allowance_tables(conn)
|
||||
|
||||
# --- 1. the RACE, unguarded: prove two parallel invites both provision -- #
|
||||
seed_member(conn, "racer-one@x.org")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
real_lock = m._GUEST_INVITE_LOCK
|
||||
m._GUEST_INVITE_LOCK = _NoLock()
|
||||
try:
|
||||
out = asyncio.run(parallel_pair(
|
||||
"racer-one@x.org", ["race-a@x.org", "race-b@x.org"]))
|
||||
finally:
|
||||
m._GUEST_INVITE_LOCK = real_lock
|
||||
provisioned = [o for o in out if isinstance(o, dict)]
|
||||
refused = [o for o in out if isinstance(o, Exception)]
|
||||
check("race (unguarded): both parallel invites slipped through "
|
||||
"(the test has teeth — this is the defect the lock closes)",
|
||||
len(provisioned) == 2 and not refused,
|
||||
f"provisioned={len(provisioned)} refused={len(refused)}")
|
||||
check("race (unguarded): two ledger rows — the rule is broken without "
|
||||
"serialization", ledger_count("racer-one@x.org") == 2,
|
||||
str(ledger_count("racer-one@x.org")))
|
||||
|
||||
# --- 2. the LOCK: the same parallel pair now yields exactly one guest --- #
|
||||
conn = m.get_db()
|
||||
seed_member(conn, "racer-two@x.org")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
calls["email"].clear()
|
||||
out = asyncio.run(parallel_pair(
|
||||
"racer-two@x.org", ["race-c@x.org", "race-d@x.org"]))
|
||||
provisioned = [o for o in out if isinstance(o, dict)]
|
||||
refused = [o for o in out if isinstance(o, Exception)]
|
||||
check("lock: exactly ONE of two parallel invites provisions",
|
||||
len(provisioned) == 1 and len(refused) == 1,
|
||||
f"provisioned={len(provisioned)} refused={len(refused)} "
|
||||
f"{[type(o).__name__ for o in out]}")
|
||||
check("lock: the refusal is the over-limit error (429, 30-day window)",
|
||||
refused and getattr(refused[0], "status_code", None) == 429
|
||||
and "30 days" in str(getattr(refused[0], "detail", "")),
|
||||
f"{refused}")
|
||||
check("lock: exactly ONE ledger row for the inviter",
|
||||
ledger_count("racer-two@x.org") == 1,
|
||||
str(ledger_count("racer-two@x.org")))
|
||||
born = sum(1 for r in ("race-c@x.org", "race-d@x.org") if row_count(r))
|
||||
check("lock: exactly ONE recipient account was created", born == 1, str(born))
|
||||
check("lock: the refused recipient is unborn (no row, no guest, no credit)",
|
||||
sum(1 for r in ("race-c@x.org", "race-d@x.org") if row_count(r) == 0) == 1)
|
||||
check("lock: exactly ONE invitation email left the building",
|
||||
len(calls["email"]) == 1, str(calls["email"]))
|
||||
|
||||
# --- 3. sequential semantics unchanged: the second invite still refuses - #
|
||||
conn = m.get_db()
|
||||
refusal = m.guest_request_guard(conn, "racer-two@x.org", "race-e@x.org")
|
||||
conn.close()
|
||||
check("aftermath: a later invite from the same member is still refused "
|
||||
"(one per 30 days, unchanged)",
|
||||
refusal is not None and "30 days" in refusal, str(refusal))
|
||||
|
||||
failed = [r for r in results if not r[1]]
|
||||
print(f"\n{len(results) - len(failed)}/{len(results)} checks passed")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user