service invites: per-room Matrix gate, resumable rate limits, Loomio record heal, 5-min sync tick
A new member could end up with a Loomio account and a Matrix account and see NOTHING: no forum group, no Space. Three defects: * sync_matrix_space() gated each member on the SPACE's membership, so an account already joined/invited to the Space was skipped for EVERY room. A member whose invite pass was interrupted (or who predates a room added to MATRIX_INVITE_ROOMS later) never received that room — and a restricted child room is invisible until you are invited to it. The gate is now per room, from a single up-front membership snapshot; a room whose membership cannot be read is left alone rather than invited into blindly. * a 429 slept for retry_after_ms (minutes) INSIDE the pass, so a scheduled run could outlive its own tick. A rate limit now ENDS the pass; the next run resumes (every invite is idempotent). * a member sitting in the Loomio group with no loomio_invited_at stamp reads as 'never invited' — the dedupe guard was only as good as the rows it had. The sync now heals the record from the live roster. And the activation window itself: provisioning skips an unactivated account by design, and activation happens later with nobody watching, so the 3x/day sweep was the pickup. New /admin/service-sync + a 5-minute scheduler task fires the same idempotent syncs while any member still awaits an invite (one SELECT when none do), and deliberately is NOT /admin/sweep — it never touches Open Collective, allowance logic or member mail. tests/test_service_invites.py proves the per-room gate, resumable rate limits, the unreadable-room skip, the record heal and the endpoint's idle/run paths.
This commit is contained in:
1 parent
de0f5c13c4
commit
0e825e6d61
6 files changed
+596
-49
No files matched your search
+177
-36
@@ -1823,49 +1823,92 @@ async def sync_matrix_space() -> dict:
|
||||
return {"skipped": True}
|
||||
H = {"Authorization": f"Bearer {MATRIX_BOT_TOKEN}", "Content-Type": "application/json"}
|
||||
|
||||
class _RateLimited(Exception):
|
||||
"""Synapse asked us to back off (429). Ends the pass, not the sync."""
|
||||
|
||||
def __init__(self, wait_ms):
|
||||
super().__init__(f"retry_after_ms={wait_ms}")
|
||||
self.wait_ms = wait_ms
|
||||
|
||||
async def invite(mxid: str, room: str) -> bool:
|
||||
import asyncio as _aio
|
||||
for attempt in range(3):
|
||||
r = await client.post(
|
||||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/invite",
|
||||
headers=H, json={"user_id": mxid},
|
||||
)
|
||||
if r.status_code == 200:
|
||||
return True
|
||||
if r.status_code == 429:
|
||||
wait = (r.json().get("retry_after_ms", 1000) / 1000) + 0.2
|
||||
await _aio.sleep(wait)
|
||||
continue
|
||||
# 403 with 'already in room' is fine
|
||||
if "already" in r.text:
|
||||
return True
|
||||
logger.warning("Matrix invite %s -> %s failed: %s %s",
|
||||
mxid, room, r.status_code, r.text[:120])
|
||||
return False
|
||||
r = await client.post(
|
||||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/invite",
|
||||
headers=H, json={"user_id": mxid},
|
||||
)
|
||||
if r.status_code == 200:
|
||||
return True
|
||||
if r.status_code == 429:
|
||||
# Synapse asks for MINUTES here, so sleeping would blow past the
|
||||
# scheduled tick that called us. End the pass instead: the next run
|
||||
# starts from the top, every already-sent invite is a cheap no-op,
|
||||
# and progress accumulates across runs.
|
||||
raise _RateLimited(r.json().get("retry_after_ms", 1000))
|
||||
# 403 with 'already in room' is fine
|
||||
if "already" in r.text:
|
||||
return True
|
||||
logger.warning("Matrix invite %s -> %s failed: %s %s",
|
||||
mxid, room, r.status_code, r.text[:120])
|
||||
return False
|
||||
|
||||
rooms = member_invite_rooms()
|
||||
invited_count = 0
|
||||
rate_limited = False
|
||||
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
# current members of the space (to avoid duplicate invites)
|
||||
r = await client.get(
|
||||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{MATRIX_SPACE_ID}/joined_members",
|
||||
headers=H)
|
||||
joined = set(r.json().get("joined", {}).keys()) if r.status_code == 200 else set()
|
||||
# also pending invites
|
||||
r = await client.get(
|
||||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{MATRIX_SPACE_ID}/members?membership=invite",
|
||||
headers=H)
|
||||
invited = {m["state_key"] for m in r.json().get("chunk", [])} if r.status_code == 200 else set()
|
||||
# Snapshot EVERY room's membership once, up front: who is joined and who
|
||||
# has a pending invite. The gate is PER ROOM on purpose — gating on the
|
||||
# Space alone meant an account already invited to the Space was skipped
|
||||
# for every other room, so a member whose invite pass was interrupted
|
||||
# (or who predates a room added to MATRIX_INVITE_ROOMS later) never got
|
||||
# that room and could not see it.
|
||||
joined_by_room: dict[str, set[str]] = {}
|
||||
invited_by_room: dict[str, set[str]] = {}
|
||||
for room in rooms:
|
||||
r = await client.get(
|
||||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/joined_members",
|
||||
headers=H)
|
||||
if r.status_code != 200:
|
||||
logger.warning(
|
||||
"Matrix sync: cannot read members of %s (%s); leaving that room alone",
|
||||
room, r.status_code)
|
||||
continue
|
||||
r2 = await client.get(
|
||||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/members",
|
||||
headers=H, params={"membership": "invite"})
|
||||
if r2.status_code != 200:
|
||||
logger.warning(
|
||||
"Matrix sync: cannot read invites of %s (%s); leaving that room alone",
|
||||
room, r2.status_code)
|
||||
continue
|
||||
joined_by_room[room] = set(r.json().get("joined", {}).keys())
|
||||
invited_by_room[room] = {m["state_key"] for m in r2.json().get("chunk", [])}
|
||||
# Rooms we could NOT inspect are skipped entirely — never invite into a
|
||||
# room whose current membership we could not read.
|
||||
readable = [room for room in rooms if room in joined_by_room]
|
||||
|
||||
# One identity per member: when a member sets a dashboard extra MXID,
|
||||
# their auto-generated local account (@username:inference.coop) should
|
||||
# NOT also sit in the Space. Track the desired set, then retire the
|
||||
# local accounts of members whose dashboard MXID replaced them.
|
||||
invited_count = 0
|
||||
async for email, mxid in matrix_member_mxids(client):
|
||||
if not mxid or mxid in joined or mxid in invited:
|
||||
if not mxid:
|
||||
continue
|
||||
for room in member_invite_rooms():
|
||||
await invite(mxid, room)
|
||||
missing = [
|
||||
room for room in readable
|
||||
if mxid not in joined_by_room[room]
|
||||
and mxid not in invited_by_room.get(room, set())
|
||||
]
|
||||
if not missing:
|
||||
continue
|
||||
try:
|
||||
for room in missing:
|
||||
await invite(mxid, room)
|
||||
except _RateLimited as exc:
|
||||
rate_limited = True
|
||||
logger.info(
|
||||
"Matrix sync: rate limited (retry_after_ms=%s); ending this pass — "
|
||||
"the next run resumes where this one stopped", exc.wait_ms)
|
||||
break
|
||||
invited_count += 1
|
||||
if invited_count:
|
||||
logger.info("Matrix space invites sent: %s", invited_count)
|
||||
@@ -1914,9 +1957,9 @@ async def sync_matrix_space() -> dict:
|
||||
# Same room set as the invite half (member_invite_rooms()) so the
|
||||
# two cannot drift apart again: a retired identity must be removed
|
||||
# from every room we invite members to, not just Space + General.
|
||||
for room in member_invite_rooms():
|
||||
for room in readable:
|
||||
enc_room = urllib.parse.quote(room, safe="")
|
||||
if local_mxid in joined:
|
||||
if local_mxid in joined_by_room.get(room, set()):
|
||||
await client.post(
|
||||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{enc_room}/kick",
|
||||
headers=H,
|
||||
@@ -1925,7 +1968,7 @@ async def sync_matrix_space() -> dict:
|
||||
)
|
||||
retired += 1
|
||||
logger.info("Matrix sync: retired local account %s (member uses %s)", local_mxid, ext_mxid)
|
||||
elif local_mxid in invited:
|
||||
elif local_mxid in invited_by_room.get(room, set()):
|
||||
await client.put(
|
||||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{enc_room}/state/m.room.member/{urllib.parse.quote(local_mxid, safe='')}",
|
||||
headers=H,
|
||||
@@ -1936,7 +1979,7 @@ async def sync_matrix_space() -> dict:
|
||||
logger.info("Matrix sync: revoked stale invite for local account %s (member uses %s)", local_mxid, ext_mxid)
|
||||
if retired:
|
||||
logger.info("Matrix sync: retired %s local account(s)/invite(s)", retired)
|
||||
return {"invited": invited_count}
|
||||
return {"invited": invited_count, "rate_limited": rate_limited}
|
||||
|
||||
|
||||
async def matrix_member_mxids(client: httpx.AsyncClient):
|
||||
@@ -2113,6 +2156,20 @@ async def sync_loomio_memberships() -> None:
|
||||
)
|
||||
return
|
||||
|
||||
# --- HEAL: reconcile our own dedupe record with the live roster. ---
|
||||
# A member sitting in the group with NO loomio_invited_at stamp (added by
|
||||
# an earlier sync generation, or by hand) reads as "never invited" to the
|
||||
# guard below. They are not: they are already in. Stamp them from the
|
||||
# roster so the record reflects reality — otherwise the guard is only as
|
||||
# good as the rows it happens to have.
|
||||
healed = 0
|
||||
for email in current:
|
||||
if not get_loomio_invited_at(email):
|
||||
mark_loomio_invited(email)
|
||||
healed += 1
|
||||
if healed:
|
||||
logger.info("Loomio sync: stamped %d already-present member(s) as invited", healed)
|
||||
|
||||
# --- REMOVE: reconcile the group to the current active set. ---
|
||||
removed = 0
|
||||
for email, mid in current.items():
|
||||
@@ -5559,6 +5616,11 @@ async def admin_model_usage(request: Request, days: int = 0):
|
||||
|
||||
_first_login_sync_inflight = False
|
||||
|
||||
# Guard for the scheduled service-sync (see /admin/service-sync). The pass does
|
||||
# full-roster reconciles, so two overlapping runs would only duplicate work and
|
||||
# burn Synapse's invite budget twice over.
|
||||
_service_sync_inflight = False
|
||||
|
||||
|
||||
def _member_slug(email: str) -> str:
|
||||
"""Member slug for log lines (slugs survive log redaction; emails don't)."""
|
||||
@@ -7737,6 +7799,85 @@ async def admin_sweep(request: Request):
|
||||
return result
|
||||
|
||||
|
||||
def _service_sync_pending() -> list[str]:
|
||||
"""Active members still waiting on their Loomio invitation.
|
||||
|
||||
Cheap by design (one indexed SELECT, no network): it is what lets the
|
||||
scheduled service-sync be a no-op in the steady state.
|
||||
|
||||
WHY LOOMIO IS THE CLOCK: a member is stamped `loomio_invited_at` only once
|
||||
the invite POST has come back 200 — so this list is non-empty for EVERY
|
||||
member who has been provisioned but not yet finished onboarding, which is
|
||||
exactly the window in which their Matrix invites also have to fire. That
|
||||
makes it the honest trigger for running the whole service sync, and it
|
||||
covers the failure this endpoint exists for: an account activated between
|
||||
two sweeps. GUEST accounts are excluded — they are never invited to Loomio
|
||||
or Matrix.
|
||||
"""
|
||||
conn = get_db()
|
||||
rows = conn.execute(
|
||||
"SELECT email FROM members "
|
||||
"WHERE active = 1 AND (account_type IS NULL OR account_type != 'guest') "
|
||||
"AND loomio_invited_at IS NULL"
|
||||
).fetchall()
|
||||
conn.close()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
@app.post("/admin/service-sync")
|
||||
@limiter.limit("20/minute")
|
||||
async def admin_service_sync(request: Request):
|
||||
"""Run the Loomio + Matrix invite syncs — the scheduler's frequent tick.
|
||||
|
||||
WHY THIS EXISTS: provisioning runs both syncs, but at provisioning time a
|
||||
new member's Cloudron account is usually not ACTIVATED yet, and the syncs
|
||||
deliberately skip an unactivated account (never invite a half-created one).
|
||||
The activation happens later, when the member follows their setup link —
|
||||
and at that moment nothing on our side is watching. Waiting for the 3x/day
|
||||
sweep left a new member without forum or Space access for hours, which
|
||||
reads as "the co-op is broken".
|
||||
|
||||
So this runs every few minutes: it fires the same idempotent syncs the
|
||||
sweep runs, ONLY while some member is still awaiting their invites, and
|
||||
costs one SELECT when nobody is. Together with the first-login hook on the
|
||||
dashboard path, a member who has just set up their account waits minutes
|
||||
at most rather than until the next sweep.
|
||||
|
||||
Authenticated by the X-Admin-Token header (same as /admin/sweep). Never
|
||||
raises out to the caller on a sync failure: the syncs are individually
|
||||
best-effort and the next tick retries.
|
||||
"""
|
||||
global _service_sync_inflight
|
||||
_verify_admin_token(request)
|
||||
|
||||
if _service_sync_inflight:
|
||||
return {"status": "busy"}
|
||||
|
||||
pending = _service_sync_pending()
|
||||
if not pending:
|
||||
return {"status": "idle", "pending": 0}
|
||||
|
||||
_service_sync_inflight = True
|
||||
result: dict = {"status": "ran", "pending": len(pending)}
|
||||
try:
|
||||
try:
|
||||
await sync_loomio_memberships()
|
||||
result["loomio"] = "ok"
|
||||
except Exception as e: # noqa: BLE001
|
||||
result["loomio"] = str(e)[:200]
|
||||
logger.warning("service-sync: Loomio sync failed: %s", e)
|
||||
try:
|
||||
result["matrix"] = await sync_matrix_space()
|
||||
except Exception as e: # noqa: BLE001
|
||||
result["matrix"] = str(e)[:200]
|
||||
logger.warning("service-sync: Matrix sync failed: %s", e)
|
||||
finally:
|
||||
_service_sync_inflight = False
|
||||
|
||||
result["pending_after"] = len(_service_sync_pending())
|
||||
return result
|
||||
|
||||
|
||||
@app.post("/admin/newsletter/reconcile")
|
||||
@limiter.limit("20/minute")
|
||||
async def admin_newsletter_reconcile(request: Request):
|
||||
|
||||
Reference in new issue
Block a user