From 0e825e6d6178b47420d928146a6baf1264445460 Mon Sep 17 00:00:00 2001 From: Inferencebot Date: Fri, 13 Nov 2026 17:37:57 +0000 Subject: [PATCH] service invites: per-room Matrix gate, resumable rate limits, Loomio record heal, 5-min sync tick MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new member could end up with a Loomio account and a Matrix account and see NOTHING: no forum group, no Space. Three defects: * sync_matrix_space() gated each member on the SPACE's membership, so an account already joined/invited to the Space was skipped for EVERY room. A member whose invite pass was interrupted (or who predates a room added to MATRIX_INVITE_ROOMS later) never received that room — and a restricted child room is invisible until you are invited to it. The gate is now per room, from a single up-front membership snapshot; a room whose membership cannot be read is left alone rather than invited into blindly. * a 429 slept for retry_after_ms (minutes) INSIDE the pass, so a scheduled run could outlive its own tick. A rate limit now ENDS the pass; the next run resumes (every invite is idempotent). * a member sitting in the Loomio group with no loomio_invited_at stamp reads as 'never invited' — the dedupe guard was only as good as the rows it had. The sync now heals the record from the live roster. And the activation window itself: provisioning skips an unactivated account by design, and activation happens later with nobody watching, so the 3x/day sweep was the pickup. New /admin/service-sync + a 5-minute scheduler task fires the same idempotent syncs while any member still awaits an invite (one SELECT when none do), and deliberately is NOT /admin/sweep — it never touches Open Collective, allowance logic or member mail. tests/test_service_invites.py proves the per-room gate, resumable rate limits, the unreadable-room skip, the record heal and the endpoint's idle/run paths. --- CloudronManifest.json | 4 + Dockerfile | 3 +- README.md | 38 ++-- app/main.py | 213 +++++++++++++++++---- service_sync.sh | 49 +++++ tests/test_service_invites.py | 338 ++++++++++++++++++++++++++++++++++ 6 files changed, 596 insertions(+), 49 deletions(-) create mode 100644 service_sync.sh create mode 100644 tests/test_service_invites.py diff --git a/CloudronManifest.json b/CloudronManifest.json index 939cb37..8e2235c 100644 --- a/CloudronManifest.json +++ b/CloudronManifest.json @@ -18,6 +18,10 @@ "schedule": "0 3,11,19 * * *", "command": "/app/code/sweep.sh" }, + "service_sync": { + "schedule": "*/5 * * * *", + "command": "/app/code/service_sync.sh" + }, "rag_refresh": { "schedule": "30 3 * * *", "command": "/app/code/rag_refresh.py" diff --git a/Dockerfile b/Dockerfile index af6548b..f8eb374 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,10 @@ RUN pip install --no-cache-dir -r requirements.txt # Copy app COPY app/ ./app/ COPY sweep.sh /app/code/sweep.sh +COPY service_sync.sh /app/code/service_sync.sh COPY rag_refresh.py /app/code/rag_refresh.py COPY metrics_snapshot.py /app/code/metrics_snapshot.py -RUN chmod +x /app/code/sweep.sh /app/code/rag_refresh.py /app/code/metrics_snapshot.py +RUN chmod +x /app/code/sweep.sh /app/code/service_sync.sh /app/code/rag_refresh.py /app/code/metrics_snapshot.py # Run as root (the base image default). Cloudron's sandboxing (read-only # rootfs, AppArmor, dropped capabilities) still applies to root inside the diff --git a/README.md b/README.md index 0a1ddc7..8d08041 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ points. Remaining work before production: ## Scheduled tasks (scheduler addon) -Three jobs run inside the container via `CloudronManifest.json` → `addons.scheduler`: +Four jobs run inside the container via `CloudronManifest.json` → `addons.scheduler`: - **membership_sweep** (`0 3,11,19 * * *` UTC — three times a day) — `sweep.sh` calls the portal's own `/admin/sweep` (auth: the `X-Admin-Token` header with @@ -182,16 +182,30 @@ Three jobs run inside the container via `CloudronManifest.json` → `addons.sche self-heals within a day. The same sweep also runs the payment-driven allowance reconciliation (below). - **First-login activation hook:** the sweep is also the backstop for service - invites, but it is no longer the only trigger. When a member first presents - themselves on the broker identity path (the dashboard's first login), the - portal checks — only if their Loomio invite is still outstanding - (`loomio_invited_at` is null) — that their Cloudron account is now activated - (`inviteAccepted`), and if so fires the same idempotent Loomio + Matrix syncs - as a background task. Established members cost one SELECT; the response never - waits on Loomio or Matrix; failures are logged and non-fatal; unactivated - accounts are never invited (the check fails closed). So a new member gets - forum + Space access at their first login instead of at the next sweep. + **Service-invite triggers:** the sweep is the backstop, not the primary path. + Two faster triggers exist, because a member who has just set up their account + should not wait hours for forum + Space access: + 1. **First-login activation hook** — when a member first presents themselves + on the broker identity path (the dashboard's first login), the portal + checks — only if their Loomio invite is still outstanding + (`loomio_invited_at` is null) — that their Cloudron account is now + activated (`inviteAccepted`), and if so fires the same idempotent Loomio + + Matrix syncs as a background task. Established members cost one SELECT; the + response never waits on Loomio or Matrix; failures are logged and + non-fatal; unactivated accounts are never invited (the check fails closed). + 2. **service_sync** (below) — the scheduled tick that covers members who never + open the dashboard. +- **service_sync** (every 5 minutes) — `service_sync.sh` calls the portal's own + `/admin/service-sync` (same `X-Admin-Token` auth as the sweep). WHY: a new + member is provisioned with an UNACTIVATED Cloudron account, and the syncs + deliberately skip an unactivated account (never invite a half-created one) — + so nobody on our side is watching at the moment the member actually activates. + This job fires the same idempotent Loomio + Matrix syncs the sweep runs, but + ONLY while some member is still awaiting their invites (one indexed SELECT + when nobody is, so it costs nothing in the steady state), and it deliberately + is NOT `/admin/sweep`: it never touches Open Collective, allowance logic or + member mail. An in-process guard drops an overlapping run. + Net effect: a member who has just set up their account waits minutes at most. - **rag_refresh** (daily 3:30am UTC) — `rag_refresh.py` pulls this repo's sibling docs from the Gitea raw API (public, no credentials), uploads changed files to Open WebUI (`OWUI_API_KEY`), attaches them to the shared knowledge @@ -217,7 +231,7 @@ Three jobs run inside the container via `CloudronManifest.json` → `addons.sche Methodology: [Energy and carbon metrics](docs/energy-metrics.md). Verify after any manifest change: the deploy log must show -`setupAddons: ... scheduler ...` listing all three tasks, and each script must +`setupAddons: ... scheduler ...` listing all four tasks, and each script must succeed when run manually inside the container. ### Allowance renewal (payment-driven, join-day anchored) diff --git a/app/main.py b/app/main.py index eb9b390..270d70e 100644 --- a/app/main.py +++ b/app/main.py @@ -1823,49 +1823,92 @@ async def sync_matrix_space() -> dict: return {"skipped": True} H = {"Authorization": f"Bearer {MATRIX_BOT_TOKEN}", "Content-Type": "application/json"} + class _RateLimited(Exception): + """Synapse asked us to back off (429). Ends the pass, not the sync.""" + + def __init__(self, wait_ms): + super().__init__(f"retry_after_ms={wait_ms}") + self.wait_ms = wait_ms + async def invite(mxid: str, room: str) -> bool: - import asyncio as _aio - for attempt in range(3): - r = await client.post( - f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/invite", - headers=H, json={"user_id": mxid}, - ) - if r.status_code == 200: - return True - if r.status_code == 429: - wait = (r.json().get("retry_after_ms", 1000) / 1000) + 0.2 - await _aio.sleep(wait) - continue - # 403 with 'already in room' is fine - if "already" in r.text: - return True - logger.warning("Matrix invite %s -> %s failed: %s %s", - mxid, room, r.status_code, r.text[:120]) - return False + r = await client.post( + f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/invite", + headers=H, json={"user_id": mxid}, + ) + if r.status_code == 200: + return True + if r.status_code == 429: + # Synapse asks for MINUTES here, so sleeping would blow past the + # scheduled tick that called us. End the pass instead: the next run + # starts from the top, every already-sent invite is a cheap no-op, + # and progress accumulates across runs. + raise _RateLimited(r.json().get("retry_after_ms", 1000)) + # 403 with 'already in room' is fine + if "already" in r.text: + return True + logger.warning("Matrix invite %s -> %s failed: %s %s", + mxid, room, r.status_code, r.text[:120]) return False + rooms = member_invite_rooms() + invited_count = 0 + rate_limited = False + async with httpx.AsyncClient(timeout=30) as client: - # current members of the space (to avoid duplicate invites) - r = await client.get( - f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{MATRIX_SPACE_ID}/joined_members", - headers=H) - joined = set(r.json().get("joined", {}).keys()) if r.status_code == 200 else set() - # also pending invites - r = await client.get( - f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{MATRIX_SPACE_ID}/members?membership=invite", - headers=H) - invited = {m["state_key"] for m in r.json().get("chunk", [])} if r.status_code == 200 else set() + # Snapshot EVERY room's membership once, up front: who is joined and who + # has a pending invite. The gate is PER ROOM on purpose — gating on the + # Space alone meant an account already invited to the Space was skipped + # for every other room, so a member whose invite pass was interrupted + # (or who predates a room added to MATRIX_INVITE_ROOMS later) never got + # that room and could not see it. + joined_by_room: dict[str, set[str]] = {} + invited_by_room: dict[str, set[str]] = {} + for room in rooms: + r = await client.get( + f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/joined_members", + headers=H) + if r.status_code != 200: + logger.warning( + "Matrix sync: cannot read members of %s (%s); leaving that room alone", + room, r.status_code) + continue + r2 = await client.get( + f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/members", + headers=H, params={"membership": "invite"}) + if r2.status_code != 200: + logger.warning( + "Matrix sync: cannot read invites of %s (%s); leaving that room alone", + room, r2.status_code) + continue + joined_by_room[room] = set(r.json().get("joined", {}).keys()) + invited_by_room[room] = {m["state_key"] for m in r2.json().get("chunk", [])} + # Rooms we could NOT inspect are skipped entirely — never invite into a + # room whose current membership we could not read. + readable = [room for room in rooms if room in joined_by_room] # One identity per member: when a member sets a dashboard extra MXID, # their auto-generated local account (@username:inference.coop) should # NOT also sit in the Space. Track the desired set, then retire the # local accounts of members whose dashboard MXID replaced them. - invited_count = 0 async for email, mxid in matrix_member_mxids(client): - if not mxid or mxid in joined or mxid in invited: + if not mxid: continue - for room in member_invite_rooms(): - await invite(mxid, room) + missing = [ + room for room in readable + if mxid not in joined_by_room[room] + and mxid not in invited_by_room.get(room, set()) + ] + if not missing: + continue + try: + for room in missing: + await invite(mxid, room) + except _RateLimited as exc: + rate_limited = True + logger.info( + "Matrix sync: rate limited (retry_after_ms=%s); ending this pass — " + "the next run resumes where this one stopped", exc.wait_ms) + break invited_count += 1 if invited_count: logger.info("Matrix space invites sent: %s", invited_count) @@ -1914,9 +1957,9 @@ async def sync_matrix_space() -> dict: # Same room set as the invite half (member_invite_rooms()) so the # two cannot drift apart again: a retired identity must be removed # from every room we invite members to, not just Space + General. - for room in member_invite_rooms(): + for room in readable: enc_room = urllib.parse.quote(room, safe="") - if local_mxid in joined: + if local_mxid in joined_by_room.get(room, set()): await client.post( f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{enc_room}/kick", headers=H, @@ -1925,7 +1968,7 @@ async def sync_matrix_space() -> dict: ) retired += 1 logger.info("Matrix sync: retired local account %s (member uses %s)", local_mxid, ext_mxid) - elif local_mxid in invited: + elif local_mxid in invited_by_room.get(room, set()): await client.put( f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{enc_room}/state/m.room.member/{urllib.parse.quote(local_mxid, safe='')}", headers=H, @@ -1936,7 +1979,7 @@ async def sync_matrix_space() -> dict: logger.info("Matrix sync: revoked stale invite for local account %s (member uses %s)", local_mxid, ext_mxid) if retired: logger.info("Matrix sync: retired %s local account(s)/invite(s)", retired) - return {"invited": invited_count} + return {"invited": invited_count, "rate_limited": rate_limited} async def matrix_member_mxids(client: httpx.AsyncClient): @@ -2113,6 +2156,20 @@ async def sync_loomio_memberships() -> None: ) return + # --- HEAL: reconcile our own dedupe record with the live roster. --- + # A member sitting in the group with NO loomio_invited_at stamp (added by + # an earlier sync generation, or by hand) reads as "never invited" to the + # guard below. They are not: they are already in. Stamp them from the + # roster so the record reflects reality — otherwise the guard is only as + # good as the rows it happens to have. + healed = 0 + for email in current: + if not get_loomio_invited_at(email): + mark_loomio_invited(email) + healed += 1 + if healed: + logger.info("Loomio sync: stamped %d already-present member(s) as invited", healed) + # --- REMOVE: reconcile the group to the current active set. --- removed = 0 for email, mid in current.items(): @@ -5559,6 +5616,11 @@ async def admin_model_usage(request: Request, days: int = 0): _first_login_sync_inflight = False +# Guard for the scheduled service-sync (see /admin/service-sync). The pass does +# full-roster reconciles, so two overlapping runs would only duplicate work and +# burn Synapse's invite budget twice over. +_service_sync_inflight = False + def _member_slug(email: str) -> str: """Member slug for log lines (slugs survive log redaction; emails don't).""" @@ -7737,6 +7799,85 @@ async def admin_sweep(request: Request): return result +def _service_sync_pending() -> list[str]: + """Active members still waiting on their Loomio invitation. + + Cheap by design (one indexed SELECT, no network): it is what lets the + scheduled service-sync be a no-op in the steady state. + + WHY LOOMIO IS THE CLOCK: a member is stamped `loomio_invited_at` only once + the invite POST has come back 200 — so this list is non-empty for EVERY + member who has been provisioned but not yet finished onboarding, which is + exactly the window in which their Matrix invites also have to fire. That + makes it the honest trigger for running the whole service sync, and it + covers the failure this endpoint exists for: an account activated between + two sweeps. GUEST accounts are excluded — they are never invited to Loomio + or Matrix. + """ + conn = get_db() + rows = conn.execute( + "SELECT email FROM members " + "WHERE active = 1 AND (account_type IS NULL OR account_type != 'guest') " + "AND loomio_invited_at IS NULL" + ).fetchall() + conn.close() + return [r[0] for r in rows] + + +@app.post("/admin/service-sync") +@limiter.limit("20/minute") +async def admin_service_sync(request: Request): + """Run the Loomio + Matrix invite syncs — the scheduler's frequent tick. + + WHY THIS EXISTS: provisioning runs both syncs, but at provisioning time a + new member's Cloudron account is usually not ACTIVATED yet, and the syncs + deliberately skip an unactivated account (never invite a half-created one). + The activation happens later, when the member follows their setup link — + and at that moment nothing on our side is watching. Waiting for the 3x/day + sweep left a new member without forum or Space access for hours, which + reads as "the co-op is broken". + + So this runs every few minutes: it fires the same idempotent syncs the + sweep runs, ONLY while some member is still awaiting their invites, and + costs one SELECT when nobody is. Together with the first-login hook on the + dashboard path, a member who has just set up their account waits minutes + at most rather than until the next sweep. + + Authenticated by the X-Admin-Token header (same as /admin/sweep). Never + raises out to the caller on a sync failure: the syncs are individually + best-effort and the next tick retries. + """ + global _service_sync_inflight + _verify_admin_token(request) + + if _service_sync_inflight: + return {"status": "busy"} + + pending = _service_sync_pending() + if not pending: + return {"status": "idle", "pending": 0} + + _service_sync_inflight = True + result: dict = {"status": "ran", "pending": len(pending)} + try: + try: + await sync_loomio_memberships() + result["loomio"] = "ok" + except Exception as e: # noqa: BLE001 + result["loomio"] = str(e)[:200] + logger.warning("service-sync: Loomio sync failed: %s", e) + try: + result["matrix"] = await sync_matrix_space() + except Exception as e: # noqa: BLE001 + result["matrix"] = str(e)[:200] + logger.warning("service-sync: Matrix sync failed: %s", e) + finally: + _service_sync_inflight = False + + result["pending_after"] = len(_service_sync_pending()) + return result + + @app.post("/admin/newsletter/reconcile") @limiter.limit("20/minute") async def admin_newsletter_reconcile(request: Request): diff --git a/service_sync.sh b/service_sync.sh new file mode 100644 index 0000000..fbfeef2 --- /dev/null +++ b/service_sync.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# Frequent service-invite sync — run by the Cloudron scheduler addon every +# few minutes. +# +# WHY: a new member is provisioned with an UNACTIVATED Cloudron account, and +# the Loomio/Matrix syncs deliberately skip an unactivated account (never +# invite a half-created one). The member activates later, by following the +# setup link in their welcome email — and until this job existed, the next +# thing to look at them was the 3x/day membership sweep, so a brand-new member +# could sit without forum or Space access for hours. +# +# Calls the portal's own /admin/service-sync endpoint (localhost), which fires +# the same idempotent Loomio + Matrix syncs the sweep runs, but ONLY while some +# member is still awaiting their invites (one SELECT when nobody is). +# +# The ADMIN_TOKEN env var is shared with the main app (the scheduler runs in +# the same launch environment), so we use it to authenticate. Admin auth is a +# header, never a URL token. +# +# Deliberately NOT the full /admin/sweep: that reconciles against Open +# Collective, runs allowance logic and can send member mail. This job must only +# ever send the invites a member is owed. + +set -e + +if [ -z "$ADMIN_TOKEN" ]; then + echo "service-sync: ADMIN_TOKEN is not set; aborting" >&2 + exit 1 +fi + +python3 - "$ADMIN_TOKEN" <<'PY' +import sys, urllib.request, urllib.error + +token = sys.argv[1] +url = "http://127.0.0.1:8000/admin/service-sync" +req = urllib.request.Request(url, method="POST", data=b"", headers={ + "Content-Type": "application/json", + "X-Admin-Token": token, +}) +try: + with urllib.request.urlopen(req, timeout=300) as resp: + print("service-sync:", resp.read().decode()) +except urllib.error.HTTPError as e: + print(f"service-sync: HTTP {e.code}: {e.read().decode()}", file=sys.stderr) + sys.exit(1) +except Exception as e: + print(f"service-sync: request failed: {e}", file=sys.stderr) + sys.exit(1) +PY diff --git a/tests/test_service_invites.py b/tests/test_service_invites.py new file mode 100644 index 0000000..4623ba0 --- /dev/null +++ b/tests/test_service_invites.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""Regression test: the service-invite sync (per-room invites + frequent tick). + +WHY THIS EXISTS + A new member got a Loomio account and a Matrix account but could see NOTHING: + no forum group, no Space. Two separate defects produced that, and a third + made it take hours. + + 1. PER-ROOM GATE (the visibility bug). sync_matrix_space() gated each member + on the SPACE's membership: anyone already joined/invited to the Space was + skipped for EVERY room. So a member whose invite pass was interrupted, or + who predates a room added to MATRIX_INVITE_ROOMS later, never received + that room's invite — and a restricted child room is invisible until you + are invited to it. + 2. RATE LIMITS ENDED NOTHING. A 429 slept for retry_after_ms (minutes) inside + the pass, so a scheduled run could outlive its own tick. A rate limit must + END the pass; the next run resumes (every invite is idempotent). + 3. THE DEDUPE RECORD DRIFTED. A member sitting in the Loomio group with no + members.loomio_invited_at stamp (added by an earlier sync generation, or + by hand) reads as "never invited" — the guard is only as good as the rows + it happens to have. The sync now heals the record from the live roster. + 4. NOTHING WATCHED ACTIVATION. Provisioning skips an unactivated account by + design, and activation happens later with nobody watching, so the 3x/day + sweep was the pickup. /admin/service-sync is the frequent tick that closes + that window; it must be a no-op (one SELECT) in the steady state. + + Network is STUBBED throughout; assertions are on what we SENT and on our own + recorded state. + +Runnable: python3 tests/test_service_invites.py (from the repo root) +""" +import os +import sys +import tempfile +import asyncio + +import httpx + +os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db") +os.environ["MATRIX_BOT_TOKEN"] = "test-bot-token" +os.environ["MATRIX_SPACE_ID"] = "!space:inference.coop" +os.environ["MATRIX_HOMESERVER"] = "https://matrix.inference.coop" +os.environ["MATRIX_INVITE_ROOMS"] = "!space:inference.coop,!general:inference.coop,!design:inference.coop" +os.environ["LOOMIO_API_KEY"] = "test-user-key" +os.environ["LOOMIO_B3_KEY"] = "test-server-key" +os.environ["LOOMIO_GROUP_ID"] = "1" +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import app.main as m # noqa: E402 + +# The app module and this test share the SAME httpx module object, so stubbing +# the client below replaces it process-wide. Keep a handle on the real class for +# the ASGI call at the end. +REAL_ASYNC_CLIENT = httpx.AsyncClient + +m.DB_PATH = os.environ["DB_PATH"] +m.MATRIX_BOT_TOKEN = "test-bot-token" +m.MATRIX_SPACE_ID = "!space:inference.coop" +m.MATRIX_HOMESERVER = "https://matrix.inference.coop" +m.MATRIX_INVITE_ROOMS = [ + "!space:inference.coop", "!general:inference.coop", "!design:inference.coop" +] +m.MATRIX_EXTRA_ROOMS = [] +m.LOOMIO_ALWAYS_KEEP = [] +m.ADMIN_TOKEN = "test-admin-token" +m.ADMIN_TOKEN_LEGACY = "" +m.WEBHOOK_TOKEN = "" + +ROOMS = ["!space:inference.coop", "!general:inference.coop", "!design:inference.coop"] +USERS = {"a@x.org": "amember", "b@x.org": "bmember", "c@x.org": "cmember"} + +ROOM_JOINED = {} # room -> set(mxid) +ROOM_INVITED = {} # room -> set(mxid) +UNREADABLE = set() # rooms whose joined_members 500s +invites_sent = [] # (room, mxid) +INVITE_429_FOR = set() # mxids whose invite comes back 429 +POSTS = [] # Loomio POST payloads + +ROSTER = {} # Loomio roster: email -> membership id + + +class FakeResp: + def __init__(self, status=200, payload=None, text=None): + self.status_code = status + self._payload = payload or {} + self.text = text if text is not None else str(payload or {}) + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + def json(self): + return self._payload + + +class FakeClient: + def __init__(self, *a, **k): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + async def get(self, url, headers=None, params=None): + if "/api/v1/users" in url: + return FakeResp(200, {"users": [ + {"email": e, "username": u, "inviteAccepted": True} + for e, u in USERS.items() + ]}) + for room in ROOMS: + if f"/rooms/{room}/joined_members" in url: + if room in UNREADABLE: + return FakeResp(500, {"error": "boom"}) + return FakeResp(200, {"joined": {m: {} for m in ROOM_JOINED.get(room, set())}}) + if f"/rooms/{room}/members" in url: + return FakeResp(200, {"chunk": [ + {"state_key": mx} for mx in ROOM_INVITED.get(room, set()) + ]}) + if "/api/b3/memberships" in url: + rows = [{"id": i, "user_email": e} for e, i in ROSTER.items()] + offset = (params or {}).get("offset", 0) + limit = (params or {}).get("limit", 50) + return FakeResp(200, {"memberships": rows[offset:offset + limit]}) + return FakeResp(404, {}) + + async def post(self, url, headers=None, json=None): + if "/invite" in url: + mxid = (json or {}).get("user_id") + if mxid in INVITE_429_FOR: + return FakeResp(429, {"retry_after_ms": 300000}, + text='{"errcode":"M_LIMIT_EXCEEDED"}') + for room in ROOMS: + if f"/rooms/{room}/invite" in url: + invites_sent.append((room, mxid)) + ROOM_INVITED.setdefault(room, set()).add(mxid) + return FakeResp(200, {}) + return FakeResp(200, {}) + if "/api/b2/memberships" in url: + POSTS.append(json or {}) + return FakeResp(200, {"added_emails": (json or {}).get("emails", [])}) + return FakeResp(200, {}) + + async def put(self, *a, **k): + return FakeResp(200, {}) + + async def delete(self, *a, **k): + return FakeResp(200, {}) + + async def patch(self, *a, **k): + return FakeResp(200, {}) + + +results = [] + + +def check(name, ok): + results.append((name, ok)) + + +def reset(): + global invites_sent + invites_sent = [] + POSTS.clear() + + +async def main(): + global ROOM_JOINED, ROOM_INVITED, UNREADABLE, INVITE_429_FOR, ROSTER + + m.get_db().close() # create + migrate the fixture schema + conn = m.get_db() + for email in USERS: + conn.execute("INSERT OR REPLACE INTO members (email, active) VALUES (?, 1)", (email,)) + conn.execute("INSERT OR REPLACE INTO members (email, active, account_type) VALUES (?, 1, 'guest')", + ("guest@x.org",)) + conn.commit() + conn.close() + + m.httpx.AsyncClient = FakeClient + + async def _active(): + return list(USERS.keys()) + m.get_active_member_emails = _active + + # --- 1. PER-ROOM GATE ------------------------------------------------- + # a@ is already in every room. b@ has the Space + General but is MISSING + # Design — the exact state an interrupted pass leaves behind. + ROOM_JOINED = {"!space:inference.coop": {"@amember:inference.coop"}, + "!general:inference.coop": {"@amember:inference.coop"}, + "!design:inference.coop": {"@amember:inference.coop"}} + ROOM_INVITED = {"!space:inference.coop": {"@bmember:inference.coop"}, + "!general:inference.coop": {"@bmember:inference.coop"}, + "!design:inference.coop": set()} + USERS["c@x.org"] = "cmember" + ROOM_JOINED["!space:inference.coop"].add("@cmember:inference.coop") + ROOM_INVITED["!space:inference.coop"].add("@cmember:inference.coop") + ROOM_JOINED["!general:inference.coop"].add("@cmember:inference.coop") + ROOM_INVITED["!general:inference.coop"].add("@cmember:inference.coop") + ROOM_JOINED["!design:inference.coop"].add("@cmember:inference.coop") + ROOM_INVITED["!design:inference.coop"].add("@cmember:inference.coop") + reset() + r = await m.sync_matrix_space() + check("per-room: only the MISSING room is invited, by room id", + invites_sent == [("!design:inference.coop", "@bmember:inference.coop")]) + check("per-room: an account already in every room is not re-invited", + not any(mx == "@amember:inference.coop" for _, mx in invites_sent)) + check("per-room: two members can be pending on different rooms", + not any(mx == "@cmember:inference.coop" for _, mx in invites_sent)) + check("return shape: the result carries invited + rate_limited", + set(r) == {"invited", "rate_limited"} and r["rate_limited"] is False) + + # --- 2. IDEMPOTENCE: a second pass sends nothing ---------------------- + reset() + await m.sync_matrix_space() + check("idempotent: a second pass sends no invites", invites_sent == []) + + # --- 3. RATE LIMIT ENDS THE PASS (no sleeping inside a scheduled tick) - + # d@ is missing all three rooms; the FIRST invite 429s. + USERS["d@x.org"] = "dmember" + m.get_db().close() + conn = m.get_db() + conn.execute("INSERT OR REPLACE INTO members (email, active) VALUES ('d@x.org', 1)") + conn.commit() + conn.close() + + async def active_with_d(): + return ["d@x.org", "a@x.org"] + m.get_active_member_emails = active_with_d + INVITE_429_FOR = {"@dmember:inference.coop"} + reset() + r = await m.sync_matrix_space() + check("rate limit: the pass reports it was rate limited", r["rate_limited"] is True) + check("rate limit: it does NOT retry in a loop inside the pass", + invites_sent == []) + INVITE_429_FOR = set() + m.get_active_member_emails = _active + + # A resumed pass DOES send the invites the limited one could not. + reset() + r = await m.sync_matrix_space() + check("rate limit: the next pass resumes and invites the missed rooms", + len(invites_sent) == 3 and {room for room, _ in invites_sent} == set(ROOMS)) + + # --- 4. A ROOM WE CANNOT READ IS LEFT ALONE --------------------------- + # e@ is only missing Design, and Design's membership cannot be read. + USERS["e@x.org"] = "emember" + conn = m.get_db() + conn.execute("INSERT OR REPLACE INTO members (email, active) VALUES ('e@x.org', 1)") + conn.commit() + conn.close() + + async def active_with_e(): + return ["e@x.org"] + m.get_active_member_emails = active_with_e + ROOM_JOINED = {r_: set() for r_ in ROOMS} + ROOM_INVITED = {"!space:inference.coop": {"@emember:inference.coop"}, + "!general:inference.coop": {"@emember:inference.coop"}, + "!design:inference.coop": set()} + UNREADABLE = {"!design:inference.coop"} + reset() + await m.sync_matrix_space() + check("unreadable room: never invited into a room whose members we could not read", + invites_sent == []) + UNREADABLE = set() + m.get_active_member_emails = _active + + # --- 5. LOOMIO RECORD SELF-HEAL -------------------------------------- + # b@ is sitting in the group with NO stamp: the guard would call them + # "never invited". The sync must heal the record from the roster, and must + # not send their address anywhere while doing it. + conn = m.get_db() + conn.execute("UPDATE members SET loomio_invited_at = NULL") + conn.commit() + conn.close() + ROSTER = {"b@x.org": 55} + reset() + await m.sync_loomio_memberships() + sent = sorted((POSTS[0].get("emails") if POSTS else []) or []) + check("heal: a present member with no stamp is stamped as invited", + m.get_loomio_invited_at("b@x.org") is not None) + check("heal: healing never re-sends a member already in the group", + "b@x.org" not in sent) + check("heal: an unstamped member who is NOT present is still invited", + sent == ["a@x.org", "c@x.org", "d@x.org", "e@x.org"]) + + # --- 6. THE FREQUENT TICK'S WAKE-UP CONDITION ------------------------- + conn = m.get_db() + conn.execute("UPDATE members SET loomio_invited_at = NULL WHERE email = 'e@x.org'") + conn.commit() + conn.close() + pending = m._service_sync_pending() + check("pending: a member still awaiting their invite is counted", pending == ["e@x.org"]) + check("pending: an invited member is NOT counted", "a@x.org" not in pending) + check("pending: guests are never counted", "guest@x.org" not in pending) + m.mark_loomio_invited("e@x.org") + check("pending: empty once every member is stamped", m._service_sync_pending() == []) + + +asyncio.run(main()) + +# --- 7. THE ENDPOINT (real ASGI call, so the wiring is exercised) ---------- + + +async def endpoint_checks(): + transport = httpx.ASGITransport(app=m.app) + + async with REAL_ASYNC_CLIENT(transport=transport, base_url="http://test") as c: + r = await c.post("/admin/service-sync") + check(f"endpoint: rejects a caller with no admin token (got {r.status_code})", + r.status_code == 403) + r = await c.post("/admin/service-sync", headers={"X-Admin-Token": "wrong"}) + check(f"endpoint: rejects a wrong admin token (got {r.status_code})", + r.status_code == 403) + r = await c.post("/admin/service-sync", headers={"X-Admin-Token": "test-admin-token"}) + check(f"endpoint: accepts the admin token (got {r.status_code})", r.status_code == 200) + body = r.json() + check(f"endpoint: is IDLE (one SELECT) when nobody is awaiting an invite (got {body})", + body.get("status") == "idle") + # Make one member await an invite again and prove the tick RUNS. + conn = m.get_db() + conn.execute("UPDATE members SET loomio_invited_at = NULL WHERE email = 'a@x.org'") + conn.commit() + conn.close() + r = await c.post("/admin/service-sync", headers={"X-Admin-Token": "test-admin-token"}) + body = r.json() + check(f"endpoint: RUNS while a member is awaiting invites (got {body})", + body.get("status") == "ran" and body.get("pending") == 1) + check(f"endpoint: runs BOTH syncs (got {body})", + body.get("loomio") == "ok" and "matrix" in body) + + +asyncio.run(endpoint_checks()) + +fails = sum(not ok for _, ok in results) +for name, ok in results: + print(f" [{'PASS' if ok else 'FAIL'}] {name}") +print(f"\n{len(results) - fails}/{len(results)} passed") +sys.exit(1 if fails else 0)