diff --git a/app/main.py b/app/main.py index 80d0325..c141008 100644 --- a/app/main.py +++ b/app/main.py @@ -234,8 +234,12 @@ PORTAL_BASE = os.environ.get("PORTAL_BASE", "https://portal.inference.coop") # Listmonk newsletter integration. The portal keeps the "Members" list in sync: # subscribe on provision, unsubscribe on deactivation. Auth uses listmonk's # native API-user token scheme (Authorization: token :). -LISTMONK_BASE = os.environ.get("LISTMONK_BASE", "https://newsletter.inference.coop") -LISTMONK_API_USER = os.environ.get("LISTMONK_API_USER", "portal-sync") +LISTMONK_BASE = os.environ.get("LISTMONK_BASE") or "https://newsletter.inference.coop" +# NOTE: `or` (not default-arg) — an env var set to EMPTY must fall back to the +# default. An empty LISTMONK_API_USER made every sync 403 silently (the token +# alone is not a valid listmonk credential) while listmonk_sync's best-effort +# error handling swallowed it — that is how stale accounts stayed subscribed. +LISTMONK_API_USER = os.environ.get("LISTMONK_API_USER") or "portal-sync" LISTMONK_API_TOKEN = os.environ.get("LISTMONK_API_TOKEN", "") LISTMONK_LIST_ID = int(os.environ.get("LISTMONK_LIST_ID", "3")) # "Members" list @@ -410,6 +414,90 @@ async def listmonk_sync(email: str, subscribe: bool) -> None: logger.warning("Listmonk sync error for %s: %s", email, e) +async def listmonk_reconcile() -> dict: + """Reconcile the Members newsletter list against the active membership. + + The newsletter list must mirror the portal's ACTIVE members. Subscribers + who are not active members are stale (deleted accounts, never-activated + test accounts, lapsed members) and are removed: + - emails that exist in the members table but are inactive are taken off + the list (the subscriber record stays as a suppression marker); + - emails that are not members at all are deleted outright (stray or + test accounts should not persist in the mailing system). + + Best-effort: never raises, so a newsletter outage can't break the sweep. + """ + if not LISTMONK_API_TOKEN: + logger.warning("Listmonk not configured; skipping newsletter reconcile") + return {"removed": 0, "emails": [], "error": "not configured"} + auth = {"Authorization": f"token {LISTMONK_API_USER}:{LISTMONK_API_TOKEN}"} + conn = get_db() + active = { + row[0].lower() + for row in conn.execute("SELECT email FROM members WHERE active = 1") + } + member_emails = { + row[0].lower() for row in conn.execute("SELECT email FROM members") + } + conn.close() + try: + async with httpx.AsyncClient(timeout=20.0) as client: + # Fetch the full subscriber set for the list (paginated). + subs, page = [], 1 + while True: + r = await client.get( + f"{LISTMONK_BASE}/api/subscribers", + headers=auth, + params={"list_id": LISTMONK_LIST_ID, "page": page, "per_page": 100}, + ) + if r.status_code != 200: + logger.warning( + "Listmonk reconcile: subscriber fetch failed: %s", r.status_code + ) + return {"removed": 0, "emails": [], "error": f"fetch {r.status_code}"} + data = r.json().get("data", {}) + results = data.get("results", []) + subs += results + if len(subs) >= (data.get("total") or 0) or not results: + break + page += 1 + removed, emails = 0, [] + for s in subs: + email = (s.get("email") or "").lower() + if not email or email in active: + continue + sid = s.get("id") + if email in member_emails: + # Inactive member: take off the list, keep the record. + r = await client.put( + f"{LISTMONK_BASE}/api/subscribers/{sid}", + headers=auth, + json={"lists": []}, + ) + else: + # Not a member at all (deleted/test account): remove it. + r = await client.delete( + f"{LISTMONK_BASE}/api/subscribers/{sid}", + headers=auth, + ) + if r.status_code in (200, 201): + removed += 1 + emails.append(email) + else: + logger.warning( + "Listmonk reconcile: remove %s failed: %s", email, r.status_code + ) + if removed: + logger.info( + "Listmonk reconcile: removed %d stale subscribers: %s", + removed, + ", ".join(emails), + ) + return {"removed": removed, "emails": emails} + except Exception as e: + logger.warning("Listmonk reconcile error: %s", e) + return {"removed": 0, "emails": [], "error": str(e)} + def _verify_portal_secret(request: Request) -> None: """Reject requests that didn't come through LibreChat (shared secret).""" @@ -3258,11 +3346,16 @@ async def reconcile_memberships() -> dict: await sync_loomio_memberships() await sync_matrix_space() + # 4. Reconcile the newsletter list against active membership (removes + # deleted accounts, never-activated test accounts, lapsed members). + newsletter = await listmonk_reconcile() + return { "status": "reconciled", "provisioned": provisioned_count, "deactivated": deactivated_count, "reinvited": reinvited_count, + "newsletter_removed": newsletter.get("removed", 0), } @@ -3281,6 +3374,22 @@ async def admin_sweep(request: Request): return result +@app.post("/admin/newsletter/reconcile") +@limiter.limit("20/minute") +async def admin_newsletter_reconcile(request: Request): + """Remove stale subscribers from the Members newsletter list. + + The newsletter list must mirror ACTIVE members: deleted accounts, + never-activated test accounts, and lapsed members are removed (see + listmonk_reconcile for the exact rules). Also runs automatically as + step 4 of the nightly membership sweep. + + Authenticated by the X-Admin-Token header. + """ + _verify_admin_token(request) + return await listmonk_reconcile() + + def _verify_sync_token(request: Request) -> None: """Authenticate the LibreChat key-sync feed via X-Sync-Token.