newsletter reconcile: Members list mirrors ACTIVE members

Root cause of test accounts receiving the newsletter: LISTMONK_API_USER was
set EMPTY in the portal env - env-present beats the code default, so every
listmonk_sync authenticated with an empty user and 403'd silently. Fixed both
the env (set to portal-sync) and the code (empty env falls back to default).

New: listmonk_reconcile() - the Members list must mirror ACTIVE members.
Stale subscribers are removed: members marked inactive are taken off the list
(record kept as suppression marker); addresses that are not members at all
(test/deleted accounts) are deleted outright. Wired as step 4 of the nightly
membership sweep + POST /admin/newsletter/reconcile for on-demand runs.

app/main.py

@@ -234,8 +234,12 @@ PORTAL_BASE = os.environ.get("PORTAL_BASE", "https://portal.inference.coop")
 # Listmonk newsletter integration. The portal keeps the "Members" list in sync:
 # subscribe on provision, unsubscribe on deactivation. Auth uses listmonk's
 # native API-user token scheme (Authorization: token <user>:<token>).
-LISTMONK_BASE = os.environ.get("LISTMONK_BASE", "https://newsletter.inference.coop")
+LISTMONK_BASE = os.environ.get("LISTMONK_BASE") or "https://newsletter.inference.coop"
-LISTMONK_API_USER = os.environ.get("LISTMONK_API_USER", "portal-sync")
+# NOTE: `or` (not default-arg) — an env var set to EMPTY must fall back to the
+# default. An empty LISTMONK_API_USER made every sync 403 silently (the token
+# alone is not a valid listmonk credential) while listmonk_sync's best-effort
+# error handling swallowed it — that is how stale accounts stayed subscribed.
+LISTMONK_API_USER = os.environ.get("LISTMONK_API_USER") or "portal-sync"
 LISTMONK_API_TOKEN = os.environ.get("LISTMONK_API_TOKEN", "")
 LISTMONK_LIST_ID = int(os.environ.get("LISTMONK_LIST_ID", "3"))  # "Members" list
 
@@ -410,6 +414,90 @@ async def listmonk_sync(email: str, subscribe: bool) -> None:
         logger.warning("Listmonk sync error for %s: %s", email, e)
 
 
+async def listmonk_reconcile() -> dict:
+    """Reconcile the Members newsletter list against the active membership.
+
+    The newsletter list must mirror the portal's ACTIVE members. Subscribers
+    who are not active members are stale (deleted accounts, never-activated
+    test accounts, lapsed members) and are removed:
+      - emails that exist in the members table but are inactive are taken off
+        the list (the subscriber record stays as a suppression marker);
+      - emails that are not members at all are deleted outright (stray or
+        test accounts should not persist in the mailing system).
+
+    Best-effort: never raises, so a newsletter outage can't break the sweep.
+    """
+    if not LISTMONK_API_TOKEN:
+        logger.warning("Listmonk not configured; skipping newsletter reconcile")
+        return {"removed": 0, "emails": [], "error": "not configured"}
+    auth = {"Authorization": f"token {LISTMONK_API_USER}:{LISTMONK_API_TOKEN}"}
+    conn = get_db()
+    active = {
+        row[0].lower()
+        for row in conn.execute("SELECT email FROM members WHERE active = 1")
+    }
+    member_emails = {
+        row[0].lower() for row in conn.execute("SELECT email FROM members")
+    }
+    conn.close()
+    try:
+        async with httpx.AsyncClient(timeout=20.0) as client:
+            # Fetch the full subscriber set for the list (paginated).
+            subs, page = [], 1
+            while True:
+                r = await client.get(
+                    f"{LISTMONK_BASE}/api/subscribers",
+                    headers=auth,
+                    params={"list_id": LISTMONK_LIST_ID, "page": page, "per_page": 100},
+                )
+                if r.status_code != 200:
+                    logger.warning(
+                        "Listmonk reconcile: subscriber fetch failed: %s", r.status_code
+                    )
+                    return {"removed": 0, "emails": [], "error": f"fetch {r.status_code}"}
+                data = r.json().get("data", {})
+                results = data.get("results", [])
+                subs += results
+                if len(subs) >= (data.get("total") or 0) or not results:
+                    break
+                page += 1
+            removed, emails = 0, []
+            for s in subs:
+                email = (s.get("email") or "").lower()
+                if not email or email in active:
+                    continue
+                sid = s.get("id")
+                if email in member_emails:
+                    # Inactive member: take off the list, keep the record.
+                    r = await client.put(
+                        f"{LISTMONK_BASE}/api/subscribers/{sid}",
+                        headers=auth,
+                        json={"lists": []},
+                    )
+                else:
+                    # Not a member at all (deleted/test account): remove it.
+                    r = await client.delete(
+                        f"{LISTMONK_BASE}/api/subscribers/{sid}",
+                        headers=auth,
+                    )
+                if r.status_code in (200, 201):
+                    removed += 1
+                    emails.append(email)
+                else:
+                    logger.warning(
+                        "Listmonk reconcile: remove %s failed: %s", email, r.status_code
+                    )
+            if removed:
+                logger.info(
+                    "Listmonk reconcile: removed %d stale subscribers: %s",
+                    removed,
+                    ", ".join(emails),
+                )
+            return {"removed": removed, "emails": emails}
+    except Exception as e:
+        logger.warning("Listmonk reconcile error: %s", e)
+        return {"removed": 0, "emails": [], "error": str(e)}
+
 
 def _verify_portal_secret(request: Request) -> None:
     """Reject requests that didn't come through LibreChat (shared secret)."""
@@ -3258,11 +3346,16 @@ async def reconcile_memberships() -> dict:
     await sync_loomio_memberships()
     await sync_matrix_space()
 
+    # 4. Reconcile the newsletter list against active membership (removes
+    #    deleted accounts, never-activated test accounts, lapsed members).
+    newsletter = await listmonk_reconcile()
+
     return {
         "status": "reconciled",
         "provisioned": provisioned_count,
         "deactivated": deactivated_count,
         "reinvited": reinvited_count,
+        "newsletter_removed": newsletter.get("removed", 0),
     }
 
 
@@ -3281,6 +3374,22 @@ async def admin_sweep(request: Request):
     return result
 
 
+@app.post("/admin/newsletter/reconcile")
+@limiter.limit("20/minute")
+async def admin_newsletter_reconcile(request: Request):
+    """Remove stale subscribers from the Members newsletter list.
+
+    The newsletter list must mirror ACTIVE members: deleted accounts,
+    never-activated test accounts, and lapsed members are removed (see
+    listmonk_reconcile for the exact rules). Also runs automatically as
+    step 4 of the nightly membership sweep.
+
+    Authenticated by the X-Admin-Token header.
+    """
+    _verify_admin_token(request)
+    return await listmonk_reconcile()
+
+
 def _verify_sync_token(request: Request) -> None:
     """Authenticate the LibreChat key-sync feed via X-Sync-Token.