unmapped backers: the paid-but-unprovisioned net moves into the portal
ci/woodpecker/push/woodpecker Pipeline was canceled
ci/woodpecker/push/woodpecker Pipeline was canceled
A contributor who pays and never gets an account is invisible to the sweep — fetch_members() drops every member node with no email, and an OC Organization never exposes one — so they wait until they ask (the mmoya case). The net for them was a daily script on the bot's host driving `cloudron exec` from outside. The portal holds both sides of that comparison, so it now owns it. reconcile_unmapped_backers() runs in the sweep and reports three classes, because each needs a different human action: an org account (link it), a VISIBLE email (the sweep should have provisioned this — check the token/sweep), and a hidden email (fetch it out-of-band). A node with no account object is attributed through same-day current orders, and stays silent when every same-day payer is mapped. MAIL POLICY (ntnsndr, 2026-11-13): only when there is a problem — and only when the finding SET changes, fingerprinted in portal_state (slug+class only, never an email), so a stuck finding does not mail three times a day while a fresh one always speaks. A clean run clears the marker. A failed read returns skipped, never a clean bill. POST /admin/unmapped-check (X-Admin-Token, ?notify=0 to stay quiet). The credit-pack reconcile now mails on the same rule. 25 checks; all 19 suites green.
This commit is contained in:
1 parent
d117da3dc6
commit
eaaabf3763
2 files changed
+506
-3
No files matched your search
+264
-3
@@ -7846,6 +7846,7 @@ CREDIT_RECONCILE = (os.environ.get("CREDIT_RECONCILE", "report") or "report").st
|
|||||||
# whole morning, so a log-derived window cannot see the failures it exists for.
|
# whole morning, so a log-derived window cannot see the failures it exists for.
|
||||||
CREDIT_RECONCILE_DAYS = int(os.environ.get("CREDIT_RECONCILE_DAYS", "30") or "30")
|
CREDIT_RECONCILE_DAYS = int(os.environ.get("CREDIT_RECONCILE_DAYS", "30") or "30")
|
||||||
CREDIT_RECONCILE_MAX_PAGES = int(os.environ.get("CREDIT_RECONCILE_MAX_PAGES", "3") or "3")
|
CREDIT_RECONCILE_MAX_PAGES = int(os.environ.get("CREDIT_RECONCILE_MAX_PAGES", "3") or "3")
|
||||||
|
CREDIT_RECONCILE_MARKER = "credit_reconcile_fingerprint"
|
||||||
|
|
||||||
|
|
||||||
async def fetch_collective_orders(per_page: int = 100, max_pages: int = 3) -> list[dict]:
|
async def fetch_collective_orders(per_page: int = 100, max_pages: int = 3) -> list[dict]:
|
||||||
@@ -8006,7 +8007,202 @@ def _nearby_book_rows(slug: str, email: str, when: datetime | None, ledger_rows:
|
|||||||
return out[:cap]
|
return out[:cap]
|
||||||
|
|
||||||
|
|
||||||
async def reconcile_credit_packs(apply: bool = False) -> dict:
|
# --- Unmapped-backer check (was a script on the bot host) ------------------- #
|
||||||
|
# The failure mode (the mmoya case, 2026-10/11): a contributor pays the
|
||||||
|
# collective, but their OC account hides their email or belongs to an
|
||||||
|
# ORGANIZATION, so reconcile_memberships() cannot map them to a portal row and
|
||||||
|
# they are never provisioned — they wait until they ask. The sweep can only
|
||||||
|
# provision what fetch_members() can SEE (it drops nodes with no email); this
|
||||||
|
# reads the RAW member nodes, including the ones the sweep must skip, and
|
||||||
|
# compares them with members.db.
|
||||||
|
#
|
||||||
|
# It used to be a daily script on the bot's host, driving `cloudron exec` from
|
||||||
|
# outside. It belongs here: the portal holds both sides of the comparison, and a
|
||||||
|
# member waiting for an account must not depend on the bot being up.
|
||||||
|
CURRENT_ORDER_STATUS = {"ACTIVE", "PAID"} # an order that is paying right now
|
||||||
|
UNMAPPED_BACKERS_MARKER = "unmapped_backers_fingerprint"
|
||||||
|
|
||||||
|
|
||||||
|
def _oc_memberish(node: dict) -> bool:
|
||||||
|
"""Mirror fetch_members()'s member definition (ADMIN, or Membership/legacy tier)."""
|
||||||
|
role = node.get("role") or ""
|
||||||
|
if role not in ("BACKER", "ADMIN"):
|
||||||
|
return False
|
||||||
|
tier = (((node.get("tier") or {}) or {}).get("name") or "").strip()
|
||||||
|
return role == "ADMIN" or not tier or "membership" in tier.lower()
|
||||||
|
|
||||||
|
|
||||||
|
async def reconcile_unmapped_backers(notify: bool = True) -> dict:
|
||||||
|
"""Paid OC contributors with no portal row. Emails info@ only when there are findings.
|
||||||
|
|
||||||
|
Silent when everything is mapped, and — once a finding has been reported — quiet
|
||||||
|
while the SAME set persists, because this runs three times a day and a stuck
|
||||||
|
finding would otherwise mail every time. The fingerprint changes when a new
|
||||||
|
finding appears or an old one is fixed, and a recurrence after a clean run
|
||||||
|
emails again (the marker is cleared on a clean run).
|
||||||
|
|
||||||
|
A read that fails reports `skipped`, never a clean result: "could not look" is
|
||||||
|
not the same as "nothing wrong".
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
nodes = await fetch_all_member_nodes()
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.warning("unmapped-backer check: member fetch failed; skipping: %s", e)
|
||||||
|
return {"status": "skipped", "reason": "member_fetch_failed", "detail": str(e)[:200]}
|
||||||
|
try:
|
||||||
|
orders = await fetch_collective_orders()
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
# A finding without dues context is still a finding; never lose it over this.
|
||||||
|
logger.warning("unmapped-backer check: order fetch failed (%s); findings omit dues", e)
|
||||||
|
orders = []
|
||||||
|
|
||||||
|
conn = get_db()
|
||||||
|
rows = [(r[0], r[1], r[2]) for r in
|
||||||
|
conn.execute("SELECT email, slug, org_slug FROM members")]
|
||||||
|
conn.close()
|
||||||
|
portal_emails = {str(e).strip().lower() for e, _s, _o in rows if e and "@" in str(e)}
|
||||||
|
portal_slugs = {str(s).strip() for _e, s, _o in rows if s}
|
||||||
|
portal_org_slugs = {str(o).strip() for _e, _s, o in rows if o}
|
||||||
|
|
||||||
|
def mapped(email, slug) -> bool:
|
||||||
|
if email and str(email).strip().lower() in portal_emails:
|
||||||
|
return True
|
||||||
|
return bool(slug) and (slug in portal_slugs or slug in portal_org_slugs)
|
||||||
|
|
||||||
|
def order_line(slug: str) -> str:
|
||||||
|
cands = [o for o in orders
|
||||||
|
if ((o.get("fromAccount") or {}).get("slug") or "") == slug]
|
||||||
|
if not cands:
|
||||||
|
return "no order found in the order list"
|
||||||
|
cands.sort(key=lambda o: str(o.get("createdAt") or ""))
|
||||||
|
ranked = ([o for o in cands if (o.get("status") or "") == "ACTIVE"]
|
||||||
|
or [o for o in cands if (o.get("status") or "") == "PAID"] or cands)
|
||||||
|
o = ranked[-1]
|
||||||
|
return (f"${(o.get('totalAmount') or {}).get('value')} {o.get('frequency')} "
|
||||||
|
f"{o.get('status')} (last order {str(o.get('createdAt') or '')[:10]})")
|
||||||
|
|
||||||
|
findings: list[dict] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
|
||||||
|
# Pass 1: account-bearing nodes (rich findings). Pass 2: null-account nodes,
|
||||||
|
# attributed through same-day orders.
|
||||||
|
for with_account in (True, False):
|
||||||
|
for n in nodes:
|
||||||
|
acct = n.get("account")
|
||||||
|
if with_account != bool(acct):
|
||||||
|
continue
|
||||||
|
if not _oc_memberish(n):
|
||||||
|
continue # e.g. a Credit-pack-only BACKER: the held-credits flow, not a member
|
||||||
|
if acct:
|
||||||
|
slug = (acct.get("slug") or "").strip()
|
||||||
|
email = (acct.get("email") or "").strip()
|
||||||
|
if mapped(email, slug):
|
||||||
|
continue
|
||||||
|
kind = (acct.get("type") or "INDIVIDUAL").upper()
|
||||||
|
name = acct.get("name") or slug or "(no name)"
|
||||||
|
if kind == "ORGANIZATION":
|
||||||
|
cls, vis = "org", "org email never visible (OC design, even for admins)"
|
||||||
|
action = ("org-backed membership — OC never exposes an org email. Link it: "
|
||||||
|
'POST /admin/org-link {"org_slug": "<slug>", "email": '
|
||||||
|
'"<preferred email>", "name": "<name>"} (X-Admin-Token)')
|
||||||
|
elif email:
|
||||||
|
cls, vis = "visible", f"VISIBLE to the admin view ({email})"
|
||||||
|
action = ("the email IS visible, so the sweep should have provisioned this "
|
||||||
|
"within 24h — check the sweep and the OC token "
|
||||||
|
"(POST /admin/provision re-provisions by hand)")
|
||||||
|
else:
|
||||||
|
cls, vis = "hidden", "HIDDEN from the admin view (null email)"
|
||||||
|
action = ("email hidden, so reconcile cannot map it. Get it out-of-band "
|
||||||
|
"(contribution page as collective admin) and provision with "
|
||||||
|
"POST /admin/provision on the operator's instruction")
|
||||||
|
key = slug or f"no-slug:{n.get('role')}"
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
findings.append({
|
||||||
|
"slug": slug, "name": name, "type": kind, "role": n.get("role"),
|
||||||
|
"tier": (((n.get("tier") or {}) or {}).get("name") or "none"),
|
||||||
|
"email_visibility": vis, "dues": order_line(slug), "action": action,
|
||||||
|
"class": cls,
|
||||||
|
"incognito_alias": slug.startswith("incognito-"),
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
# No account object even in the ADMIN view (deleted, or
|
||||||
|
# incognito-hidden). Attribute via same-day current orders; stay
|
||||||
|
# silent when every same-day payer is already mapped.
|
||||||
|
day = str(n.get("createdAt") or "")[:10]
|
||||||
|
current = [o for o in orders
|
||||||
|
if str(o.get("createdAt") or "")[:10] == day
|
||||||
|
and (o.get("status") or "") in CURRENT_ORDER_STATUS]
|
||||||
|
unmapped = sorted({
|
||||||
|
((o.get("fromAccount") or {}).get("slug") or "")
|
||||||
|
for o in current
|
||||||
|
if not mapped(None, ((o.get("fromAccount") or {}).get("slug") or ""))})
|
||||||
|
if current and not unmapped:
|
||||||
|
continue
|
||||||
|
key = f"null-account:{day}"
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
findings.append({
|
||||||
|
"slug": "", "name": "(no account object in the admin view)",
|
||||||
|
"type": "UNKNOWN", "role": n.get("role"), "tier": "none",
|
||||||
|
"email_visibility": "no account object returned",
|
||||||
|
"dues": ", ".join(s for s in unmapped if s) or "no same-day order identified",
|
||||||
|
"action": ("a member node with no account object — attribute it from the "
|
||||||
|
"same-day orders above and provision by hand"),
|
||||||
|
"class": "null-account",
|
||||||
|
"incognito_alias": False,
|
||||||
|
})
|
||||||
|
|
||||||
|
# Fingerprint the SET (class + slug only: never a member's email) so the same
|
||||||
|
# unresolved finding is reported once, and a fixed one is not remembered.
|
||||||
|
fingerprint = "|".join(sorted(f"{f['slug']}:{f['class']}" for f in findings))[:400]
|
||||||
|
conn = get_db()
|
||||||
|
previous = _state_get(conn, UNMAPPED_BACKERS_MARKER, "")
|
||||||
|
if fingerprint != previous:
|
||||||
|
_state_set(conn, UNMAPPED_BACKERS_MARKER, fingerprint)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
if findings:
|
||||||
|
logger.warning("unmapped backers: %d paid contributor(s) with no portal row "
|
||||||
|
"(set %s): %s", len(findings), fingerprint or "-",
|
||||||
|
"; ".join(f["slug"] or f["name"] for f in findings))
|
||||||
|
|
||||||
|
reported = False
|
||||||
|
if findings and fingerprint != previous and notify:
|
||||||
|
lines = []
|
||||||
|
for f in findings:
|
||||||
|
tag = " [incognito alias]" if f["incognito_alias"] else ""
|
||||||
|
lines.append(
|
||||||
|
f"- slug={f['slug'] or '?'}{tag} ({f['name']}) {f['type']} "
|
||||||
|
f"role={f['role']} tier={f['tier']}\n"
|
||||||
|
f" email: {f['email_visibility']}\n"
|
||||||
|
f" dues: {f['dues']}\n"
|
||||||
|
f" action: {f['action']}")
|
||||||
|
try:
|
||||||
|
send_email(
|
||||||
|
"info@inference.coop",
|
||||||
|
f"Inference Cooperative — {len(findings)} paid contributor(s) with no account",
|
||||||
|
"These Open Collective contributors look like members, but no portal row "
|
||||||
|
"matches them, so nothing has provisioned an account:\n\n"
|
||||||
|
+ "\n".join(lines)
|
||||||
|
+ "\n\nThis is sent when the SET changes, not on every sweep. A contributor "
|
||||||
|
"who has just joined may also be caught by the next membership sweep "
|
||||||
|
"(3x/day); if they are still listed after that, the sweep cannot see them "
|
||||||
|
"and someone has to act.\n",
|
||||||
|
)
|
||||||
|
reported = True
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.warning("unmapped-backer notice email failed: %s", e)
|
||||||
|
|
||||||
|
return {"status": "ok", "findings": len(findings), "classes":
|
||||||
|
sorted({f["class"] for f in findings}), "fingerprint": fingerprint,
|
||||||
|
"reported": reported, "detail": findings}
|
||||||
|
|
||||||
|
|
||||||
|
async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict:
|
||||||
"""Credit PAID credit packs that never reached our books (issue #7).
|
"""Credit PAID credit packs that never reached our books (issue #7).
|
||||||
|
|
||||||
`apply=False` (report) is the default and changes NOTHING; granting also
|
`apply=False` (report) is the default and changes NOTHING; granting also
|
||||||
@@ -8131,6 +8327,47 @@ async def reconcile_credit_packs(apply: bool = False) -> dict:
|
|||||||
if c.get("nearby_book_rows") else "")
|
if c.get("nearby_book_rows") else "")
|
||||||
for c in candidates))
|
for c in candidates))
|
||||||
|
|
||||||
|
# Tell a human when money needed attention — and ONLY when the set changes, so
|
||||||
|
# a finding that stays put does not mail three times a day. Silence when there
|
||||||
|
# is nothing to say, which is the normal case.
|
||||||
|
fingerprint = "|".join(sorted(
|
||||||
|
[f"{c['order_id']}:grant" for c in candidates]
|
||||||
|
+ [f"{c['order_id']}:hold" for c in needs_hold]))[:400]
|
||||||
|
conn = get_db()
|
||||||
|
previous = _state_get(conn, CREDIT_RECONCILE_MARKER, "")
|
||||||
|
if fingerprint != previous:
|
||||||
|
_state_set(conn, CREDIT_RECONCILE_MARKER, fingerprint)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
notified = False
|
||||||
|
if fingerprint and fingerprint != previous and notify:
|
||||||
|
granted = "\n".join(
|
||||||
|
f" granted ${c['amount']:.2f} to {c['slug']} for order {c['order_id']} "
|
||||||
|
f"({c['created_at']}) — {c.get('result', 'not applied (report mode)')}"
|
||||||
|
for c in candidates) or " (nothing granted)"
|
||||||
|
held = "\n".join(
|
||||||
|
f" NOT granted: ${c['amount']:.2f} bought by {c['slug']} for order "
|
||||||
|
f"{c['order_id']} ({c['created_at']}) — {c['reason']}" for c in needs_hold)
|
||||||
|
try:
|
||||||
|
send_email(
|
||||||
|
"info@inference.coop",
|
||||||
|
f"Inference Cooperative — paid credit pack(s) missing from the ledger "
|
||||||
|
f"({len(candidates)}/{len(needs_hold)})",
|
||||||
|
"A credit pack that Open Collective recorded as PAID had no matching entry "
|
||||||
|
"in our books. Credits are normally granted by the Open Collective webhook; "
|
||||||
|
"this pass reads Open Collective's own order list every sweep, so a purchase "
|
||||||
|
"whose delivery failed is credited without anyone having to notice.\n\n"
|
||||||
|
+ granted
|
||||||
|
+ (("\n\n" + held + "\n\n(These buyers have no active membership, so they keep "
|
||||||
|
"the hold-and-email path — nothing was granted.)")
|
||||||
|
if needs_hold else "")
|
||||||
|
+ "\n\nThis notice is sent when the set changes, not on every sweep.\n",
|
||||||
|
)
|
||||||
|
notified = True
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.warning("Credit reconcile: notice email failed: %s", e)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
"dry_run": not acting,
|
"dry_run": not acting,
|
||||||
@@ -8141,6 +8378,7 @@ async def reconcile_credit_packs(apply: bool = False) -> dict:
|
|||||||
"candidates": candidates,
|
"candidates": candidates,
|
||||||
"needs_hold": needs_hold,
|
"needs_hold": needs_hold,
|
||||||
"applied": applied,
|
"applied": applied,
|
||||||
|
"notified": notified,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -8351,6 +8589,15 @@ async def admin_sweep(request: Request):
|
|||||||
except Exception as exc: # noqa: BLE001
|
except Exception as exc: # noqa: BLE001
|
||||||
result["credit_reconcile"] = {"error": str(exc)[:200]}
|
result["credit_reconcile"] = {"error": str(exc)[:200]}
|
||||||
|
|
||||||
|
# Paid Open Collective contributors with no portal row — the net for members
|
||||||
|
# the sweep cannot see (hidden email, org account). Emails info@ only when it
|
||||||
|
# finds something, and only when the set changes. Was a daily script on the
|
||||||
|
# bot's host.
|
||||||
|
try:
|
||||||
|
result["unmapped_backers"] = await reconcile_unmapped_backers()
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
result["unmapped_backers"] = {"error": str(exc)[:200]}
|
||||||
|
|
||||||
# Deliveries the portal refused. The portal records these itself (see
|
# Deliveries the portal refused. The portal records these itself (see
|
||||||
# record_webhook_failure) and reports them here ONCE per batch, so the alert
|
# record_webhook_failure) and reports them here ONCE per batch, so the alert
|
||||||
# for a failed delivery belongs to the system rather than to a log scraper on
|
# for a failed delivery belongs to the system rather than to a log scraper on
|
||||||
@@ -8365,7 +8612,7 @@ async def admin_sweep(request: Request):
|
|||||||
|
|
||||||
@app.post("/admin/credit-pack-reconcile")
|
@app.post("/admin/credit-pack-reconcile")
|
||||||
@limiter.limit("20/minute")
|
@limiter.limit("20/minute")
|
||||||
async def admin_credit_pack_reconcile(request: Request, apply: int = 0):
|
async def admin_credit_pack_reconcile(request: Request, apply: int = 0, notify: int = 1):
|
||||||
"""Reconcile PAID credit packs against our books — DRY RUN by default.
|
"""Reconcile PAID credit packs against our books — DRY RUN by default.
|
||||||
|
|
||||||
Authenticated by the X-Admin-Token header, like the other /admin/* routes.
|
Authenticated by the X-Admin-Token header, like the other /admin/* routes.
|
||||||
@@ -8374,9 +8621,23 @@ async def admin_credit_pack_reconcile(request: Request, apply: int = 0):
|
|||||||
this pass's output in production before any member's money is touched. Every
|
this pass's output in production before any member's money is touched. Every
|
||||||
grant uses the same per-order reference the webhook writes, so running it
|
grant uses the same per-order reference the webhook writes, so running it
|
||||||
twice — or running it after a late delivery — credits a purchase once.
|
twice — or running it after a late delivery — credits a purchase once.
|
||||||
|
|
||||||
|
`?notify=0` suppresses the info@ notice (for a check that should not mail).
|
||||||
"""
|
"""
|
||||||
_verify_admin_token(request)
|
_verify_admin_token(request)
|
||||||
return await reconcile_credit_packs(apply=bool(apply))
|
return await reconcile_credit_packs(apply=bool(apply), notify=bool(notify))
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/admin/unmapped-check")
|
||||||
|
@limiter.limit("20/minute")
|
||||||
|
async def admin_unmapped_check(request: Request, notify: int = 1):
|
||||||
|
"""Run the paid-but-unprovisioned check on demand (X-Admin-Token).
|
||||||
|
|
||||||
|
Emits info@ only when there are findings AND the set has changed, so this is
|
||||||
|
safe to call repeatedly; `?notify=0` suppresses the notice entirely.
|
||||||
|
"""
|
||||||
|
_verify_admin_token(request)
|
||||||
|
return await reconcile_unmapped_backers(notify=bool(notify))
|
||||||
|
|
||||||
|
|
||||||
def _service_sync_pending() -> list[str]:
|
def _service_sync_pending() -> list[str]:
|
||||||
|
|||||||
@@ -0,0 +1,242 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Regression test: paid OC contributors with no portal row (the unmapped-backer net).
|
||||||
|
|
||||||
|
WHY THIS EXISTS
|
||||||
|
A contributor pays the collective and never gets an account, because our sweep
|
||||||
|
cannot SEE them: fetch_members() drops every member node with no email, and an
|
||||||
|
OC Organization never exposes one at all. The mmoya case (2026-10/11) was found
|
||||||
|
weeks late, by the member asking. The net ran as a script on the bot's host,
|
||||||
|
driving `cloudron exec` from outside; it now lives in the portal, where both
|
||||||
|
sides of the comparison already are.
|
||||||
|
|
||||||
|
WHAT IS PINNED DOWN HERE
|
||||||
|
* the member definition matches fetch_members()'s (ADMIN, or Membership/legacy
|
||||||
|
tier — a Credit-pack-only BACKER is the held-credits flow, not a member),
|
||||||
|
* the three classes of finding are told apart (org / visible email / hidden
|
||||||
|
email) because each needs a DIFFERENT action from a human,
|
||||||
|
* a member node with no account object is attributed through same-day current
|
||||||
|
orders, and stays silent when every same-day payer is already mapped,
|
||||||
|
* info@ is told ONLY when there are findings, and only when the SET changes —
|
||||||
|
it runs three times a day, and a stuck finding must not mail every time,
|
||||||
|
* a fetch failure reports `skipped`, never a clean bill of health.
|
||||||
|
|
||||||
|
Runnable: python3 tests/test_unmapped_backers.py (from the repo root)
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import asyncio
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
import app.main as m # noqa: E402
|
||||||
|
|
||||||
|
m.DB_PATH = os.environ["DB_PATH"]
|
||||||
|
m.ADMIN_TOKEN = "test-admin-token"
|
||||||
|
m.ADMIN_TOKEN_LEGACY = ""
|
||||||
|
|
||||||
|
SENT = []
|
||||||
|
|
||||||
|
|
||||||
|
def fake_send(to, subject, body, *a, **k):
|
||||||
|
SENT.append({"to": to, "subject": subject, "body": body})
|
||||||
|
|
||||||
|
|
||||||
|
FAIL_NODES = False
|
||||||
|
NODES: list = []
|
||||||
|
ORDERS: list = []
|
||||||
|
|
||||||
|
|
||||||
|
async def fake_nodes():
|
||||||
|
if FAIL_NODES:
|
||||||
|
raise RuntimeError("OC member fetch blew up")
|
||||||
|
return list(NODES)
|
||||||
|
|
||||||
|
|
||||||
|
async def fake_orders(per_page=100, max_pages=3):
|
||||||
|
return list(ORDERS)
|
||||||
|
|
||||||
|
|
||||||
|
def node(role, tier, slug=None, email=None, atype="INDIVIDUAL", name=None,
|
||||||
|
created="2026-11-01T00:00:00Z", account=True):
|
||||||
|
return {"role": role, "tier": ({"name": tier} if tier else None),
|
||||||
|
"createdAt": created,
|
||||||
|
"account": ({"name": name or (slug or "?"), "slug": slug, "type": atype,
|
||||||
|
"email": email} if account else None)}
|
||||||
|
|
||||||
|
|
||||||
|
def order(slug, status="ACTIVE", day="2026-11-01", amount=15):
|
||||||
|
return {"id": f"or_{slug}_{day}", "status": status, "createdAt": f"{day}T10:00:00Z",
|
||||||
|
"frequency": "MONTHLY", "amount": {"value": amount},
|
||||||
|
"totalAmount": {"value": amount}, "fromAccount": {"slug": slug, "name": slug}}
|
||||||
|
|
||||||
|
|
||||||
|
results = []
|
||||||
|
|
||||||
|
|
||||||
|
def check(name, ok):
|
||||||
|
results.append((name, bool(ok)))
|
||||||
|
|
||||||
|
|
||||||
|
def seed():
|
||||||
|
conn = m.get_db()
|
||||||
|
conn.execute("DELETE FROM members")
|
||||||
|
conn.execute("DELETE FROM portal_state")
|
||||||
|
for email, slug, org in (("alice@x.org", "alice", None),
|
||||||
|
("admin@x.org", "adminco", None),
|
||||||
|
("contact@beta.example", "beta-contact", "betaorg")):
|
||||||
|
conn.execute("INSERT INTO members (email, slug, org_slug, active, account_type, "
|
||||||
|
"credit_balance) VALUES (?, ?, ?, 1, 'member', 0)", (email, slug, org))
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
global FAIL_NODES
|
||||||
|
m.fetch_all_member_nodes = fake_nodes
|
||||||
|
m.fetch_collective_orders = fake_orders
|
||||||
|
m.send_email = fake_send
|
||||||
|
|
||||||
|
seed()
|
||||||
|
NODES[:] = [
|
||||||
|
# mapped by email -> not a finding
|
||||||
|
node("BACKER", "Membership", "alice", "alice@x.org", name="Alice"),
|
||||||
|
# mapped ADMIN -> not a finding
|
||||||
|
node("ADMIN", "Membership", "adminco", "admin@x.org", name="Co-op"),
|
||||||
|
# NOT a member: a credit-pack-only BACKER takes the held-credits path
|
||||||
|
node("BACKER", "Credit pack", "packbuyer", "pack@x.org", name="Pack Buyer"),
|
||||||
|
# not a member role
|
||||||
|
node("MEMBER", "Membership", "roleonly", "role@x.org", name="Role Only"),
|
||||||
|
# FINDING: visible email -> the sweep should have provisioned this
|
||||||
|
node("BACKER", "Membership", "vic", "vic@x.org", name="Visible Vic"),
|
||||||
|
# FINDING: email hidden even from the admin view
|
||||||
|
node("BACKER", "Membership", "han", None, name="Hidden Han"),
|
||||||
|
# a MAPPED Organization (its org_slug is on a portal row) -> not a finding
|
||||||
|
node("BACKER", "Membership", "betaorg", None, atype="ORGANIZATION", name="Beta Org"),
|
||||||
|
# FINDING: an Organization OC never exposes an email for
|
||||||
|
node("BACKER", "Membership", "gammaorg", None, atype="ORGANIZATION", name="Gamma Org"),
|
||||||
|
# null-account node on a day whose only payer is MAPPED -> silent
|
||||||
|
node("BACKER", "Membership", account=False, created="2026-11-01T00:00:00Z"),
|
||||||
|
# null-account node on a day with an UNMAPPED payer -> finding
|
||||||
|
node("BACKER", "Membership", account=False, created="2026-11-02T00:00:00Z"),
|
||||||
|
]
|
||||||
|
ORDERS[:] = [
|
||||||
|
order("alice", day="2026-11-01"),
|
||||||
|
order("ghost", day="2026-11-02"),
|
||||||
|
order("vic", day="2026-11-03", amount=25),
|
||||||
|
]
|
||||||
|
|
||||||
|
SENT.clear()
|
||||||
|
res = await m.reconcile_unmapped_backers()
|
||||||
|
check(f"the check runs (got {res.get('status')})", res.get("status") == "ok")
|
||||||
|
classes = sorted(res.get("classes") or [])
|
||||||
|
check(f"four findings in three classes (got {res.get('findings')} {classes})",
|
||||||
|
res.get("findings") == 4 and classes == ["hidden", "null-account", "org", "visible"])
|
||||||
|
slugs = sorted(f["slug"] or "(null-account)" for f in res["detail"])
|
||||||
|
check(f"exactly the unmapped member-ish nodes are findings (got {slugs})",
|
||||||
|
slugs == ["(null-account)", "gammaorg", "han", "vic"])
|
||||||
|
check("a Credit-pack-only BACKER is not a member finding",
|
||||||
|
not any(f["slug"] == "packbuyer" for f in res["detail"]))
|
||||||
|
check("a MEMBER-role node is not a member finding",
|
||||||
|
not any(f["slug"] == "roleonly" for f in res["detail"]))
|
||||||
|
vis = {f["slug"]: f["email_visibility"] for f in res["detail"]}
|
||||||
|
check("the VISIBLE case tells the operator the sweep should have caught it",
|
||||||
|
"VISIBLE" in vis.get("vic", "") and "sweep" in
|
||||||
|
next(f["action"] for f in res["detail"] if f["slug"] == "vic"))
|
||||||
|
check("the ORG case points at /admin/org-link",
|
||||||
|
"/admin/org-link" in next(f["action"] for f in res["detail"] if f["slug"] == "gammaorg"))
|
||||||
|
check("the HIDDEN case says the email must be fetched out-of-band",
|
||||||
|
"out-of-band" in next(f["action"] for f in res["detail"] if f["slug"] == "han"))
|
||||||
|
check(f"findings carry the payer's dues (got {next(f['dues'] for f in res['detail'] if f['slug'] == 'vic')})",
|
||||||
|
"$25 MONTHLY ACTIVE" in next(f["dues"] for f in res["detail"] if f["slug"] == "vic"))
|
||||||
|
check(f"the null-account finding names the unmapped same-day payer (got "
|
||||||
|
f"{next(f['dues'] for f in res['detail'] if f['class'] == 'null-account')})",
|
||||||
|
"ghost" in next(f["dues"] for f in res["detail"] if f["class"] == "null-account"))
|
||||||
|
check(f"info@ is told once, with every class in the body (got {len(SENT)})",
|
||||||
|
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
|
||||||
|
and "org-link" in SENT[0]["body"] and "out-of-band" in SENT[0]["body"])
|
||||||
|
check("it says it is sent on change, not every sweep",
|
||||||
|
"not on every sweep" in SENT[0]["body"])
|
||||||
|
|
||||||
|
# Same set again: a report, but NO second email.
|
||||||
|
res2 = await m.reconcile_unmapped_backers()
|
||||||
|
check(f"the same finding set is not re-emailed (got {len(SENT)} emails)",
|
||||||
|
res2.get("findings") == 4 and res2.get("reported") is False and len(SENT) == 1)
|
||||||
|
|
||||||
|
# The set CHANGES (one finding resolved): mail again.
|
||||||
|
NODES[:] = [n for n in NODES
|
||||||
|
if ((n.get("account") or {}) or {}).get("slug") != "han"]
|
||||||
|
res3 = await m.reconcile_unmapped_backers()
|
||||||
|
check(f"a CHANGED set is reported again (got {res3.get('findings')} findings, {len(SENT)} emails)",
|
||||||
|
res3.get("findings") == 3 and len(SENT) == 2)
|
||||||
|
|
||||||
|
# Fully clear: silent, and the marker is cleared so a recurrence speaks up.
|
||||||
|
NODES[:] = [node("BACKER", "Membership", "alice", "alice@x.org", name="Alice")]
|
||||||
|
res4 = await m.reconcile_unmapped_backers()
|
||||||
|
check(f"a clean run says nothing (got {res4.get('findings')}, {len(SENT)} emails)",
|
||||||
|
res4.get("findings") == 0 and res4.get("reported") is False and len(SENT) == 2)
|
||||||
|
conn = m.get_db()
|
||||||
|
marker = conn.execute("SELECT value FROM portal_state WHERE key = ?",
|
||||||
|
(m.UNMAPPED_BACKERS_MARKER,)).fetchone()
|
||||||
|
conn.close()
|
||||||
|
check(f"the marker is cleared on a clean run (got {marker})", not (marker and marker[0]))
|
||||||
|
|
||||||
|
NODES[:] = [node("BACKER", "Membership", "vic", "vic@x.org", name="Visible Vic")]
|
||||||
|
res5 = await m.reconcile_unmapped_backers()
|
||||||
|
check(f"a finding after a clean run speaks up again (got {len(SENT)} emails)",
|
||||||
|
res5.get("findings") == 1 and len(SENT) == 3)
|
||||||
|
|
||||||
|
# Notify can be suppressed (on-demand checks must not mail).
|
||||||
|
NODES[:] = [node("BACKER", "Membership", "newguy", "new@x.org", name="New Guy")]
|
||||||
|
res6 = await m.reconcile_unmapped_backers(notify=False)
|
||||||
|
check(f"notify=False suppresses the mail (got {res6.get('findings')} findings, {len(SENT)} emails)",
|
||||||
|
res6.get("findings") == 1 and res6.get("reported") is False and len(SENT) == 3)
|
||||||
|
|
||||||
|
# A failed read is UNKNOWN, not silence.
|
||||||
|
FAIL_NODES = True
|
||||||
|
res7 = await m.reconcile_unmapped_backers()
|
||||||
|
check(f"a failed member fetch is skipped, not clean (got {res7})",
|
||||||
|
res7.get("status") == "skipped" and res7.get("reason") == "member_fetch_failed")
|
||||||
|
FAIL_NODES = False
|
||||||
|
|
||||||
|
# An order-fetch failure must not lose the finding.
|
||||||
|
async def boom_orders(per_page=100, max_pages=3):
|
||||||
|
raise RuntimeError("orders unavailable")
|
||||||
|
|
||||||
|
m.fetch_collective_orders = boom_orders
|
||||||
|
res8 = await m.reconcile_unmapped_backers()
|
||||||
|
check(f"a failed ORDER fetch still reports the finding (got {res8.get('findings')})",
|
||||||
|
res8.get("findings") == 1)
|
||||||
|
check("...and says so in the dues line",
|
||||||
|
"no order found" in res8["detail"][0]["dues"])
|
||||||
|
m.fetch_collective_orders = fake_orders
|
||||||
|
|
||||||
|
# --- the endpoint ---
|
||||||
|
# A fresh finding set, so the endpoint run has something NEW to report
|
||||||
|
# (the marker is per-set: the same set would rightly stay quiet).
|
||||||
|
NODES[:] = [node("BACKER", "Membership", "endpointcase", "ep@x.org", name="EP")]
|
||||||
|
SENT.clear()
|
||||||
|
transport = httpx.ASGITransport(app=m.app)
|
||||||
|
async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:
|
||||||
|
r = await c.post("/admin/unmapped-check")
|
||||||
|
check(f"endpoint: needs the admin token (got {r.status_code})", r.status_code in (401, 403))
|
||||||
|
r = await c.post("/admin/unmapped-check", headers={"X-Admin-Token": "test-admin-token"})
|
||||||
|
body = r.json()
|
||||||
|
check(f"endpoint: runs the check (got {body.get('findings')} findings)",
|
||||||
|
r.status_code == 200 and body.get("findings") == 1)
|
||||||
|
check(f"endpoint: the run mails once for the new set (got {len(SENT)})", len(SENT) == 1)
|
||||||
|
r = await c.post("/admin/unmapped-check?notify=0",
|
||||||
|
headers={"X-Admin-Token": "test-admin-token"})
|
||||||
|
check(f"endpoint: ?notify=0 suppresses the mail (got {len(SENT)} emails)",
|
||||||
|
r.status_code == 200 and len(SENT) == 1)
|
||||||
|
|
||||||
|
|
||||||
|
asyncio.run(main())
|
||||||
|
|
||||||
|
fails = sum(not ok for _, ok in results)
|
||||||
|
for name, ok in results:
|
||||||
|
print(f" [{'PASS' if ok else 'FAIL'}] {name}")
|
||||||
|
print(f"\n{len(results) - fails}/{len(results)} passed")
|
||||||
|
sys.exit(1 if fails else 0)
|
||||||
Reference in new issue
Block a user