Files
member-portal/tests/test_unmapped_backers.py
T
inference-admin bf1e5f013b
ci/woodpecker/push/woodpecker Pipeline was canceled
unmapped backers: stop the phantom finding (the members query had no createdAt)
The first production run reported a null-account node as an uncredited payer. The
cause was in my port, not the data: the members query selected id-less,
date-less nodes, so every null-account node fell into the date "" bucket, matched
no same-day order, and was reported as unattributable — while the host-side script
(correctly) said clean.

Two fixes: select id + createdAt in fetch_all_member_nodes, and never emit a
null-account finding when the node carries no date — there is nothing to attribute
it against, and guessing is what produced the phantom. 27 checks in the suite.
2026-11-13 20:35:28 +00:00

258 lines
13 KiB
Python

#!/usr/bin/env python3
"""Regression test: paid OC contributors with no portal row (the unmapped-backer net).
WHY THIS EXISTS
A contributor pays the collective and never gets an account, because our sweep
cannot SEE them: fetch_members() drops every member node with no email, and an
OC Organization never exposes one at all. The mmoya case (2026-10/11) was found
weeks late, by the member asking. The net ran as a script on the bot's host,
driving `cloudron exec` from outside; it now lives in the portal, where both
sides of the comparison already are.
WHAT IS PINNED DOWN HERE
* the member definition matches fetch_members()'s (ADMIN, or Membership/legacy
tier — a Credit-pack-only BACKER is the held-credits flow, not a member),
* the three classes of finding are told apart (org / visible email / hidden
email) because each needs a DIFFERENT action from a human,
* a member node with no account object is attributed through same-day current
orders, and stays silent when every same-day payer is already mapped,
* info@ is told ONLY when there are findings, and only when the SET changes —
it runs three times a day, and a stuck finding must not mail every time,
* a fetch failure reports `skipped`, never a clean bill of health.
Runnable: python3 tests/test_unmapped_backers.py (from the repo root)
"""
import os
import sys
import asyncio
import tempfile
import httpx
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import app.main as m # noqa: E402
m.DB_PATH = os.environ["DB_PATH"]
m.ADMIN_TOKEN = "test-admin-token"
m.ADMIN_TOKEN_LEGACY = ""
SENT = []
def fake_send(to, subject, body, *a, **k):
SENT.append({"to": to, "subject": subject, "body": body})
FAIL_NODES = False
NODES: list = []
ORDERS: list = []
async def fake_nodes():
if FAIL_NODES:
raise RuntimeError("OC member fetch blew up")
return list(NODES)
async def fake_orders(per_page=100, max_pages=3):
return list(ORDERS)
def node(role, tier, slug=None, email=None, atype="INDIVIDUAL", name=None,
created="2026-11-01T00:00:00Z", account=True):
return {"role": role, "tier": ({"name": tier} if tier else None),
"createdAt": created,
"account": ({"name": name or (slug or "?"), "slug": slug, "type": atype,
"email": email} if account else None)}
def order(slug, status="ACTIVE", day="2026-11-01", amount=15):
return {"id": f"or_{slug}_{day}", "status": status, "createdAt": f"{day}T10:00:00Z",
"frequency": "MONTHLY", "amount": {"value": amount},
"totalAmount": {"value": amount}, "fromAccount": {"slug": slug, "name": slug}}
results = []
def check(name, ok):
results.append((name, bool(ok)))
def seed():
conn = m.get_db()
conn.execute("DELETE FROM members")
conn.execute("DELETE FROM portal_state")
for email, slug, org in (("alice@x.org", "alice", None),
("admin@x.org", "adminco", None),
("contact@beta.example", "beta-contact", "betaorg")):
conn.execute("INSERT INTO members (email, slug, org_slug, active, account_type, "
"credit_balance) VALUES (?, ?, ?, 1, 'member', 0)", (email, slug, org))
conn.commit()
conn.close()
async def main():
global FAIL_NODES
m.fetch_all_member_nodes = fake_nodes
m.fetch_collective_orders = fake_orders
m.send_email = fake_send
seed()
NODES[:] = [
# mapped by email -> not a finding
node("BACKER", "Membership", "alice", "alice@x.org", name="Alice"),
# mapped ADMIN -> not a finding
node("ADMIN", "Membership", "adminco", "admin@x.org", name="Co-op"),
# NOT a member: a credit-pack-only BACKER takes the held-credits path
node("BACKER", "Credit pack", "packbuyer", "pack@x.org", name="Pack Buyer"),
# not a member role
node("MEMBER", "Membership", "roleonly", "role@x.org", name="Role Only"),
# FINDING: visible email -> the sweep should have provisioned this
node("BACKER", "Membership", "vic", "vic@x.org", name="Visible Vic"),
# FINDING: email hidden even from the admin view
node("BACKER", "Membership", "han", None, name="Hidden Han"),
# a MAPPED Organization (its org_slug is on a portal row) -> not a finding
node("BACKER", "Membership", "betaorg", None, atype="ORGANIZATION", name="Beta Org"),
# FINDING: an Organization OC never exposes an email for
node("BACKER", "Membership", "gammaorg", None, atype="ORGANIZATION", name="Gamma Org"),
# null-account node on a day whose only payer is MAPPED -> silent
node("BACKER", "Membership", account=False, created="2026-11-01T00:00:00Z"),
# null-account node on a day with an UNMAPPED payer -> finding
node("BACKER", "Membership", account=False, created="2026-11-02T00:00:00Z"),
# null-account node with NO date -> NOT a finding: there is nothing to
# attribute it against, and reporting it is how a phantom appears (the
# members query did not select createdAt, so every such node looked like an
# uncredited payer on the day "").
{"role": "BACKER", "tier": {"name": "Membership"}, "createdAt": None, "account": None},
]
ORDERS[:] = [
order("alice", day="2026-11-01"),
order("ghost", day="2026-11-02"),
order("vic", day="2026-11-03", amount=25),
]
SENT.clear()
res = await m.reconcile_unmapped_backers()
check(f"the check runs (got {res.get('status')})", res.get("status") == "ok")
classes = sorted(res.get("classes") or [])
check(f"four findings in three classes (got {res.get('findings')} {classes})",
res.get("findings") == 4 and classes == ["hidden", "null-account", "org", "visible"])
check("a node with no createdAt is NOT reported (nothing to attribute it against)",
sum(1 for f in res["detail"] if f["class"] == "null-account") == 1)
slugs = sorted(f["slug"] or "(null-account)" for f in res["detail"])
check(f"exactly the unmapped member-ish nodes are findings (got {slugs})",
slugs == ["(null-account)", "gammaorg", "han", "vic"])
check("a Credit-pack-only BACKER is not a member finding",
not any(f["slug"] == "packbuyer" for f in res["detail"]))
check("a MEMBER-role node is not a member finding",
not any(f["slug"] == "roleonly" for f in res["detail"]))
vis = {f["slug"]: f["email_visibility"] for f in res["detail"]}
check("the VISIBLE case tells the operator the sweep should have caught it",
"VISIBLE" in vis.get("vic", "") and "sweep" in
next(f["action"] for f in res["detail"] if f["slug"] == "vic"))
check("the ORG case points at /admin/org-link",
"/admin/org-link" in next(f["action"] for f in res["detail"] if f["slug"] == "gammaorg"))
check("the HIDDEN case says the email must be fetched out-of-band",
"out-of-band" in next(f["action"] for f in res["detail"] if f["slug"] == "han"))
check(f"findings carry the payer's dues (got {next(f['dues'] for f in res['detail'] if f['slug'] == 'vic')})",
"$25 MONTHLY ACTIVE" in next(f["dues"] for f in res["detail"] if f["slug"] == "vic"))
check(f"the null-account finding names the unmapped same-day payer (got "
f"{next(f['dues'] for f in res['detail'] if f['class'] == 'null-account')})",
"ghost" in next(f["dues"] for f in res["detail"] if f["class"] == "null-account"))
check(f"info@ is told once, with every class in the body (got {len(SENT)})",
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
and "org-link" in SENT[0]["body"] and "out-of-band" in SENT[0]["body"])
check("it says the notice covers what is new, not every sweep",
"not every sweep" in SENT[0]["body"])
# Same set again: a report, but NO second email.
res2 = await m.reconcile_unmapped_backers()
check(f"the same finding set is not re-emailed (got {len(SENT)} emails)",
res2.get("findings") == 4 and res2.get("reported") is False and len(SENT) == 1)
# The set CHANGES (one finding resolved): mail again.
NODES[:] = [n for n in NODES
if ((n.get("account") or {}) or {}).get("slug") != "han"]
res3 = await m.reconcile_unmapped_backers()
# A set that SHRINKS (one finding resolved) must NOT re-mail the survivors: a
# person already told about a finding is not told again just because the list
# got shorter. Only genuinely NEW keys speak.
check(f"a smaller set re-mails nobody (got {res3.get('findings')} findings, "
f"{res3.get('new')} new, {len(SENT)} emails)",
res3.get("findings") == 3 and res3.get("new") == 0 and len(SENT) == 1)
# Fully clear: silent, and the marker is cleared so a recurrence speaks up.
NODES[:] = [node("BACKER", "Membership", "alice", "alice@x.org", name="Alice")]
res4 = await m.reconcile_unmapped_backers()
check(f"a clean run says nothing (got {res4.get('findings')}, {len(SENT)} emails)",
res4.get("findings") == 0 and res4.get("reported") is False and len(SENT) == 1)
conn = m.get_db()
marker = conn.execute("SELECT value FROM portal_state WHERE key = ?",
(m.UNMAPPED_BACKERS_MARKER,)).fetchone()
conn.close()
check(f"the marker is cleared on a clean run (got {marker})", not (marker and marker[0]))
NODES[:] = [node("BACKER", "Membership", "vic", "vic@x.org", name="Visible Vic")]
res5 = await m.reconcile_unmapped_backers()
check(f"a finding after a clean run speaks up again (got {len(SENT)} emails)",
res5.get("findings") == 1 and res5.get("new") == 1 and len(SENT) == 2)
# Notify can be suppressed (on-demand checks must not mail).
NODES[:] = [node("BACKER", "Membership", "newguy", "new@x.org", name="New Guy")]
res6 = await m.reconcile_unmapped_backers(notify=False)
check(f"notify=False suppresses the mail (got {res6.get('findings')} findings, {len(SENT)} emails)",
res6.get("findings") == 1 and res6.get("reported") is False and len(SENT) == 2)
# ...and a look must not CONSUME the alert: the next notifying run still speaks.
res6b = await m.reconcile_unmapped_backers()
check(f"a silent look does not swallow the alert (got {res6b.get('new')} new, {len(SENT)} emails)",
res6b.get("reported") is True and len(SENT) == 3)
# A failed read is UNKNOWN, not silence.
FAIL_NODES = True
res7 = await m.reconcile_unmapped_backers()
check(f"a failed member fetch is skipped, not clean (got {res7})",
res7.get("status") == "skipped" and res7.get("reason") == "member_fetch_failed")
FAIL_NODES = False
# An order-fetch failure must not lose the finding.
async def boom_orders(per_page=100, max_pages=3):
raise RuntimeError("orders unavailable")
m.fetch_collective_orders = boom_orders
res8 = await m.reconcile_unmapped_backers()
check(f"a failed ORDER fetch still reports the finding (got {res8.get('findings')})",
res8.get("findings") == 1)
check("...and says so in the dues line",
"no order found" in res8["detail"][0]["dues"])
m.fetch_collective_orders = fake_orders
# --- the endpoint ---
# A fresh finding set, so the endpoint run has something NEW to report
# (the marker is per-set: the same set would rightly stay quiet).
NODES[:] = [node("BACKER", "Membership", "endpointcase", "ep@x.org", name="EP")]
SENT.clear()
transport = httpx.ASGITransport(app=m.app)
async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:
r = await c.post("/admin/unmapped-check")
check(f"endpoint: needs the admin token (got {r.status_code})", r.status_code in (401, 403))
r = await c.post("/admin/unmapped-check", headers={"X-Admin-Token": "test-admin-token"})
body = r.json()
check(f"endpoint: runs the check (got {body.get('findings')} findings)",
r.status_code == 200 and body.get("findings") == 1)
check(f"endpoint: the run mails once for the new set (got {len(SENT)})", len(SENT) == 1)
r = await c.post("/admin/unmapped-check?notify=0",
headers={"X-Admin-Token": "test-admin-token"})
check(f"endpoint: ?notify=0 suppresses the mail (got {len(SENT)} emails)",
r.status_code == 200 and len(SENT) == 1)
asyncio.run(main())
fails = sum(not ok for _, ok in results)
for name, ok in results:
print(f" [{'PASS' if ok else 'FAIL'}] {name}")
print(f"\n{len(results) - fails}/{len(results)} passed")
sys.exit(1 if fails else 0)