notices: tell a human about what is NEW, and only when a notice actually went out
ci/woodpecker/push/woodpecker Pipeline was successful

Three fixes to the 'email only when there is a problem' rule, found by testing it:

1. The marker recorded the CURRENT set, so a set that SHRANK looked changed — one
   purchase granted dropped out, the remaining hold looked new, and it was mailed
   about twice. What a human has been TOLD is now a growing SET of keys
   (order:grant / order:hold, slug:class for backers): only keys not in it are
   mailed, and the union is stored.
2. A notify=False run advanced the marker, so a verification look CONSUMED the
   only alert a finding gets. The marker now advances only when the notice was
   actually sent (a failed send retries next sweep), and a clean pass clears it so
   a recurrence is news again.
3. The credit reconcile had the same shape of bug.

Tests updated to pin the new semantics: a smaller set re-mails nobody, a silent
look does not swallow the alert, a clean run clears the marker and a recurrence
speaks. 26 + 40 checks in those suites; all 19 suites green.
This commit is contained in:
inference-admin committed 2026-11-13 20:34:24 +00:00
1 parent f258a22aa3
commit 13b6131652
3 files changed
+108 -44

No files matched your search

+76 -37
View File
@@ -3845,9 +3845,6 @@ def webhook_failure_report(notify: bool = True) -> dict:
for r in conn.execute(
"SELECT id, at, status, event_type, detail FROM webhook_failures "
"WHERE id > ? ORDER BY id", (last_id,))]
if rows:
_state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"])
conn.commit()
conn.close()
except Exception as e: # noqa: BLE001
logger.warning("webhook failure report failed: %s", e)
@@ -3864,7 +3861,11 @@ def webhook_failure_report(notify: bool = True) -> dict:
]
logger.error("OC webhook: %d delivery(ies) were not accepted since the last report:\n%s",
len(rows), "\n".join(lines))
if notify:
if not notify:
# A look, not a notice: logged, told to nobody, and the marker is NOT
# advanced — a silent run must never swallow the alert a failure gets.
return {"status": "observed", "new_failures": len(rows), "failures": rows}
body = (
"The portal did not accept these Open Collective deliveries:\n\n"
+ "\n".join(lines)
@@ -3884,6 +3885,17 @@ def webhook_failure_report(notify: bool = True) -> dict:
logger.warning("webhook failure notice email failed: %s", e)
return {"status": "reported_no_email", "new_failures": len(rows),
"failures": rows, "email_error": str(e)[:120]}
# The marker records what someone was TOLD — advanced only after a notice is
# actually sent, so a failed send retries on the next sweep instead of the
# failure going quiet forever.
try:
conn = get_db()
_state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"])
conn.commit()
conn.close()
except Exception as e: # noqa: BLE001
logger.warning("webhook failure marker write failed: %s", e)
return {"status": "reported", "new_failures": len(rows), "failures": rows}
@@ -8155,25 +8167,25 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
"incognito_alias": False,
})
# Fingerprint the SET (class + slug only: never a member's email) so the same
# unresolved finding is reported once, and a fixed one is not remembered.
fingerprint = "|".join(sorted(f"{f['slug']}:{f['class']}" for f in findings))[:400]
# Keys, not a fingerprint: what has been TOLD is a set that grows, so a finding
# that stays put is never re-mailed while a genuinely new one always speaks.
keys = {f"{f['slug']}:{f['class']}" for f in findings}
conn = get_db()
previous = _state_get(conn, UNMAPPED_BACKERS_MARKER, "")
if fingerprint != previous:
_state_set(conn, UNMAPPED_BACKERS_MARKER, fingerprint)
conn.commit()
reported_keys = {k for k in (_state_get(conn, UNMAPPED_BACKERS_MARKER, "") or "").split(",") if k}
conn.close()
new_keys = sorted(keys - reported_keys)
if findings:
logger.warning("unmapped backers: %d paid contributor(s) with no portal row "
"(set %s): %s", len(findings), fingerprint or "-",
"(%d new): %s", len(findings), len(new_keys),
"; ".join(f["slug"] or f["name"] for f in findings))
reported = False
if findings and fingerprint != previous and notify:
if new_keys and notify:
lines = []
for f in findings:
if f"{f['slug']}:{f['class']}" not in new_keys:
continue
tag = " [incognito alias]" if f["incognito_alias"] else ""
lines.append(
f"- slug={f['slug'] or '?'}{tag} ({f['name']}) {f['type']} "
@@ -8184,22 +8196,37 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
try:
send_email(
"info@inference.coop",
f"Inference Cooperative — {len(findings)} paid contributor(s) with no account",
f"Inference Cooperative — {len(new_keys)} paid contributor(s) with no account",
"These Open Collective contributors look like members, but no portal row "
"matches them, so nothing has provisioned an account:\n\n"
+ "\n".join(lines)
+ "\n\nThis is sent when the SET changes, not on every sweep. A contributor "
"who has just joined may also be caught by the next membership sweep "
"(3x/day); if they are still listed after that, the sweep cannot see them "
"and someone has to act.\n",
+ "\n\nThis covers what is NEW since the last notice, not every sweep. A "
"contributor who has just joined may also be caught by the next membership "
"sweep (3x/day); if they are still listed after that, the sweep cannot see "
"them and someone has to act.\n",
)
reported = True
except Exception as e: # noqa: BLE001
logger.warning("unmapped-backer notice email failed: %s", e)
return {"status": "ok", "findings": len(findings), "classes":
sorted({f["class"] for f in findings}), "fingerprint": fingerprint,
"reported": reported, "detail": findings}
# Advance the marker only when someone was actually TOLD. A run with notify=False
# is a look, not a notice: advancing here would let a silent check swallow the
# only alert a finding ever gets. A failed send retries on the next sweep. A
# CLEAN run clears it, so a recurrence is news again.
if reported:
conn = get_db()
_state_set(conn, UNMAPPED_BACKERS_MARKER, ",".join(sorted(reported_keys | keys))[:400])
conn.commit()
conn.close()
elif not findings:
conn = get_db()
_state_set(conn, UNMAPPED_BACKERS_MARKER, "")
conn.commit()
conn.close()
return {"status": "ok", "findings": len(findings), "new": len(new_keys), "classes":
sorted({f["class"] for f in findings}), "reported": reported,
"detail": findings}
async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict:
@@ -8327,33 +8354,31 @@ async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> di
if c.get("nearby_book_rows") else "")
for c in candidates))
# Tell a human when money needed attention — and ONLY when the set changes, so
# a finding that stays put does not mail three times a day. Silence when there
# is nothing to say, which is the normal case.
fingerprint = "|".join(sorted(
[f"{c['order_id']}:grant" for c in candidates]
+ [f"{c['order_id']}:hold" for c in needs_hold]))[:400]
# What a human has already been TOLD is a SET of keys, not a fingerprint of the
# current set: a purchase drops out of the set once it is granted, and a
# fingerprint would make the remaining item look new and mail about it twice.
keys = {f"{c['order_id']}:grant" for c in candidates} | \
{f"{c['order_id']}:hold" for c in needs_hold}
conn = get_db()
previous = _state_get(conn, CREDIT_RECONCILE_MARKER, "")
if fingerprint != previous:
_state_set(conn, CREDIT_RECONCILE_MARKER, fingerprint)
conn.commit()
reported_keys = {k for k in (_state_get(conn, CREDIT_RECONCILE_MARKER, "") or "").split(",") if k}
conn.close()
new_keys = keys - reported_keys
notified = False
if fingerprint and fingerprint != previous and notify:
if new_keys and notify:
granted = "\n".join(
f" granted ${c['amount']:.2f} to {c['slug']} for order {c['order_id']} "
f"({c['created_at']}) — {c.get('result', 'not applied (report mode)')}"
for c in candidates) or " (nothing granted)"
for c in candidates if f"{c['order_id']}:grant" in new_keys) or " (nothing granted)"
held = "\n".join(
f" NOT granted: ${c['amount']:.2f} bought by {c['slug']} for order "
f"{c['order_id']} ({c['created_at']}) — {c['reason']}" for c in needs_hold)
f"{c['order_id']} ({c['created_at']}) — {c['reason']}" for c in needs_hold
if f"{c['order_id']}:hold" in new_keys)
try:
send_email(
"info@inference.coop",
f"Inference Cooperative — paid credit pack(s) missing from the ledger "
f"({len(candidates)}/{len(needs_hold)})",
f"({len(new_keys)} new)",
"A credit pack that Open Collective recorded as PAID had no matching entry "
"in our books. Credits are normally granted by the Open Collective webhook; "
"this pass reads Open Collective's own order list every sweep, so a purchase "
@@ -8361,13 +8386,27 @@ async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> di
+ granted
+ (("\n\n" + held + "\n\n(These buyers have no active membership, so they keep "
"the hold-and-email path — nothing was granted.)")
if needs_hold else "")
+ "\n\nThis notice is sent when the set changes, not on every sweep.\n",
if held else "")
+ "\n\nThis notice covers what is NEW since the last one, not every sweep.\n",
)
notified = True
except Exception as e: # noqa: BLE001
logger.warning("Credit reconcile: notice email failed: %s", e)
# Same rule as the other checks: the marker records what someone was TOLD. A
# silent (notify=False) run must not swallow a real notice, and a clean pass
# clears it so the next occurrence is news again.
if notified:
conn = get_db()
_state_set(conn, CREDIT_RECONCILE_MARKER, ",".join(sorted(reported_keys | keys))[:400])
conn.commit()
conn.close()
elif not keys:
conn = get_db()
_state_set(conn, CREDIT_RECONCILE_MARKER, "")
conn.commit()
conn.close()
return {
"status": "ok",
"dry_run": not acting,
+17
View File
@@ -110,6 +110,11 @@ async def fake_index():
BUDGET_PUSHES = []
SENT = []
def fake_send(to, subject, body, *a, **k):
SENT.append({"to": to, "subject": subject, "body": body})
async def fake_budget(email):
@@ -145,6 +150,9 @@ def seed():
conn.execute("DELETE FROM members")
conn.execute("DELETE FROM credit_ledger")
conn.execute("DELETE FROM held_credits")
# A fresh world: the report markers are part of the state under test, so a
# leftover fingerprint from an earlier section would silence a real notice.
conn.execute("DELETE FROM portal_state")
for email, slug, active in (("alice@x.org", "alice", 1), ("bob@x.org", "bob", 1),
("carol@x.org", "carol", 1), ("dave@x.org", "dave", 0),
("erin@x.org", "erin", 1), ("frank@x.org", "frank", 1)):
@@ -179,6 +187,7 @@ async def main():
m.fetch_collective_orders = fake_orders
m.fetch_member_index = fake_index
m.sync_team_budget_from_balance = fake_budget
m.send_email = fake_send
# ---------- dry run: report only, change nothing --------------------------
seed()
@@ -248,6 +257,7 @@ async def main():
# ---------- CREDIT_RECONCILE=on: grant exactly once ----------------------
seed()
SENT.clear()
m.CREDIT_RECONCILE = "on"
res = await m.reconcile_credit_packs(apply=True)
check(f"apply grants the one candidate (got {res.get('applied')})",
@@ -265,6 +275,12 @@ async def main():
check("the out-of-window member was NOT credited",
balance("frank@x.org") == 0.0)
# The money notice: someone is told when a purchase had to be recovered —
# and NOT told again about a set they have already heard about.
check(f"the grant mails a notice naming the order (got {len(SENT)})",
len(SENT) == 1 and "or_CCC" in SENT[0]["body"] and "carol" in SENT[0]["body"]
and SENT[0]["to"] == "info@inference.coop")
# the webhook and this pass must dedupe each other, whichever runs first
check("the webhook would now answer already_credited for this order",
m.credit_reference_exists(m.credit_pack_reference("carol", {"idV2": "or_CCC"})))
@@ -276,6 +292,7 @@ async def main():
balance("carol@x.org") == 10.0)
check(f"still one row for the order (got {len(ledger_rows())})",
len(ledger_rows()) == 4)
check(f"a re-run with nothing to report mails nothing (got {len(SENT)})", len(SENT) == 1)
# ---------- a candidate must SHOW the nearby rows a human needs ----------
# A manual grant can be real and carry no per-order reference: ntnsndr's three
+15 -7
View File
@@ -157,8 +157,8 @@ async def main():
check(f"info@ is told once, with every class in the body (got {len(SENT)})",
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
and "org-link" in SENT[0]["body"] and "out-of-band" in SENT[0]["body"])
check("it says it is sent on change, not every sweep",
"not on every sweep" in SENT[0]["body"])
check("it says the notice covers what is new, not every sweep",
"not every sweep" in SENT[0]["body"])
# Same set again: a report, but NO second email.
res2 = await m.reconcile_unmapped_backers()
@@ -169,14 +169,18 @@ async def main():
NODES[:] = [n for n in NODES
if ((n.get("account") or {}) or {}).get("slug") != "han"]
res3 = await m.reconcile_unmapped_backers()
check(f"a CHANGED set is reported again (got {res3.get('findings')} findings, {len(SENT)} emails)",
res3.get("findings") == 3 and len(SENT) == 2)
# A set that SHRINKS (one finding resolved) must NOT re-mail the survivors: a
# person already told about a finding is not told again just because the list
# got shorter. Only genuinely NEW keys speak.
check(f"a smaller set re-mails nobody (got {res3.get('findings')} findings, "
f"{res3.get('new')} new, {len(SENT)} emails)",
res3.get("findings") == 3 and res3.get("new") == 0 and len(SENT) == 1)
# Fully clear: silent, and the marker is cleared so a recurrence speaks up.
NODES[:] = [node("BACKER", "Membership", "alice", "alice@x.org", name="Alice")]
res4 = await m.reconcile_unmapped_backers()
check(f"a clean run says nothing (got {res4.get('findings')}, {len(SENT)} emails)",
res4.get("findings") == 0 and res4.get("reported") is False and len(SENT) == 2)
res4.get("findings") == 0 and res4.get("reported") is False and len(SENT) == 1)
conn = m.get_db()
marker = conn.execute("SELECT value FROM portal_state WHERE key = ?",
(m.UNMAPPED_BACKERS_MARKER,)).fetchone()
@@ -186,13 +190,17 @@ async def main():
NODES[:] = [node("BACKER", "Membership", "vic", "vic@x.org", name="Visible Vic")]
res5 = await m.reconcile_unmapped_backers()
check(f"a finding after a clean run speaks up again (got {len(SENT)} emails)",
res5.get("findings") == 1 and len(SENT) == 3)
res5.get("findings") == 1 and res5.get("new") == 1 and len(SENT) == 2)
# Notify can be suppressed (on-demand checks must not mail).
NODES[:] = [node("BACKER", "Membership", "newguy", "new@x.org", name="New Guy")]
res6 = await m.reconcile_unmapped_backers(notify=False)
check(f"notify=False suppresses the mail (got {res6.get('findings')} findings, {len(SENT)} emails)",
res6.get("findings") == 1 and res6.get("reported") is False and len(SENT) == 3)
res6.get("findings") == 1 and res6.get("reported") is False and len(SENT) == 2)
# ...and a look must not CONSUME the alert: the next notifying run still speaks.
res6b = await m.reconcile_unmapped_backers()
check(f"a silent look does not swallow the alert (got {res6b.get('new')} new, {len(SENT)} emails)",
res6b.get("reported") is True and len(SENT) == 3)
# A failed read is UNKNOWN, not silence.
FAIL_NODES = True