notices: tell a human about what is NEW, and only when a notice actually went out
ci/woodpecker/push/woodpecker Pipeline was successful

Three fixes to the 'email only when there is a problem' rule, found by testing it:

1. The marker recorded the CURRENT set, so a set that SHRANK looked changed — one
   purchase granted dropped out, the remaining hold looked new, and it was mailed
   about twice. What a human has been TOLD is now a growing SET of keys
   (order:grant / order:hold, slug:class for backers): only keys not in it are
   mailed, and the union is stored.
2. A notify=False run advanced the marker, so a verification look CONSUMED the
   only alert a finding gets. The marker now advances only when the notice was
   actually sent (a failed send retries next sweep), and a clean pass clears it so
   a recurrence is news again.
3. The credit reconcile had the same shape of bug.

Tests updated to pin the new semantics: a smaller set re-mails nobody, a silent
look does not swallow the alert, a clean run clears the marker and a recurrence
speaks. 26 + 40 checks in those suites; all 19 suites green.
This commit is contained in:
inference-admin committed 2026-11-13 20:34:24 +00:00
1 parent f258a22aa3
commit 13b6131652
3 files changed
+127 -63

No files matched your search

+95 -56
View File
@@ -3845,9 +3845,6 @@ def webhook_failure_report(notify: bool = True) -> dict:
for r in conn.execute(
"SELECT id, at, status, event_type, detail FROM webhook_failures "
"WHERE id > ? ORDER BY id", (last_id,))]
if rows:
_state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"])
conn.commit()
conn.close()
except Exception as e: # noqa: BLE001
logger.warning("webhook failure report failed: %s", e)
@@ -3864,26 +3861,41 @@ def webhook_failure_report(notify: bool = True) -> dict:
]
logger.error("OC webhook: %d delivery(ies) were not accepted since the last report:\n%s",
len(rows), "\n".join(lines))
if notify:
body = (
"The portal did not accept these Open Collective deliveries:\n\n"
+ "\n".join(lines)
+ "\n\nA 401/403/404 means the webhook URL and WEBHOOK_TOKEN disagree — the "
"delivery never reached the handler. Any other status means we took it and "
"then errored; if the payload was an order, confirm the member was credited. "
"The sweep's credit-pack reconciliation reads Open Collective's own orders "
"and grants anything missing, so a failed purchase repairs itself; this notice "
"is for the failures it cannot see.\n"
)
try:
send_email("info@inference.coop",
f"Inference Cooperative — {len(rows)} Open Collective "
f"delivery(ies) failed",
body)
except Exception as e: # noqa: BLE001
logger.warning("webhook failure notice email failed: %s", e)
return {"status": "reported_no_email", "new_failures": len(rows),
"failures": rows, "email_error": str(e)[:120]}
if not notify:
# A look, not a notice: logged, told to nobody, and the marker is NOT
# advanced — a silent run must never swallow the alert a failure gets.
return {"status": "observed", "new_failures": len(rows), "failures": rows}
body = (
"The portal did not accept these Open Collective deliveries:\n\n"
+ "\n".join(lines)
+ "\n\nA 401/403/404 means the webhook URL and WEBHOOK_TOKEN disagree — the "
"delivery never reached the handler. Any other status means we took it and "
"then errored; if the payload was an order, confirm the member was credited. "
"The sweep's credit-pack reconciliation reads Open Collective's own orders "
"and grants anything missing, so a failed purchase repairs itself; this notice "
"is for the failures it cannot see.\n"
)
try:
send_email("info@inference.coop",
f"Inference Cooperative — {len(rows)} Open Collective "
f"delivery(ies) failed",
body)
except Exception as e: # noqa: BLE001
logger.warning("webhook failure notice email failed: %s", e)
return {"status": "reported_no_email", "new_failures": len(rows),
"failures": rows, "email_error": str(e)[:120]}
# The marker records what someone was TOLD — advanced only after a notice is
# actually sent, so a failed send retries on the next sweep instead of the
# failure going quiet forever.
try:
conn = get_db()
_state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"])
conn.commit()
conn.close()
except Exception as e: # noqa: BLE001
logger.warning("webhook failure marker write failed: %s", e)
return {"status": "reported", "new_failures": len(rows), "failures": rows}
@@ -8155,25 +8167,25 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
"incognito_alias": False,
})
# Fingerprint the SET (class + slug only: never a member's email) so the same
# unresolved finding is reported once, and a fixed one is not remembered.
fingerprint = "|".join(sorted(f"{f['slug']}:{f['class']}" for f in findings))[:400]
# Keys, not a fingerprint: what has been TOLD is a set that grows, so a finding
# that stays put is never re-mailed while a genuinely new one always speaks.
keys = {f"{f['slug']}:{f['class']}" for f in findings}
conn = get_db()
previous = _state_get(conn, UNMAPPED_BACKERS_MARKER, "")
if fingerprint != previous:
_state_set(conn, UNMAPPED_BACKERS_MARKER, fingerprint)
conn.commit()
reported_keys = {k for k in (_state_get(conn, UNMAPPED_BACKERS_MARKER, "") or "").split(",") if k}
conn.close()
new_keys = sorted(keys - reported_keys)
if findings:
logger.warning("unmapped backers: %d paid contributor(s) with no portal row "
"(set %s): %s", len(findings), fingerprint or "-",
"(%d new): %s", len(findings), len(new_keys),
"; ".join(f["slug"] or f["name"] for f in findings))
reported = False
if findings and fingerprint != previous and notify:
if new_keys and notify:
lines = []
for f in findings:
if f"{f['slug']}:{f['class']}" not in new_keys:
continue
tag = " [incognito alias]" if f["incognito_alias"] else ""
lines.append(
f"- slug={f['slug'] or '?'}{tag} ({f['name']}) {f['type']} "
@@ -8184,22 +8196,37 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
try:
send_email(
"info@inference.coop",
f"Inference Cooperative — {len(findings)} paid contributor(s) with no account",
f"Inference Cooperative — {len(new_keys)} paid contributor(s) with no account",
"These Open Collective contributors look like members, but no portal row "
"matches them, so nothing has provisioned an account:\n\n"
+ "\n".join(lines)
+ "\n\nThis is sent when the SET changes, not on every sweep. A contributor "
"who has just joined may also be caught by the next membership sweep "
"(3x/day); if they are still listed after that, the sweep cannot see them "
"and someone has to act.\n",
+ "\n\nThis covers what is NEW since the last notice, not every sweep. A "
"contributor who has just joined may also be caught by the next membership "
"sweep (3x/day); if they are still listed after that, the sweep cannot see "
"them and someone has to act.\n",
)
reported = True
except Exception as e: # noqa: BLE001
logger.warning("unmapped-backer notice email failed: %s", e)
return {"status": "ok", "findings": len(findings), "classes":
sorted({f["class"] for f in findings}), "fingerprint": fingerprint,
"reported": reported, "detail": findings}
# Advance the marker only when someone was actually TOLD. A run with notify=False
# is a look, not a notice: advancing here would let a silent check swallow the
# only alert a finding ever gets. A failed send retries on the next sweep. A
# CLEAN run clears it, so a recurrence is news again.
if reported:
conn = get_db()
_state_set(conn, UNMAPPED_BACKERS_MARKER, ",".join(sorted(reported_keys | keys))[:400])
conn.commit()
conn.close()
elif not findings:
conn = get_db()
_state_set(conn, UNMAPPED_BACKERS_MARKER, "")
conn.commit()
conn.close()
return {"status": "ok", "findings": len(findings), "new": len(new_keys), "classes":
sorted({f["class"] for f in findings}), "reported": reported,
"detail": findings}
async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict:
@@ -8327,33 +8354,31 @@ async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> di
if c.get("nearby_book_rows") else "")
for c in candidates))
# Tell a human when money needed attention — and ONLY when the set changes, so
# a finding that stays put does not mail three times a day. Silence when there
# is nothing to say, which is the normal case.
fingerprint = "|".join(sorted(
[f"{c['order_id']}:grant" for c in candidates]
+ [f"{c['order_id']}:hold" for c in needs_hold]))[:400]
# What a human has already been TOLD is a SET of keys, not a fingerprint of the
# current set: a purchase drops out of the set once it is granted, and a
# fingerprint would make the remaining item look new and mail about it twice.
keys = {f"{c['order_id']}:grant" for c in candidates} | \
{f"{c['order_id']}:hold" for c in needs_hold}
conn = get_db()
previous = _state_get(conn, CREDIT_RECONCILE_MARKER, "")
if fingerprint != previous:
_state_set(conn, CREDIT_RECONCILE_MARKER, fingerprint)
conn.commit()
reported_keys = {k for k in (_state_get(conn, CREDIT_RECONCILE_MARKER, "") or "").split(",") if k}
conn.close()
new_keys = keys - reported_keys
notified = False
if fingerprint and fingerprint != previous and notify:
if new_keys and notify:
granted = "\n".join(
f" granted ${c['amount']:.2f} to {c['slug']} for order {c['order_id']} "
f"({c['created_at']}) — {c.get('result', 'not applied (report mode)')}"
for c in candidates) or " (nothing granted)"
for c in candidates if f"{c['order_id']}:grant" in new_keys) or " (nothing granted)"
held = "\n".join(
f" NOT granted: ${c['amount']:.2f} bought by {c['slug']} for order "
f"{c['order_id']} ({c['created_at']}) — {c['reason']}" for c in needs_hold)
f"{c['order_id']} ({c['created_at']}) — {c['reason']}" for c in needs_hold
if f"{c['order_id']}:hold" in new_keys)
try:
send_email(
"info@inference.coop",
f"Inference Cooperative — paid credit pack(s) missing from the ledger "
f"({len(candidates)}/{len(needs_hold)})",
f"({len(new_keys)} new)",
"A credit pack that Open Collective recorded as PAID had no matching entry "
"in our books. Credits are normally granted by the Open Collective webhook; "
"this pass reads Open Collective's own order list every sweep, so a purchase "
@@ -8361,13 +8386,27 @@ async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> di
+ granted
+ (("\n\n" + held + "\n\n(These buyers have no active membership, so they keep "
"the hold-and-email path — nothing was granted.)")
if needs_hold else "")
+ "\n\nThis notice is sent when the set changes, not on every sweep.\n",
if held else "")
+ "\n\nThis notice covers what is NEW since the last one, not every sweep.\n",
)
notified = True
except Exception as e: # noqa: BLE001
logger.warning("Credit reconcile: notice email failed: %s", e)
# Same rule as the other checks: the marker records what someone was TOLD. A
# silent (notify=False) run must not swallow a real notice, and a clean pass
# clears it so the next occurrence is news again.
if notified:
conn = get_db()
_state_set(conn, CREDIT_RECONCILE_MARKER, ",".join(sorted(reported_keys | keys))[:400])
conn.commit()
conn.close()
elif not keys:
conn = get_db()
_state_set(conn, CREDIT_RECONCILE_MARKER, "")
conn.commit()
conn.close()
return {
"status": "ok",
"dry_run": not acting,