notices: tell a human about what is NEW, and only when a notice actually went out
ci/woodpecker/push/woodpecker Pipeline was successful
ci/woodpecker/push/woodpecker Pipeline was successful
Three fixes to the 'email only when there is a problem' rule, found by testing it: 1. The marker recorded the CURRENT set, so a set that SHRANK looked changed — one purchase granted dropped out, the remaining hold looked new, and it was mailed about twice. What a human has been TOLD is now a growing SET of keys (order:grant / order:hold, slug:class for backers): only keys not in it are mailed, and the union is stored. 2. A notify=False run advanced the marker, so a verification look CONSUMED the only alert a finding gets. The marker now advances only when the notice was actually sent (a failed send retries next sweep), and a clean pass clears it so a recurrence is news again. 3. The credit reconcile had the same shape of bug. Tests updated to pin the new semantics: a smaller set re-mails nobody, a silent look does not swallow the alert, a clean run clears the marker and a recurrence speaks. 26 + 40 checks in those suites; all 19 suites green.
This commit is contained in:
1 parent
f258a22aa3
commit
13b6131652
3 files changed
+127
-63
No files matched your search
+95
-56
@@ -3845,9 +3845,6 @@ def webhook_failure_report(notify: bool = True) -> dict:
|
||||
for r in conn.execute(
|
||||
"SELECT id, at, status, event_type, detail FROM webhook_failures "
|
||||
"WHERE id > ? ORDER BY id", (last_id,))]
|
||||
if rows:
|
||||
_state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("webhook failure report failed: %s", e)
|
||||
@@ -3864,26 +3861,41 @@ def webhook_failure_report(notify: bool = True) -> dict:
|
||||
]
|
||||
logger.error("OC webhook: %d delivery(ies) were not accepted since the last report:\n%s",
|
||||
len(rows), "\n".join(lines))
|
||||
if notify:
|
||||
body = (
|
||||
"The portal did not accept these Open Collective deliveries:\n\n"
|
||||
+ "\n".join(lines)
|
||||
+ "\n\nA 401/403/404 means the webhook URL and WEBHOOK_TOKEN disagree — the "
|
||||
"delivery never reached the handler. Any other status means we took it and "
|
||||
"then errored; if the payload was an order, confirm the member was credited. "
|
||||
"The sweep's credit-pack reconciliation reads Open Collective's own orders "
|
||||
"and grants anything missing, so a failed purchase repairs itself; this notice "
|
||||
"is for the failures it cannot see.\n"
|
||||
)
|
||||
try:
|
||||
send_email("info@inference.coop",
|
||||
f"Inference Cooperative — {len(rows)} Open Collective "
|
||||
f"delivery(ies) failed",
|
||||
body)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("webhook failure notice email failed: %s", e)
|
||||
return {"status": "reported_no_email", "new_failures": len(rows),
|
||||
"failures": rows, "email_error": str(e)[:120]}
|
||||
if not notify:
|
||||
# A look, not a notice: logged, told to nobody, and the marker is NOT
|
||||
# advanced — a silent run must never swallow the alert a failure gets.
|
||||
return {"status": "observed", "new_failures": len(rows), "failures": rows}
|
||||
|
||||
body = (
|
||||
"The portal did not accept these Open Collective deliveries:\n\n"
|
||||
+ "\n".join(lines)
|
||||
+ "\n\nA 401/403/404 means the webhook URL and WEBHOOK_TOKEN disagree — the "
|
||||
"delivery never reached the handler. Any other status means we took it and "
|
||||
"then errored; if the payload was an order, confirm the member was credited. "
|
||||
"The sweep's credit-pack reconciliation reads Open Collective's own orders "
|
||||
"and grants anything missing, so a failed purchase repairs itself; this notice "
|
||||
"is for the failures it cannot see.\n"
|
||||
)
|
||||
try:
|
||||
send_email("info@inference.coop",
|
||||
f"Inference Cooperative — {len(rows)} Open Collective "
|
||||
f"delivery(ies) failed",
|
||||
body)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("webhook failure notice email failed: %s", e)
|
||||
return {"status": "reported_no_email", "new_failures": len(rows),
|
||||
"failures": rows, "email_error": str(e)[:120]}
|
||||
|
||||
# The marker records what someone was TOLD — advanced only after a notice is
|
||||
# actually sent, so a failed send retries on the next sweep instead of the
|
||||
# failure going quiet forever.
|
||||
try:
|
||||
conn = get_db()
|
||||
_state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("webhook failure marker write failed: %s", e)
|
||||
return {"status": "reported", "new_failures": len(rows), "failures": rows}
|
||||
|
||||
|
||||
@@ -8155,25 +8167,25 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
|
||||
"incognito_alias": False,
|
||||
})
|
||||
|
||||
# Fingerprint the SET (class + slug only: never a member's email) so the same
|
||||
# unresolved finding is reported once, and a fixed one is not remembered.
|
||||
fingerprint = "|".join(sorted(f"{f['slug']}:{f['class']}" for f in findings))[:400]
|
||||
# Keys, not a fingerprint: what has been TOLD is a set that grows, so a finding
|
||||
# that stays put is never re-mailed while a genuinely new one always speaks.
|
||||
keys = {f"{f['slug']}:{f['class']}" for f in findings}
|
||||
conn = get_db()
|
||||
previous = _state_get(conn, UNMAPPED_BACKERS_MARKER, "")
|
||||
if fingerprint != previous:
|
||||
_state_set(conn, UNMAPPED_BACKERS_MARKER, fingerprint)
|
||||
conn.commit()
|
||||
reported_keys = {k for k in (_state_get(conn, UNMAPPED_BACKERS_MARKER, "") or "").split(",") if k}
|
||||
conn.close()
|
||||
new_keys = sorted(keys - reported_keys)
|
||||
|
||||
if findings:
|
||||
logger.warning("unmapped backers: %d paid contributor(s) with no portal row "
|
||||
"(set %s): %s", len(findings), fingerprint or "-",
|
||||
"(%d new): %s", len(findings), len(new_keys),
|
||||
"; ".join(f["slug"] or f["name"] for f in findings))
|
||||
|
||||
reported = False
|
||||
if findings and fingerprint != previous and notify:
|
||||
if new_keys and notify:
|
||||
lines = []
|
||||
for f in findings:
|
||||
if f"{f['slug']}:{f['class']}" not in new_keys:
|
||||
continue
|
||||
tag = " [incognito alias]" if f["incognito_alias"] else ""
|
||||
lines.append(
|
||||
f"- slug={f['slug'] or '?'}{tag} ({f['name']}) {f['type']} "
|
||||
@@ -8184,22 +8196,37 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
|
||||
try:
|
||||
send_email(
|
||||
"info@inference.coop",
|
||||
f"Inference Cooperative — {len(findings)} paid contributor(s) with no account",
|
||||
f"Inference Cooperative — {len(new_keys)} paid contributor(s) with no account",
|
||||
"These Open Collective contributors look like members, but no portal row "
|
||||
"matches them, so nothing has provisioned an account:\n\n"
|
||||
+ "\n".join(lines)
|
||||
+ "\n\nThis is sent when the SET changes, not on every sweep. A contributor "
|
||||
"who has just joined may also be caught by the next membership sweep "
|
||||
"(3x/day); if they are still listed after that, the sweep cannot see them "
|
||||
"and someone has to act.\n",
|
||||
+ "\n\nThis covers what is NEW since the last notice, not every sweep. A "
|
||||
"contributor who has just joined may also be caught by the next membership "
|
||||
"sweep (3x/day); if they are still listed after that, the sweep cannot see "
|
||||
"them and someone has to act.\n",
|
||||
)
|
||||
reported = True
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("unmapped-backer notice email failed: %s", e)
|
||||
|
||||
return {"status": "ok", "findings": len(findings), "classes":
|
||||
sorted({f["class"] for f in findings}), "fingerprint": fingerprint,
|
||||
"reported": reported, "detail": findings}
|
||||
# Advance the marker only when someone was actually TOLD. A run with notify=False
|
||||
# is a look, not a notice: advancing here would let a silent check swallow the
|
||||
# only alert a finding ever gets. A failed send retries on the next sweep. A
|
||||
# CLEAN run clears it, so a recurrence is news again.
|
||||
if reported:
|
||||
conn = get_db()
|
||||
_state_set(conn, UNMAPPED_BACKERS_MARKER, ",".join(sorted(reported_keys | keys))[:400])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
elif not findings:
|
||||
conn = get_db()
|
||||
_state_set(conn, UNMAPPED_BACKERS_MARKER, "")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
return {"status": "ok", "findings": len(findings), "new": len(new_keys), "classes":
|
||||
sorted({f["class"] for f in findings}), "reported": reported,
|
||||
"detail": findings}
|
||||
|
||||
|
||||
async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict:
|
||||
@@ -8327,33 +8354,31 @@ async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> di
|
||||
if c.get("nearby_book_rows") else "")
|
||||
for c in candidates))
|
||||
|
||||
# Tell a human when money needed attention — and ONLY when the set changes, so
|
||||
# a finding that stays put does not mail three times a day. Silence when there
|
||||
# is nothing to say, which is the normal case.
|
||||
fingerprint = "|".join(sorted(
|
||||
[f"{c['order_id']}:grant" for c in candidates]
|
||||
+ [f"{c['order_id']}:hold" for c in needs_hold]))[:400]
|
||||
# What a human has already been TOLD is a SET of keys, not a fingerprint of the
|
||||
# current set: a purchase drops out of the set once it is granted, and a
|
||||
# fingerprint would make the remaining item look new and mail about it twice.
|
||||
keys = {f"{c['order_id']}:grant" for c in candidates} | \
|
||||
{f"{c['order_id']}:hold" for c in needs_hold}
|
||||
conn = get_db()
|
||||
previous = _state_get(conn, CREDIT_RECONCILE_MARKER, "")
|
||||
if fingerprint != previous:
|
||||
_state_set(conn, CREDIT_RECONCILE_MARKER, fingerprint)
|
||||
conn.commit()
|
||||
reported_keys = {k for k in (_state_get(conn, CREDIT_RECONCILE_MARKER, "") or "").split(",") if k}
|
||||
conn.close()
|
||||
new_keys = keys - reported_keys
|
||||
|
||||
notified = False
|
||||
if fingerprint and fingerprint != previous and notify:
|
||||
if new_keys and notify:
|
||||
granted = "\n".join(
|
||||
f" granted ${c['amount']:.2f} to {c['slug']} for order {c['order_id']} "
|
||||
f"({c['created_at']}) — {c.get('result', 'not applied (report mode)')}"
|
||||
for c in candidates) or " (nothing granted)"
|
||||
for c in candidates if f"{c['order_id']}:grant" in new_keys) or " (nothing granted)"
|
||||
held = "\n".join(
|
||||
f" NOT granted: ${c['amount']:.2f} bought by {c['slug']} for order "
|
||||
f"{c['order_id']} ({c['created_at']}) — {c['reason']}" for c in needs_hold)
|
||||
f"{c['order_id']} ({c['created_at']}) — {c['reason']}" for c in needs_hold
|
||||
if f"{c['order_id']}:hold" in new_keys)
|
||||
try:
|
||||
send_email(
|
||||
"info@inference.coop",
|
||||
f"Inference Cooperative — paid credit pack(s) missing from the ledger "
|
||||
f"({len(candidates)}/{len(needs_hold)})",
|
||||
f"({len(new_keys)} new)",
|
||||
"A credit pack that Open Collective recorded as PAID had no matching entry "
|
||||
"in our books. Credits are normally granted by the Open Collective webhook; "
|
||||
"this pass reads Open Collective's own order list every sweep, so a purchase "
|
||||
@@ -8361,13 +8386,27 @@ async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> di
|
||||
+ granted
|
||||
+ (("\n\n" + held + "\n\n(These buyers have no active membership, so they keep "
|
||||
"the hold-and-email path — nothing was granted.)")
|
||||
if needs_hold else "")
|
||||
+ "\n\nThis notice is sent when the set changes, not on every sweep.\n",
|
||||
if held else "")
|
||||
+ "\n\nThis notice covers what is NEW since the last one, not every sweep.\n",
|
||||
)
|
||||
notified = True
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("Credit reconcile: notice email failed: %s", e)
|
||||
|
||||
# Same rule as the other checks: the marker records what someone was TOLD. A
|
||||
# silent (notify=False) run must not swallow a real notice, and a clean pass
|
||||
# clears it so the next occurrence is news again.
|
||||
if notified:
|
||||
conn = get_db()
|
||||
_state_set(conn, CREDIT_RECONCILE_MARKER, ",".join(sorted(reported_keys | keys))[:400])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
elif not keys:
|
||||
conn = get_db()
|
||||
_state_set(conn, CREDIT_RECONCILE_MARKER, "")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"dry_run": not acting,
|
||||
|
||||
Reference in new issue
Block a user