notices: tell a human about what is NEW, and only when a notice actually went out
ci/woodpecker/push/woodpecker Pipeline was successful
ci/woodpecker/push/woodpecker Pipeline was successful
Three fixes to the 'email only when there is a problem' rule, found by testing it: 1. The marker recorded the CURRENT set, so a set that SHRANK looked changed — one purchase granted dropped out, the remaining hold looked new, and it was mailed about twice. What a human has been TOLD is now a growing SET of keys (order:grant / order:hold, slug:class for backers): only keys not in it are mailed, and the union is stored. 2. A notify=False run advanced the marker, so a verification look CONSUMED the only alert a finding gets. The marker now advances only when the notice was actually sent (a failed send retries next sweep), and a clean pass clears it so a recurrence is news again. 3. The credit reconcile had the same shape of bug. Tests updated to pin the new semantics: a smaller set re-mails nobody, a silent look does not swallow the alert, a clean run clears the marker and a recurrence speaks. 26 + 40 checks in those suites; all 19 suites green.
This commit is contained in:
1 parent
f258a22aa3
commit
13b6131652
3 files changed
+127
-63
No files matched your search
+95
-56
@@ -3845,9 +3845,6 @@ def webhook_failure_report(notify: bool = True) -> dict:
|
||||
for r in conn.execute(
|
||||
"SELECT id, at, status, event_type, detail FROM webhook_failures "
|
||||
"WHERE id > ? ORDER BY id", (last_id,))]
|
||||
if rows:
|
||||
_state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("webhook failure report failed: %s", e)
|
||||
@@ -3864,26 +3861,41 @@ def webhook_failure_report(notify: bool = True) -> dict:
|
||||
]
|
||||
logger.error("OC webhook: %d delivery(ies) were not accepted since the last report:\n%s",
|
||||
len(rows), "\n".join(lines))
|
||||
if notify:
|
||||
body = (
|
||||
"The portal did not accept these Open Collective deliveries:\n\n"
|
||||
+ "\n".join(lines)
|
||||
+ "\n\nA 401/403/404 means the webhook URL and WEBHOOK_TOKEN disagree — the "
|
||||
"delivery never reached the handler. Any other status means we took it and "
|
||||
"then errored; if the payload was an order, confirm the member was credited. "
|
||||
"The sweep's credit-pack reconciliation reads Open Collective's own orders "
|
||||
"and grants anything missing, so a failed purchase repairs itself; this notice "
|
||||
"is for the failures it cannot see.\n"
|
||||
)
|
||||
try:
|
||||
send_email("info@inference.coop",
|
||||
f"Inference Cooperative — {len(rows)} Open Collective "
|
||||
f"delivery(ies) failed",
|
||||
body)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("webhook failure notice email failed: %s", e)
|
||||
return {"status": "reported_no_email", "new_failures": len(rows),
|
||||
"failures": rows, "email_error": str(e)[:120]}
|
||||
if not notify:
|
||||
# A look, not a notice: logged, told to nobody, and the marker is NOT
|
||||
# advanced — a silent run must never swallow the alert a failure gets.
|
||||
return {"status": "observed", "new_failures": len(rows), "failures": rows}
|
||||
|
||||
body = (
|
||||
"The portal did not accept these Open Collective deliveries:\n\n"
|
||||
+ "\n".join(lines)
|
||||
+ "\n\nA 401/403/404 means the webhook URL and WEBHOOK_TOKEN disagree — the "
|
||||
"delivery never reached the handler. Any other status means we took it and "
|
||||
"then errored; if the payload was an order, confirm the member was credited. "
|
||||
"The sweep's credit-pack reconciliation reads Open Collective's own orders "
|
||||
"and grants anything missing, so a failed purchase repairs itself; this notice "
|
||||
"is for the failures it cannot see.\n"
|
||||
)
|
||||
try:
|
||||
send_email("info@inference.coop",
|
||||
f"Inference Cooperative — {len(rows)} Open Collective "
|
||||
f"delivery(ies) failed",
|
||||
body)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("webhook failure notice email failed: %s", e)
|
||||
return {"status": "reported_no_email", "new_failures": len(rows),
|
||||
"failures": rows, "email_error": str(e)[:120]}
|
||||
|
||||
# The marker records what someone was TOLD — advanced only after a notice is
|
||||
# actually sent, so a failed send retries on the next sweep instead of the
|
||||
# failure going quiet forever.
|
||||
try:
|
||||
conn = get_db()
|
||||
_state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("webhook failure marker write failed: %s", e)
|
||||
return {"status": "reported", "new_failures": len(rows), "failures": rows}
|
||||
|
||||
|
||||
@@ -8155,25 +8167,25 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
|
||||
"incognito_alias": False,
|
||||
})
|
||||
|
||||
# Fingerprint the SET (class + slug only: never a member's email) so the same
|
||||
# unresolved finding is reported once, and a fixed one is not remembered.
|
||||
fingerprint = "|".join(sorted(f"{f['slug']}:{f['class']}" for f in findings))[:400]
|
||||
# Keys, not a fingerprint: what has been TOLD is a set that grows, so a finding
|
||||
# that stays put is never re-mailed while a genuinely new one always speaks.
|
||||
keys = {f"{f['slug']}:{f['class']}" for f in findings}
|
||||
conn = get_db()
|
||||
previous = _state_get(conn, UNMAPPED_BACKERS_MARKER, "")
|
||||
if fingerprint != previous:
|
||||
_state_set(conn, UNMAPPED_BACKERS_MARKER, fingerprint)
|
||||
conn.commit()
|
||||
reported_keys = {k for k in (_state_get(conn, UNMAPPED_BACKERS_MARKER, "") or "").split(",") if k}
|
||||
conn.close()
|
||||
new_keys = sorted(keys - reported_keys)
|
||||
|
||||
if findings:
|
||||
logger.warning("unmapped backers: %d paid contributor(s) with no portal row "
|
||||
"(set %s): %s", len(findings), fingerprint or "-",
|
||||
"(%d new): %s", len(findings), len(new_keys),
|
||||
"; ".join(f["slug"] or f["name"] for f in findings))
|
||||
|
||||
reported = False
|
||||
if findings and fingerprint != previous and notify:
|
||||
if new_keys and notify:
|
||||
lines = []
|
||||
for f in findings:
|
||||
if f"{f['slug']}:{f['class']}" not in new_keys:
|
||||
continue
|
||||
tag = " [incognito alias]" if f["incognito_alias"] else ""
|
||||
lines.append(
|
||||
f"- slug={f['slug'] or '?'}{tag} ({f['name']}) {f['type']} "
|
||||
@@ -8184,22 +8196,37 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
|
||||
try:
|
||||
send_email(
|
||||
"info@inference.coop",
|
||||
f"Inference Cooperative — {len(findings)} paid contributor(s) with no account",
|
||||
f"Inference Cooperative — {len(new_keys)} paid contributor(s) with no account",
|
||||
"These Open Collective contributors look like members, but no portal row "
|
||||
"matches them, so nothing has provisioned an account:\n\n"
|
||||
+ "\n".join(lines)
|
||||
+ "\n\nThis is sent when the SET changes, not on every sweep. A contributor "
|
||||
"who has just joined may also be caught by the next membership sweep "
|
||||
"(3x/day); if they are still listed after that, the sweep cannot see them "
|
||||
"and someone has to act.\n",
|
||||
+ "\n\nThis covers what is NEW since the last notice, not every sweep. A "
|
||||
"contributor who has just joined may also be caught by the next membership "
|
||||
"sweep (3x/day); if they are still listed after that, the sweep cannot see "
|
||||
"them and someone has to act.\n",
|
||||
)
|
||||
reported = True
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("unmapped-backer notice email failed: %s", e)
|
||||
|
||||
return {"status": "ok", "findings": len(findings), "classes":
|
||||
sorted({f["class"] for f in findings}), "fingerprint": fingerprint,
|
||||
"reported": reported, "detail": findings}
|
||||
# Advance the marker only when someone was actually TOLD. A run with notify=False
|
||||
# is a look, not a notice: advancing here would let a silent check swallow the
|
||||
# only alert a finding ever gets. A failed send retries on the next sweep. A
|
||||
# CLEAN run clears it, so a recurrence is news again.
|
||||
if reported:
|
||||
conn = get_db()
|
||||
_state_set(conn, UNMAPPED_BACKERS_MARKER, ",".join(sorted(reported_keys | keys))[:400])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
elif not findings:
|
||||
conn = get_db()
|
||||
_state_set(conn, UNMAPPED_BACKERS_MARKER, "")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
return {"status": "ok", "findings": len(findings), "new": len(new_keys), "classes":
|
||||
sorted({f["class"] for f in findings}), "reported": reported,
|
||||
"detail": findings}
|
||||
|
||||
|
||||
async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict:
|
||||
@@ -8327,33 +8354,31 @@ async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> di
|
||||
if c.get("nearby_book_rows") else "")
|
||||
for c in candidates))
|
||||
|
||||
# Tell a human when money needed attention — and ONLY when the set changes, so
|
||||
# a finding that stays put does not mail three times a day. Silence when there
|
||||
# is nothing to say, which is the normal case.
|
||||
fingerprint = "|".join(sorted(
|
||||
[f"{c['order_id']}:grant" for c in candidates]
|
||||
+ [f"{c['order_id']}:hold" for c in needs_hold]))[:400]
|
||||
# What a human has already been TOLD is a SET of keys, not a fingerprint of the
|
||||
# current set: a purchase drops out of the set once it is granted, and a
|
||||
# fingerprint would make the remaining item look new and mail about it twice.
|
||||
keys = {f"{c['order_id']}:grant" for c in candidates} | \
|
||||
{f"{c['order_id']}:hold" for c in needs_hold}
|
||||
conn = get_db()
|
||||
previous = _state_get(conn, CREDIT_RECONCILE_MARKER, "")
|
||||
if fingerprint != previous:
|
||||
_state_set(conn, CREDIT_RECONCILE_MARKER, fingerprint)
|
||||
conn.commit()
|
||||
reported_keys = {k for k in (_state_get(conn, CREDIT_RECONCILE_MARKER, "") or "").split(",") if k}
|
||||
conn.close()
|
||||
new_keys = keys - reported_keys
|
||||
|
||||
notified = False
|
||||
if fingerprint and fingerprint != previous and notify:
|
||||
if new_keys and notify:
|
||||
granted = "\n".join(
|
||||
f" granted ${c['amount']:.2f} to {c['slug']} for order {c['order_id']} "
|
||||
f"({c['created_at']}) — {c.get('result', 'not applied (report mode)')}"
|
||||
for c in candidates) or " (nothing granted)"
|
||||
for c in candidates if f"{c['order_id']}:grant" in new_keys) or " (nothing granted)"
|
||||
held = "\n".join(
|
||||
f" NOT granted: ${c['amount']:.2f} bought by {c['slug']} for order "
|
||||
f"{c['order_id']} ({c['created_at']}) — {c['reason']}" for c in needs_hold)
|
||||
f"{c['order_id']} ({c['created_at']}) — {c['reason']}" for c in needs_hold
|
||||
if f"{c['order_id']}:hold" in new_keys)
|
||||
try:
|
||||
send_email(
|
||||
"info@inference.coop",
|
||||
f"Inference Cooperative — paid credit pack(s) missing from the ledger "
|
||||
f"({len(candidates)}/{len(needs_hold)})",
|
||||
f"({len(new_keys)} new)",
|
||||
"A credit pack that Open Collective recorded as PAID had no matching entry "
|
||||
"in our books. Credits are normally granted by the Open Collective webhook; "
|
||||
"this pass reads Open Collective's own order list every sweep, so a purchase "
|
||||
@@ -8361,13 +8386,27 @@ async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> di
|
||||
+ granted
|
||||
+ (("\n\n" + held + "\n\n(These buyers have no active membership, so they keep "
|
||||
"the hold-and-email path — nothing was granted.)")
|
||||
if needs_hold else "")
|
||||
+ "\n\nThis notice is sent when the set changes, not on every sweep.\n",
|
||||
if held else "")
|
||||
+ "\n\nThis notice covers what is NEW since the last one, not every sweep.\n",
|
||||
)
|
||||
notified = True
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("Credit reconcile: notice email failed: %s", e)
|
||||
|
||||
# Same rule as the other checks: the marker records what someone was TOLD. A
|
||||
# silent (notify=False) run must not swallow a real notice, and a clean pass
|
||||
# clears it so the next occurrence is news again.
|
||||
if notified:
|
||||
conn = get_db()
|
||||
_state_set(conn, CREDIT_RECONCILE_MARKER, ",".join(sorted(reported_keys | keys))[:400])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
elif not keys:
|
||||
conn = get_db()
|
||||
_state_set(conn, CREDIT_RECONCILE_MARKER, "")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"dry_run": not acting,
|
||||
|
||||
@@ -110,6 +110,11 @@ async def fake_index():
|
||||
|
||||
|
||||
BUDGET_PUSHES = []
|
||||
SENT = []
|
||||
|
||||
|
||||
def fake_send(to, subject, body, *a, **k):
|
||||
SENT.append({"to": to, "subject": subject, "body": body})
|
||||
|
||||
|
||||
async def fake_budget(email):
|
||||
@@ -145,6 +150,9 @@ def seed():
|
||||
conn.execute("DELETE FROM members")
|
||||
conn.execute("DELETE FROM credit_ledger")
|
||||
conn.execute("DELETE FROM held_credits")
|
||||
# A fresh world: the report markers are part of the state under test, so a
|
||||
# leftover fingerprint from an earlier section would silence a real notice.
|
||||
conn.execute("DELETE FROM portal_state")
|
||||
for email, slug, active in (("alice@x.org", "alice", 1), ("bob@x.org", "bob", 1),
|
||||
("carol@x.org", "carol", 1), ("dave@x.org", "dave", 0),
|
||||
("erin@x.org", "erin", 1), ("frank@x.org", "frank", 1)):
|
||||
@@ -179,6 +187,7 @@ async def main():
|
||||
m.fetch_collective_orders = fake_orders
|
||||
m.fetch_member_index = fake_index
|
||||
m.sync_team_budget_from_balance = fake_budget
|
||||
m.send_email = fake_send
|
||||
|
||||
# ---------- dry run: report only, change nothing --------------------------
|
||||
seed()
|
||||
@@ -248,6 +257,7 @@ async def main():
|
||||
|
||||
# ---------- CREDIT_RECONCILE=on: grant exactly once ----------------------
|
||||
seed()
|
||||
SENT.clear()
|
||||
m.CREDIT_RECONCILE = "on"
|
||||
res = await m.reconcile_credit_packs(apply=True)
|
||||
check(f"apply grants the one candidate (got {res.get('applied')})",
|
||||
@@ -265,6 +275,12 @@ async def main():
|
||||
check("the out-of-window member was NOT credited",
|
||||
balance("frank@x.org") == 0.0)
|
||||
|
||||
# The money notice: someone is told when a purchase had to be recovered —
|
||||
# and NOT told again about a set they have already heard about.
|
||||
check(f"the grant mails a notice naming the order (got {len(SENT)})",
|
||||
len(SENT) == 1 and "or_CCC" in SENT[0]["body"] and "carol" in SENT[0]["body"]
|
||||
and SENT[0]["to"] == "info@inference.coop")
|
||||
|
||||
# the webhook and this pass must dedupe each other, whichever runs first
|
||||
check("the webhook would now answer already_credited for this order",
|
||||
m.credit_reference_exists(m.credit_pack_reference("carol", {"idV2": "or_CCC"})))
|
||||
@@ -276,6 +292,7 @@ async def main():
|
||||
balance("carol@x.org") == 10.0)
|
||||
check(f"still one row for the order (got {len(ledger_rows())})",
|
||||
len(ledger_rows()) == 4)
|
||||
check(f"a re-run with nothing to report mails nothing (got {len(SENT)})", len(SENT) == 1)
|
||||
|
||||
# ---------- a candidate must SHOW the nearby rows a human needs ----------
|
||||
# A manual grant can be real and carry no per-order reference: ntnsndr's three
|
||||
|
||||
@@ -157,8 +157,8 @@ async def main():
|
||||
check(f"info@ is told once, with every class in the body (got {len(SENT)})",
|
||||
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
|
||||
and "org-link" in SENT[0]["body"] and "out-of-band" in SENT[0]["body"])
|
||||
check("it says it is sent on change, not every sweep",
|
||||
"not on every sweep" in SENT[0]["body"])
|
||||
check("it says the notice covers what is new, not every sweep",
|
||||
"not every sweep" in SENT[0]["body"])
|
||||
|
||||
# Same set again: a report, but NO second email.
|
||||
res2 = await m.reconcile_unmapped_backers()
|
||||
@@ -169,14 +169,18 @@ async def main():
|
||||
NODES[:] = [n for n in NODES
|
||||
if ((n.get("account") or {}) or {}).get("slug") != "han"]
|
||||
res3 = await m.reconcile_unmapped_backers()
|
||||
check(f"a CHANGED set is reported again (got {res3.get('findings')} findings, {len(SENT)} emails)",
|
||||
res3.get("findings") == 3 and len(SENT) == 2)
|
||||
# A set that SHRINKS (one finding resolved) must NOT re-mail the survivors: a
|
||||
# person already told about a finding is not told again just because the list
|
||||
# got shorter. Only genuinely NEW keys speak.
|
||||
check(f"a smaller set re-mails nobody (got {res3.get('findings')} findings, "
|
||||
f"{res3.get('new')} new, {len(SENT)} emails)",
|
||||
res3.get("findings") == 3 and res3.get("new") == 0 and len(SENT) == 1)
|
||||
|
||||
# Fully clear: silent, and the marker is cleared so a recurrence speaks up.
|
||||
NODES[:] = [node("BACKER", "Membership", "alice", "alice@x.org", name="Alice")]
|
||||
res4 = await m.reconcile_unmapped_backers()
|
||||
check(f"a clean run says nothing (got {res4.get('findings')}, {len(SENT)} emails)",
|
||||
res4.get("findings") == 0 and res4.get("reported") is False and len(SENT) == 2)
|
||||
res4.get("findings") == 0 and res4.get("reported") is False and len(SENT) == 1)
|
||||
conn = m.get_db()
|
||||
marker = conn.execute("SELECT value FROM portal_state WHERE key = ?",
|
||||
(m.UNMAPPED_BACKERS_MARKER,)).fetchone()
|
||||
@@ -186,13 +190,17 @@ async def main():
|
||||
NODES[:] = [node("BACKER", "Membership", "vic", "vic@x.org", name="Visible Vic")]
|
||||
res5 = await m.reconcile_unmapped_backers()
|
||||
check(f"a finding after a clean run speaks up again (got {len(SENT)} emails)",
|
||||
res5.get("findings") == 1 and len(SENT) == 3)
|
||||
res5.get("findings") == 1 and res5.get("new") == 1 and len(SENT) == 2)
|
||||
|
||||
# Notify can be suppressed (on-demand checks must not mail).
|
||||
NODES[:] = [node("BACKER", "Membership", "newguy", "new@x.org", name="New Guy")]
|
||||
res6 = await m.reconcile_unmapped_backers(notify=False)
|
||||
check(f"notify=False suppresses the mail (got {res6.get('findings')} findings, {len(SENT)} emails)",
|
||||
res6.get("findings") == 1 and res6.get("reported") is False and len(SENT) == 3)
|
||||
res6.get("findings") == 1 and res6.get("reported") is False and len(SENT) == 2)
|
||||
# ...and a look must not CONSUME the alert: the next notifying run still speaks.
|
||||
res6b = await m.reconcile_unmapped_backers()
|
||||
check(f"a silent look does not swallow the alert (got {res6b.get('new')} new, {len(SENT)} emails)",
|
||||
res6b.get("reported") is True and len(SENT) == 3)
|
||||
|
||||
# A failed read is UNKNOWN, not silence.
|
||||
FAIL_NODES = True
|
||||
|
||||
Reference in new issue
Block a user