allowances: credit the shortfall when OC's billing day truncates a paid cycle

Open Collective aligns recurring charges onto the collective's common billing
day, so a mid-month joiner is charged again before a full month is up: they pay
for a full month and receive less than one. Credit the missing ALLOWANCE (the
flat MEMBER_BUDGET pro-rated for the days lost, unrounded) as non-expiring
credits.

- derived from the collective's real per-order charge history, on every sweep;
  any consecutive pair on the same order < a full month apart is truncated
- idempotent on the OC charge reference (dedupe on the charge, not the member)
- skips and logs any cycle whose dates cannot be determined
- ledger reason names the cycle; the notice is staged and defaults OFF
- [redacted member reference]'s remediated charge is excluded by ref

Tests: tests/test_allowance_shortfall.py (pure derivation).
This commit is contained in:
inference-bot committed 2026-11-04 20:01:10 +00:00
1 parent c468b61d3e
commit db64dac235
3 files changed
+529

No files matched your search

+387
View File
@@ -4791,6 +4791,383 @@ async def reconcile_allowances(send: bool = False) -> dict:
}
# --------------------------------------------------------------------------- #
# Allowance shortfall — OC's billing-day alignment truncates a paid cycle. #
# --------------------------------------------------------------------------- #
# Open Collective aligns every recurring charge onto the collective's COMMON
# billing day, so a member who joins mid-month is charged again well before a
# full month has elapsed: they pay for a full month and receive less than one.
# Operator decision (2026-11-04): make the member whole by crediting the missing
# ALLOWANCE — the flat MEMBER_BUDGET pro-rated for the days lost — as
# non-expiring credits (the same balance that extends the allowance and is spent
# only after it). It is NOT a share of what they paid, and it is NOT rounded.
#
# Scope: EVERY monthly paid cycle shorter than a full month, for any member and
# any cycle (not only the first — OC can re-align an existing charge date too).
# Derived FROM DATA (each recurring order's real charge history); members are
# never asked to claim it. Idempotent: each credit is keyed to the specific OC
# charge that caused it, so a re-run credits nothing twice. A charge whose date
# cannot be determined is SKIPPED and logged — paying a credit that is not owed
# is worse than paying one late.
#
# Member notices go out automatically (the sweep does both in one pass), but they
# default OFF (ALLOWANCE_SHORTFALL_EMAILS=off) so the wording can be reviewed
# before any member is written to about billing. Setting it to `on` sends the
# staged notice on the next sweep; nothing is re-sent per charge.
FULL_PAID_MONTH_DAYS = 30 # the allowance is a flat monthly amount => "a month" = 30 days
ALLOWANCE_SHORTFALL_EMAILS = (os.environ.get("ALLOWANCE_SHORTFALL_EMAILS", "off") or "off").strip().lower()
def shortfall_reason(prev: str, cur: str) -> str:
"""Human-readable ledger reason — a member's question is answerable from it."""
return f"allowance shortfall, truncated paid cycle {prev}→{cur} (OC billing alignment)"
# Members already made whole OUTSIDE this mechanism, keyed by (email, OC charge
# reference). [redacted member reference] received a manual allowance reset
# on 2026-11-04 for the cycle OC truncated ([redacted timeline]); that
# specific charge is excluded so a re-run never credits it again. Later charges
# form new cycles and are credited normally.
SHORTFALL_REMEDIATED: set[tuple[str, str]] = {
("[redacted member reference]", "[redacted member reference]"),
}
def _usd(x: float) -> str:
"""Format a credit amount for a member without inventing cents (2 -> $2)."""
return f"${x:g}"
def _ensure_shortfall_tables(conn) -> None:
"""Records which charge-triggered notices have actually been sent.
Kept apart from the credit ledger on purpose: the ledger answers "what did we
credit, and why", this answers "what left the building". A notice is sent at
most once per charge reference.
"""
conn.execute(
"CREATE TABLE IF NOT EXISTS shortfall_notices ("
" reference TEXT PRIMARY KEY, email TEXT NOT NULL, amount REAL NOT NULL,"
" sent_at TEXT NOT NULL DEFAULT (datetime('now')))"
)
conn.commit()
def credit_reference_exists(reference: str) -> bool:
"""Has a credit already been recorded against this OC charge reference?"""
conn = get_db()
row = conn.execute(
"SELECT 1 FROM credit_ledger WHERE reference = ? LIMIT 1", (reference,)
).fetchone()
conn.close()
return bool(row)
def shortfall_notice_sent(reference: str) -> bool:
conn = get_db()
row = conn.execute(
"SELECT 1 FROM shortfall_notices WHERE reference = ? LIMIT 1", (reference,)
).fetchone()
conn.close()
return bool(row)
def mark_shortfall_notice_sent(reference: str, email: str, amount: float) -> None:
conn = get_db()
_ensure_shortfall_tables(conn)
conn.execute(
"INSERT OR REPLACE INTO shortfall_notices (reference, email, amount, sent_at)"
" VALUES (?, ?, ?, datetime('now'))",
(reference, email, amount),
)
conn.commit()
conn.close()
async def fetch_oc_monthly_order_charges() -> list[dict]:
"""Every MONTHLY order's real contribution charges — one record per order.
A record is {"slug", "order_id", "charges": [(datetime, publicId), ...]} from
ONE order. Truncation is judged per order: merging the charges of two
DIFFERENT subscriptions a member happens to hold invents a short gap that is
not a truncated cycle (one member started a second monthly order six days
after the first; that is not OC re-aligning a charge). Raises on failure so
the caller skips rather than mis-decides. Charges are the CONTRIBUTION
amounts INTO the collective (a non-refund CREDIT), one per charge.
"""
if not OC_PERSONAL_TOKEN:
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch order charges")
query = (
'{ collective(slug: "%s") { orders(limit: 200) { totalCount nodes { '
"id frequency fromAccount { slug } "
"transactions { publicId createdAt type kind isRefund } } } } }"
% OC_COLLECTIVE_SLUG
)
async with httpx.AsyncClient(timeout=30.0) as client:
r = await client.post(
OC_GRAPHQL_URL,
headers={"Content-Type": "application/json",
"Authorization": f"Bearer {OC_PERSONAL_TOKEN}",
"Api-Version": "2022-01-01",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
json={"query": query},
)
if r.status_code != 200:
raise RuntimeError(f"OC order-charge fetch failed: HTTP {r.status_code}")
body = r.json()
if body.get("errors"):
raise RuntimeError(f"OC order-charge fetch errors: {str(body['errors'])[:200]}")
nodes = (((body.get("data") or {}).get("collective") or {}).get("orders") or {}).get("nodes") or []
out: list[dict] = []
for o in nodes:
if (o.get("frequency") or "").upper() != "MONTHLY":
continue
slug = ((o.get("fromAccount") or {}).get("slug") or "").strip()
if not slug:
continue
charges = []
for t in o.get("transactions") or []:
if t.get("kind") != "CONTRIBUTION" or t.get("isRefund"):
continue
if (t.get("type") or "").upper() != "CREDIT":
continue
dt = _parse_oc_dt(t.get("createdAt"))
if dt and t.get("publicId"):
charges.append((dt, t["publicId"]))
charges.sort()
if charges:
out.append({"slug": slug, "order_id": o.get("id"), "charges": charges})
return out
def derive_shortfalls(order_records: list[dict], budget: float | None = None,
full_month_days: int = FULL_PAID_MONTH_DAYS) -> list[dict]:
"""Pure: truncated paid cycles from per-order monthly charge ladders.
For each consecutive pair of charges on the SAME order, a gap shorter than a
full month means the member paid for a full month and received fewer days of
allowance. Credit = budget * days_lost / full_month_days (no rounding). One
record per truncated cycle; `reference` is the early charge that truncated it
(globally unique, so it is the idempotence key).
"""
budget = MEMBER_BUDGET if budget is None else budget
out: list[dict] = []
for rec in order_records:
charges = rec.get("charges") or []
for i in range(1, len(charges)):
prev_dt, _ = charges[i - 1]
cur_dt, cur_id = charges[i]
gap = (cur_dt - prev_dt).days
if gap >= full_month_days:
continue
lost = full_month_days - gap
out.append({
"slug": rec.get("slug"), "order_id": rec.get("order_id"),
"prev": prev_dt.date().isoformat(), "cur": cur_dt.date().isoformat(),
"gap_days": gap, "days_lost": lost,
"amount": budget * lost / full_month_days, "reference": cur_id,
})
return out
def shortfall_email(name: str, amount: float, prev: str, cur: str,
gap_days: int, days_lost: int, pack_url: str) -> tuple[str, str, str]:
"""Build the shortfall credit notice (subject, text, html).
Same shell as welcome_email()/budget_notice_email(): cream background, white
card, forest-green button, inline pine-tree mark, table-based, no external
assets. Plain and direct — says what happened, why, the amount, and how to get
more. No filler, no apology theatre.
"""
subject = "A credit for a short allowance cycle"
text = (
f"Hi {name},\n\n"
"Your Open Collective contribution is charged on the collective's billing "
"day, not on the day you joined. That can make a charge arrive less than a "
"month after the one before it, so a month you paid for turns out shorter "
"than a month of allowance.\n\n"
f"That happened on your account: charges on {prev} and {cur} were only "
f"{gap_days} days apart, so you paid for a full month and received "
f"{days_lost} days less than one.\n\n"
f"To make that up, we've added {_usd(amount)} to your account as credits. "
"Credits don't expire, and they're used after your monthly allowance.\n\n"
"If you'd like to add more, you can buy credit packs here:\n\n"
f"{pack_url}\n\n"
"Questions? Reply to this email or write to info@inference.coop.\n\n"
"— The Inference Cooperative\n"
)
html = f"""<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
<tr>
<td align="center">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
<tr>
<td align="center" style="padding-bottom:24px;">
{PINE_LOGO_SVG}
</td>
</tr>
<tr>
<td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">A credit for a short allowance cycle</h1>
<p style="margin:0 0 8px;font-size:16px;color:#6b7a62;line-height:1.5;">Your Open Collective contribution is charged on the collective's billing day, not on the day you joined. That can make a charge arrive less than a month after the one before it, so a month you paid for turns out shorter than a month of allowance.</p>
<p style="margin:0 0 20px;font-size:15px;color:#6b7a62;line-height:1.5;">That happened on your account: charges on {prev} and {cur} were only {gap_days} days apart, so you paid for a full month and received {days_lost} days less than one.</p>
<p style="margin:0 0 24px;font-size:15px;color:#2d3327;line-height:1.5;"><strong>We've added {_usd(amount)} to your account as credits.</strong> Credits don't expire, and they're used after your monthly allowance.</p>
<a href="{pack_url}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:999px;font-size:16px;font-weight:600;">Buy credit packs</a>
<p style="margin:28px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">If you'd like to add more, you can buy credit packs at the button above.</p>
</td>
</tr>
<tr>
<td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
Questions? Reply to this email or write to <a href="mailto:info@inference.coop" style="color:#6b7a62;">info@inference.coop</a>.<br>
<span style="color:#2d3327;">The Inference Cooperative</span> &middot; inference.coop
</td>
</tr>
</table>
</td>
</tr>
</table>
</body>
</html>"""
return subject, text, html
async def reconcile_allowance_shortfalls(send: bool = False) -> dict:
"""Credit members whose PAID cycle Open Collective's billing-day alignment truncated.
`send=False` (report) computes the set and changes NOTHING. `send=True`
(honoured only when ALLOWANCE_POLICY == "on") applies one credit per truncated
charge — idempotent on the charge reference — and, when
ALLOWANCE_SHORTFALL_EMAILS == "on", sends the member notice (once per charge).
Emails default OFF so the wording is reviewed before any member is written to.
"""
try:
records = await fetch_oc_monthly_order_charges()
except Exception as e:
logger.warning("Allowance shortfall: charge fetch failed; skipping (fail-safe): %s", e)
return {"status": "skipped", "reason": "charge_fetch_failed", "detail": str(e)[:200]}
try:
index = await fetch_member_index()
except Exception as e:
logger.warning("Allowance shortfall: member index fetch failed; skipping (fail-safe): %s", e)
return {"status": "skipped", "reason": "member_index_failed", "detail": str(e)[:200]}
cycles = derive_shortfalls(records)
# slug -> email from the LIVE OC member list; only provisioned, active members
# are credited. Orders are matched by the slug resolved from the member's email
# (the stored column can hold a stale guest-… slug from first provisioning).
slug2email: dict[str, str] = {}
for e, rec in index.items():
s = rec.get("slug") or ""
if s:
slug2email.setdefault(s, e)
conn = get_db()
_ensure_shortfall_tables(conn)
active = {r[0].strip().lower() for r in conn.execute("SELECT email FROM members WHERE active = 1")}
meta = {r[0].strip().lower(): {"slug": r[1], "org_slug": r[2]}
for r in conn.execute("SELECT email, slug, org_slug FROM members")}
conn.close()
acting = bool(send and ALLOWANCE_POLICY == "on")
mail_on = ALLOWANCE_SHORTFALL_EMAILS == "on"
pack_url = credit_pack_url()
credited, staged, skipped = [], [], []
seen: set[str] = set()
for c in cycles:
if c["reference"] in seen: # a charge truncates exactly one cycle
continue
email = slug2email.get(c["slug"])
if not email:
skipped.append({**c, "why": "no Open Collective account email for this charge's slug"})
continue
if email not in active:
skipped.append({**c, "email": email, "why": "not an active portal member"})
continue
m = meta.get(email) or {}
if email in MANUAL_MEMBERS or m.get("org_slug"):
skipped.append({**c, "email": email, "why": "exempt member (manual / organization-backed)"})
continue
rec = index.get(email)
if rec and rec.get("role") == "ADMIN":
skipped.append({**c, "email": email, "why": "operator account"})
continue
if (email, c["reference"]) in SHORTFALL_REMEDIATED:
skipped.append({**c, "email": email,
"why": "already remediated by a manual allowance reset (2026-11-04)"})
continue
seen.add(c["reference"])
entry = {**c, "email": email, "name": (m.get("slug") or "there")}
already = credit_reference_exists(c["reference"])
entry["already_credited"] = already
if not already and acting:
add_credits(email, c["amount"],
reason=shortfall_reason(c["prev"], c["cur"]),
reference=c["reference"])
try:
await sync_team_budget_from_balance(email)
except Exception as e: # budget push is best-effort
logger.warning("Shortfall: team budget sync failed for %s: %s", email, e)
entry["credited"] = True
elif not already:
entry["credited"] = False # would-credit (report mode / policy off)
if mail_on and not shortfall_notice_sent(c["reference"]):
subject, text_body, html_body = shortfall_email(
entry["name"], c["amount"], c["prev"], c["cur"], c["gap_days"],
c["days_lost"], pack_url)
if send_email(email, subject, text_body, html_body):
mark_shortfall_notice_sent(c["reference"], email, c["amount"])
entry["notified"] = True
else:
entry["notified"] = False
elif not mail_on:
entry["notice"] = "staged (ALLOWANCE_SHORTFALL_EMAILS is off)"
(credited if entry.get("credited") or entry.get("already_credited") else staged).append(entry)
newly = sum(e["amount"] for e in credited if e.get("credited"))
total = sum(e["amount"] for e in credited)
return {
"status": "ok",
"mode": ALLOWANCE_POLICY,
"acting": acting,
"emails": ALLOWANCE_SHORTFALL_EMAILS,
"full_month_days": FULL_PAID_MONTH_DAYS,
"affected": len(credited),
"newly_credited": sum(1 for e in credited if e.get("credited")),
"already_credited": sum(1 for e in credited if e.get("already_credited")),
"new_credit_total": round(newly, 10),
"affected_credit_total": round(total, 10),
"credits": credited,
"would_credit": staged,
"skipped": skipped,
}
@app.post("/admin/shortfall-credits")
@limiter.limit("10/minute")
async def admin_shortfall_credits(request: Request, mode: str = "report"):
"""Preview (default) or apply allowance-shortfall credits.
`mode=report` computes the affected set and changes nothing. `mode=send`
applies the credits (requires ALLOWANCE_POLICY=on) — idempotent on the OC
charge reference, so a second call credits nothing again. Member notices are
sent only when ALLOWANCE_SHORTFALL_EMAILS=on.
"""
_verify_admin_token(request)
if mode == "send":
if ALLOWANCE_POLICY != "on":
return {"refused": f"ALLOWANCE_POLICY is '{ALLOWANCE_POLICY}', not 'on'"}
return await reconcile_allowance_shortfalls(send=True)
return await reconcile_allowance_shortfalls(send=False)
@app.post("/admin/allowance")
@limiter.limit("10/minute")
async def admin_allowance(request: Request, mode: str = "report", to: str | None = None):
@@ -4846,6 +5223,16 @@ async def admin_sweep(request: Request):
except Exception as exc: # noqa: BLE001
result["allowances"] = {"error": str(exc)[:200]}
# Allowance-shortfall credits: OC's billing-day alignment truncates some paid
# cycles. Rides the same sweep so a new member is compensated automatically;
# credits are idempotent on the OC charge reference. Wrapped so it can never
# break reconciliation; notices only go out when ALLOWANCE_SHORTFALL_EMAILS=on.
try:
result["shortfall_credits"] = await reconcile_allowance_shortfalls(
send=(ALLOWANCE_POLICY == "on"))
except Exception as exc: # noqa: BLE001
result["shortfall_credits"] = {"error": str(exc)[:200]}
return result