metrics: membership history from list-join dates, not welcome_sent_at

The backfill's first real curve was wrong: built on welcome_sent_at it showed
34 members on 19 October, when the co-op had ~65. Most members were provisioned
before the portal sent welcome emails, so that field lags by dozens of people —
and a chart built on it would have misrepresented the co-op's growth.

Membership now comes from Listmonk's per-subscription subscription_created_at
on the members list, which the portal's nightly reconcile keeps mirroring
active members. That reproduces the real curve (13 members mid-September → 71
today). Past days are the cumulative joins of today's members — an honest floor,
documented as such, since members who left are absent.
This commit is contained in:
inference-bot committed 2026-10-28 03:56:06 +00:00
1 parent c5d5fa2fac
commit 1cf78c4e56
1 file changed
+55 -10
+55 -10
View File
@@ -70,6 +70,42 @@ def _get_json(url: str, headers: dict | None = None):
# ---------------------------------------------------------------- sources
def listmonk_joins() -> list[str] | None:
"""The date each current member joined the members list (YYYY-MM-DD).
This is the honest source for a membership curve: the list is maintained by
the portal's nightly reconcile to mirror ACTIVE members, and each
subscription carries `subscription_created_at`. `welcome_sent_at` in the
members table is NOT equivalent — most members were provisioned before the
portal sent welcome emails, so a curve built on it lags by dozens of people.
"""
token, user = os.environ.get("LISTMONK_API_TOKEN"), os.environ.get("LISTMONK_API_USER") or "portal-sync"
if not token:
print("metrics: LISTMONK_API_TOKEN not set — membership history unavailable")
return None
joins, page = [], 1
try:
while True:
data = _get_json(
f"{LISTMONK_BASE}/api/subscribers?list_id={LISTMONK_LIST_ID}&per_page=100&page={page}",
{"Authorization": f"token {user}:{token}"},
) or {}
rows = ((data.get("data") or {}).get("results")) or []
for s in rows:
for membership in s.get("lists") or []:
if membership.get("id") == LISTMONK_LIST_ID:
stamp = membership.get("subscription_created_at") or s.get("created_at")
if stamp:
joins.append(str(stamp)[:10])
if not rows or len(rows) < 100 or page >= 20:
break
page += 1
except Exception as exc: # noqa: BLE001
print(f"metrics: listmonk joins read failed ({exc})")
return None
return joins or None
def from_sqlite() -> dict:
"""Members, plus the provisioning dates that give the growth curve."""
conn = connect()
@@ -230,13 +266,14 @@ def snapshot(day: str | None = None) -> dict:
day = day or today()
m = from_sqlite()
subs = from_listmonk()
joins = listmonk_joins()
space = from_matrix()
spend = spend_by_day(since=dt.date.fromisoformat(day) - dt.timedelta(days=1))
today_spend = spend.get(day, (None, None))
row = {
"members_active": m["active"],
"members_total": m["total"],
"members_new": sum(1 for j in m["joins"] if str(j)[:10] == day),
"members_new": (sum(1 for j in joins if j == day) if joins is not None else None),
"newsletter_subs": subs,
"matrix_space_members": space,
"spend_usd": today_spend[0],
@@ -248,29 +285,37 @@ def snapshot(day: str | None = None) -> dict:
def backfill() -> None:
"""Rebuild history: cumulative members from provisioning dates, spend by day."""
"""Rebuild history: membership from list-join dates, usage from spend logs.
Membership uses Listmonk's per-subscription `subscription_created_at` — the
join date of every CURRENT member. Members who joined and later left are not
in it (their departure is likewise absent), so past days are the cumulative
joins of today's members: an honest floor for "how many of us were here by
that day", not a reconstruction of the roster at the time.
"""
m = from_sqlite()
joins = sorted(str(j)[:10] for j in m["joins"])
joins = sorted(listmonk_joins() or [])
spend = spend_by_day()
if not joins:
print("metrics: no membership history (listmonk unavailable) — past days left NULL for members")
if not joins and not spend:
print("metrics: backfill found no history")
print("metrics: backfill found no history at all")
return
first = min([d for d in joins] + ([min(spend)] if spend else []))
candidates = list(joins) + list(spend)
first = min(candidates)
last = today()
print(
f"metrics: backfilling {first}..{last} "
f"({len(joins)} provisioning dates, {len(spend)} spend days)"
f"({len(joins)} join dates, {len(spend)} spend days)"
)
day = dt.date.fromisoformat(first)
end = dt.date.fromisoformat(last)
written = 0
while day <= end:
key = day.isoformat()
# Cumulative members AS OF this day, from provisioning dates. Members
# provisioned without a welcome email (manual/founder adds) are not in
# this history, so backfilled days are a LOWER BOUND — today's row below
# carries the exact count.
# Cumulative members AS OF this day: current members whose list-join date
# is on or before it. Days before anyone joined stay 0.
members_total = sum(1 for j in joins if j <= key) if joins else None
row = {
"members_total": members_total,