metrics: membership history from list-join dates, not welcome_sent_at
The backfill's first real curve was wrong: built on welcome_sent_at it showed 34 members on 19 October, when the co-op had ~65. Most members were provisioned before the portal sent welcome emails, so that field lags by dozens of people — and a chart built on it would have misrepresented the co-op's growth. Membership now comes from Listmonk's per-subscription subscription_created_at on the members list, which the portal's nightly reconcile keeps mirroring active members. That reproduces the real curve (13 members mid-September → 71 today). Past days are the cumulative joins of today's members — an honest floor, documented as such, since members who left are absent.
This commit is contained in:
1 parent
c5d5fa2fac
commit
1cf78c4e56
1 file changed
+55
-10
+55
-10
@@ -70,6 +70,42 @@ def _get_json(url: str, headers: dict | None = None):
|
||||
|
||||
# ---------------------------------------------------------------- sources
|
||||
|
||||
def listmonk_joins() -> list[str] | None:
|
||||
"""The date each current member joined the members list (YYYY-MM-DD).
|
||||
|
||||
This is the honest source for a membership curve: the list is maintained by
|
||||
the portal's nightly reconcile to mirror ACTIVE members, and each
|
||||
subscription carries `subscription_created_at`. `welcome_sent_at` in the
|
||||
members table is NOT equivalent — most members were provisioned before the
|
||||
portal sent welcome emails, so a curve built on it lags by dozens of people.
|
||||
"""
|
||||
token, user = os.environ.get("LISTMONK_API_TOKEN"), os.environ.get("LISTMONK_API_USER") or "portal-sync"
|
||||
if not token:
|
||||
print("metrics: LISTMONK_API_TOKEN not set — membership history unavailable")
|
||||
return None
|
||||
joins, page = [], 1
|
||||
try:
|
||||
while True:
|
||||
data = _get_json(
|
||||
f"{LISTMONK_BASE}/api/subscribers?list_id={LISTMONK_LIST_ID}&per_page=100&page={page}",
|
||||
{"Authorization": f"token {user}:{token}"},
|
||||
) or {}
|
||||
rows = ((data.get("data") or {}).get("results")) or []
|
||||
for s in rows:
|
||||
for membership in s.get("lists") or []:
|
||||
if membership.get("id") == LISTMONK_LIST_ID:
|
||||
stamp = membership.get("subscription_created_at") or s.get("created_at")
|
||||
if stamp:
|
||||
joins.append(str(stamp)[:10])
|
||||
if not rows or len(rows) < 100 or page >= 20:
|
||||
break
|
||||
page += 1
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"metrics: listmonk joins read failed ({exc})")
|
||||
return None
|
||||
return joins or None
|
||||
|
||||
|
||||
def from_sqlite() -> dict:
|
||||
"""Members, plus the provisioning dates that give the growth curve."""
|
||||
conn = connect()
|
||||
@@ -230,13 +266,14 @@ def snapshot(day: str | None = None) -> dict:
|
||||
day = day or today()
|
||||
m = from_sqlite()
|
||||
subs = from_listmonk()
|
||||
joins = listmonk_joins()
|
||||
space = from_matrix()
|
||||
spend = spend_by_day(since=dt.date.fromisoformat(day) - dt.timedelta(days=1))
|
||||
today_spend = spend.get(day, (None, None))
|
||||
row = {
|
||||
"members_active": m["active"],
|
||||
"members_total": m["total"],
|
||||
"members_new": sum(1 for j in m["joins"] if str(j)[:10] == day),
|
||||
"members_new": (sum(1 for j in joins if j == day) if joins is not None else None),
|
||||
"newsletter_subs": subs,
|
||||
"matrix_space_members": space,
|
||||
"spend_usd": today_spend[0],
|
||||
@@ -248,29 +285,37 @@ def snapshot(day: str | None = None) -> dict:
|
||||
|
||||
|
||||
def backfill() -> None:
|
||||
"""Rebuild history: cumulative members from provisioning dates, spend by day."""
|
||||
"""Rebuild history: membership from list-join dates, usage from spend logs.
|
||||
|
||||
Membership uses Listmonk's per-subscription `subscription_created_at` — the
|
||||
join date of every CURRENT member. Members who joined and later left are not
|
||||
in it (their departure is likewise absent), so past days are the cumulative
|
||||
joins of today's members: an honest floor for "how many of us were here by
|
||||
that day", not a reconstruction of the roster at the time.
|
||||
"""
|
||||
m = from_sqlite()
|
||||
joins = sorted(str(j)[:10] for j in m["joins"])
|
||||
joins = sorted(listmonk_joins() or [])
|
||||
spend = spend_by_day()
|
||||
if not joins:
|
||||
print("metrics: no membership history (listmonk unavailable) — past days left NULL for members")
|
||||
if not joins and not spend:
|
||||
print("metrics: backfill found no history")
|
||||
print("metrics: backfill found no history at all")
|
||||
return
|
||||
|
||||
first = min([d for d in joins] + ([min(spend)] if spend else []))
|
||||
candidates = list(joins) + list(spend)
|
||||
first = min(candidates)
|
||||
last = today()
|
||||
print(
|
||||
f"metrics: backfilling {first}..{last} "
|
||||
f"({len(joins)} provisioning dates, {len(spend)} spend days)"
|
||||
f"({len(joins)} join dates, {len(spend)} spend days)"
|
||||
)
|
||||
day = dt.date.fromisoformat(first)
|
||||
end = dt.date.fromisoformat(last)
|
||||
written = 0
|
||||
while day <= end:
|
||||
key = day.isoformat()
|
||||
# Cumulative members AS OF this day, from provisioning dates. Members
|
||||
# provisioned without a welcome email (manual/founder adds) are not in
|
||||
# this history, so backfilled days are a LOWER BOUND — today's row below
|
||||
# carries the exact count.
|
||||
# Cumulative members AS OF this day: current members whose list-join date
|
||||
# is on or before it. Days before anyone joined stay 0.
|
||||
members_total = sum(1 for j in joins if j <= key) if joins else None
|
||||
row = {
|
||||
"members_total": members_total,
|
||||
|
||||
Reference in new issue
Block a user