From 1cf78c4e56a47dc5d84a962bb7caf04293420838 Mon Sep 17 00:00:00 2001 From: Inferencebot Date: Wed, 28 Oct 2026 03:56:06 +0000 Subject: [PATCH] metrics: membership history from list-join dates, not welcome_sent_at MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backfill's first real curve was wrong: built on welcome_sent_at it showed 34 members on 19 October, when the co-op had ~65. Most members were provisioned before the portal sent welcome emails, so that field lags by dozens of people — and a chart built on it would have misrepresented the co-op's growth. Membership now comes from Listmonk's per-subscription subscription_created_at on the members list, which the portal's nightly reconcile keeps mirroring active members. That reproduces the real curve (13 members mid-September → 71 today). Past days are the cumulative joins of today's members — an honest floor, documented as such, since members who left are absent. --- metrics_snapshot.py | 65 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/metrics_snapshot.py b/metrics_snapshot.py index ee216a9..f2ea1a8 100644 --- a/metrics_snapshot.py +++ b/metrics_snapshot.py @@ -70,6 +70,42 @@ def _get_json(url: str, headers: dict | None = None): # ---------------------------------------------------------------- sources +def listmonk_joins() -> list[str] | None: + """The date each current member joined the members list (YYYY-MM-DD). + + This is the honest source for a membership curve: the list is maintained by + the portal's nightly reconcile to mirror ACTIVE members, and each + subscription carries `subscription_created_at`. `welcome_sent_at` in the + members table is NOT equivalent — most members were provisioned before the + portal sent welcome emails, so a curve built on it lags by dozens of people. + """ + token, user = os.environ.get("LISTMONK_API_TOKEN"), os.environ.get("LISTMONK_API_USER") or "portal-sync" + if not token: + print("metrics: LISTMONK_API_TOKEN not set — membership history unavailable") + return None + joins, page = [], 1 + try: + while True: + data = _get_json( + f"{LISTMONK_BASE}/api/subscribers?list_id={LISTMONK_LIST_ID}&per_page=100&page={page}", + {"Authorization": f"token {user}:{token}"}, + ) or {} + rows = ((data.get("data") or {}).get("results")) or [] + for s in rows: + for membership in s.get("lists") or []: + if membership.get("id") == LISTMONK_LIST_ID: + stamp = membership.get("subscription_created_at") or s.get("created_at") + if stamp: + joins.append(str(stamp)[:10]) + if not rows or len(rows) < 100 or page >= 20: + break + page += 1 + except Exception as exc: # noqa: BLE001 + print(f"metrics: listmonk joins read failed ({exc})") + return None + return joins or None + + def from_sqlite() -> dict: """Members, plus the provisioning dates that give the growth curve.""" conn = connect() @@ -230,13 +266,14 @@ def snapshot(day: str | None = None) -> dict: day = day or today() m = from_sqlite() subs = from_listmonk() + joins = listmonk_joins() space = from_matrix() spend = spend_by_day(since=dt.date.fromisoformat(day) - dt.timedelta(days=1)) today_spend = spend.get(day, (None, None)) row = { "members_active": m["active"], "members_total": m["total"], - "members_new": sum(1 for j in m["joins"] if str(j)[:10] == day), + "members_new": (sum(1 for j in joins if j == day) if joins is not None else None), "newsletter_subs": subs, "matrix_space_members": space, "spend_usd": today_spend[0], @@ -248,29 +285,37 @@ def snapshot(day: str | None = None) -> dict: def backfill() -> None: - """Rebuild history: cumulative members from provisioning dates, spend by day.""" + """Rebuild history: membership from list-join dates, usage from spend logs. + + Membership uses Listmonk's per-subscription `subscription_created_at` — the + join date of every CURRENT member. Members who joined and later left are not + in it (their departure is likewise absent), so past days are the cumulative + joins of today's members: an honest floor for "how many of us were here by + that day", not a reconstruction of the roster at the time. + """ m = from_sqlite() - joins = sorted(str(j)[:10] for j in m["joins"]) + joins = sorted(listmonk_joins() or []) spend = spend_by_day() + if not joins: + print("metrics: no membership history (listmonk unavailable) — past days left NULL for members") if not joins and not spend: - print("metrics: backfill found no history") + print("metrics: backfill found no history at all") return - first = min([d for d in joins] + ([min(spend)] if spend else [])) + candidates = list(joins) + list(spend) + first = min(candidates) last = today() print( f"metrics: backfilling {first}..{last} " - f"({len(joins)} provisioning dates, {len(spend)} spend days)" + f"({len(joins)} join dates, {len(spend)} spend days)" ) day = dt.date.fromisoformat(first) end = dt.date.fromisoformat(last) written = 0 while day <= end: key = day.isoformat() - # Cumulative members AS OF this day, from provisioning dates. Members - # provisioned without a welcome email (manual/founder adds) are not in - # this history, so backfilled days are a LOWER BOUND — today's row below - # carries the exact count. + # Cumulative members AS OF this day: current members whose list-join date + # is on or before it. Days before anyone joined stay 0. members_total = sum(1 for j in joins if j <= key) if joins else None row = { "members_total": members_total,