metrics: membership history from list-join dates, not welcome_sent_at
The backfill's first real curve was wrong: built on welcome_sent_at it showed 34 members on 19 October, when the co-op had ~65. Most members were provisioned before the portal sent welcome emails, so that field lags by dozens of people — and a chart built on it would have misrepresented the co-op's growth. Membership now comes from Listmonk's per-subscription subscription_created_at on the members list, which the portal's nightly reconcile keeps mirroring active members. That reproduces the real curve (13 members mid-September → 71 today). Past days are the cumulative joins of today's members — an honest floor, documented as such, since members who left are absent.
This commit is contained in:
1 parent
c5d5fa2fac
commit
1cf78c4e56
1 file changed
+55
-10
+55
-10
@@ -70,6 +70,42 @@ def _get_json(url: str, headers: dict | None = None):
|
|||||||
|
|
||||||
# ---------------------------------------------------------------- sources
|
# ---------------------------------------------------------------- sources
|
||||||
|
|
||||||
|
def listmonk_joins() -> list[str] | None:
|
||||||
|
"""The date each current member joined the members list (YYYY-MM-DD).
|
||||||
|
|
||||||
|
This is the honest source for a membership curve: the list is maintained by
|
||||||
|
the portal's nightly reconcile to mirror ACTIVE members, and each
|
||||||
|
subscription carries `subscription_created_at`. `welcome_sent_at` in the
|
||||||
|
members table is NOT equivalent — most members were provisioned before the
|
||||||
|
portal sent welcome emails, so a curve built on it lags by dozens of people.
|
||||||
|
"""
|
||||||
|
token, user = os.environ.get("LISTMONK_API_TOKEN"), os.environ.get("LISTMONK_API_USER") or "portal-sync"
|
||||||
|
if not token:
|
||||||
|
print("metrics: LISTMONK_API_TOKEN not set — membership history unavailable")
|
||||||
|
return None
|
||||||
|
joins, page = [], 1
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
data = _get_json(
|
||||||
|
f"{LISTMONK_BASE}/api/subscribers?list_id={LISTMONK_LIST_ID}&per_page=100&page={page}",
|
||||||
|
{"Authorization": f"token {user}:{token}"},
|
||||||
|
) or {}
|
||||||
|
rows = ((data.get("data") or {}).get("results")) or []
|
||||||
|
for s in rows:
|
||||||
|
for membership in s.get("lists") or []:
|
||||||
|
if membership.get("id") == LISTMONK_LIST_ID:
|
||||||
|
stamp = membership.get("subscription_created_at") or s.get("created_at")
|
||||||
|
if stamp:
|
||||||
|
joins.append(str(stamp)[:10])
|
||||||
|
if not rows or len(rows) < 100 or page >= 20:
|
||||||
|
break
|
||||||
|
page += 1
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
print(f"metrics: listmonk joins read failed ({exc})")
|
||||||
|
return None
|
||||||
|
return joins or None
|
||||||
|
|
||||||
|
|
||||||
def from_sqlite() -> dict:
|
def from_sqlite() -> dict:
|
||||||
"""Members, plus the provisioning dates that give the growth curve."""
|
"""Members, plus the provisioning dates that give the growth curve."""
|
||||||
conn = connect()
|
conn = connect()
|
||||||
@@ -230,13 +266,14 @@ def snapshot(day: str | None = None) -> dict:
|
|||||||
day = day or today()
|
day = day or today()
|
||||||
m = from_sqlite()
|
m = from_sqlite()
|
||||||
subs = from_listmonk()
|
subs = from_listmonk()
|
||||||
|
joins = listmonk_joins()
|
||||||
space = from_matrix()
|
space = from_matrix()
|
||||||
spend = spend_by_day(since=dt.date.fromisoformat(day) - dt.timedelta(days=1))
|
spend = spend_by_day(since=dt.date.fromisoformat(day) - dt.timedelta(days=1))
|
||||||
today_spend = spend.get(day, (None, None))
|
today_spend = spend.get(day, (None, None))
|
||||||
row = {
|
row = {
|
||||||
"members_active": m["active"],
|
"members_active": m["active"],
|
||||||
"members_total": m["total"],
|
"members_total": m["total"],
|
||||||
"members_new": sum(1 for j in m["joins"] if str(j)[:10] == day),
|
"members_new": (sum(1 for j in joins if j == day) if joins is not None else None),
|
||||||
"newsletter_subs": subs,
|
"newsletter_subs": subs,
|
||||||
"matrix_space_members": space,
|
"matrix_space_members": space,
|
||||||
"spend_usd": today_spend[0],
|
"spend_usd": today_spend[0],
|
||||||
@@ -248,29 +285,37 @@ def snapshot(day: str | None = None) -> dict:
|
|||||||
|
|
||||||
|
|
||||||
def backfill() -> None:
|
def backfill() -> None:
|
||||||
"""Rebuild history: cumulative members from provisioning dates, spend by day."""
|
"""Rebuild history: membership from list-join dates, usage from spend logs.
|
||||||
|
|
||||||
|
Membership uses Listmonk's per-subscription `subscription_created_at` — the
|
||||||
|
join date of every CURRENT member. Members who joined and later left are not
|
||||||
|
in it (their departure is likewise absent), so past days are the cumulative
|
||||||
|
joins of today's members: an honest floor for "how many of us were here by
|
||||||
|
that day", not a reconstruction of the roster at the time.
|
||||||
|
"""
|
||||||
m = from_sqlite()
|
m = from_sqlite()
|
||||||
joins = sorted(str(j)[:10] for j in m["joins"])
|
joins = sorted(listmonk_joins() or [])
|
||||||
spend = spend_by_day()
|
spend = spend_by_day()
|
||||||
|
if not joins:
|
||||||
|
print("metrics: no membership history (listmonk unavailable) — past days left NULL for members")
|
||||||
if not joins and not spend:
|
if not joins and not spend:
|
||||||
print("metrics: backfill found no history")
|
print("metrics: backfill found no history at all")
|
||||||
return
|
return
|
||||||
|
|
||||||
first = min([d for d in joins] + ([min(spend)] if spend else []))
|
candidates = list(joins) + list(spend)
|
||||||
|
first = min(candidates)
|
||||||
last = today()
|
last = today()
|
||||||
print(
|
print(
|
||||||
f"metrics: backfilling {first}..{last} "
|
f"metrics: backfilling {first}..{last} "
|
||||||
f"({len(joins)} provisioning dates, {len(spend)} spend days)"
|
f"({len(joins)} join dates, {len(spend)} spend days)"
|
||||||
)
|
)
|
||||||
day = dt.date.fromisoformat(first)
|
day = dt.date.fromisoformat(first)
|
||||||
end = dt.date.fromisoformat(last)
|
end = dt.date.fromisoformat(last)
|
||||||
written = 0
|
written = 0
|
||||||
while day <= end:
|
while day <= end:
|
||||||
key = day.isoformat()
|
key = day.isoformat()
|
||||||
# Cumulative members AS OF this day, from provisioning dates. Members
|
# Cumulative members AS OF this day: current members whose list-join date
|
||||||
# provisioned without a welcome email (manual/founder adds) are not in
|
# is on or before it. Days before anyone joined stay 0.
|
||||||
# this history, so backfilled days are a LOWER BOUND — today's row below
|
|
||||||
# carries the exact count.
|
|
||||||
members_total = sum(1 for j in joins if j <= key) if joins else None
|
members_total = sum(1 for j in joins if j <= key) if joins else None
|
||||||
row = {
|
row = {
|
||||||
"members_total": members_total,
|
"members_total": members_total,
|
||||||
|
|||||||
Reference in new issue
Block a user