privacy: redact member emails from portal logs (audit M1)
The _RedactWebhookPath filter now also strips email-shaped strings from every log record — messages, %-format args (recursing into tuple/list/ dict args), and any exception text already attached to the record — the same retention discipline the webhook token got in ops/security#2. _send_email recipients, listmonk failures, Loomio invite lists and provisioning/credit lines carry member addresses into container logs, which persist on the host and ride the backups. MXIDs, counts, slugs and amounts stay readable; the daily digest's own sanitisation is unchanged. Filter keeps its never-break-logging guarantee (all redaction inside try/except; non-string args untouched). Live pre-state measured: 99 email-bearing lines in the current app log.
This commit is contained in:
1 parent
2b501b75be
commit
d63f702ec3
2 files changed
+97
-6
No files matched your search
+41
-6
@@ -114,24 +114,59 @@ logger = logging.getLogger("member-portal")
|
|||||||
# sit on the access logger and on the root handler so both uvicorn's line and any
|
# sit on the access logger and on the root handler so both uvicorn's line and any
|
||||||
# string the app itself logs get scrubbed.
|
# string the app itself logs get scrubbed.
|
||||||
_WEBHOOK_TOKEN_RE = re.compile(r"(/webhook/opencollective/)[^\s\"'?&]+")
|
_WEBHOOK_TOKEN_RE = re.compile(r"(/webhook/opencollective/)[^\s\"'?&]+")
|
||||||
|
# M1 (audit 2026-11-05): member emails persist in portal logs — send_email
|
||||||
|
# recipients, listmonk failures, Loomio invite lists, provisioning and credit
|
||||||
|
# lines (17 sites). Container logs are readable via `cloudron logs`, persist
|
||||||
|
# on the host and ride the backups, so an email is redacted from every record
|
||||||
|
# the same way the webhook token is. Logs stay useful: counts, slugs, MXIDs
|
||||||
|
# (matrix @user:server has no address shape) and amounts all survive; only
|
||||||
|
# the address is dropped. The daily digest's own sanitisation is unchanged.
|
||||||
|
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+")
|
||||||
|
|
||||||
|
|
||||||
def _redact_webhook_path(value):
|
def _redact_emails(value):
|
||||||
|
if isinstance(value, str) and "@" in value:
|
||||||
|
return _EMAIL_RE.sub("<email>", value)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_log_value(value):
|
||||||
if isinstance(value, str) and "/webhook/opencollective/" in value:
|
if isinstance(value, str) and "/webhook/opencollective/" in value:
|
||||||
return _WEBHOOK_TOKEN_RE.sub(r"\1<redacted>", value)
|
value = _WEBHOOK_TOKEN_RE.sub(r"\1<redacted>", value)
|
||||||
|
return _redact_emails(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_deep(value):
|
||||||
|
"""Redact every string in a log argument, preserving the argument's type."""
|
||||||
|
if isinstance(value, str):
|
||||||
|
return _redact_log_value(value)
|
||||||
|
if isinstance(value, tuple):
|
||||||
|
return tuple(_redact_deep(v) for v in value)
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [_redact_deep(v) for v in value]
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {k: _redact_deep(v) for k, v in value.items()}
|
||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
class _RedactWebhookPath(logging.Filter):
|
class _RedactWebhookPath(logging.Filter):
|
||||||
"""Replace the webhook token in any logged request path with <redacted>."""
|
"""Scrub secrets and member emails out of any log record.
|
||||||
|
|
||||||
|
Replaces the webhook token in a logged request path (ops/security#2) and
|
||||||
|
redacts email-shaped strings from messages, %-format args (recursing into
|
||||||
|
tuple/list/dict args) and any exception text already attached to the
|
||||||
|
record. A logging filter must never break logging.
|
||||||
|
"""
|
||||||
|
|
||||||
def filter(self, record: logging.LogRecord) -> bool:
|
def filter(self, record: logging.LogRecord) -> bool:
|
||||||
try:
|
try:
|
||||||
record.msg = _redact_webhook_path(record.msg)
|
record.msg = _redact_log_value(record.msg)
|
||||||
if isinstance(record.args, tuple):
|
if isinstance(record.args, tuple):
|
||||||
record.args = tuple(_redact_webhook_path(a) for a in record.args)
|
record.args = tuple(_redact_deep(a) for a in record.args)
|
||||||
elif isinstance(record.args, dict):
|
elif isinstance(record.args, dict):
|
||||||
record.args = {k: _redact_webhook_path(v) for k, v in record.args.items()}
|
record.args = {k: _redact_deep(v) for k, v in record.args.items()}
|
||||||
|
if isinstance(record.exc_text, str):
|
||||||
|
record.exc_text = _redact_log_value(record.exc_text)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass # a logging filter must never break logging
|
pass # a logging filter must never break logging
|
||||||
return True
|
return True
|
||||||
|
|||||||
@@ -205,5 +205,61 @@ try:
|
|||||||
finally:
|
finally:
|
||||||
urllib.request.urlopen = _real_urlopen
|
urllib.request.urlopen = _real_urlopen
|
||||||
|
|
||||||
|
# ------------------------------------------------------- M1: log redaction ---
|
||||||
|
# The webhook token and member emails must not survive into portal logs. The
|
||||||
|
# guard must FIRE: feed a record with both a token and an email and read back
|
||||||
|
# the formatted line.
|
||||||
|
import logging # noqa: E402
|
||||||
|
|
||||||
|
from app.main import _EMAIL_RE, _RedactWebhookPath # noqa: E402
|
||||||
|
|
||||||
|
flt = _RedactWebhookPath()
|
||||||
|
rec = logging.LogRecord("t", logging.INFO, "p", 1,
|
||||||
|
"Credit pack: +%s for %s (balance %s)", (20.0, "member@example.org", 30.0), None)
|
||||||
|
flt.filter(rec)
|
||||||
|
check("email in %-args redacted", rec.getMessage(), "Credit pack: +20.0 for <email> (balance 30.0)")
|
||||||
|
check("argument tuple preserved", isinstance(rec.args, tuple), True)
|
||||||
|
|
||||||
|
rec = logging.LogRecord("t", logging.INFO, "p", 1,
|
||||||
|
"Provisioned member hidden@example.org ok", (), None)
|
||||||
|
flt.filter(rec)
|
||||||
|
check("email embedded via f-string (msg) redacted", rec.getMessage(), "Provisioned member <email> ok")
|
||||||
|
|
||||||
|
rec = logging.LogRecord("t", logging.INFO, "p", 1, "Loomio invites: %s",
|
||||||
|
(["a@example.org", "b@example.org"],), None)
|
||||||
|
flt.filter(rec)
|
||||||
|
check("list args redacted, list type kept",
|
||||||
|
(rec.getMessage(), type(rec.args[0]).__name__), ("Loomio invites: ['<email>', '<email>']", "list"))
|
||||||
|
|
||||||
|
rec = logging.LogRecord("t", logging.INFO, "p", 1,
|
||||||
|
'POST /webhook/opencollective/sekret123?x=1 HTTP/1.1', (), None)
|
||||||
|
filt_result = flt.filter(rec)
|
||||||
|
check("webhook token redacted, filter returns True",
|
||||||
|
(rec.getMessage(), filt_result), ("POST /webhook/opencollective/<redacted>?x=1 HTTP/1.1", True))
|
||||||
|
rec = logging.LogRecord("t", logging.INFO, "p", 1,
|
||||||
|
"Credit pack for member@example.org via /webhook/opencollective/tok9", (), None)
|
||||||
|
flt.filter(rec)
|
||||||
|
check("token and email redacted in one line",
|
||||||
|
"<email>" in rec.getMessage() and "<redacted>" in rec.getMessage()
|
||||||
|
and "member@example.org" not in rec.getMessage() and "tok9" not in rec.getMessage(), True)
|
||||||
|
|
||||||
|
rec = logging.LogRecord("t", logging.INFO, "p", 1, "Matrix invite for @someuser:inference.coop", (), None)
|
||||||
|
flt.filter(rec)
|
||||||
|
check("MXIDs are not emails and stay readable",
|
||||||
|
rec.getMessage(), "Matrix invite for @someuser:inference.coop")
|
||||||
|
|
||||||
|
rec = logging.LogRecord("t", logging.INFO, "p", 1, "counter=%d", (7,), None)
|
||||||
|
flt.filter(rec)
|
||||||
|
check("non-string args untouched", rec.getMessage(), "counter=7")
|
||||||
|
|
||||||
|
rec = logging.LogRecord("t", logging.INFO, "p", 1, "boom", (), None)
|
||||||
|
rec.exc_text = "ValueError: No member account for lost@example.org"
|
||||||
|
flt.filter(rec)
|
||||||
|
check("already-formatted exception text redacted",
|
||||||
|
rec.exc_text, "ValueError: No member account for <email>")
|
||||||
|
|
||||||
|
check("regex matches a real-shaped address",
|
||||||
|
bool(_EMAIL_RE.search("contact info@a.b-stuff.coop today")), True)
|
||||||
|
|
||||||
print(f"\n{'FAILED' if failures else 'OK'} — {failures} failure(s)")
|
print(f"\n{'FAILED' if failures else 'OK'} — {failures} failure(s)")
|
||||||
sys.exit(1 if failures else 0)
|
sys.exit(1 if failures else 0)
|
||||||
Reference in new issue
Block a user