Move docs-RAG refresh into the portal container (Gitea raw API + scheduler addon, 3:30am daily)

rag_refresh.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Nightly docs-RAG refresh — server-side (portal container).
+
+Fetches the co-op docs from the Gitea raw API (public repo, no credentials),
+uploads changed files to Open WebUI via the internal network (OWUI_API_KEY,
+already in the portal env), attaches them to the shared knowledge collection,
+reindexes, and keeps the members group grant intact.
+
+Replaces the bot-hosted refresh script (same logic, no bot dependency).
+Silent-ish: prints a one-line summary; exits non-zero on hard failure so the
+Cloudron scheduler log captures it.
+"""
+import json, os, sys, time, hashlib, urllib.request, urllib.error
+
+GITEA_RAW = "https://git.inference.coop/co-op/docs/raw/branch/main/"
+OWUI_BASE = os.environ.get("OWUI_BASE", "http://chat:8080")
+OWUI_KEY = os.environ.get("OWUI_API_KEY", "")
+KB_ID = os.environ.get("DOCS_KNOWLEDGE_ID", "")
+GID = "eb348e1a-059a-4a8f-8920-0f73942f6272"  # OWUI members group
+DOCS = ["README.md", "getting-started.md", "models.md", "api-access.md",
+        "terms-of-service.md", "privacy-policy.md", "charter.md",
+        "infrastructure.md", "open-questions.md"]
+STATE_PATH = "/app/data/rag_refresh_state.json"
+
+if not OWUI_KEY or not KB_ID:
+    print("rag_refresh: OWUI_API_KEY / DOCS_KNOWLEDGE_ID not set", file=sys.stderr)
+    sys.exit(1)
+
+def api(path, body=None, method=None, key=None, base=None, timeout=180):
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(
+        f"{base or OWUI_BASE}{path}", data=data, method=method or ("POST" if data else "GET"),
+        headers={"Authorization": f"Bearer {key or OWUI_KEY}", "Content-Type": "application/json"})
+    try:
+        r = urllib.request.urlopen(req, timeout=timeout)
+        return r.status, json.loads(r.read())
+    except urllib.error.HTTPError as e:
+        return e.code, e.read().decode()[:200]
+
+def multipart_upload(filename, content):
+    boundary = "----ragrefresh9x"
+    part = (f"--{boundary}\r\n"
+            f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n'
+            f"Content-Type: text/markdown\r\n\r\n").encode() + content.encode() + \
+           f"\r\n--{boundary}--\r\n".encode()
+    req = urllib.request.Request(
+        f"{OWUI_BASE}/api/v1/files/", data=part, method="POST",
+        headers={"Authorization": f"Bearer {OWUI_KEY}",
+                 "Content-Type": f"multipart/form-data; boundary={boundary}"})
+    r = urllib.request.urlopen(req, timeout=180)
+    return json.loads(r.read())
+
+def fetch_doc(name):
+    req = urllib.request.Request(GITEA_RAW + name)
+    try:
+        return urllib.request.urlopen(req, timeout=30).read().decode()
+    except Exception:
+        return None
+
+state = {}
+if os.path.exists(STATE_PATH):
+    try:
+        state = json.load(open(STATE_PATH))
+    except Exception:
+        state = {}
+
+changed = 0
+skipped = 0
+for name in DOCS:
+    content = fetch_doc(name)
+    if content is None:
+        print(f"rag_refresh: WARN could not fetch {name}", file=sys.stderr)
+        continue
+    newhash = hashlib.sha256(content.encode()).hexdigest()
+    if state.get(name) == newhash:
+        skipped += 1
+        continue
+
+    # delete stale file rows with this name, then upload fresh + attach
+    code, files = api(f"/api/v1/files/?name={name}")
+    if code == 200 and isinstance(files, list):
+        for f in files:
+            if f.get("filename") == name:
+                api(f"/api/v1/files/{f['id']}", method="DELETE")
+
+    try:
+        resp = multipart_upload(name, content)
+    except Exception as e:
+        print(f"rag_refresh: upload failed for {name}: {e}", file=sys.stderr)
+        continue
+    fid = resp.get("id")
+    code, _ = api(f"/api/v1/knowledge/{KB_ID}/file/add", {"file_id": fid})
+    if code != 200:
+        print(f"rag_refresh: attach failed for {name}: {code}", file=sys.stderr)
+        continue
+    state[name] = newhash
+    changed += 1
+
+# reindex the KB (chunk copies at attach time go stale otherwise)
+if changed:
+    api("/api/v1/knowledge/reindex", {"knowledge_id": KB_ID}, timeout=300)
+
+# keep the members-group grant intact (guards against grant loss)
+api(f"/api/v1/knowledge/{KB_ID}/access/update",
+    {"access_grants": [{"principal_type": "group", "principal_id": GID, "permission": "read"}]})
+
+json.dump(state, open(STATE_PATH, "w"))
+print(f"rag_refresh: changed={changed} skipped={skipped}")
+sys.exit(0)