diff --git a/CloudronManifest.json b/CloudronManifest.json index 796d5f1..f39a17c 100644 --- a/CloudronManifest.json +++ b/CloudronManifest.json @@ -14,10 +14,14 @@ }, "sendmail": {}, "scheduler": { - "membership_sweep": { - "schedule": "0 3 * * *", - "command": "/app/code/sweep.sh" - } + "membership_sweep": { + "schedule": "0 3 * * *", + "command": "/app/code/sweep.sh" + }, + "rag_refresh": { + "schedule": "30 3 * * *", + "command": "/app/code/rag_refresh.py" + } } }, "manifestVersion": 2, diff --git a/Dockerfile b/Dockerfile index d2606ba..e138779 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ RUN pip install --no-cache-dir -r requirements.txt # Copy app COPY app/ ./app/ COPY sweep.sh /app/code/sweep.sh -RUN chmod +x /app/code/sweep.sh +COPY rag_refresh.py /app/code/rag_refresh.py +RUN chmod +x /app/code/sweep.sh /app/code/rag_refresh.py # Run as root (the base image default). Cloudron's sandboxing (read-only # rootfs, AppArmor, dropped capabilities) still applies to root inside the diff --git a/rag_refresh.py b/rag_refresh.py new file mode 100644 index 0000000..3c05c42 --- /dev/null +++ b/rag_refresh.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Nightly docs-RAG refresh — server-side (portal container). + +Fetches the co-op docs from the Gitea raw API (public repo, no credentials), +uploads changed files to Open WebUI via the internal network (OWUI_API_KEY, +already in the portal env), attaches them to the shared knowledge collection, +reindexes, and keeps the members group grant intact. + +Replaces the bot-hosted refresh script (same logic, no bot dependency). +Silent-ish: prints a one-line summary; exits non-zero on hard failure so the +Cloudron scheduler log captures it. +""" +import json, os, sys, time, hashlib, urllib.request, urllib.error + +GITEA_RAW = "https://git.inference.coop/co-op/docs/raw/branch/main/" +OWUI_BASE = os.environ.get("OWUI_BASE", "http://chat:8080") +OWUI_KEY = os.environ.get("OWUI_API_KEY", "") +KB_ID = os.environ.get("DOCS_KNOWLEDGE_ID", "") +GID = "eb348e1a-059a-4a8f-8920-0f73942f6272" # OWUI members group +DOCS = ["README.md", "getting-started.md", "models.md", "api-access.md", + "terms-of-service.md", "privacy-policy.md", "charter.md", + "infrastructure.md", "open-questions.md"] +STATE_PATH = "/app/data/rag_refresh_state.json" + +if not OWUI_KEY or not KB_ID: + print("rag_refresh: OWUI_API_KEY / DOCS_KNOWLEDGE_ID not set", file=sys.stderr) + sys.exit(1) + +def api(path, body=None, method=None, key=None, base=None, timeout=180): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request( + f"{base or OWUI_BASE}{path}", data=data, method=method or ("POST" if data else "GET"), + headers={"Authorization": f"Bearer {key or OWUI_KEY}", "Content-Type": "application/json"}) + try: + r = urllib.request.urlopen(req, timeout=timeout) + return r.status, json.loads(r.read()) + except urllib.error.HTTPError as e: + return e.code, e.read().decode()[:200] + +def multipart_upload(filename, content): + boundary = "----ragrefresh9x" + part = (f"--{boundary}\r\n" + f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n' + f"Content-Type: text/markdown\r\n\r\n").encode() + content.encode() + \ + f"\r\n--{boundary}--\r\n".encode() + req = urllib.request.Request( + f"{OWUI_BASE}/api/v1/files/", data=part, method="POST", + headers={"Authorization": f"Bearer {OWUI_KEY}", + "Content-Type": f"multipart/form-data; boundary={boundary}"}) + r = urllib.request.urlopen(req, timeout=180) + return json.loads(r.read()) + +def fetch_doc(name): + req = urllib.request.Request(GITEA_RAW + name) + try: + return urllib.request.urlopen(req, timeout=30).read().decode() + except Exception: + return None + +state = {} +if os.path.exists(STATE_PATH): + try: + state = json.load(open(STATE_PATH)) + except Exception: + state = {} + +changed = 0 +skipped = 0 +for name in DOCS: + content = fetch_doc(name) + if content is None: + print(f"rag_refresh: WARN could not fetch {name}", file=sys.stderr) + continue + newhash = hashlib.sha256(content.encode()).hexdigest() + if state.get(name) == newhash: + skipped += 1 + continue + + # delete stale file rows with this name, then upload fresh + attach + code, files = api(f"/api/v1/files/?name={name}") + if code == 200 and isinstance(files, list): + for f in files: + if f.get("filename") == name: + api(f"/api/v1/files/{f['id']}", method="DELETE") + + try: + resp = multipart_upload(name, content) + except Exception as e: + print(f"rag_refresh: upload failed for {name}: {e}", file=sys.stderr) + continue + fid = resp.get("id") + code, _ = api(f"/api/v1/knowledge/{KB_ID}/file/add", {"file_id": fid}) + if code != 200: + print(f"rag_refresh: attach failed for {name}: {code}", file=sys.stderr) + continue + state[name] = newhash + changed += 1 + +# reindex the KB (chunk copies at attach time go stale otherwise) +if changed: + api("/api/v1/knowledge/reindex", {"knowledge_id": KB_ID}, timeout=300) + +# keep the members-group grant intact (guards against grant loss) +api(f"/api/v1/knowledge/{KB_ID}/access/update", + {"access_grants": [{"principal_type": "group", "principal_id": GID, "permission": "read"}]}) + +json.dump(state, open(STATE_PATH, "w")) +print(f"rag_refresh: changed={changed} skipped={skipped}") +sys.exit(0)