112 lines
4.4 KiB
Python
112 lines
4.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Nightly docs-RAG refresh — server-side (portal container).
|
|
|
|
Fetches the co-op docs from the Gitea raw API (public repo, no credentials),
|
|
uploads changed files to Open WebUI via the internal network (OWUI_API_KEY,
|
|
already in the portal env), attaches them to the shared knowledge collection,
|
|
reindexes, and keeps the members group grant intact.
|
|
|
|
Replaces the bot-hosted refresh script (same logic, no bot dependency).
|
|
Silent-ish: prints a one-line summary; exits non-zero on hard failure so the
|
|
Cloudron scheduler log captures it.
|
|
"""
|
|
import json, os, sys, time, hashlib, urllib.request, urllib.error
|
|
|
|
GITEA_RAW = "https://git.inference.coop/co-op/docs/raw/branch/main/"
|
|
OWUI_BASE = os.environ.get("OWUI_BASE", "http://chat:8080")
|
|
OWUI_KEY = os.environ.get("OWUI_API_KEY", "")
|
|
KB_ID = os.environ.get("DOCS_KNOWLEDGE_ID", "")
|
|
GID = "eb348e1a-059a-4a8f-8920-0f73942f6272" # OWUI members group
|
|
DOCS = ["README.md", "getting-started.md", "models.md", "api-access.md",
|
|
"terms-of-service.md", "privacy-policy.md", "charter.md",
|
|
"infrastructure.md", "open-questions.md"]
|
|
STATE_PATH = "/app/data/rag_refresh_state.json"
|
|
|
|
if not OWUI_KEY or not KB_ID:
|
|
print("rag_refresh: OWUI_API_KEY / DOCS_KNOWLEDGE_ID not set", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
def api(path, body=None, method=None, key=None, base=None, timeout=180):
|
|
data = json.dumps(body).encode() if body is not None else None
|
|
req = urllib.request.Request(
|
|
f"{base or OWUI_BASE}{path}", data=data, method=method or ("POST" if data else "GET"),
|
|
headers={"Authorization": f"Bearer {key or OWUI_KEY}", "Content-Type": "application/json"})
|
|
try:
|
|
r = urllib.request.urlopen(req, timeout=timeout)
|
|
return r.status, json.loads(r.read())
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, e.read().decode()[:200]
|
|
|
|
def multipart_upload(filename, content):
|
|
boundary = "----ragrefresh9x"
|
|
part = (f"--{boundary}\r\n"
|
|
f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n'
|
|
f"Content-Type: text/markdown\r\n\r\n").encode() + content.encode() + \
|
|
f"\r\n--{boundary}--\r\n".encode()
|
|
req = urllib.request.Request(
|
|
f"{OWUI_BASE}/api/v1/files/", data=part, method="POST",
|
|
headers={"Authorization": f"Bearer {OWUI_KEY}",
|
|
"Content-Type": f"multipart/form-data; boundary={boundary}"})
|
|
r = urllib.request.urlopen(req, timeout=180)
|
|
return json.loads(r.read())
|
|
|
|
def fetch_doc(name):
|
|
req = urllib.request.Request(GITEA_RAW + name)
|
|
try:
|
|
return urllib.request.urlopen(req, timeout=30).read().decode()
|
|
except Exception:
|
|
return None
|
|
|
|
state = {}
|
|
if os.path.exists(STATE_PATH):
|
|
try:
|
|
state = json.load(open(STATE_PATH))
|
|
except Exception:
|
|
state = {}
|
|
|
|
changed = 0
|
|
skipped = 0
|
|
for name in DOCS:
|
|
content = fetch_doc(name)
|
|
if content is None:
|
|
print(f"rag_refresh: WARN could not fetch {name}", file=sys.stderr)
|
|
continue
|
|
newhash = hashlib.sha256(content.encode()).hexdigest()
|
|
if state.get(name) == newhash:
|
|
skipped += 1
|
|
continue
|
|
|
|
# delete stale file rows with this name, then upload fresh + attach
|
|
code, files = api("/api/v1/files/?page=1")
|
|
if code == 200 and isinstance(files, dict):
|
|
files = files.get("items", []) # OWUI returns {items: [...]}
|
|
if code == 200 and isinstance(files, list):
|
|
for f in files:
|
|
if f.get("filename") == name:
|
|
api(f"/api/v1/files/{f['id']}", method="DELETE")
|
|
|
|
try:
|
|
resp = multipart_upload(name, content)
|
|
except Exception as e:
|
|
print(f"rag_refresh: upload failed for {name}: {e}", file=sys.stderr)
|
|
continue
|
|
fid = resp.get("id")
|
|
code, _ = api(f"/api/v1/knowledge/{KB_ID}/file/add", {"file_id": fid})
|
|
if code != 200:
|
|
print(f"rag_refresh: attach failed for {name}: {code}", file=sys.stderr)
|
|
continue
|
|
state[name] = newhash
|
|
changed += 1
|
|
|
|
# reindex the KB (chunk copies at attach time go stale otherwise)
|
|
if changed:
|
|
api("/api/v1/knowledge/reindex", {"knowledge_id": KB_ID}, timeout=300)
|
|
|
|
# keep the members-group grant intact (guards against grant loss)
|
|
api(f"/api/v1/knowledge/{KB_ID}/access/update",
|
|
{"access_grants": [{"principal_type": "group", "principal_id": GID, "permission": "read"}]})
|
|
|
|
json.dump(state, open(STATE_PATH, "w"))
|
|
print(f"rag_refresh: changed={changed} skipped={skipped}")
|
|
sys.exit(0)
|