commit 5f88cebedeb950bbcc013e0ff250a0ab99b697a3 Author: Inferencebot Date: Thu Nov 19 22:59:22 2026 +0000 Page fetch service: Keenable-shaped API, SSRF guard, no access log diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..00f2d38 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +.venv/ diff --git a/CloudronManifest.json b/CloudronManifest.json new file mode 100644 index 0000000..6b1fd51 --- /dev/null +++ b/CloudronManifest.json @@ -0,0 +1,20 @@ +{ + "id": "coop.inference.fetcher", + "title": "Page Fetcher", + "author": "Inference Cooperative", + "description": "Internal page-fetch service for LibreChat web search: fetches result pages and returns readable markdown. Admin-visible only; called server-to-server with an API key.", + "tagline": "Readable pages for co-op web search", + "version": "0.1.0", + "healthCheckPath": "/health", + "httpPort": 8000, + "manifestVersion": 2, + "website": "https://inference.coop", + "contactEmail": "info@inference.coop", + "tags": [ + "internal", + "search", + "fetcher" + ], + "memoryLimit": 536870912, + "dockerImage": "registry.inference.coop/coop.inference.fetcher:dev" +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c4c0378 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.12-slim + +WORKDIR /app/code +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 + +COPY requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY app/ ./app/ + +EXPOSE 8000 + +# No access log: the path carries the fetched URL, which is member search +# output. Errors still reach Cloudron's log stream; URLs do not. +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..b7b0b5c --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +# Page Fetcher + +Internal page-fetch service for the co-op's LibreChat web search. It replaces +the third-party Keenable fetch: LibreChat sends the result URL here, gets +readable markdown back, and the model cites the page. + +## Contract + +``` +GET /v1/fetch?url= (X-API-Key required) +GET /v1/fetch/public?url= (same; path LibreChat uses keyless) +GET /health (no key — Cloudron health check) +``` + +Response (Keenable-compatible shape, so LibreChat needs no config fork): + +```json +{"content": "markdown…", "title": "…", "description": "…", "url": "https://final.example/page"} +``` + +Errors are JSON `{"detail": "…"}` with 4xx/5xx. `FETCHER_KEY` unset means the +service refuses every fetch (503). + +## Where it sits + +``` +members → LibreChat → SearXNG (search) → Page Fetcher (this app) → page text + ↘ gateway (answer), with citations +``` + +LibreChat reaches it on the internal Cloudron network +(`http://fetch:8000`). No member can reach it: the public app is restricted to +the `managers` Cloudron group, there is no UI, and every fetch requires +`X-API-Key`. + +## Privacy choices + +- **No access log.** The request path carries the fetched URL, which is member + search output. Uvicorn runs with `--no-access-log`; errors log the host, never + the full URL. Nothing about a fetch is stored. +- **SSRF guard.** http/https only, ports 80/443 only, credentials in the URL + refused, private/loopback/link-local/reserved/multicast addresses refused, + redirects re-validated at every hop, 2 MB body cap, 10 s timeout. +- **No browser.** JS-rendered pages yield little or no text; the fetch fails + cleanly rather than returning junk. + +## Deploy + +```sh +# build (Cloudron builder) +~/.hermes/scripts/cloudron_builder_build.sh coop.inference.fetcher . + +# pin CloudronManifest.json dockerImage to the new tag, commit, push +# install once: +cloudron install --app fetch.inference.coop --location fetch + +# configure: FETCHER_KEY (32+ random bytes), then restrict access to managers +``` + +## Not in scope + +- Search itself stays on SearXNG. +- No crawling, no index, no cache, no UI. diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..9563aa9 --- /dev/null +++ b/app/main.py @@ -0,0 +1,150 @@ +"""Inference Cooperative page-fetch service. + +A small internal service that LibreChat's web-search scraper calls instead of a +third-party page fetcher. It implements the shape Keenable's fetch API returns +(`{content, title, description, url}`) so LibreChat can use it through +`KEENABLE_FETCH_URL` with no config fork. + +Privacy posture: +- API-key authenticated; fail closed when FETCHER_KEY is unset. +- No access log: full result URLs are not written anywhere. Only errors are + logged, and they name the host, never the query. +- SSRF guard: http/https only, ports 80/443 only, no credentials in the URL, + private/loopback/link-local/reserved/multicast addresses refused, redirects + re-validated hop by hop, 2 MB body cap, 10 s timeout. +""" + +import ipaddress +import os +import re +import secrets +import socket +import urllib.parse +from typing import Optional + +import httpx +import trafilatura +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse + +FETCHER_KEY = os.environ.get("FETCHER_KEY", "") +TIMEOUT = float(os.environ.get("FETCH_TIMEOUT_SECONDS", "10")) +MAX_BYTES = int(os.environ.get("FETCH_MAX_BYTES", "2000000")) +MAX_REDIRECTS = 3 +ALLOWED_PORTS = {80, 443} +ALLOWED_TYPES = ("text/html", "application/xhtml+xml", "text/plain") + +app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) +_client = httpx.AsyncClient( + timeout=httpx.Timeout(TIMEOUT, connect=5.0), + follow_redirects=False, + headers={"User-Agent": "InferenceCoop-Fetcher/0.1 (+https://inference.coop)"}, +) + + +def _require_key(request: Request) -> None: + if not FETCHER_KEY: + raise HTTPException(503, "fetcher key not configured") + provided = request.headers.get("x-api-key", "") + if not secrets.compare_digest(provided, FETCHER_KEY): + raise HTTPException(401, "invalid api key") + + +def _check_url(url: str) -> urllib.parse.SplitResult: + parsed = urllib.parse.urlsplit(url.strip()) + if parsed.scheme not in ("http", "https"): + raise HTTPException(400, "only http/https URLs are supported") + if parsed.username or parsed.password: + raise HTTPException(400, "credentials in URL are refused") + if not parsed.hostname: + raise HTTPException(400, "URL has no host") + port = parsed.port or (443 if parsed.scheme == "https" else 80) + if port not in ALLOWED_PORTS: + raise HTTPException(400, "only ports 80 and 443 are allowed") + try: + infos = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM) + except socket.gaierror: + raise HTTPException(400, "host does not resolve") + for info in infos: + ip = ipaddress.ip_address(info[4][0]) + if ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_reserved + or ip.is_multicast + or ip.is_unspecified + ): + raise HTTPException(400, "host resolves to a non-public address") + return parsed + + +async def _fetch(url: str) -> tuple[str, str]: + current = url + for _ in range(MAX_REDIRECTS + 1): + _check_url(current) + async with _client.stream("GET", current) as resp: + if resp.status_code in (301, 302, 303, 307, 308): + location = resp.headers.get("location") + if not location: + raise HTTPException(502, "redirect without location") + current = urllib.parse.urljoin(current, location) + continue + if resp.status_code != 200: + raise HTTPException(502, f"upstream returned {resp.status_code}") + ctype = (resp.headers.get("content-type") or "").split(";")[0].strip().lower() + if ctype not in ALLOWED_TYPES: + raise HTTPException(415, f"unsupported content type: {ctype or 'unknown'}") + chunks = [] + size = 0 + async for chunk in resp.aiter_bytes(): + size += len(chunk) + if size > MAX_BYTES: + raise HTTPException(413, "page exceeds the size limit") + chunks.append(chunk) + return current, b"".join(chunks).decode(resp.encoding or "utf-8", "replace") + raise HTTPException(508, "too many redirects") + + +def _extract(html: str, url: str) -> dict: + content = trafilatura.extract( + html, + url=url, + output_format="markdown", + include_links=False, + include_comments=False, + include_tables=True, + favor_recall=True, + ) + if not content: + raise HTTPException(422, "no readable text found") + meta = trafilatura.extract_metadata(html) + return { + "content": content, + "title": getattr(meta, "title", None) or "", + "description": getattr(meta, "description", None) or "", + "url": url, + } + + +@app.get("/health") +async def health() -> dict: + return {"status": "ok"} + + +@app.get("/v1/fetch") +@app.get("/v1/fetch/public") +async def fetch(request: Request, url: str) -> JSONResponse: + _require_key(request) + try: + final_url, html = await _fetch(url) + return JSONResponse(_extract(html, final_url)) + except HTTPException: + raise + except httpx.TimeoutException: + raise HTTPException(504, "fetch timed out") + except httpx.HTTPError as exc: + # Name the host only; never echo the full URL (it carries member search + # results) into logs or responses. + host = urllib.parse.urlsplit(url).hostname or "unknown" + raise HTTPException(502, f"fetch failed for {host}: {exc.__class__.__name__}") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..32af9e0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +trafilatura==2.3.1 +fastapi==0.165.0 +uvicorn[standard]==0.54.0 +httpx==0.28.1