Page fetch service: Keenable-shaped API, SSRF guard, no access log

This commit is contained in:
inference-admin committed 2026-11-19 22:59:22 +00:00
commit 5f88cebede
6 files changed
+255

No files matched your search

+3
View File
@@ -0,0 +1,3 @@
__pycache__/
*.pyc
.venv/
+20
View File
@@ -0,0 +1,20 @@
{
"id": "coop.inference.fetcher",
"title": "Page Fetcher",
"author": "Inference Cooperative",
"description": "Internal page-fetch service for LibreChat web search: fetches result pages and returns readable markdown. Admin-visible only; called server-to-server with an API key.",
"tagline": "Readable pages for co-op web search",
"version": "0.1.0",
"healthCheckPath": "/health",
"httpPort": 8000,
"manifestVersion": 2,
"website": "https://inference.coop",
"contactEmail": "info@inference.coop",
"tags": [
"internal",
"search",
"fetcher"
],
"memoryLimit": 536870912,
"dockerImage": "registry.inference.coop/coop.inference.fetcher:dev"
}
+15
View File
@@ -0,0 +1,15 @@
FROM python:3.12-slim
WORKDIR /app/code
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt
COPY app/ ./app/
EXPOSE 8000
# No access log: the path carries the fetched URL, which is member search
# output. Errors still reach Cloudron's log stream; URLs do not.
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
+63
View File
@@ -0,0 +1,63 @@
# Page Fetcher
Internal page-fetch service for the co-op's LibreChat web search. It replaces
the third-party Keenable fetch: LibreChat sends the result URL here, gets
readable markdown back, and the model cites the page.
## Contract
```
GET /v1/fetch?url=<page> (X-API-Key required)
GET /v1/fetch/public?url=<page> (same; path LibreChat uses keyless)
GET /health (no key — Cloudron health check)
```
Response (Keenable-compatible shape, so LibreChat needs no config fork):
```json
{"content": "markdown…", "title": "…", "description": "…", "url": "https://final.example/page"}
```
Errors are JSON `{"detail": "…"}` with 4xx/5xx. `FETCHER_KEY` unset means the
service refuses every fetch (503).
## Where it sits
```
members → LibreChat → SearXNG (search) → Page Fetcher (this app) → page text
↘ gateway (answer), with citations
```
LibreChat reaches it on the internal Cloudron network
(`http://fetch:8000`). No member can reach it: the public app is restricted to
the `managers` Cloudron group, there is no UI, and every fetch requires
`X-API-Key`.
## Privacy choices
- **No access log.** The request path carries the fetched URL, which is member
search output. Uvicorn runs with `--no-access-log`; errors log the host, never
the full URL. Nothing about a fetch is stored.
- **SSRF guard.** http/https only, ports 80/443 only, credentials in the URL
refused, private/loopback/link-local/reserved/multicast addresses refused,
redirects re-validated at every hop, 2 MB body cap, 10 s timeout.
- **No browser.** JS-rendered pages yield little or no text; the fetch fails
cleanly rather than returning junk.
## Deploy
```sh
# build (Cloudron builder)
~/.hermes/scripts/cloudron_builder_build.sh coop.inference.fetcher <tag> .
# pin CloudronManifest.json dockerImage to the new tag, commit, push
# install once:
cloudron install --app fetch.inference.coop --location fetch
# configure: FETCHER_KEY (32+ random bytes), then restrict access to managers
```
## Not in scope
- Search itself stays on SearXNG.
- No crawling, no index, no cache, no UI.
+150
View File
@@ -0,0 +1,150 @@
"""Inference Cooperative page-fetch service.
A small internal service that LibreChat's web-search scraper calls instead of a
third-party page fetcher. It implements the shape Keenable's fetch API returns
(`{content, title, description, url}`) so LibreChat can use it through
`KEENABLE_FETCH_URL` with no config fork.
Privacy posture:
- API-key authenticated; fail closed when FETCHER_KEY is unset.
- No access log: full result URLs are not written anywhere. Only errors are
logged, and they name the host, never the query.
- SSRF guard: http/https only, ports 80/443 only, no credentials in the URL,
private/loopback/link-local/reserved/multicast addresses refused, redirects
re-validated hop by hop, 2 MB body cap, 10 s timeout.
"""
import ipaddress
import os
import re
import secrets
import socket
import urllib.parse
from typing import Optional
import httpx
import trafilatura
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse
FETCHER_KEY = os.environ.get("FETCHER_KEY", "")
TIMEOUT = float(os.environ.get("FETCH_TIMEOUT_SECONDS", "10"))
MAX_BYTES = int(os.environ.get("FETCH_MAX_BYTES", "2000000"))
MAX_REDIRECTS = 3
ALLOWED_PORTS = {80, 443}
ALLOWED_TYPES = ("text/html", "application/xhtml+xml", "text/plain")
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
_client = httpx.AsyncClient(
timeout=httpx.Timeout(TIMEOUT, connect=5.0),
follow_redirects=False,
headers={"User-Agent": "InferenceCoop-Fetcher/0.1 (+https://inference.coop)"},
)
def _require_key(request: Request) -> None:
if not FETCHER_KEY:
raise HTTPException(503, "fetcher key not configured")
provided = request.headers.get("x-api-key", "")
if not secrets.compare_digest(provided, FETCHER_KEY):
raise HTTPException(401, "invalid api key")
def _check_url(url: str) -> urllib.parse.SplitResult:
parsed = urllib.parse.urlsplit(url.strip())
if parsed.scheme not in ("http", "https"):
raise HTTPException(400, "only http/https URLs are supported")
if parsed.username or parsed.password:
raise HTTPException(400, "credentials in URL are refused")
if not parsed.hostname:
raise HTTPException(400, "URL has no host")
port = parsed.port or (443 if parsed.scheme == "https" else 80)
if port not in ALLOWED_PORTS:
raise HTTPException(400, "only ports 80 and 443 are allowed")
try:
infos = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM)
except socket.gaierror:
raise HTTPException(400, "host does not resolve")
for info in infos:
ip = ipaddress.ip_address(info[4][0])
if (
ip.is_private
or ip.is_loopback
or ip.is_link_local
or ip.is_reserved
or ip.is_multicast
or ip.is_unspecified
):
raise HTTPException(400, "host resolves to a non-public address")
return parsed
async def _fetch(url: str) -> tuple[str, str]:
current = url
for _ in range(MAX_REDIRECTS + 1):
_check_url(current)
async with _client.stream("GET", current) as resp:
if resp.status_code in (301, 302, 303, 307, 308):
location = resp.headers.get("location")
if not location:
raise HTTPException(502, "redirect without location")
current = urllib.parse.urljoin(current, location)
continue
if resp.status_code != 200:
raise HTTPException(502, f"upstream returned {resp.status_code}")
ctype = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
if ctype not in ALLOWED_TYPES:
raise HTTPException(415, f"unsupported content type: {ctype or 'unknown'}")
chunks = []
size = 0
async for chunk in resp.aiter_bytes():
size += len(chunk)
if size > MAX_BYTES:
raise HTTPException(413, "page exceeds the size limit")
chunks.append(chunk)
return current, b"".join(chunks).decode(resp.encoding or "utf-8", "replace")
raise HTTPException(508, "too many redirects")
def _extract(html: str, url: str) -> dict:
content = trafilatura.extract(
html,
url=url,
output_format="markdown",
include_links=False,
include_comments=False,
include_tables=True,
favor_recall=True,
)
if not content:
raise HTTPException(422, "no readable text found")
meta = trafilatura.extract_metadata(html)
return {
"content": content,
"title": getattr(meta, "title", None) or "",
"description": getattr(meta, "description", None) or "",
"url": url,
}
@app.get("/health")
async def health() -> dict:
return {"status": "ok"}
@app.get("/v1/fetch")
@app.get("/v1/fetch/public")
async def fetch(request: Request, url: str) -> JSONResponse:
_require_key(request)
try:
final_url, html = await _fetch(url)
return JSONResponse(_extract(html, final_url))
except HTTPException:
raise
except httpx.TimeoutException:
raise HTTPException(504, "fetch timed out")
except httpx.HTTPError as exc:
# Name the host only; never echo the full URL (it carries member search
# results) into logs or responses.
host = urllib.parse.urlsplit(url).hostname or "unknown"
raise HTTPException(502, f"fetch failed for {host}: {exc.__class__.__name__}")
+4
View File
@@ -0,0 +1,4 @@
trafilatura==2.3.1
fastapi==0.165.0
uvicorn[standard]==0.54.0
httpx==0.28.1