Page fetch service: Keenable-shaped API, SSRF guard, no access log
This commit is contained in:
commit
5f88cebede
6 files changed
+255
No files matched your search
@@ -0,0 +1,3 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.venv/
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"id": "coop.inference.fetcher",
|
||||
"title": "Page Fetcher",
|
||||
"author": "Inference Cooperative",
|
||||
"description": "Internal page-fetch service for LibreChat web search: fetches result pages and returns readable markdown. Admin-visible only; called server-to-server with an API key.",
|
||||
"tagline": "Readable pages for co-op web search",
|
||||
"version": "0.1.0",
|
||||
"healthCheckPath": "/health",
|
||||
"httpPort": 8000,
|
||||
"manifestVersion": 2,
|
||||
"website": "https://inference.coop",
|
||||
"contactEmail": "info@inference.coop",
|
||||
"tags": [
|
||||
"internal",
|
||||
"search",
|
||||
"fetcher"
|
||||
],
|
||||
"memoryLimit": 536870912,
|
||||
"dockerImage": "registry.inference.coop/coop.inference.fetcher:dev"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app/code
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app/ ./app/
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
# No access log: the path carries the fetched URL, which is member search
|
||||
# output. Errors still reach Cloudron's log stream; URLs do not.
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
|
||||
@@ -0,0 +1,63 @@
|
||||
# Page Fetcher
|
||||
|
||||
Internal page-fetch service for the co-op's LibreChat web search. It replaces
|
||||
the third-party Keenable fetch: LibreChat sends the result URL here, gets
|
||||
readable markdown back, and the model cites the page.
|
||||
|
||||
## Contract
|
||||
|
||||
```
|
||||
GET /v1/fetch?url=<page> (X-API-Key required)
|
||||
GET /v1/fetch/public?url=<page> (same; path LibreChat uses keyless)
|
||||
GET /health (no key — Cloudron health check)
|
||||
```
|
||||
|
||||
Response (Keenable-compatible shape, so LibreChat needs no config fork):
|
||||
|
||||
```json
|
||||
{"content": "markdown…", "title": "…", "description": "…", "url": "https://final.example/page"}
|
||||
```
|
||||
|
||||
Errors are JSON `{"detail": "…"}` with 4xx/5xx. `FETCHER_KEY` unset means the
|
||||
service refuses every fetch (503).
|
||||
|
||||
## Where it sits
|
||||
|
||||
```
|
||||
members → LibreChat → SearXNG (search) → Page Fetcher (this app) → page text
|
||||
↘ gateway (answer), with citations
|
||||
```
|
||||
|
||||
LibreChat reaches it on the internal Cloudron network
|
||||
(`http://fetch:8000`). No member can reach it: the public app is restricted to
|
||||
the `managers` Cloudron group, there is no UI, and every fetch requires
|
||||
`X-API-Key`.
|
||||
|
||||
## Privacy choices
|
||||
|
||||
- **No access log.** The request path carries the fetched URL, which is member
|
||||
search output. Uvicorn runs with `--no-access-log`; errors log the host, never
|
||||
the full URL. Nothing about a fetch is stored.
|
||||
- **SSRF guard.** http/https only, ports 80/443 only, credentials in the URL
|
||||
refused, private/loopback/link-local/reserved/multicast addresses refused,
|
||||
redirects re-validated at every hop, 2 MB body cap, 10 s timeout.
|
||||
- **No browser.** JS-rendered pages yield little or no text; the fetch fails
|
||||
cleanly rather than returning junk.
|
||||
|
||||
## Deploy
|
||||
|
||||
```sh
|
||||
# build (Cloudron builder)
|
||||
~/.hermes/scripts/cloudron_builder_build.sh coop.inference.fetcher <tag> .
|
||||
|
||||
# pin CloudronManifest.json dockerImage to the new tag, commit, push
|
||||
# install once:
|
||||
cloudron install --app fetch.inference.coop --location fetch
|
||||
|
||||
# configure: FETCHER_KEY (32+ random bytes), then restrict access to managers
|
||||
```
|
||||
|
||||
## Not in scope
|
||||
|
||||
- Search itself stays on SearXNG.
|
||||
- No crawling, no index, no cache, no UI.
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
"""Inference Cooperative page-fetch service.
|
||||
|
||||
A small internal service that LibreChat's web-search scraper calls instead of a
|
||||
third-party page fetcher. It implements the shape Keenable's fetch API returns
|
||||
(`{content, title, description, url}`) so LibreChat can use it through
|
||||
`KEENABLE_FETCH_URL` with no config fork.
|
||||
|
||||
Privacy posture:
|
||||
- API-key authenticated; fail closed when FETCHER_KEY is unset.
|
||||
- No access log: full result URLs are not written anywhere. Only errors are
|
||||
logged, and they name the host, never the query.
|
||||
- SSRF guard: http/https only, ports 80/443 only, no credentials in the URL,
|
||||
private/loopback/link-local/reserved/multicast addresses refused, redirects
|
||||
re-validated hop by hop, 2 MB body cap, 10 s timeout.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import socket
|
||||
import urllib.parse
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
import trafilatura
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
FETCHER_KEY = os.environ.get("FETCHER_KEY", "")
|
||||
TIMEOUT = float(os.environ.get("FETCH_TIMEOUT_SECONDS", "10"))
|
||||
MAX_BYTES = int(os.environ.get("FETCH_MAX_BYTES", "2000000"))
|
||||
MAX_REDIRECTS = 3
|
||||
ALLOWED_PORTS = {80, 443}
|
||||
ALLOWED_TYPES = ("text/html", "application/xhtml+xml", "text/plain")
|
||||
|
||||
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
|
||||
_client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(TIMEOUT, connect=5.0),
|
||||
follow_redirects=False,
|
||||
headers={"User-Agent": "InferenceCoop-Fetcher/0.1 (+https://inference.coop)"},
|
||||
)
|
||||
|
||||
|
||||
def _require_key(request: Request) -> None:
|
||||
if not FETCHER_KEY:
|
||||
raise HTTPException(503, "fetcher key not configured")
|
||||
provided = request.headers.get("x-api-key", "")
|
||||
if not secrets.compare_digest(provided, FETCHER_KEY):
|
||||
raise HTTPException(401, "invalid api key")
|
||||
|
||||
|
||||
def _check_url(url: str) -> urllib.parse.SplitResult:
|
||||
parsed = urllib.parse.urlsplit(url.strip())
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
raise HTTPException(400, "only http/https URLs are supported")
|
||||
if parsed.username or parsed.password:
|
||||
raise HTTPException(400, "credentials in URL are refused")
|
||||
if not parsed.hostname:
|
||||
raise HTTPException(400, "URL has no host")
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
if port not in ALLOWED_PORTS:
|
||||
raise HTTPException(400, "only ports 80 and 443 are allowed")
|
||||
try:
|
||||
infos = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM)
|
||||
except socket.gaierror:
|
||||
raise HTTPException(400, "host does not resolve")
|
||||
for info in infos:
|
||||
ip = ipaddress.ip_address(info[4][0])
|
||||
if (
|
||||
ip.is_private
|
||||
or ip.is_loopback
|
||||
or ip.is_link_local
|
||||
or ip.is_reserved
|
||||
or ip.is_multicast
|
||||
or ip.is_unspecified
|
||||
):
|
||||
raise HTTPException(400, "host resolves to a non-public address")
|
||||
return parsed
|
||||
|
||||
|
||||
async def _fetch(url: str) -> tuple[str, str]:
|
||||
current = url
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
_check_url(current)
|
||||
async with _client.stream("GET", current) as resp:
|
||||
if resp.status_code in (301, 302, 303, 307, 308):
|
||||
location = resp.headers.get("location")
|
||||
if not location:
|
||||
raise HTTPException(502, "redirect without location")
|
||||
current = urllib.parse.urljoin(current, location)
|
||||
continue
|
||||
if resp.status_code != 200:
|
||||
raise HTTPException(502, f"upstream returned {resp.status_code}")
|
||||
ctype = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
|
||||
if ctype not in ALLOWED_TYPES:
|
||||
raise HTTPException(415, f"unsupported content type: {ctype or 'unknown'}")
|
||||
chunks = []
|
||||
size = 0
|
||||
async for chunk in resp.aiter_bytes():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise HTTPException(413, "page exceeds the size limit")
|
||||
chunks.append(chunk)
|
||||
return current, b"".join(chunks).decode(resp.encoding or "utf-8", "replace")
|
||||
raise HTTPException(508, "too many redirects")
|
||||
|
||||
|
||||
def _extract(html: str, url: str) -> dict:
|
||||
content = trafilatura.extract(
|
||||
html,
|
||||
url=url,
|
||||
output_format="markdown",
|
||||
include_links=False,
|
||||
include_comments=False,
|
||||
include_tables=True,
|
||||
favor_recall=True,
|
||||
)
|
||||
if not content:
|
||||
raise HTTPException(422, "no readable text found")
|
||||
meta = trafilatura.extract_metadata(html)
|
||||
return {
|
||||
"content": content,
|
||||
"title": getattr(meta, "title", None) or "",
|
||||
"description": getattr(meta, "description", None) or "",
|
||||
"url": url,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health() -> dict:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/v1/fetch")
|
||||
@app.get("/v1/fetch/public")
|
||||
async def fetch(request: Request, url: str) -> JSONResponse:
|
||||
_require_key(request)
|
||||
try:
|
||||
final_url, html = await _fetch(url)
|
||||
return JSONResponse(_extract(html, final_url))
|
||||
except HTTPException:
|
||||
raise
|
||||
except httpx.TimeoutException:
|
||||
raise HTTPException(504, "fetch timed out")
|
||||
except httpx.HTTPError as exc:
|
||||
# Name the host only; never echo the full URL (it carries member search
|
||||
# results) into logs or responses.
|
||||
host = urllib.parse.urlsplit(url).hostname or "unknown"
|
||||
raise HTTPException(502, f"fetch failed for {host}: {exc.__class__.__name__}")
|
||||
@@ -0,0 +1,4 @@
|
||||
trafilatura==2.3.1
|
||||
fastapi==0.165.0
|
||||
uvicorn[standard]==0.54.0
|
||||
httpx==0.28.1
|
||||
Reference in new issue
Block a user