From e56d0456bb03a7932cf0ead04ea1b4f86f3b69d7 Mon Sep 17 00:00:00 2001
From: Inferencebot <inference-bot@inference.coop>
Date: Sat, 10 Oct 2026 13:41:20 -0600
Subject: [PATCH] Chat models: silent auto-RAG via legacy FC (no knowledge
 tool, no narration)

---
 app/main.py | 47 +++++++++++++++++++++++++++--------------------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/app/main.py b/app/main.py
index fd3e421..22058b6 100644
--- a/app/main.py
+++ b/app/main.py
@@ -2024,31 +2024,38 @@ async def list_models(request: Request):
                 if kb_entry not in model["info"]["meta"]["knowledge"]:
                     model["info"]["meta"]["knowledge"].append(kb_entry)
 
-                # Steering: with knowledge attached, the model gets a
-                # query_knowledge_files tool and tends to call it on every
-                # prompt — including questions unrelated to the co-op — which
-                # adds an embedding call + vector search (latency) before
-                # answering. A soft hint proved insufficient (member test
-                # 2026-10-09: off-topic question still queried the docs), so
-                # this is a firm rule with explicit negative instruction.
-                # params.system is merged ahead of any system content by Open
-                # WebUI's middleware.
+                # SILENT AUTO-RAG (legacy function-calling mode), chosen 2026-10-09:
+                # with native FC the model gets a query_knowledge_files tool and
+                # calls it on every prompt — including off-topic ones — with
+                # visible "let me check the docs" narration. Prompt steering
+                # failed twice (soft + firm). Legacy mode instead auto-injects
+                # relevant docs chunks into context on every message: silent,
+                # no tool-calls, no announcements, grounding always on. Cost:
+                # one embedding + vector search per message (~300ms, paid even
+                # on off-topic messages). builtinTools.knowledge=false removes
+                # the knowledge tool + <attached_knowledge> tag so the model
+                # can't narrate or fumble; the injected sources still ground
+                # co-op answers.
                 model.setdefault("params", {})
+                model["params"]["function_calling"] = "legacy"
+                model["info"]["meta"]["builtinTools"] = {"knowledge": False}
+
+                # Steering: sources arrive as plain context. Tell the model the
+                # docs may be present but are co-op-only, so it never cites or
+                # mentions them for off-topic answers.
                 steering = (
-                    "About the attached knowledge base: it contains ONLY the "
-                    "Inference Cooperative's own documentation (membership, "
-                    "models, API access, governance). It is NOT general "
-                    "knowledge. Do NOT search or query it unless the user "
-                    "explicitly asks about the Inference Cooperative, its "
-                    "services, its models, its pricing, or its governance. For "
-                    "ANY other topic — general knowledge, how-tos, opinions, "
-                    "world facts — answer directly from your own knowledge "
-                    "without touching the knowledge base. When in doubt, do "
-                    "not query it."
+                    "Some messages include excerpts from the Inference "
+                    "Cooperative's own documentation (membership, models, API, "
+                    "governance), added automatically as background context. "
+                    "Use those excerpts only when they are relevant to the "
+                    "user's actual question. For general questions, ignore them "
+                    "and answer from your own knowledge — do not mention the "
+                    "co-op, its documentation, or any knowledge search unless "
+                    "the user explicitly asks about the co-op."
                 )
                 existing = model["params"].get("system")
                 if existing:
-                    if "Inference Cooperative's" not in existing:
+                    if "Inference Cooperative" not in existing:
                         model["params"]["system"] = existing + "\n\n" + steering
                 else:
                     model["params"]["system"] = steering