338 lines
9.7 KiB
TypeScript
338 lines
9.7 KiB
TypeScript
import * as vscode from "vscode";
|
|
|
|
const BASE_URL = "https://gateway.inference.coop/v1";
|
|
const PREFIX = "inference-coop/";
|
|
const SECRET_KEY = "inferenceCoop.apiKey";
|
|
|
|
type CoopModel = {
|
|
id: string;
|
|
name: string;
|
|
family: string;
|
|
version: string;
|
|
maxInputTokens: number;
|
|
maxOutputTokens: number;
|
|
capabilities: { toolCalling: boolean; imageInput: boolean };
|
|
};
|
|
|
|
export function activate(context: vscode.ExtensionContext) {
|
|
const out = vscode.window.createOutputChannel("Inference Coop");
|
|
context.subscriptions.push(out);
|
|
|
|
const changed = new vscode.EventEmitter<void>();
|
|
context.subscriptions.push(changed);
|
|
|
|
let cached: CoopModel[] = [];
|
|
let lastFetch = 0;
|
|
|
|
async function refresh(force = false): Promise<CoopModel[]> {
|
|
if (!force && Date.now() - lastFetch < 30_000 && cached.length) {
|
|
return cached;
|
|
}
|
|
const key = await context.secrets.get(SECRET_KEY);
|
|
if (!key) {
|
|
return cached;
|
|
}
|
|
try {
|
|
const res = await fetch(`${BASE_URL}/models`, {
|
|
headers: { Authorization: `Bearer ${key}` },
|
|
});
|
|
out.appendLine(`refresh: /models ${res.status}`);
|
|
if (!res.ok) {
|
|
return cached;
|
|
}
|
|
const body = (await res.json()) as {
|
|
data: Array<{ id: string; mode?: string }>;
|
|
};
|
|
const next: CoopModel[] = body.data
|
|
.filter((m) => !m.mode || m.mode === "chat")
|
|
.map((m) => ({
|
|
id: `${PREFIX}${m.id}`,
|
|
name: m.id,
|
|
family: "inference-coop",
|
|
version: "1.0.0",
|
|
maxInputTokens: 131072,
|
|
maxOutputTokens: 8192,
|
|
capabilities: { toolCalling: true, imageInput: false },
|
|
}));
|
|
lastFetch = Date.now();
|
|
if (JSON.stringify(next) !== JSON.stringify(cached)) {
|
|
cached = next;
|
|
changed.fire();
|
|
out.appendLine(`refresh: ${cached.length} models, fired change event`);
|
|
}
|
|
return cached;
|
|
} catch (e) {
|
|
out.appendLine(`refresh failed: ${e}`);
|
|
return cached;
|
|
}
|
|
}
|
|
|
|
const provider: vscode.LanguageModelChatProvider = {
|
|
onDidChangeLanguageModelChatInformation: changed.event,
|
|
|
|
async provideLanguageModelChatInformation() {
|
|
out.appendLine("provideLanguageModelChatInformation: called");
|
|
if (cached.length) {
|
|
void refresh(); // refresh in background, answer now
|
|
return cached;
|
|
}
|
|
return await refresh(true); // first call: must wait
|
|
},
|
|
|
|
async provideLanguageModelChatResponse(
|
|
model,
|
|
messages,
|
|
options,
|
|
progress,
|
|
token,
|
|
) {
|
|
out.appendLine(`provideLanguageModelChatResponse: ${model.id}`);
|
|
out.appendLine(` tools offered: ${options.tools?.length ?? 0}`);
|
|
|
|
const key = await context.secrets.get(SECRET_KEY);
|
|
|
|
const body: Record<string, unknown> = {
|
|
model: model.id.replace(PREFIX, ""),
|
|
stream: true,
|
|
messages: toOpenAIMessages(messages, out),
|
|
};
|
|
|
|
if (options.tools?.length) {
|
|
body.tools = options.tools.map((t) => ({
|
|
type: "function",
|
|
function: {
|
|
name: t.name,
|
|
description: t.description,
|
|
parameters: t.inputSchema ?? { type: "object", properties: {} },
|
|
},
|
|
}));
|
|
body.tool_choice = "auto";
|
|
}
|
|
|
|
const res = await fetch(`${BASE_URL}/chat/completions`, {
|
|
method: "POST",
|
|
headers: {
|
|
Authorization: `Bearer ${key}`,
|
|
"Content-Type": "application/json",
|
|
},
|
|
body: JSON.stringify(body),
|
|
signal: token.isCancellationRequested ? AbortSignal.abort() : undefined,
|
|
});
|
|
out.appendLine(` /chat/completions status: ${res.status}`);
|
|
if (!res.ok || !res.body) {
|
|
throw new Error(
|
|
`Inference Coop: HTTP ${res.status} — ${await res.text()}`,
|
|
);
|
|
}
|
|
|
|
const reader = res.body.getReader();
|
|
const decoder = new TextDecoder();
|
|
let buf = "";
|
|
let textChunks = 0;
|
|
const pending = new Map<
|
|
number,
|
|
{ id?: string; name: string; args: string }
|
|
>();
|
|
|
|
const flushTools = () => {
|
|
for (const call of pending.values()) {
|
|
let input: object = {};
|
|
try {
|
|
input = JSON.parse(call.args || "{}");
|
|
} catch {
|
|
out.appendLine(
|
|
` WARN bad tool args for ${call.name}: ${call.args}`,
|
|
);
|
|
}
|
|
out.appendLine(` tool call: ${call.name} ${JSON.stringify(input)}`);
|
|
progress.report(
|
|
new vscode.LanguageModelToolCallPart(
|
|
call.id ?? `call_${Math.random().toString(36).slice(2)}`,
|
|
call.name,
|
|
input,
|
|
),
|
|
);
|
|
}
|
|
pending.clear();
|
|
};
|
|
|
|
while (true) {
|
|
const { done, value } = await reader.read();
|
|
if (done) {
|
|
break;
|
|
}
|
|
buf += decoder.decode(value, { stream: true });
|
|
const frames = buf.split("\n\n");
|
|
buf = frames.pop() ?? "";
|
|
|
|
for (const frame of frames) {
|
|
for (const line of frame.split("\n")) {
|
|
if (!line.startsWith("data:")) {
|
|
continue;
|
|
}
|
|
const data = line.slice(5).trim();
|
|
if (data === "[DONE]") {
|
|
if (pending.size) {
|
|
flushTools();
|
|
}
|
|
out.appendLine(` done, ${textChunks} text chunks`);
|
|
return;
|
|
}
|
|
let json: any;
|
|
try {
|
|
json = JSON.parse(data);
|
|
} catch {
|
|
continue;
|
|
}
|
|
const choice = json.choices?.[0];
|
|
const delta = choice?.delta;
|
|
if (!delta) {
|
|
continue;
|
|
}
|
|
|
|
if (typeof delta.content === "string" && delta.content) {
|
|
textChunks++;
|
|
progress.report(new vscode.LanguageModelTextPart(delta.content));
|
|
}
|
|
|
|
for (const tc of delta.tool_calls ?? []) {
|
|
const i = tc.index ?? 0;
|
|
const acc = pending.get(i) ?? {
|
|
id: undefined,
|
|
name: "",
|
|
args: "",
|
|
};
|
|
acc.id = tc.id ?? acc.id;
|
|
acc.name = tc.function?.name ?? acc.name;
|
|
acc.args += tc.function?.arguments ?? "";
|
|
pending.set(i, acc);
|
|
}
|
|
|
|
if (choice?.finish_reason) {
|
|
out.appendLine(` finish_reason: ${choice.finish_reason}`);
|
|
if (pending.size) {
|
|
flushTools();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (pending.size) {
|
|
flushTools();
|
|
}
|
|
out.appendLine(` stream ended, ${textChunks} text chunks`);
|
|
},
|
|
|
|
async provideTokenCount() {
|
|
return 0;
|
|
},
|
|
};
|
|
|
|
context.subscriptions.push(
|
|
vscode.lm.registerLanguageModelChatProvider("inference-coop", provider),
|
|
);
|
|
|
|
context.subscriptions.push(
|
|
vscode.commands.registerCommand("inference-coop-copilot.helloWorld", () => {
|
|
vscode.window.showInformationMessage("Inference Coop extension loaded.");
|
|
}),
|
|
);
|
|
|
|
context.subscriptions.push(
|
|
vscode.commands.registerCommand("inferenceCoop.setKey", async () => {
|
|
const key = await vscode.window.showInputBox({
|
|
title: "Inference Coop API key",
|
|
prompt: "Paste your key from the co-op dashboard.",
|
|
password: true,
|
|
ignoreFocusOut: true,
|
|
validateInput: (v) =>
|
|
v.trim().length < 16
|
|
? "That looks too short to be an API key."
|
|
: undefined,
|
|
});
|
|
if (!key) {return;}
|
|
await context.secrets.store(SECRET_KEY, key.trim());
|
|
cached = [];
|
|
lastFetch = 0;
|
|
await refresh(true);
|
|
vscode.window.showInformationMessage(
|
|
`Inference Coop: signed in, ${cached.length} models available.`,
|
|
);
|
|
}),
|
|
);
|
|
|
|
context.subscriptions.push(
|
|
vscode.commands.registerCommand("inferenceCoop.clearKey", async () => {
|
|
await context.secrets.delete(SECRET_KEY);
|
|
cached = [];
|
|
lastFetch = 0;
|
|
changed.fire();
|
|
vscode.window.showInformationMessage("Inference Coop: signed out.");
|
|
}),
|
|
);
|
|
}
|
|
|
|
function toOpenAIMessages(
|
|
messages: readonly vscode.LanguageModelChatRequestMessage[],
|
|
out: vscode.OutputChannel,
|
|
): object[] {
|
|
const result: object[] = [];
|
|
|
|
for (const msg of messages) {
|
|
const text: string[] = [];
|
|
const toolCalls: object[] = [];
|
|
const toolResults: object[] = [];
|
|
|
|
for (const part of msg.content ?? []) {
|
|
if (part instanceof vscode.LanguageModelTextPart) {
|
|
text.push(part.value);
|
|
} else if (part instanceof vscode.LanguageModelToolCallPart) {
|
|
toolCalls.push({
|
|
id: part.callId,
|
|
type: "function",
|
|
function: {
|
|
name: part.name,
|
|
arguments: JSON.stringify(part.input ?? {}),
|
|
},
|
|
});
|
|
} else if (part instanceof vscode.LanguageModelToolResultPart) {
|
|
toolResults.push({
|
|
role: "tool",
|
|
tool_call_id: part.callId,
|
|
content: part.content
|
|
.map((c: unknown) =>
|
|
c instanceof vscode.LanguageModelTextPart
|
|
? c.value
|
|
: JSON.stringify(c),
|
|
)
|
|
.join("\n"),
|
|
});
|
|
}
|
|
}
|
|
|
|
const role =
|
|
msg.role === vscode.LanguageModelChatMessageRole.Assistant
|
|
? "assistant"
|
|
: msg.role === vscode.LanguageModelChatMessageRole.User
|
|
? "user"
|
|
: "system";
|
|
|
|
if (text.length || toolCalls.length) {
|
|
const m: Record<string, unknown> = {
|
|
role,
|
|
content: text.length ? text.join("\n") : null,
|
|
};
|
|
if (toolCalls.length) {
|
|
m.tool_calls = toolCalls;
|
|
}
|
|
result.push(m);
|
|
}
|
|
result.push(...toolResults);
|
|
}
|
|
|
|
out.appendLine(` translated ${result.length} messages`);
|
|
return result;
|
|
}
|
|
|
|
export function deactivate() {}
|