Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8f1d71d913 | ||
|
|
e9a060af0f |
No files matched your search
@@ -1,8 +1,9 @@
|
|||||||
# Inference Coop for Copilot Chat (unofficial)
|
# Inference Coop for Copilot Chat
|
||||||
|
|
||||||
Community-built model provider for [inference.coop](https://inference.coop),
|
Official model provider for [inference.coop](https://inference.coop), usable
|
||||||
usable in GitHub Copilot Chat. Not affiliated with or endorsed by the
|
in GitHub Copilot Chat. An official project of the Inference Cooperative,
|
||||||
Inference Cooperative.
|
maintained in the co-op's Git at
|
||||||
|
[code/inference-coop-copilot](https://git.inference.coop/code/inference-coop-copilot).
|
||||||
|
|
||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
@@ -30,7 +31,6 @@ To use a different gateway, set `inferenceCoop.baseUrl`. To sign out, run
|
|||||||
context budgeting is approximate.
|
context budgeting is approximate.
|
||||||
- No inline code completions. Copilot does not support third-party providers
|
- No inline code completions. Copilot does not support third-party providers
|
||||||
for completions; this extension only adds chat models.
|
for completions; this extension only adds chat models.
|
||||||
- Unofficial client. Report extension bugs here, not to the co-op.
|
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
+127
-94
@@ -109,118 +109,151 @@ export function activate(context: vscode.ExtensionContext) {
|
|||||||
body.tool_choice = "auto";
|
body.tool_choice = "auto";
|
||||||
}
|
}
|
||||||
|
|
||||||
const res = await fetch(`${BASE_URL}/chat/completions`, {
|
if (token.isCancellationRequested) {
|
||||||
method: "POST",
|
out.appendLine(" request cancelled before it started");
|
||||||
headers: {
|
return;
|
||||||
Authorization: `Bearer ${key}`,
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
},
|
|
||||||
body: JSON.stringify(body),
|
|
||||||
signal: token.isCancellationRequested ? AbortSignal.abort() : undefined,
|
|
||||||
});
|
|
||||||
out.appendLine(` /chat/completions status: ${res.status}`);
|
|
||||||
if (!res.ok || !res.body) {
|
|
||||||
throw new Error(
|
|
||||||
`Inference Coop: HTTP ${res.status} — ${await res.text()}`,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const reader = res.body.getReader();
|
// Abort the request as soon as the user cancels, so a cancelled chat
|
||||||
const decoder = new TextDecoder();
|
// stops generating (and stops billing) server-side.
|
||||||
let buf = "";
|
const abort = new AbortController();
|
||||||
let textChunks = 0;
|
const cancelSubscription = token.onCancellationRequested(() =>
|
||||||
const pending = new Map<
|
abort.abort(),
|
||||||
number,
|
);
|
||||||
{ id?: string; name: string; args: string }
|
|
||||||
>();
|
|
||||||
|
|
||||||
const flushTools = () => {
|
try {
|
||||||
for (const call of pending.values()) {
|
const res = await fetch(`${BASE_URL}/chat/completions`, {
|
||||||
let input: object = {};
|
method: "POST",
|
||||||
try {
|
headers: {
|
||||||
input = JSON.parse(call.args || "{}");
|
Authorization: `Bearer ${key}`,
|
||||||
} catch {
|
"Content-Type": "application/json",
|
||||||
out.appendLine(
|
},
|
||||||
` WARN bad tool args for ${call.name}: ${call.args}`,
|
body: JSON.stringify(body),
|
||||||
);
|
signal: abort.signal,
|
||||||
}
|
});
|
||||||
out.appendLine(` tool call: ${call.name} ${JSON.stringify(input)}`);
|
out.appendLine(` /chat/completions status: ${res.status}`);
|
||||||
progress.report(
|
if (!res.ok || !res.body) {
|
||||||
new vscode.LanguageModelToolCallPart(
|
throw new Error(
|
||||||
call.id ?? `call_${Math.random().toString(36).slice(2)}`,
|
`Inference Coop: HTTP ${res.status} — ${await res.text()}`,
|
||||||
call.name,
|
|
||||||
input,
|
|
||||||
),
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
pending.clear();
|
|
||||||
};
|
|
||||||
|
|
||||||
while (true) {
|
const reader = res.body.getReader();
|
||||||
const { done, value } = await reader.read();
|
const decoder = new TextDecoder();
|
||||||
if (done) {
|
let buf = "";
|
||||||
break;
|
let textChunks = 0;
|
||||||
}
|
const pending = new Map<
|
||||||
buf += decoder.decode(value, { stream: true });
|
number,
|
||||||
const frames = buf.split("\n\n");
|
{ id?: string; name: string; args: string }
|
||||||
buf = frames.pop() ?? "";
|
>();
|
||||||
|
|
||||||
for (const frame of frames) {
|
const flushTools = () => {
|
||||||
for (const line of frame.split("\n")) {
|
for (const call of pending.values()) {
|
||||||
if (!line.startsWith("data:")) {
|
let input: object = {};
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const data = line.slice(5).trim();
|
|
||||||
if (data === "[DONE]") {
|
|
||||||
if (pending.size) {
|
|
||||||
flushTools();
|
|
||||||
}
|
|
||||||
out.appendLine(` done, ${textChunks} text chunks`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let json: any;
|
|
||||||
try {
|
try {
|
||||||
json = JSON.parse(data);
|
input = JSON.parse(call.args || "{}");
|
||||||
} catch {
|
} catch {
|
||||||
continue;
|
out.appendLine(
|
||||||
}
|
` WARN bad tool args for ${call.name}: ${call.args}`,
|
||||||
const choice = json.choices?.[0];
|
);
|
||||||
const delta = choice?.delta;
|
|
||||||
if (!delta) {
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
out.appendLine(` tool call: ${call.name} ${JSON.stringify(input)}`);
|
||||||
|
progress.report(
|
||||||
|
new vscode.LanguageModelToolCallPart(
|
||||||
|
call.id ?? `call_${Math.random().toString(36).slice(2)}`,
|
||||||
|
call.name,
|
||||||
|
input,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
pending.clear();
|
||||||
|
};
|
||||||
|
|
||||||
if (typeof delta.content === "string" && delta.content) {
|
while (true) {
|
||||||
textChunks++;
|
if (token.isCancellationRequested) {
|
||||||
progress.report(new vscode.LanguageModelTextPart(delta.content));
|
out.appendLine(" cancelled mid-stream; aborting request");
|
||||||
}
|
abort.abort(); // close the connection so the gateway stops generating
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
buf += decoder.decode(value, { stream: true });
|
||||||
|
const frames = buf.split("\n\n");
|
||||||
|
buf = frames.pop() ?? "";
|
||||||
|
|
||||||
for (const tc of delta.tool_calls ?? []) {
|
for (const frame of frames) {
|
||||||
const i = tc.index ?? 0;
|
for (const line of frame.split("\n")) {
|
||||||
const acc = pending.get(i) ?? {
|
if (!line.startsWith("data:")) {
|
||||||
id: undefined,
|
continue;
|
||||||
name: "",
|
}
|
||||||
args: "",
|
const data = line.slice(5).trim();
|
||||||
};
|
if (data === "[DONE]") {
|
||||||
acc.id = tc.id ?? acc.id;
|
if (pending.size) {
|
||||||
acc.name = tc.function?.name ?? acc.name;
|
flushTools();
|
||||||
acc.args += tc.function?.arguments ?? "";
|
}
|
||||||
pending.set(i, acc);
|
out.appendLine(` done, ${textChunks} text chunks`);
|
||||||
}
|
return;
|
||||||
|
}
|
||||||
|
let json: any;
|
||||||
|
try {
|
||||||
|
json = JSON.parse(data);
|
||||||
|
} catch {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const choice = json.choices?.[0];
|
||||||
|
const delta = choice?.delta;
|
||||||
|
if (!delta) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
if (choice?.finish_reason) {
|
if (typeof delta.content === "string" && delta.content) {
|
||||||
out.appendLine(` finish_reason: ${choice.finish_reason}`);
|
textChunks++;
|
||||||
if (pending.size) {
|
progress.report(new vscode.LanguageModelTextPart(delta.content));
|
||||||
flushTools();
|
}
|
||||||
|
|
||||||
|
for (const tc of delta.tool_calls ?? []) {
|
||||||
|
const i = tc.index ?? 0;
|
||||||
|
const acc = pending.get(i) ?? {
|
||||||
|
id: undefined,
|
||||||
|
name: "",
|
||||||
|
args: "",
|
||||||
|
};
|
||||||
|
acc.id = tc.id ?? acc.id;
|
||||||
|
acc.name = tc.function?.name ?? acc.name;
|
||||||
|
acc.args += tc.function?.arguments ?? "";
|
||||||
|
pending.set(i, acc);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (choice?.finish_reason) {
|
||||||
|
out.appendLine(` finish_reason: ${choice.finish_reason}`);
|
||||||
|
if (pending.size) {
|
||||||
|
flushTools();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (pending.size) {
|
||||||
|
flushTools();
|
||||||
|
}
|
||||||
|
out.appendLine(` stream ended, ${textChunks} text chunks`);
|
||||||
|
} catch (e) {
|
||||||
|
// A cancellation aborts fetch (before headers) or the pending stream
|
||||||
|
// read (mid-response) - both surface as AbortError. That is the user
|
||||||
|
// pressing stop, not a failure: stop quietly instead of erroring.
|
||||||
|
if (
|
||||||
|
token.isCancellationRequested ||
|
||||||
|
(e instanceof Error && e.name === "AbortError")
|
||||||
|
) {
|
||||||
|
out.appendLine(" stream aborted after cancellation");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw e;
|
||||||
|
} finally {
|
||||||
|
cancelSubscription.dispose();
|
||||||
}
|
}
|
||||||
if (pending.size) {
|
|
||||||
flushTools();
|
|
||||||
}
|
|
||||||
out.appendLine(` stream ended, ${textChunks} text chunks`);
|
|
||||||
},
|
},
|
||||||
|
|
||||||
async provideTokenCount() {
|
async provideTokenCount() {
|
||||||
|
|||||||
Reference in new issue
Block a user