Abort in-flight requests when the chat is cancelled #1

Merged
evanp merged 1 commits from code/inference-coop-copilot:fix/abort-on-cancel into main 2026-11-07 04:07:20 +00:00
Showing only changes of commit 9798a41745 - Show all commits
+127 -94
View File
@@ -109,118 +109,151 @@ export function activate(context: vscode.ExtensionContext) {
body.tool_choice = "auto";
}
const res = await fetch(`${BASE_URL}/chat/completions`, {
method: "POST",
headers: {
Authorization: `Bearer ${key}`,
"Content-Type": "application/json",
},
body: JSON.stringify(body),
signal: token.isCancellationRequested ? AbortSignal.abort() : undefined,
});
out.appendLine(` /chat/completions status: ${res.status}`);
if (!res.ok || !res.body) {
throw new Error(
`Inference Coop: HTTP ${res.status} — ${await res.text()}`,
);
if (token.isCancellationRequested) {
out.appendLine(" request cancelled before it started");
return;
}
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buf = "";
let textChunks = 0;
const pending = new Map<
number,
{ id?: string; name: string; args: string }
>();
// Abort the request as soon as the user cancels, so a cancelled chat
// stops generating (and stops billing) server-side.
const abort = new AbortController();
const cancelSubscription = token.onCancellationRequested(() =>
abort.abort(),
);
const flushTools = () => {
for (const call of pending.values()) {
let input: object = {};
try {
input = JSON.parse(call.args || "{}");
} catch {
out.appendLine(
` WARN bad tool args for ${call.name}: ${call.args}`,
);
}
out.appendLine(` tool call: ${call.name} ${JSON.stringify(input)}`);
progress.report(
new vscode.LanguageModelToolCallPart(
call.id ?? `call_${Math.random().toString(36).slice(2)}`,
call.name,
input,
),
try {
const res = await fetch(`${BASE_URL}/chat/completions`, {
method: "POST",
headers: {
Authorization: `Bearer ${key}`,
"Content-Type": "application/json",
},
body: JSON.stringify(body),
signal: abort.signal,
});
out.appendLine(` /chat/completions status: ${res.status}`);
if (!res.ok || !res.body) {
throw new Error(
`Inference Coop: HTTP ${res.status} — ${await res.text()}`,
);
}
pending.clear();
};
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
buf += decoder.decode(value, { stream: true });
const frames = buf.split("\n\n");
buf = frames.pop() ?? "";
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buf = "";
let textChunks = 0;
const pending = new Map<
number,
{ id?: string; name: string; args: string }
>();
for (const frame of frames) {
for (const line of frame.split("\n")) {
if (!line.startsWith("data:")) {
continue;
}
const data = line.slice(5).trim();
if (data === "[DONE]") {
if (pending.size) {
flushTools();
}
out.appendLine(` done, ${textChunks} text chunks`);
return;
}
let json: any;
const flushTools = () => {
for (const call of pending.values()) {
let input: object = {};
try {
json = JSON.parse(data);
input = JSON.parse(call.args || "{}");
} catch {
continue;
}
const choice = json.choices?.[0];
const delta = choice?.delta;
if (!delta) {
continue;
out.appendLine(
` WARN bad tool args for ${call.name}: ${call.args}`,
);
}
out.appendLine(` tool call: ${call.name} ${JSON.stringify(input)}`);
progress.report(
new vscode.LanguageModelToolCallPart(
call.id ?? `call_${Math.random().toString(36).slice(2)}`,
call.name,
input,
),
);
}
pending.clear();
};
if (typeof delta.content === "string" && delta.content) {
textChunks++;
progress.report(new vscode.LanguageModelTextPart(delta.content));
}
while (true) {
if (token.isCancellationRequested) {
out.appendLine(" cancelled mid-stream; aborting request");
abort.abort(); // close the connection so the gateway stops generating
return;
}
const { done, value } = await reader.read();
if (done) {
break;
}
buf += decoder.decode(value, { stream: true });
const frames = buf.split("\n\n");
buf = frames.pop() ?? "";
for (const tc of delta.tool_calls ?? []) {
const i = tc.index ?? 0;
const acc = pending.get(i) ?? {
id: undefined,
name: "",
args: "",
};
acc.id = tc.id ?? acc.id;
acc.name = tc.function?.name ?? acc.name;
acc.args += tc.function?.arguments ?? "";
pending.set(i, acc);
}
for (const frame of frames) {
for (const line of frame.split("\n")) {
if (!line.startsWith("data:")) {
continue;
}
const data = line.slice(5).trim();
if (data === "[DONE]") {
if (pending.size) {
flushTools();
}
out.appendLine(` done, ${textChunks} text chunks`);
return;
}
let json: any;
try {
json = JSON.parse(data);
} catch {
continue;
}
const choice = json.choices?.[0];
const delta = choice?.delta;
if (!delta) {
continue;
}
if (choice?.finish_reason) {
out.appendLine(` finish_reason: ${choice.finish_reason}`);
if (pending.size) {
flushTools();
if (typeof delta.content === "string" && delta.content) {
textChunks++;
progress.report(new vscode.LanguageModelTextPart(delta.content));
}
for (const tc of delta.tool_calls ?? []) {
const i = tc.index ?? 0;
const acc = pending.get(i) ?? {
id: undefined,
name: "",
args: "",
};
acc.id = tc.id ?? acc.id;
acc.name = tc.function?.name ?? acc.name;
acc.args += tc.function?.arguments ?? "";
pending.set(i, acc);
}
if (choice?.finish_reason) {
out.appendLine(` finish_reason: ${choice.finish_reason}`);
if (pending.size) {
flushTools();
}
}
}
}
}
if (pending.size) {
flushTools();
}
out.appendLine(` stream ended, ${textChunks} text chunks`);
} catch (e) {
// A cancellation aborts fetch (before headers) or the pending stream
// read (mid-response) - both surface as AbortError. That is the user
// pressing stop, not a failure: stop quietly instead of erroring.
if (
token.isCancellationRequested ||
(e instanceof Error && e.name === "AbortError")
) {
out.appendLine(" stream aborted after cancellation");
return;
}
throw e;
} finally {
cancelSubscription.dispose();
}
if (pending.size) {
flushTools();
}
out.appendLine(` stream ended, ${textChunks} text chunks`);
},
async provideTokenCount() {