fix: abort in-flight chat requests on cancellation
Cancellation was only checked at POST time; once the SSE stream was open the reader loop never looked at the token again, so a stopped chat kept generating server-side (and billing) until the response ended. Wire an AbortController to token.onCancellationRequested, pass its signal to fetch, and check token.isCancellationRequested at the top of the reader loop so the connection closes as soon as the user stops the request. AbortError is swallowed (a stop is not a failure); real HTTP errors still surface, and the cancellation listener is disposed in a finally.
This commit is contained in:
1 parent
46b8d7e888
commit
9798a41745
1 file changed
+127
-94
+127
-94
@@ -109,118 +109,151 @@ export function activate(context: vscode.ExtensionContext) {
|
||||
body.tool_choice = "auto";
|
||||
}
|
||||
|
||||
const res = await fetch(`${BASE_URL}/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${key}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: token.isCancellationRequested ? AbortSignal.abort() : undefined,
|
||||
});
|
||||
out.appendLine(` /chat/completions status: ${res.status}`);
|
||||
if (!res.ok || !res.body) {
|
||||
throw new Error(
|
||||
`Inference Coop: HTTP ${res.status} — ${await res.text()}`,
|
||||
);
|
||||
if (token.isCancellationRequested) {
|
||||
out.appendLine(" request cancelled before it started");
|
||||
return;
|
||||
}
|
||||
|
||||
const reader = res.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buf = "";
|
||||
let textChunks = 0;
|
||||
const pending = new Map<
|
||||
number,
|
||||
{ id?: string; name: string; args: string }
|
||||
>();
|
||||
// Abort the request as soon as the user cancels, so a cancelled chat
|
||||
// stops generating (and stops billing) server-side.
|
||||
const abort = new AbortController();
|
||||
const cancelSubscription = token.onCancellationRequested(() =>
|
||||
abort.abort(),
|
||||
);
|
||||
|
||||
const flushTools = () => {
|
||||
for (const call of pending.values()) {
|
||||
let input: object = {};
|
||||
try {
|
||||
input = JSON.parse(call.args || "{}");
|
||||
} catch {
|
||||
out.appendLine(
|
||||
` WARN bad tool args for ${call.name}: ${call.args}`,
|
||||
);
|
||||
}
|
||||
out.appendLine(` tool call: ${call.name} ${JSON.stringify(input)}`);
|
||||
progress.report(
|
||||
new vscode.LanguageModelToolCallPart(
|
||||
call.id ?? `call_${Math.random().toString(36).slice(2)}`,
|
||||
call.name,
|
||||
input,
|
||||
),
|
||||
try {
|
||||
const res = await fetch(`${BASE_URL}/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${key}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: abort.signal,
|
||||
});
|
||||
out.appendLine(` /chat/completions status: ${res.status}`);
|
||||
if (!res.ok || !res.body) {
|
||||
throw new Error(
|
||||
`Inference Coop: HTTP ${res.status} — ${await res.text()}`,
|
||||
);
|
||||
}
|
||||
pending.clear();
|
||||
};
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
buf += decoder.decode(value, { stream: true });
|
||||
const frames = buf.split("\n\n");
|
||||
buf = frames.pop() ?? "";
|
||||
const reader = res.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buf = "";
|
||||
let textChunks = 0;
|
||||
const pending = new Map<
|
||||
number,
|
||||
{ id?: string; name: string; args: string }
|
||||
>();
|
||||
|
||||
for (const frame of frames) {
|
||||
for (const line of frame.split("\n")) {
|
||||
if (!line.startsWith("data:")) {
|
||||
continue;
|
||||
}
|
||||
const data = line.slice(5).trim();
|
||||
if (data === "[DONE]") {
|
||||
if (pending.size) {
|
||||
flushTools();
|
||||
}
|
||||
out.appendLine(` done, ${textChunks} text chunks`);
|
||||
return;
|
||||
}
|
||||
let json: any;
|
||||
const flushTools = () => {
|
||||
for (const call of pending.values()) {
|
||||
let input: object = {};
|
||||
try {
|
||||
json = JSON.parse(data);
|
||||
input = JSON.parse(call.args || "{}");
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const choice = json.choices?.[0];
|
||||
const delta = choice?.delta;
|
||||
if (!delta) {
|
||||
continue;
|
||||
out.appendLine(
|
||||
` WARN bad tool args for ${call.name}: ${call.args}`,
|
||||
);
|
||||
}
|
||||
out.appendLine(` tool call: ${call.name} ${JSON.stringify(input)}`);
|
||||
progress.report(
|
||||
new vscode.LanguageModelToolCallPart(
|
||||
call.id ?? `call_${Math.random().toString(36).slice(2)}`,
|
||||
call.name,
|
||||
input,
|
||||
),
|
||||
);
|
||||
}
|
||||
pending.clear();
|
||||
};
|
||||
|
||||
if (typeof delta.content === "string" && delta.content) {
|
||||
textChunks++;
|
||||
progress.report(new vscode.LanguageModelTextPart(delta.content));
|
||||
}
|
||||
while (true) {
|
||||
if (token.isCancellationRequested) {
|
||||
out.appendLine(" cancelled mid-stream; aborting request");
|
||||
abort.abort(); // close the connection so the gateway stops generating
|
||||
return;
|
||||
}
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
buf += decoder.decode(value, { stream: true });
|
||||
const frames = buf.split("\n\n");
|
||||
buf = frames.pop() ?? "";
|
||||
|
||||
for (const tc of delta.tool_calls ?? []) {
|
||||
const i = tc.index ?? 0;
|
||||
const acc = pending.get(i) ?? {
|
||||
id: undefined,
|
||||
name: "",
|
||||
args: "",
|
||||
};
|
||||
acc.id = tc.id ?? acc.id;
|
||||
acc.name = tc.function?.name ?? acc.name;
|
||||
acc.args += tc.function?.arguments ?? "";
|
||||
pending.set(i, acc);
|
||||
}
|
||||
for (const frame of frames) {
|
||||
for (const line of frame.split("\n")) {
|
||||
if (!line.startsWith("data:")) {
|
||||
continue;
|
||||
}
|
||||
const data = line.slice(5).trim();
|
||||
if (data === "[DONE]") {
|
||||
if (pending.size) {
|
||||
flushTools();
|
||||
}
|
||||
out.appendLine(` done, ${textChunks} text chunks`);
|
||||
return;
|
||||
}
|
||||
let json: any;
|
||||
try {
|
||||
json = JSON.parse(data);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const choice = json.choices?.[0];
|
||||
const delta = choice?.delta;
|
||||
if (!delta) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (choice?.finish_reason) {
|
||||
out.appendLine(` finish_reason: ${choice.finish_reason}`);
|
||||
if (pending.size) {
|
||||
flushTools();
|
||||
if (typeof delta.content === "string" && delta.content) {
|
||||
textChunks++;
|
||||
progress.report(new vscode.LanguageModelTextPart(delta.content));
|
||||
}
|
||||
|
||||
for (const tc of delta.tool_calls ?? []) {
|
||||
const i = tc.index ?? 0;
|
||||
const acc = pending.get(i) ?? {
|
||||
id: undefined,
|
||||
name: "",
|
||||
args: "",
|
||||
};
|
||||
acc.id = tc.id ?? acc.id;
|
||||
acc.name = tc.function?.name ?? acc.name;
|
||||
acc.args += tc.function?.arguments ?? "";
|
||||
pending.set(i, acc);
|
||||
}
|
||||
|
||||
if (choice?.finish_reason) {
|
||||
out.appendLine(` finish_reason: ${choice.finish_reason}`);
|
||||
if (pending.size) {
|
||||
flushTools();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (pending.size) {
|
||||
flushTools();
|
||||
}
|
||||
out.appendLine(` stream ended, ${textChunks} text chunks`);
|
||||
} catch (e) {
|
||||
// A cancellation aborts fetch (before headers) or the pending stream
|
||||
// read (mid-response) - both surface as AbortError. That is the user
|
||||
// pressing stop, not a failure: stop quietly instead of erroring.
|
||||
if (
|
||||
token.isCancellationRequested ||
|
||||
(e instanceof Error && e.name === "AbortError")
|
||||
) {
|
||||
out.appendLine(" stream aborted after cancellation");
|
||||
return;
|
||||
}
|
||||
throw e;
|
||||
} finally {
|
||||
cancelSubscription.dispose();
|
||||
}
|
||||
if (pending.size) {
|
||||
flushTools();
|
||||
}
|
||||
out.appendLine(` stream ended, ${textChunks} text chunks`);
|
||||
},
|
||||
|
||||
async provideTokenCount() {
|
||||
|
||||
Reference in new issue
Block a user