forked from evanp/inference-coop-copilot
Merge pull request 'Abort in-flight requests when the chat is cancelled' (#1) from code/inference-coop-copilot:fix/abort-on-cancel into main
Reviewed-on: evanp/inference-coop-copilot#1
This commit is contained in:
commit
4fdaa0ca9d
1 file changed
+34
-1
+34
-1
@@ -109,6 +109,19 @@ export function activate(context: vscode.ExtensionContext) {
|
|||||||
body.tool_choice = "auto";
|
body.tool_choice = "auto";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (token.isCancellationRequested) {
|
||||||
|
out.appendLine(" request cancelled before it started");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Abort the request as soon as the user cancels, so a cancelled chat
|
||||||
|
// stops generating (and stops billing) server-side.
|
||||||
|
const abort = new AbortController();
|
||||||
|
const cancelSubscription = token.onCancellationRequested(() =>
|
||||||
|
abort.abort(),
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
const res = await fetch(`${BASE_URL}/chat/completions`, {
|
const res = await fetch(`${BASE_URL}/chat/completions`, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: {
|
headers: {
|
||||||
@@ -116,7 +129,7 @@ export function activate(context: vscode.ExtensionContext) {
|
|||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
},
|
},
|
||||||
body: JSON.stringify(body),
|
body: JSON.stringify(body),
|
||||||
signal: token.isCancellationRequested ? AbortSignal.abort() : undefined,
|
signal: abort.signal,
|
||||||
});
|
});
|
||||||
out.appendLine(` /chat/completions status: ${res.status}`);
|
out.appendLine(` /chat/completions status: ${res.status}`);
|
||||||
if (!res.ok || !res.body) {
|
if (!res.ok || !res.body) {
|
||||||
@@ -157,6 +170,11 @@ export function activate(context: vscode.ExtensionContext) {
|
|||||||
};
|
};
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
|
if (token.isCancellationRequested) {
|
||||||
|
out.appendLine(" cancelled mid-stream; aborting request");
|
||||||
|
abort.abort(); // close the connection so the gateway stops generating
|
||||||
|
return;
|
||||||
|
}
|
||||||
const { done, value } = await reader.read();
|
const { done, value } = await reader.read();
|
||||||
if (done) {
|
if (done) {
|
||||||
break;
|
break;
|
||||||
@@ -221,6 +239,21 @@ export function activate(context: vscode.ExtensionContext) {
|
|||||||
flushTools();
|
flushTools();
|
||||||
}
|
}
|
||||||
out.appendLine(` stream ended, ${textChunks} text chunks`);
|
out.appendLine(` stream ended, ${textChunks} text chunks`);
|
||||||
|
} catch (e) {
|
||||||
|
// A cancellation aborts fetch (before headers) or the pending stream
|
||||||
|
// read (mid-response) - both surface as AbortError. That is the user
|
||||||
|
// pressing stop, not a failure: stop quietly instead of erroring.
|
||||||
|
if (
|
||||||
|
token.isCancellationRequested ||
|
||||||
|
(e instanceof Error && e.name === "AbortError")
|
||||||
|
) {
|
||||||
|
out.appendLine(" stream aborted after cancellation");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
throw e;
|
||||||
|
} finally {
|
||||||
|
cancelSubscription.dispose();
|
||||||
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
async provideTokenCount() {
|
async provideTokenCount() {
|
||||||
|
|||||||
Reference in new issue
Block a user