diff --git a/src/extension.ts b/src/extension.ts index e5ad164..be97992 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -109,118 +109,151 @@ export function activate(context: vscode.ExtensionContext) { body.tool_choice = "auto"; } - const res = await fetch(`${BASE_URL}/chat/completions`, { - method: "POST", - headers: { - Authorization: `Bearer ${key}`, - "Content-Type": "application/json", - }, - body: JSON.stringify(body), - signal: token.isCancellationRequested ? AbortSignal.abort() : undefined, - }); - out.appendLine(` /chat/completions status: ${res.status}`); - if (!res.ok || !res.body) { - throw new Error( - `Inference Coop: HTTP ${res.status} — ${await res.text()}`, - ); + if (token.isCancellationRequested) { + out.appendLine(" request cancelled before it started"); + return; } - const reader = res.body.getReader(); - const decoder = new TextDecoder(); - let buf = ""; - let textChunks = 0; - const pending = new Map< - number, - { id?: string; name: string; args: string } - >(); + // Abort the request as soon as the user cancels, so a cancelled chat + // stops generating (and stops billing) server-side. + const abort = new AbortController(); + const cancelSubscription = token.onCancellationRequested(() => + abort.abort(), + ); - const flushTools = () => { - for (const call of pending.values()) { - let input: object = {}; - try { - input = JSON.parse(call.args || "{}"); - } catch { - out.appendLine( - ` WARN bad tool args for ${call.name}: ${call.args}`, - ); - } - out.appendLine(` tool call: ${call.name} ${JSON.stringify(input)}`); - progress.report( - new vscode.LanguageModelToolCallPart( - call.id ?? `call_${Math.random().toString(36).slice(2)}`, - call.name, - input, - ), + try { + const res = await fetch(`${BASE_URL}/chat/completions`, { + method: "POST", + headers: { + Authorization: `Bearer ${key}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + signal: abort.signal, + }); + out.appendLine(` /chat/completions status: ${res.status}`); + if (!res.ok || !res.body) { + throw new Error( + `Inference Coop: HTTP ${res.status} — ${await res.text()}`, ); } - pending.clear(); - }; - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - buf += decoder.decode(value, { stream: true }); - const frames = buf.split("\n\n"); - buf = frames.pop() ?? ""; + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buf = ""; + let textChunks = 0; + const pending = new Map< + number, + { id?: string; name: string; args: string } + >(); - for (const frame of frames) { - for (const line of frame.split("\n")) { - if (!line.startsWith("data:")) { - continue; - } - const data = line.slice(5).trim(); - if (data === "[DONE]") { - if (pending.size) { - flushTools(); - } - out.appendLine(` done, ${textChunks} text chunks`); - return; - } - let json: any; + const flushTools = () => { + for (const call of pending.values()) { + let input: object = {}; try { - json = JSON.parse(data); + input = JSON.parse(call.args || "{}"); } catch { - continue; - } - const choice = json.choices?.[0]; - const delta = choice?.delta; - if (!delta) { - continue; + out.appendLine( + ` WARN bad tool args for ${call.name}: ${call.args}`, + ); } + out.appendLine(` tool call: ${call.name} ${JSON.stringify(input)}`); + progress.report( + new vscode.LanguageModelToolCallPart( + call.id ?? `call_${Math.random().toString(36).slice(2)}`, + call.name, + input, + ), + ); + } + pending.clear(); + }; - if (typeof delta.content === "string" && delta.content) { - textChunks++; - progress.report(new vscode.LanguageModelTextPart(delta.content)); - } + while (true) { + if (token.isCancellationRequested) { + out.appendLine(" cancelled mid-stream; aborting request"); + abort.abort(); // close the connection so the gateway stops generating + return; + } + const { done, value } = await reader.read(); + if (done) { + break; + } + buf += decoder.decode(value, { stream: true }); + const frames = buf.split("\n\n"); + buf = frames.pop() ?? ""; - for (const tc of delta.tool_calls ?? []) { - const i = tc.index ?? 0; - const acc = pending.get(i) ?? { - id: undefined, - name: "", - args: "", - }; - acc.id = tc.id ?? acc.id; - acc.name = tc.function?.name ?? acc.name; - acc.args += tc.function?.arguments ?? ""; - pending.set(i, acc); - } + for (const frame of frames) { + for (const line of frame.split("\n")) { + if (!line.startsWith("data:")) { + continue; + } + const data = line.slice(5).trim(); + if (data === "[DONE]") { + if (pending.size) { + flushTools(); + } + out.appendLine(` done, ${textChunks} text chunks`); + return; + } + let json: any; + try { + json = JSON.parse(data); + } catch { + continue; + } + const choice = json.choices?.[0]; + const delta = choice?.delta; + if (!delta) { + continue; + } - if (choice?.finish_reason) { - out.appendLine(` finish_reason: ${choice.finish_reason}`); - if (pending.size) { - flushTools(); + if (typeof delta.content === "string" && delta.content) { + textChunks++; + progress.report(new vscode.LanguageModelTextPart(delta.content)); + } + + for (const tc of delta.tool_calls ?? []) { + const i = tc.index ?? 0; + const acc = pending.get(i) ?? { + id: undefined, + name: "", + args: "", + }; + acc.id = tc.id ?? acc.id; + acc.name = tc.function?.name ?? acc.name; + acc.args += tc.function?.arguments ?? ""; + pending.set(i, acc); + } + + if (choice?.finish_reason) { + out.appendLine(` finish_reason: ${choice.finish_reason}`); + if (pending.size) { + flushTools(); + } } } } } + if (pending.size) { + flushTools(); + } + out.appendLine(` stream ended, ${textChunks} text chunks`); + } catch (e) { + // A cancellation aborts fetch (before headers) or the pending stream + // read (mid-response) - both surface as AbortError. That is the user + // pressing stop, not a failure: stop quietly instead of erroring. + if ( + token.isCancellationRequested || + (e instanceof Error && e.name === "AbortError") + ) { + out.appendLine(" stream aborted after cancellation"); + return; + } + throw e; + } finally { + cancelSubscription.dispose(); } - if (pending.size) { - flushTools(); - } - out.appendLine(` stream ended, ${textChunks} text chunks`); }, async provideTokenCount() {