From 90b661583245b7804bf3ad5e24c615ed0a661531 Mon Sep 17 00:00:00 2001 From: Aaron Munger Date: Thu, 10 Sep 2026 15:48:06 -0700 Subject: [PATCH 1/4] agentHost: scope edit timing correlations to active turns Release completed-turn SDK and interaction IDs instead of retaining them for the lifetime of a Copilot session. Keep steering and late-event routing safe through turn-owned correlation sets. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../node/copilot/copilotAgentSession.ts | 48 +++----- .../test/node/copilotAgentSession.test.ts | 107 +++++++++++++++++- 2 files changed, 119 insertions(+), 36 deletions(-) diff --git a/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts b/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts index 71e418aee6eff..609bc9735c58c 100644 --- a/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts +++ b/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts @@ -705,6 +705,9 @@ class CopilotTurn extends Disposable { */ readonly toolCounts = new Map(); readonly mainModelCallIds = new Set(); + /** Root SDK correlations are valid only while their owning protocol turn is active. */ + readonly sdkTurnIds = new Set(); + readonly interactionIds = new Set(); toolCallRounds = 0; totalToolCalls = 0; parallelToolCallRounds = 0; @@ -783,6 +786,8 @@ class CopilotTurn extends Disposable { * Rejects {@link eventId} before disposal so pending fork-boundary checks do not hang. */ override dispose(): void { + this.sdkTurnIds.clear(); + this.interactionIds.clear(); if (!this._eventId.isSettled) { this._eventId.error(new Error(`Turn ${this.id} was disposed before its SDK event id was recorded`)); } @@ -829,10 +834,6 @@ export class CopilotAgentSession extends Disposable { * the same id, so mappings live until session teardown. */ private readonly _parentToolCallIdsByAgentId = new Map(); - /** Maps SDK root-agent turn ids to their owning host protocol turn ids. */ - private readonly _hostTurnIdsBySdkTurnId = new Map(); - /** Maps runtime interactions to their owning host protocol turn ids. */ - private readonly _hostTurnIdsByInteractionId = new Map(); private _activeRootSdkTurnId: string | undefined; private readonly _rootTurnIdBySubagentToolCallId = new Map(); readonly modelCallTurnCorrelation = new ModelCallTurnCorrelation(); @@ -1367,13 +1368,14 @@ export class CopilotAgentSession extends Disposable { this._logService.trace(`[Copilot:${this.sessionId}] Ignoring unroutable subagent model.call_finished: agentId=${event.agentId}, sdkTurnId=${event.data.turnId}`); return; } + const turn = this._currentTurn.value; let turnId: string | undefined; if (event.agentId) { turnId = this._turnId; } else if (event.data.interactionId) { - turnId = this._hostTurnIdsByInteractionId.get(event.data.interactionId); + turnId = turn?.interactionIds.has(event.data.interactionId) ? turn.id : undefined; } else { - turnId = this._hostTurnIdsBySdkTurnId.get(event.data.turnId); + turnId = turn?.sdkTurnIds.has(event.data.turnId) ? turn.id : undefined; } if (!turnId) { this._logService.trace(`[Copilot:${this.sessionId}] Ignoring model.call_finished without a host turn mapping: sdkTurnId=${event.data.turnId}`); @@ -1393,23 +1395,10 @@ export class CopilotAgentSession extends Disposable { } /** - * Promotes a pending steering message into its own protocol turn: - * closes the in-flight turn (so its responseParts settle into history) - * and dispatches {@link ActionType.ChatTurnStarted} for a fresh - * turn whose user message is the steering content. The action's - * `queuedMessageId` atomically clears the corresponding pending - * steering message from the session state. - * - * All subsequent SDK events (message deltas, tool calls, …) emitted - * by the agent now reference the new `_turnId`, so the steering - * response lands in the new turn rather than being folded into the - * original. - * - * Returns the new turn id so callers (notably the `user.message` - * handler) can associate the SDK event id with the steering turn for - * history.truncate / sessions.fork mapping. + * Closes the in-flight protocol turn and promotes a pending steering message into its own turn. + * Carries the active SDK turn association forward so subsequent events can target the steering turn. */ - private _beginSteeringTurn(steering: PendingMessage): string { + private _beginSteeringTurn(steering: PendingMessage): void { this._completeActiveTurn(); const newTurnId = generateUuid(); this._emitAction({ @@ -1432,10 +1421,9 @@ export class CopilotAgentSession extends Disposable { turn.messageCharLen = steering.message.text.length; turn.markRunning(); } - if (this._activeRootSdkTurnId) { - this._hostTurnIdsBySdkTurnId.set(this._activeRootSdkTurnId, newTurnId); + if (this._activeRootSdkTurnId && turn) { + turn.sdkTurnIds.add(this._activeRootSdkTurnId); } - return newTurnId; } /** @@ -5025,9 +5013,9 @@ export class CopilotAgentSession extends Disposable { this._currentTurn.value?.markRunning(); const steering = this._takeMatchingPendingSteering(e.data.content); if (steering) { - const turnId = this._beginSteeringTurn(steering); + this._beginSteeringTurn(steering); if (e.data.interactionId) { - this._hostTurnIdsByInteractionId.set(e.data.interactionId, turnId); + this._currentTurn.value?.interactionIds.add(e.data.interactionId); } } if (this._turnId) { @@ -6725,10 +6713,10 @@ export class CopilotAgentSession extends Disposable { this._resumeSubagentForEvent(e); if (!e.agentId) { this._activeRootSdkTurnId = e.data.turnId; - if (this._currentTurn.value) { - this._hostTurnIdsBySdkTurnId.set(e.data.turnId, this._currentTurn.value.id); + if (turn) { + turn.sdkTurnIds.add(e.data.turnId); if (e.data.interactionId) { - this._hostTurnIdsByInteractionId.set(e.data.interactionId, this._currentTurn.value.id); + turn.interactionIds.add(e.data.interactionId); } } const telemetryMessageId = this._currentTurn.value?.id ?? e.data.turnId; diff --git a/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts b/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts index fbe286966e07d..421953cfcc34d 100644 --- a/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts +++ b/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts @@ -6657,15 +6657,17 @@ Use the attached image as context. assert.strictEqual(steeringCompletions.length, 0, 'an aborted steering turn must not be completed'); }); - for (const { name, interactionId, expectedOriginalTurn } of [ - { name: 'new steering interaction before the next assistant.turn_start', interactionId: 'interaction-steer', expectedOriginalTurn: false }, - { name: 'stale original interaction after steering promotion', interactionId: 'interaction-original', expectedOriginalTurn: true }, - { name: 'missing interaction fallback after steering promotion', interactionId: undefined, expectedOriginalTurn: false }, + for (const { name, interactionId, expectedDropped } of [ + { name: 'new steering interaction before the next assistant.turn_start', interactionId: 'interaction-steer', expectedDropped: false }, + { name: 'stale original interaction after steering promotion', interactionId: 'interaction-original', expectedDropped: true }, + { name: 'missing interaction fallback after steering promotion', interactionId: undefined, expectedDropped: false }, ]) { test(`maps model-call lifecycle events with ${name}`, async () => { const { session, mockSession, signals } = await createAgentSession(disposables); session.resetTurnState('turn-original'); mockSession.fire('assistant.turn_start', { turnId: 'sdk-0', interactionId: 'interaction-original' }); + const originalTurn = session['_currentTurn'].value; + assert.ok(originalTurn); await session.sendSteering({ id: 'steer-1', message: { text: 'focus on tests', origin: { kind: MessageKind.User } } }); mockSession.fire('user.message', { @@ -6714,16 +6716,18 @@ Use the attached image as context. } assert.deepStrictEqual({ + originalCorrelations: [originalTurn.sdkTurnIds.size, originalTurn.interactionIds.size], modelCallTurnIds: signals.filter(signal => signal.kind === 'model_call_finished').map(signal => signal.turnId), completedTurns: telemetryService.events.filter(event => event.eventName === 'agentHost.turnCompleted').map(event => { const data = event.data as { turnId: string; timeToFirstEdit?: number }; return { turnId: data.turnId, timeToFirstEdit: data.timeToFirstEdit }; }), }, { - modelCallTurnIds: [expectedOriginalTurn ? 'turn-original' : steeringTurnId], + originalCorrelations: [0, 0], + modelCallTurnIds: expectedDropped ? [] : [steeringTurnId], completedTurns: [ { turnId: 'turn-original', timeToFirstEdit: undefined }, - { turnId: steeringTurnId, timeToFirstEdit: expectedOriginalTurn ? undefined : 250 }, + { turnId: steeringTurnId, timeToFirstEdit: expectedDropped ? undefined : 250 }, ], }); }); @@ -7627,6 +7631,97 @@ Use the attached image as context. ); }); + for (const ending of ['complete', 'abort', 'fail', 'discard', 'replace', 'dispose'] as const) { + test(`releases model-call correlations when a host turn ends via ${ending}`, async () => { + const { session, mockSession, signals } = await createAgentSession(disposables); + const counts: { sdkTurnIds: number; interactionIds: number }[] = []; + const iterations = ending === 'dispose' ? 1 : 25; + for (let i = 0; i < iterations; i++) { + session.resetTurnState(`host-turn-${i}`); + mockSession.fire('assistant.turn_start', { turnId: `sdk-turn-${i}`, interactionId: `interaction-${i}` }); + mockSession.fire('assistant.turn_start', { turnId: `sdk-turn-${i}-next`, interactionId: `interaction-${i}` }); + const turn = session['_currentTurn'].value; + assert.ok(turn); + assert.deepStrictEqual([turn.sdkTurnIds.size, turn.interactionIds.size], [2, 1]); + + switch (ending) { + case 'complete': + mockSession.fire('session.idle', {}); + break; + case 'abort': + mockSession.fire('session.idle', { aborted: true }); + break; + case 'fail': + session.failActiveTurn({ errorType: 'test', message: 'failure' }); + break; + case 'discard': + session.discardActiveTurn(); + break; + case 'replace': + session.resetTurnState('replacement'); + break; + case 'dispose': + session.dispose(); + break; + } + + counts.push({ sdkTurnIds: turn.sdkTurnIds.size, interactionIds: turn.interactionIds.size }); + if (ending !== 'dispose') { + session.resetTurnState('replacement'); + mockSession.fire('assistant.turn_start', { turnId: `sdk-turn-${i}-next`, interactionId: 'replacement-interaction' }); + for (const interactionId of [`interaction-${i}`, undefined]) { + mockSession.fireRaw({ + type: 'model.call_finished', + ephemeral: true, + id: `late-call-${i}-${interactionId}`, + data: { + turnId: interactionId ? `sdk-turn-${i}-next` : `sdk-turn-${i}`, + interactionId, + dispatchDurationMs: 250, + outcome: 'success', + containsBuiltInFileEditRequest: true, + editClassifierVersion: 1, + }, + }); + } + } + } + + assert.deepStrictEqual({ + counts, + modelCalls: signals.filter(signal => signal.kind === 'model_call_finished'), + }, { + counts: Array.from({ length: iterations }, () => ({ sdkTurnIds: 0, interactionIds: 0 })), + modelCalls: [], + }); + }); + } + + test('keeps model-call correlations through SDK turn end until the host turn ends', async () => { + const { session, mockSession, signals } = await createAgentSession(disposables); + session.resetTurnState('host-turn'); + for (const sdkTurnId of ['0', '1']) { + mockSession.fire('assistant.turn_start', { turnId: sdkTurnId, interactionId: 'interaction' }); + mockSession.fire('assistant.turn_end', { turnId: sdkTurnId }); + } + for (const interactionId of ['interaction', undefined]) { + mockSession.fireRaw({ + type: 'model.call_finished', + ephemeral: true, + id: `model-call-${interactionId}`, + data: { + turnId: '0', + interactionId, + dispatchDurationMs: 250, + outcome: 'success', + containsBuiltInFileEditRequest: true, + editClassifierVersion: 1, + }, + }); + } + assert.deepStrictEqual(signals.filter(signal => signal.kind === 'model_call_finished').map(signal => signal.turnId), ['host-turn', 'host-turn']); + }); + test('resumes a subagent on turn start before mapping model.call_finished', async () => { const { session, mockSession, signals } = await createAgentSession(disposables); session.resetTurnState('host-turn-1'); From a1dd1c923d04d771701814c2f2a6363bc35e447e Mon Sep 17 00:00:00 2001 From: Aaron Munger Date: Thu, 10 Sep 2026 15:51:20 -0700 Subject: [PATCH 2/4] agentHost: avoid shadowed turn in correlation registration Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../platform/agentHost/node/copilot/copilotAgentSession.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts b/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts index 609bc9735c58c..c90a248dccfc8 100644 --- a/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts +++ b/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts @@ -6713,10 +6713,10 @@ export class CopilotAgentSession extends Disposable { this._resumeSubagentForEvent(e); if (!e.agentId) { this._activeRootSdkTurnId = e.data.turnId; - if (turn) { - turn.sdkTurnIds.add(e.data.turnId); + if (this._currentTurn.value) { + this._currentTurn.value.sdkTurnIds.add(e.data.turnId); if (e.data.interactionId) { - turn.interactionIds.add(e.data.interactionId); + this._currentTurn.value.interactionIds.add(e.data.interactionId); } } const telemetryMessageId = this._currentTurn.value?.id ?? e.data.turnId; From 478aaf63d44cc66a5ca10028e437f6ef471e96c1 Mon Sep 17 00:00:00 2001 From: Aaron Munger Date: Fri, 11 Sep 2026 08:31:28 -0700 Subject: [PATCH 3/4] agentHost: expire active SDK turn IDs with their host turn Keep steering from inheriting an SDK turn ID left behind by an earlier completed, failed, aborted, discarded, or replaced host turn. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../node/copilot/copilotAgentSession.ts | 16 +++--- .../test/node/copilotAgentSession.test.ts | 52 ++++++++++++++++--- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts b/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts index c90a248dccfc8..d079ba9423f00 100644 --- a/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts +++ b/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts @@ -708,6 +708,7 @@ class CopilotTurn extends Disposable { /** Root SDK correlations are valid only while their owning protocol turn is active. */ readonly sdkTurnIds = new Set(); readonly interactionIds = new Set(); + activeSdkTurnId: string | undefined; toolCallRounds = 0; totalToolCalls = 0; parallelToolCallRounds = 0; @@ -788,6 +789,7 @@ class CopilotTurn extends Disposable { override dispose(): void { this.sdkTurnIds.clear(); this.interactionIds.clear(); + this.activeSdkTurnId = undefined; if (!this._eventId.isSettled) { this._eventId.error(new Error(`Turn ${this.id} was disposed before its SDK event id was recorded`)); } @@ -834,7 +836,6 @@ export class CopilotAgentSession extends Disposable { * the same id, so mappings live until session teardown. */ private readonly _parentToolCallIdsByAgentId = new Map(); - private _activeRootSdkTurnId: string | undefined; private readonly _rootTurnIdBySubagentToolCallId = new Map(); readonly modelCallTurnCorrelation = new ModelCallTurnCorrelation(); private readonly _subagentDirectUsageByToolCallId = new Map(); @@ -1399,6 +1400,7 @@ export class CopilotAgentSession extends Disposable { * Carries the active SDK turn association forward so subsequent events can target the steering turn. */ private _beginSteeringTurn(steering: PendingMessage): void { + const activeSdkTurnId = this._currentTurn.value?.activeSdkTurnId; this._completeActiveTurn(); const newTurnId = generateUuid(); this._emitAction({ @@ -1421,8 +1423,9 @@ export class CopilotAgentSession extends Disposable { turn.messageCharLen = steering.message.text.length; turn.markRunning(); } - if (this._activeRootSdkTurnId && turn) { - turn.sdkTurnIds.add(this._activeRootSdkTurnId); + if (activeSdkTurnId && turn) { + turn.activeSdkTurnId = activeSdkTurnId; + turn.sdkTurnIds.add(activeSdkTurnId); } } @@ -6712,8 +6715,8 @@ export class CopilotAgentSession extends Disposable { this._logService.trace(`[Copilot:${sessionId}] Turn started: ${e.data.turnId}`); this._resumeSubagentForEvent(e); if (!e.agentId) { - this._activeRootSdkTurnId = e.data.turnId; if (this._currentTurn.value) { + this._currentTurn.value.activeSdkTurnId = e.data.turnId; this._currentTurn.value.sdkTurnIds.add(e.data.turnId); if (e.data.interactionId) { this._currentTurn.value.interactionIds.add(e.data.interactionId); @@ -6759,8 +6762,9 @@ export class CopilotAgentSession extends Disposable { this._register(wrapper.onTurnEnd(e => { this._logService.trace(`[Copilot:${sessionId}] Turn ended: ${e.data.turnId}`); - if (!e.agentId && this._activeRootSdkTurnId === e.data.turnId) { - this._activeRootSdkTurnId = undefined; + const turn = this._currentTurn.value; + if (!e.agentId && turn?.activeSdkTurnId === e.data.turnId) { + turn.activeSdkTurnId = undefined; } })); diff --git a/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts b/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts index 421953cfcc34d..c478bde1b261e 100644 --- a/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts +++ b/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts @@ -6716,14 +6716,14 @@ Use the attached image as context. } assert.deepStrictEqual({ - originalCorrelations: [originalTurn.sdkTurnIds.size, originalTurn.interactionIds.size], + originalCorrelations: [originalTurn.sdkTurnIds.size, originalTurn.interactionIds.size, originalTurn.activeSdkTurnId], modelCallTurnIds: signals.filter(signal => signal.kind === 'model_call_finished').map(signal => signal.turnId), completedTurns: telemetryService.events.filter(event => event.eventName === 'agentHost.turnCompleted').map(event => { const data = event.data as { turnId: string; timeToFirstEdit?: number }; return { turnId: data.turnId, timeToFirstEdit: data.timeToFirstEdit }; }), }, { - originalCorrelations: [0, 0], + originalCorrelations: [0, 0, undefined], modelCallTurnIds: expectedDropped ? [] : [steeringTurnId], completedTurns: [ { turnId: 'turn-original', timeToFirstEdit: undefined }, @@ -6733,6 +6733,46 @@ Use the attached image as context. }); } + for (const ending of ['complete', 'abort', 'fail', 'discard', 'replace'] as const) { + test(`does not carry an old SDK turn into steering after ${ending}`, async () => { + const { session, mockSession, signals } = await createAgentSession(disposables); + session.resetTurnState('old-turn'); + mockSession.fire('assistant.turn_start', { turnId: 'old-sdk-turn', interactionId: 'old-interaction' }); + switch (ending) { + case 'complete': + mockSession.fire('session.idle', {}); + break; + case 'abort': + mockSession.fire('session.idle', { aborted: true }); + break; + case 'fail': + session.failActiveTurn({ errorType: 'test', message: 'failure' }); + break; + case 'discard': + session.discardActiveTurn(); + break; + case 'replace': + break; + } + session.resetTurnState('new-turn'); + await session.sendSteering({ id: 'steer', message: { text: 'follow up', origin: { kind: MessageKind.User } } }); + mockSession.fire('user.message', { content: 'follow up', interactionId: 'steering-interaction' }); + mockSession.fireRaw({ + type: 'model.call_finished', + ephemeral: true, + id: 'late-old-call', + data: { + turnId: 'old-sdk-turn', + dispatchDurationMs: 250, + outcome: 'success', + containsBuiltInFileEditRequest: true, + editClassifierVersion: 1, + }, + }); + assert.deepStrictEqual(signals.filter(signal => signal.kind === 'model_call_finished'), []); + }); + } + test('does not signal cleanup when send fails', async () => { const { session, mockSession, signals } = await createAgentSession(disposables); @@ -7634,7 +7674,7 @@ Use the attached image as context. for (const ending of ['complete', 'abort', 'fail', 'discard', 'replace', 'dispose'] as const) { test(`releases model-call correlations when a host turn ends via ${ending}`, async () => { const { session, mockSession, signals } = await createAgentSession(disposables); - const counts: { sdkTurnIds: number; interactionIds: number }[] = []; + const counts: { sdkTurnIds: number; interactionIds: number; activeSdkTurnId: string | undefined }[] = []; const iterations = ending === 'dispose' ? 1 : 25; for (let i = 0; i < iterations; i++) { session.resetTurnState(`host-turn-${i}`); @@ -7642,7 +7682,7 @@ Use the attached image as context. mockSession.fire('assistant.turn_start', { turnId: `sdk-turn-${i}-next`, interactionId: `interaction-${i}` }); const turn = session['_currentTurn'].value; assert.ok(turn); - assert.deepStrictEqual([turn.sdkTurnIds.size, turn.interactionIds.size], [2, 1]); + assert.deepStrictEqual([turn.sdkTurnIds.size, turn.interactionIds.size, turn.activeSdkTurnId], [2, 1, `sdk-turn-${i}-next`]); switch (ending) { case 'complete': @@ -7665,7 +7705,7 @@ Use the attached image as context. break; } - counts.push({ sdkTurnIds: turn.sdkTurnIds.size, interactionIds: turn.interactionIds.size }); + counts.push({ sdkTurnIds: turn.sdkTurnIds.size, interactionIds: turn.interactionIds.size, activeSdkTurnId: turn.activeSdkTurnId }); if (ending !== 'dispose') { session.resetTurnState('replacement'); mockSession.fire('assistant.turn_start', { turnId: `sdk-turn-${i}-next`, interactionId: 'replacement-interaction' }); @@ -7691,7 +7731,7 @@ Use the attached image as context. counts, modelCalls: signals.filter(signal => signal.kind === 'model_call_finished'), }, { - counts: Array.from({ length: iterations }, () => ({ sdkTurnIds: 0, interactionIds: 0 })), + counts: Array.from({ length: iterations }, () => ({ sdkTurnIds: 0, interactionIds: 0, activeSdkTurnId: undefined })), modelCalls: [], }); }); From 7807e5ba56c17908abcf506fde726725fcca5b87 Mon Sep 17 00:00:00 2001 From: Aaron Munger Date: Fri, 11 Sep 2026 15:26:46 -0700 Subject: [PATCH 4/4] agentHost: identify steering in turn telemetry Distinguish provider-promoted steering turns from turns that receive steering, retaining time-to-first-edit measurements for both so queries can select the desired population. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../node/agentHostTelemetryReporter.ts | 8 + .../agentHost/node/agentHostTurnTracker.ts | 17 ++ .../agentHost/node/agentSideEffects.ts | 4 + .../builtInChatContributions.ts | 2 + .../steeringTelemetryContribution.ts | 38 +++++ .../test/node/agentHostTurnTelemetry.test.ts | 159 +++++++++++++++++- 6 files changed, 223 insertions(+), 5 deletions(-) create mode 100644 src/vs/platform/agentHost/node/chatContributions/steeringTelemetry/steeringTelemetryContribution.ts diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts index 6dd1315a98f36..14a6d5a5ee17a 100644 --- a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts +++ b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts @@ -219,6 +219,8 @@ export interface IAgentHostTurnCompletedEvent extends IAgentHostEventTelemetry { timeToFirstProgress: number | undefined; timeToFirstEdit: number | undefined; timeToFirstEditClassifierVersion: number | undefined; + startedWithSteering: boolean; + receivedSteering: boolean; totalTime: number; result: AgentHostTurnResult; model: string | TelemetryTrustedValue | undefined; @@ -251,6 +253,8 @@ export type IAgentHostTurnCompletedClassification = IAgentHostEventClassificatio timeToFirstProgress: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Time in milliseconds from turn start to the first visible progress (text delta, response part, tool call start, or reasoning).' }; timeToFirstEdit: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Cumulative provider-dispatch time in milliseconds through the first accepted response that requests a built-in file edit. Excludes prompt construction, retry backoff, tool execution, confirmations, and post-response processing.' }; timeToFirstEditClassifierVersion: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Version of the built-in file-edit request classifier used for timeToFirstEdit.' }; + startedWithSteering: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the provider promoted a steering message into this turn. Time to first edit remains a per-turn measurement, not a cumulative measurement across the preceding turn.' }; + receivedSteering: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether a steering message was submitted to this chat while this turn was active, regardless of whether the provider consumed it. Previously recorded time to first edit is preserved.' }; totalTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Total time in milliseconds from turn start to turn completion.' }; result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the turn completed successfully, with an error, or was cancelled.' }; model: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The trusted provider model identifier selected at turn start, or a generic value for BYOK and unknown models.' }; @@ -327,6 +331,8 @@ export interface IAgentHostTurnCompletedReport extends IAgentHostTurnAttributedR timeToFirstProgress: number | undefined; timeToFirstEditMs: number | undefined; timeToFirstEditClassifierVersion: number | undefined; + startedWithSteering: boolean; + receivedSteering: boolean; totalTime: number; result: AgentHostTurnResult; model: string | undefined; @@ -1352,6 +1358,8 @@ export class AgentHostTelemetryReporter { timeToFirstProgress: report.timeToFirstProgress, timeToFirstEdit: report.timeToFirstEditMs, timeToFirstEditClassifierVersion: report.timeToFirstEditClassifierVersion, + startedWithSteering: report.startedWithSteering, + receivedSteering: report.receivedSteering, totalTime: report.totalTime, result: report.result, model, diff --git a/src/vs/platform/agentHost/node/agentHostTurnTracker.ts b/src/vs/platform/agentHost/node/agentHostTurnTracker.ts index 56d90576dd388..63a3587472814 100644 --- a/src/vs/platform/agentHost/node/agentHostTurnTracker.ts +++ b/src/vs/platform/agentHost/node/agentHostTurnTracker.ts @@ -81,6 +81,8 @@ interface ITurnTiming { modelCallDispatchDurationMs: number; timeToFirstEditMs: number | undefined; timeToFirstEditClassifierVersion: number | undefined; + startedWithSteering: boolean; + receivedSteering: boolean; firstProgressMs: number | undefined; currentStage: AgentHostTurnFailureStage; @@ -201,6 +203,8 @@ export class AgentHostTurnTracker extends Disposable { modelCallDispatchDurationMs: 0, timeToFirstEditMs: undefined, timeToFirstEditClassifierVersion: undefined, + startedWithSteering: false, + receivedSteering: false, firstProgressMs: undefined, currentStage: 'validation', quietStopWatch: StopWatch.create(false), @@ -378,6 +382,17 @@ export class AgentHostTurnTracker extends Disposable { this._turnTimings.get(this._key(session, turnId))?.completedModelCallIds.add(modelCallId); } + markSteering(session: string, turnId: string, kind: 'started' | 'received'): void { + const timing = this._turnTimings.get(this._key(session, turnId)); + if (timing) { + if (kind === 'started') { + timing.startedWithSteering = true; + } else { + timing.receivedSteering = true; + } + } + } + modelCallFinished(session: string, turnId: string, modelCallId: string, dispatchDurationMs: number, outcome: AgentModelCallFinishedOutcome, containsBuiltInFileEditRequest: boolean | undefined, editClassifierVersion: number): void { const timing = this._turnTimings.get(this._key(session, turnId)); if (!timing || timing.finishedModelCallIds.has(modelCallId)) { @@ -452,6 +467,8 @@ export class AgentHostTurnTracker extends Disposable { timeToFirstProgress: timing.firstProgressMs, timeToFirstEditMs: timing.timeToFirstEditMs, timeToFirstEditClassifierVersion: timing.timeToFirstEditClassifierVersion, + startedWithSteering: timing.startedWithSteering, + receivedSteering: timing.receivedSteering, totalTime, result, model: timing.model, diff --git a/src/vs/platform/agentHost/node/agentSideEffects.ts b/src/vs/platform/agentHost/node/agentSideEffects.ts index eca19deeb1a80..1360b92c48b26 100644 --- a/src/vs/platform/agentHost/node/agentSideEffects.ts +++ b/src/vs/platform/agentHost/node/agentSideEffects.ts @@ -724,6 +724,10 @@ export class AgentSideEffects extends Disposable { hostLaunchKind: this._options.hostLaunchKind ?? AgentHostLaunchKind.Unknown, }; this._turnTracker.turnStarted(agent, sessionKey, action.turnId, model, modelTelemetryKind, modelSelectionKind, permissionLevel, interactionMode, clientContext, undefined, undefined, undefined, getMessageOriginTelemetryKind(action.message, this._stateManager.isEphemeralSession(sessionChannel))); + // Queue-drained starts are host-owned; a provider start consuming a pending message is steering. + if (action.queuedMessageId !== undefined) { + this._turnTracker.markSteering(sessionKey, action.turnId, 'started'); + } this._turnTracker.setCurrentStage(sessionKey, action.turnId, 'provider'); } else if (action.type === ActionType.ChatTurnComplete) { this._runTurnCompleteSideEffects(sessionKey, undefined); diff --git a/src/vs/platform/agentHost/node/chatContributions/builtInChatContributions.ts b/src/vs/platform/agentHost/node/chatContributions/builtInChatContributions.ts index b9d740c87452a..660453792c9a8 100644 --- a/src/vs/platform/agentHost/node/chatContributions/builtInChatContributions.ts +++ b/src/vs/platform/agentHost/node/chatContributions/builtInChatContributions.ts @@ -21,6 +21,7 @@ import { SessionFlagsContribution } from './sessionFlags/sessionFlagsContributio import { SessionInputNeededContribution } from './sessionInputNeeded/sessionInputNeededContribution.js'; import { SessionTitleContribution } from './sessionTitle/sessionTitleContribution.js'; import { SideChatContribution } from './sideChat/sideChatContribution.js'; +import { SteeringTelemetryContribution } from './steeringTelemetry/steeringTelemetryContribution.js'; import { TurnAdmissionContribution } from './turnAdmission/turnAdmissionContribution.js'; import { TurnDelegationContribution } from './turnDelegation/turnDelegationContribution.js'; import { WorktreeAnnouncementContribution } from './worktreeAnnouncement/worktreeAnnouncementContribution.js'; @@ -35,6 +36,7 @@ export function registerBuiltInChatContributions( registrations.add(contributions.registerContribution(PullRequestChatContribution)); registrations.add(contributions.registerContribution(TurnDelegationContribution)); registrations.add(contributions.registerContribution(PersistedTurnUsageContribution)); + registrations.add(contributions.registerContribution(SteeringTelemetryContribution)); registrations.add(contributions.registerContribution(WorktreeAnnouncementContribution)); registrations.add(contributions.registerContribution(CheckpointAndChangesetContribution)); registrations.add(contributions.registerContribution(SessionWorkspaceConversionContribution)); diff --git a/src/vs/platform/agentHost/node/chatContributions/steeringTelemetry/steeringTelemetryContribution.ts b/src/vs/platform/agentHost/node/chatContributions/steeringTelemetry/steeringTelemetryContribution.ts new file mode 100644 index 0000000000000..8c7ee826e3599 --- /dev/null +++ b/src/vs/platform/agentHost/node/chatContributions/steeringTelemetry/steeringTelemetryContribution.ts @@ -0,0 +1,38 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Disposable } from '../../../../../base/common/lifecycle.js'; +import type { IAgentHostChatContribution, IAgentHostChatContributionContext, IDispatchedAction } from '../../../common/agentHostChatContributionsService.js'; +import { ActionType } from '../../../common/state/sessionActions.js'; +import { isAhpChatChannel, PendingMessageKind } from '../../../common/state/sessionState.js'; +import { AgentHostStateManager, IAgentHostStateManager } from '../../agentHostStateManager.js'; +import { AgentHostTurnTracker, IAgentHostTurnTracker } from '../../agentHostTurnTracker.js'; + +export class SteeringTelemetryContribution extends Disposable implements IAgentHostChatContribution { + static readonly id = 'steeringTelemetry'; + readonly order = 150; + + constructor( + _context: IAgentHostChatContributionContext, + @IAgentHostStateManager private readonly _stateManager: AgentHostStateManager, + @IAgentHostTurnTracker private readonly _turnTracker: AgentHostTurnTracker, + ) { + super(); + } + + onDidDispatchAction({ channel, action, rejectionReason }: IDispatchedAction): void { + if (rejectionReason !== undefined || !isAhpChatChannel(channel) + || action.type !== ActionType.ChatPendingMessageSet || action.kind !== PendingMessageKind.Steering) { + return; + } + if (this._stateManager.getChatState(channel)?.steeringMessage?.id !== action.id) { + return; + } + const turnId = this._stateManager.getActiveTurnId(channel); + if (turnId) { + this._turnTracker.markSteering(channel, turnId, 'received'); + } + } +} diff --git a/src/vs/platform/agentHost/test/node/agentHostTurnTelemetry.test.ts b/src/vs/platform/agentHost/test/node/agentHostTurnTelemetry.test.ts index 5937ffbd6563f..842c4f8197bad 100644 --- a/src/vs/platform/agentHost/test/node/agentHostTurnTelemetry.test.ts +++ b/src/vs/platform/agentHost/test/node/agentHostTurnTelemetry.test.ts @@ -26,7 +26,7 @@ import { SessionConfigKey } from '../../common/sessionConfigKeys.js'; import { ActionType, type ChatAction, type ChatUsageAction } from '../../common/state/sessionActions.js'; import { withEphemeralSessionMeta } from '../../common/meta/agentEphemeralSessionMeta.js'; import { toAgentMergeMessageMeta } from '../../common/meta/agentMergeMessageMeta.js'; -import { buildDefaultChatUri, buildSubagentChatUri, createErrorResponsePart, type Message, MessageKind, PendingMessageKind, ResponsePartKind, SessionStatus } from '../../common/state/sessionState.js'; +import { buildChatUri, buildDefaultChatUri, buildSubagentChatUri, createErrorResponsePart, type Message, MessageKind, PendingMessageKind, ResponsePartKind, SessionStatus } from '../../common/state/sessionState.js'; import { IAgentHostCheckpointService, NULL_CHECKPOINT_SERVICE } from '../../common/agentHostCheckpointService.js'; import { IAgentHostChatContributions } from '../../common/agentHostChatContributionsService.js'; import { IAgentHostTerminalManager } from '../../node/agentHostTerminalManager.js'; @@ -120,6 +120,7 @@ suite('AgentSideEffects — turn tracker telemetry', () => { let telemetry: CapturingTelemetryService; let logService: NullLogService; let turnTracker: AgentHostTurnTracker; + let chatContributions: AgentHostChatContributions; const sessionUri = AgentSession.uri('mock', 'session-1'); const sessionKey = sessionUri.toString(); @@ -265,7 +266,7 @@ suite('AgentSideEffects — turn tracker telemetry', () => { }], ); const instantiationService = disposables.add(new InstantiationService(services, /*strict*/ true)); - const chatContributions = disposables.add(new AgentHostChatContributions(logService, instantiationService)); + chatContributions = disposables.add(new AgentHostChatContributions(logService, instantiationService)); services.set(IAgentHostChatContributions, chatContributions); services.set(IAgentHostTurnService, new AgentHostTurnService(stateManager, chatContributions, instantiationService)); services.set(IAgentHostSessionTitleController, disposables.add(new AgentHostSessionTitleController(stateManager, { sessionDataService }, logService))); @@ -425,9 +426,17 @@ suite('AgentSideEffects — turn tracker telemetry', () => { setSessionConfig({ autoApprove: 'autopilot', mode: 'interactive' }); startTurn('turn-original'); await new Promise(resolve => setTimeout(resolve, 0)); - fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-original', duration: 1000 }); + fireModelCallFinished('turn-original', 'call-before-steering', 100, 'success', true); + let previousTurnId = 'turn-original'; for (const turnId of ['turn-steering-1', 'turn-steering-2']) { + stateManager.dispatchClientAction(defaultChatUri, { + type: ActionType.ChatPendingMessageSet, + kind: PendingMessageKind.Steering, + id: `queued-${turnId}`, + message: { text: 'edit the file', origin: { kind: MessageKind.User } }, + }, { clientId: 'test', clientSeq: 2 }); + fire({ type: ActionType.ChatTurnComplete, turnId: previousTurnId, duration: 1000 }); fire({ type: ActionType.ChatTurnStarted, turnId, @@ -436,8 +445,9 @@ suite('AgentSideEffects — turn tracker telemetry', () => { queuedMessageId: `queued-${turnId}`, }); fireModelCallFinished(turnId, `call-${turnId}`, 250, 'success', true); - fire({ type: ActionType.ChatTurnComplete, turnId, duration: 1000 }); + previousTurnId = turnId; } + fire({ type: ActionType.ChatTurnComplete, turnId: previousTurnId, duration: 1000 }); await new Promise(resolve => setTimeout(resolve, 0)); assert.deepStrictEqual({ @@ -447,6 +457,9 @@ suite('AgentSideEffects — turn tracker telemetry', () => { return { turnId: data.turnId, timeToFirstEdit: data.timeToFirstEdit, + timeToFirstEditClassifierVersion: data.timeToFirstEditClassifierVersion, + startedWithSteering: data.startedWithSteering, + receivedSteering: data.receivedSteering, hostLaunchKind: data.hostLaunchKind, permissionLevel: data.permissionLevel, interactionMode: data.interactionMode, @@ -457,7 +470,10 @@ suite('AgentSideEffects — turn tracker telemetry', () => { sentPrompts: ['hello'], completed: ['turn-original', 'turn-steering-1', 'turn-steering-2'].map(turnId => ({ turnId, - timeToFirstEdit: turnId === 'turn-original' ? undefined : 250, + timeToFirstEdit: turnId === 'turn-original' ? 100 : 250, + timeToFirstEditClassifierVersion: 1, + startedWithSteering: turnId !== 'turn-original', + receivedSteering: turnId !== 'turn-steering-2', hostLaunchKind: undefined, permissionLevel: 'autopilot', interactionMode: 'interactive', @@ -466,6 +482,139 @@ suite('AgentSideEffects — turn tracker telemetry', () => { }); }); + for (const result of ['success', 'cancelled', 'error'] as const) { + test(`preserves steering submission and first edit on ${result} even if steering is removed`, () => { + setupSession(); + startTurn('turn-steered'); + fireModelCallFinished('turn-steered', 'edit', 150, 'success', true); + stateManager.dispatchClientAction(defaultChatUri, { + type: ActionType.ChatPendingMessageSet, + kind: PendingMessageKind.Steering, + id: 'steer', + message: { text: 'change direction', origin: { kind: MessageKind.User } }, + }, { clientId: 'test', clientSeq: 2 }); + stateManager.dispatchClientAction(defaultChatUri, { + type: ActionType.ChatPendingMessageRemoved, + kind: PendingMessageKind.Steering, + id: 'steer', + }, { clientId: 'test', clientSeq: 3 }); + if (result === 'error') { + fire({ type: ActionType.ChatError, turnId: 'turn-steered', duration: 1000, part: createErrorResponsePart({ errorType: 'test', message: 'failed' }) }); + } else { + fire({ type: result === 'success' ? ActionType.ChatTurnComplete : ActionType.ChatTurnCancelled, turnId: 'turn-steered', duration: 1000 }); + } + startTurn('turn-next'); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-next', duration: 1000 }); + + assert.deepStrictEqual(completedEvents().map(event => { + const data = event.data as Record; + return { + turnId: data.turnId, + result: data.result, + timeToFirstEdit: data.timeToFirstEdit, + startedWithSteering: data.startedWithSteering, + receivedSteering: data.receivedSteering, + }; + }), [ + { turnId: 'turn-steered', result, timeToFirstEdit: 150, startedWithSteering: false, receivedSteering: true }, + { turnId: 'turn-next', result: 'success', timeToFirstEdit: undefined, startedWithSteering: false, receivedSteering: false }, + ]); + }); + } + + test('does not classify queued follow-ups as steering', () => { + setupSession(); + startTurn('turn-original'); + const queued: ChatAction = { + type: ActionType.ChatPendingMessageSet, + kind: PendingMessageKind.Queued, + id: 'queued-follow-up', + message: { text: 'follow up', origin: { kind: MessageKind.User } }, + }; + stateManager.dispatchClientAction(defaultChatUri, queued, { clientId: 'test', clientSeq: 2 }); + sideEffects.handleAction(defaultChatUri, queued); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-original', duration: 1000 }); + const queuedTurnId = stateManager.getActiveTurnId(defaultChatUri); + assert.ok(queuedTurnId); + fireModelCallFinished(queuedTurnId, 'edit', 200, 'success', true); + fire({ type: ActionType.ChatTurnComplete, turnId: queuedTurnId, duration: 1000 }); + + assert.deepStrictEqual(completedEvents().map(event => { + const data = event.data as Record; + return { + timeToFirstEdit: data.timeToFirstEdit, + startedWithSteering: data.startedWithSteering, + receivedSteering: data.receivedSteering, + }; + }), [ + { timeToFirstEdit: undefined, startedWithSteering: false, receivedSteering: false }, + { timeToFirstEdit: 200, startedWithSteering: false, receivedSteering: false }, + ]); + }); + + test('ignores rejected steering actions and submissions made while idle', () => { + setupSession(); + const action: ChatAction = { + type: ActionType.ChatPendingMessageSet, + kind: PendingMessageKind.Steering, + id: 'steer-idle', + message: { text: 'change direction', origin: { kind: MessageKind.User } }, + }; + stateManager.dispatchClientAction(defaultChatUri, action, { clientId: 'test', clientSeq: 1 }); + startTurn('turn-after-idle'); + chatContributions.didDispatchAction({ channel: defaultChatUri, session: sessionKey, action, rejectionReason: 'rejected' }); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-after-idle', duration: 1000 }); + + const data = completedEvents()[0].data as Record; + assert.deepStrictEqual({ + startedWithSteering: data.startedWithSteering, + receivedSteering: data.receivedSteering, + }, { startedWithSteering: false, receivedSteering: false }); + }); + + test('attributes steering only to the targeted peer chat', () => { + setupSession(); + const peerChatUri = buildChatUri(sessionUri, 'peer'); + stateManager.addChat(sessionKey, peerChatUri); + startTurn('turn-default'); + startTurn('turn-peer', 'hello peer', undefined, peerChatUri); + stateManager.dispatchClientAction(peerChatUri, { + type: ActionType.ChatPendingMessageSet, + kind: PendingMessageKind.Steering, + id: 'steer-peer', + message: { text: 'change direction', origin: { kind: MessageKind.User } }, + }, { clientId: 'test', clientSeq: 2 }); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-peer', duration: 1000 }, peerChatUri); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-default', duration: 1000 }); + + assert.deepStrictEqual(completedEvents().map(event => { + const data = event.data as Record; + return { turnId: data.turnId, startedWithSteering: data.startedWithSteering, receivedSteering: data.receivedSteering }; + }), [ + { turnId: 'turn-peer', startedWithSteering: false, receivedSteering: true }, + { turnId: 'turn-default', startedWithSteering: false, receivedSteering: false }, + ]); + }); + + test('does not classify provider starts without a pending message as steering', () => { + setupSession(); + fire({ + type: ActionType.ChatTurnStarted, + turnId: 'turn-notification', + startedAt: new Date().toISOString(), + message: { text: 'background task done', origin: { kind: MessageKind.SystemNotification } }, + }); + fireModelCallFinished('turn-notification', 'edit', 300, 'success', true); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-notification', duration: 1000 }); + + const data = completedEvents()[0].data as Record; + assert.deepStrictEqual({ + timeToFirstEdit: data.timeToFirstEdit, + startedWithSteering: data.startedWithSteering, + receivedSteering: data.receivedSteering, + }, { timeToFirstEdit: 300, startedWithSteering: false, receivedSteering: false }); + }); + test('deduplicates model-call attempts and leaves time to first edit absent when no edit is requested', () => { setupSession(); startTurn('turn-no-edit');