diff --git a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts index 6dd1315a98f36b..14a6d5a5ee17a2 100644 --- a/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts +++ b/src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts @@ -219,6 +219,8 @@ export interface IAgentHostTurnCompletedEvent extends IAgentHostEventTelemetry { timeToFirstProgress: number | undefined; timeToFirstEdit: number | undefined; timeToFirstEditClassifierVersion: number | undefined; + startedWithSteering: boolean; + receivedSteering: boolean; totalTime: number; result: AgentHostTurnResult; model: string | TelemetryTrustedValue | undefined; @@ -251,6 +253,8 @@ export type IAgentHostTurnCompletedClassification = IAgentHostEventClassificatio timeToFirstProgress: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Time in milliseconds from turn start to the first visible progress (text delta, response part, tool call start, or reasoning).' }; timeToFirstEdit: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Cumulative provider-dispatch time in milliseconds through the first accepted response that requests a built-in file edit. Excludes prompt construction, retry backoff, tool execution, confirmations, and post-response processing.' }; timeToFirstEditClassifierVersion: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Version of the built-in file-edit request classifier used for timeToFirstEdit.' }; + startedWithSteering: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the provider promoted a steering message into this turn. Time to first edit remains a per-turn measurement, not a cumulative measurement across the preceding turn.' }; + receivedSteering: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether a steering message was submitted to this chat while this turn was active, regardless of whether the provider consumed it. Previously recorded time to first edit is preserved.' }; totalTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Total time in milliseconds from turn start to turn completion.' }; result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the turn completed successfully, with an error, or was cancelled.' }; model: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The trusted provider model identifier selected at turn start, or a generic value for BYOK and unknown models.' }; @@ -327,6 +331,8 @@ export interface IAgentHostTurnCompletedReport extends IAgentHostTurnAttributedR timeToFirstProgress: number | undefined; timeToFirstEditMs: number | undefined; timeToFirstEditClassifierVersion: number | undefined; + startedWithSteering: boolean; + receivedSteering: boolean; totalTime: number; result: AgentHostTurnResult; model: string | undefined; @@ -1352,6 +1358,8 @@ export class AgentHostTelemetryReporter { timeToFirstProgress: report.timeToFirstProgress, timeToFirstEdit: report.timeToFirstEditMs, timeToFirstEditClassifierVersion: report.timeToFirstEditClassifierVersion, + startedWithSteering: report.startedWithSteering, + receivedSteering: report.receivedSteering, totalTime: report.totalTime, result: report.result, model, diff --git a/src/vs/platform/agentHost/node/agentHostTurnTracker.ts b/src/vs/platform/agentHost/node/agentHostTurnTracker.ts index 56d90576dd388e..63a35874728143 100644 --- a/src/vs/platform/agentHost/node/agentHostTurnTracker.ts +++ b/src/vs/platform/agentHost/node/agentHostTurnTracker.ts @@ -81,6 +81,8 @@ interface ITurnTiming { modelCallDispatchDurationMs: number; timeToFirstEditMs: number | undefined; timeToFirstEditClassifierVersion: number | undefined; + startedWithSteering: boolean; + receivedSteering: boolean; firstProgressMs: number | undefined; currentStage: AgentHostTurnFailureStage; @@ -201,6 +203,8 @@ export class AgentHostTurnTracker extends Disposable { modelCallDispatchDurationMs: 0, timeToFirstEditMs: undefined, timeToFirstEditClassifierVersion: undefined, + startedWithSteering: false, + receivedSteering: false, firstProgressMs: undefined, currentStage: 'validation', quietStopWatch: StopWatch.create(false), @@ -378,6 +382,17 @@ export class AgentHostTurnTracker extends Disposable { this._turnTimings.get(this._key(session, turnId))?.completedModelCallIds.add(modelCallId); } + markSteering(session: string, turnId: string, kind: 'started' | 'received'): void { + const timing = this._turnTimings.get(this._key(session, turnId)); + if (timing) { + if (kind === 'started') { + timing.startedWithSteering = true; + } else { + timing.receivedSteering = true; + } + } + } + modelCallFinished(session: string, turnId: string, modelCallId: string, dispatchDurationMs: number, outcome: AgentModelCallFinishedOutcome, containsBuiltInFileEditRequest: boolean | undefined, editClassifierVersion: number): void { const timing = this._turnTimings.get(this._key(session, turnId)); if (!timing || timing.finishedModelCallIds.has(modelCallId)) { @@ -452,6 +467,8 @@ export class AgentHostTurnTracker extends Disposable { timeToFirstProgress: timing.firstProgressMs, timeToFirstEditMs: timing.timeToFirstEditMs, timeToFirstEditClassifierVersion: timing.timeToFirstEditClassifierVersion, + startedWithSteering: timing.startedWithSteering, + receivedSteering: timing.receivedSteering, totalTime, result, model: timing.model, diff --git a/src/vs/platform/agentHost/node/agentSideEffects.ts b/src/vs/platform/agentHost/node/agentSideEffects.ts index eca19deeb1a80c..1360b92c48b260 100644 --- a/src/vs/platform/agentHost/node/agentSideEffects.ts +++ b/src/vs/platform/agentHost/node/agentSideEffects.ts @@ -724,6 +724,10 @@ export class AgentSideEffects extends Disposable { hostLaunchKind: this._options.hostLaunchKind ?? AgentHostLaunchKind.Unknown, }; this._turnTracker.turnStarted(agent, sessionKey, action.turnId, model, modelTelemetryKind, modelSelectionKind, permissionLevel, interactionMode, clientContext, undefined, undefined, undefined, getMessageOriginTelemetryKind(action.message, this._stateManager.isEphemeralSession(sessionChannel))); + // Queue-drained starts are host-owned; a provider start consuming a pending message is steering. + if (action.queuedMessageId !== undefined) { + this._turnTracker.markSteering(sessionKey, action.turnId, 'started'); + } this._turnTracker.setCurrentStage(sessionKey, action.turnId, 'provider'); } else if (action.type === ActionType.ChatTurnComplete) { this._runTurnCompleteSideEffects(sessionKey, undefined); diff --git a/src/vs/platform/agentHost/node/chatContributions/builtInChatContributions.ts b/src/vs/platform/agentHost/node/chatContributions/builtInChatContributions.ts index b9d740c87452ab..660453792c9a87 100644 --- a/src/vs/platform/agentHost/node/chatContributions/builtInChatContributions.ts +++ b/src/vs/platform/agentHost/node/chatContributions/builtInChatContributions.ts @@ -21,6 +21,7 @@ import { SessionFlagsContribution } from './sessionFlags/sessionFlagsContributio import { SessionInputNeededContribution } from './sessionInputNeeded/sessionInputNeededContribution.js'; import { SessionTitleContribution } from './sessionTitle/sessionTitleContribution.js'; import { SideChatContribution } from './sideChat/sideChatContribution.js'; +import { SteeringTelemetryContribution } from './steeringTelemetry/steeringTelemetryContribution.js'; import { TurnAdmissionContribution } from './turnAdmission/turnAdmissionContribution.js'; import { TurnDelegationContribution } from './turnDelegation/turnDelegationContribution.js'; import { WorktreeAnnouncementContribution } from './worktreeAnnouncement/worktreeAnnouncementContribution.js'; @@ -35,6 +36,7 @@ export function registerBuiltInChatContributions( registrations.add(contributions.registerContribution(PullRequestChatContribution)); registrations.add(contributions.registerContribution(TurnDelegationContribution)); registrations.add(contributions.registerContribution(PersistedTurnUsageContribution)); + registrations.add(contributions.registerContribution(SteeringTelemetryContribution)); registrations.add(contributions.registerContribution(WorktreeAnnouncementContribution)); registrations.add(contributions.registerContribution(CheckpointAndChangesetContribution)); registrations.add(contributions.registerContribution(SessionWorkspaceConversionContribution)); diff --git a/src/vs/platform/agentHost/node/chatContributions/steeringTelemetry/steeringTelemetryContribution.ts b/src/vs/platform/agentHost/node/chatContributions/steeringTelemetry/steeringTelemetryContribution.ts new file mode 100644 index 00000000000000..8c7ee826e3599d --- /dev/null +++ b/src/vs/platform/agentHost/node/chatContributions/steeringTelemetry/steeringTelemetryContribution.ts @@ -0,0 +1,38 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Disposable } from '../../../../../base/common/lifecycle.js'; +import type { IAgentHostChatContribution, IAgentHostChatContributionContext, IDispatchedAction } from '../../../common/agentHostChatContributionsService.js'; +import { ActionType } from '../../../common/state/sessionActions.js'; +import { isAhpChatChannel, PendingMessageKind } from '../../../common/state/sessionState.js'; +import { AgentHostStateManager, IAgentHostStateManager } from '../../agentHostStateManager.js'; +import { AgentHostTurnTracker, IAgentHostTurnTracker } from '../../agentHostTurnTracker.js'; + +export class SteeringTelemetryContribution extends Disposable implements IAgentHostChatContribution { + static readonly id = 'steeringTelemetry'; + readonly order = 150; + + constructor( + _context: IAgentHostChatContributionContext, + @IAgentHostStateManager private readonly _stateManager: AgentHostStateManager, + @IAgentHostTurnTracker private readonly _turnTracker: AgentHostTurnTracker, + ) { + super(); + } + + onDidDispatchAction({ channel, action, rejectionReason }: IDispatchedAction): void { + if (rejectionReason !== undefined || !isAhpChatChannel(channel) + || action.type !== ActionType.ChatPendingMessageSet || action.kind !== PendingMessageKind.Steering) { + return; + } + if (this._stateManager.getChatState(channel)?.steeringMessage?.id !== action.id) { + return; + } + const turnId = this._stateManager.getActiveTurnId(channel); + if (turnId) { + this._turnTracker.markSteering(channel, turnId, 'received'); + } + } +} diff --git a/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts b/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts index 71e418aee6eff9..d079ba9423f000 100644 --- a/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts +++ b/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts @@ -705,6 +705,10 @@ class CopilotTurn extends Disposable { */ readonly toolCounts = new Map(); readonly mainModelCallIds = new Set(); + /** Root SDK correlations are valid only while their owning protocol turn is active. */ + readonly sdkTurnIds = new Set(); + readonly interactionIds = new Set(); + activeSdkTurnId: string | undefined; toolCallRounds = 0; totalToolCalls = 0; parallelToolCallRounds = 0; @@ -783,6 +787,9 @@ class CopilotTurn extends Disposable { * Rejects {@link eventId} before disposal so pending fork-boundary checks do not hang. */ override dispose(): void { + this.sdkTurnIds.clear(); + this.interactionIds.clear(); + this.activeSdkTurnId = undefined; if (!this._eventId.isSettled) { this._eventId.error(new Error(`Turn ${this.id} was disposed before its SDK event id was recorded`)); } @@ -829,11 +836,6 @@ export class CopilotAgentSession extends Disposable { * the same id, so mappings live until session teardown. */ private readonly _parentToolCallIdsByAgentId = new Map(); - /** Maps SDK root-agent turn ids to their owning host protocol turn ids. */ - private readonly _hostTurnIdsBySdkTurnId = new Map(); - /** Maps runtime interactions to their owning host protocol turn ids. */ - private readonly _hostTurnIdsByInteractionId = new Map(); - private _activeRootSdkTurnId: string | undefined; private readonly _rootTurnIdBySubagentToolCallId = new Map(); readonly modelCallTurnCorrelation = new ModelCallTurnCorrelation(); private readonly _subagentDirectUsageByToolCallId = new Map(); @@ -1367,13 +1369,14 @@ export class CopilotAgentSession extends Disposable { this._logService.trace(`[Copilot:${this.sessionId}] Ignoring unroutable subagent model.call_finished: agentId=${event.agentId}, sdkTurnId=${event.data.turnId}`); return; } + const turn = this._currentTurn.value; let turnId: string | undefined; if (event.agentId) { turnId = this._turnId; } else if (event.data.interactionId) { - turnId = this._hostTurnIdsByInteractionId.get(event.data.interactionId); + turnId = turn?.interactionIds.has(event.data.interactionId) ? turn.id : undefined; } else { - turnId = this._hostTurnIdsBySdkTurnId.get(event.data.turnId); + turnId = turn?.sdkTurnIds.has(event.data.turnId) ? turn.id : undefined; } if (!turnId) { this._logService.trace(`[Copilot:${this.sessionId}] Ignoring model.call_finished without a host turn mapping: sdkTurnId=${event.data.turnId}`); @@ -1393,23 +1396,11 @@ export class CopilotAgentSession extends Disposable { } /** - * Promotes a pending steering message into its own protocol turn: - * closes the in-flight turn (so its responseParts settle into history) - * and dispatches {@link ActionType.ChatTurnStarted} for a fresh - * turn whose user message is the steering content. The action's - * `queuedMessageId` atomically clears the corresponding pending - * steering message from the session state. - * - * All subsequent SDK events (message deltas, tool calls, …) emitted - * by the agent now reference the new `_turnId`, so the steering - * response lands in the new turn rather than being folded into the - * original. - * - * Returns the new turn id so callers (notably the `user.message` - * handler) can associate the SDK event id with the steering turn for - * history.truncate / sessions.fork mapping. + * Closes the in-flight protocol turn and promotes a pending steering message into its own turn. + * Carries the active SDK turn association forward so subsequent events can target the steering turn. */ - private _beginSteeringTurn(steering: PendingMessage): string { + private _beginSteeringTurn(steering: PendingMessage): void { + const activeSdkTurnId = this._currentTurn.value?.activeSdkTurnId; this._completeActiveTurn(); const newTurnId = generateUuid(); this._emitAction({ @@ -1432,10 +1423,10 @@ export class CopilotAgentSession extends Disposable { turn.messageCharLen = steering.message.text.length; turn.markRunning(); } - if (this._activeRootSdkTurnId) { - this._hostTurnIdsBySdkTurnId.set(this._activeRootSdkTurnId, newTurnId); + if (activeSdkTurnId && turn) { + turn.activeSdkTurnId = activeSdkTurnId; + turn.sdkTurnIds.add(activeSdkTurnId); } - return newTurnId; } /** @@ -5025,9 +5016,9 @@ export class CopilotAgentSession extends Disposable { this._currentTurn.value?.markRunning(); const steering = this._takeMatchingPendingSteering(e.data.content); if (steering) { - const turnId = this._beginSteeringTurn(steering); + this._beginSteeringTurn(steering); if (e.data.interactionId) { - this._hostTurnIdsByInteractionId.set(e.data.interactionId, turnId); + this._currentTurn.value?.interactionIds.add(e.data.interactionId); } } if (this._turnId) { @@ -6724,11 +6715,11 @@ export class CopilotAgentSession extends Disposable { this._logService.trace(`[Copilot:${sessionId}] Turn started: ${e.data.turnId}`); this._resumeSubagentForEvent(e); if (!e.agentId) { - this._activeRootSdkTurnId = e.data.turnId; if (this._currentTurn.value) { - this._hostTurnIdsBySdkTurnId.set(e.data.turnId, this._currentTurn.value.id); + this._currentTurn.value.activeSdkTurnId = e.data.turnId; + this._currentTurn.value.sdkTurnIds.add(e.data.turnId); if (e.data.interactionId) { - this._hostTurnIdsByInteractionId.set(e.data.interactionId, this._currentTurn.value.id); + this._currentTurn.value.interactionIds.add(e.data.interactionId); } } const telemetryMessageId = this._currentTurn.value?.id ?? e.data.turnId; @@ -6771,8 +6762,9 @@ export class CopilotAgentSession extends Disposable { this._register(wrapper.onTurnEnd(e => { this._logService.trace(`[Copilot:${sessionId}] Turn ended: ${e.data.turnId}`); - if (!e.agentId && this._activeRootSdkTurnId === e.data.turnId) { - this._activeRootSdkTurnId = undefined; + const turn = this._currentTurn.value; + if (!e.agentId && turn?.activeSdkTurnId === e.data.turnId) { + turn.activeSdkTurnId = undefined; } })); diff --git a/src/vs/platform/agentHost/test/node/agentHostTurnTelemetry.test.ts b/src/vs/platform/agentHost/test/node/agentHostTurnTelemetry.test.ts index 5937ffbd6563fa..842c4f8197badc 100644 --- a/src/vs/platform/agentHost/test/node/agentHostTurnTelemetry.test.ts +++ b/src/vs/platform/agentHost/test/node/agentHostTurnTelemetry.test.ts @@ -26,7 +26,7 @@ import { SessionConfigKey } from '../../common/sessionConfigKeys.js'; import { ActionType, type ChatAction, type ChatUsageAction } from '../../common/state/sessionActions.js'; import { withEphemeralSessionMeta } from '../../common/meta/agentEphemeralSessionMeta.js'; import { toAgentMergeMessageMeta } from '../../common/meta/agentMergeMessageMeta.js'; -import { buildDefaultChatUri, buildSubagentChatUri, createErrorResponsePart, type Message, MessageKind, PendingMessageKind, ResponsePartKind, SessionStatus } from '../../common/state/sessionState.js'; +import { buildChatUri, buildDefaultChatUri, buildSubagentChatUri, createErrorResponsePart, type Message, MessageKind, PendingMessageKind, ResponsePartKind, SessionStatus } from '../../common/state/sessionState.js'; import { IAgentHostCheckpointService, NULL_CHECKPOINT_SERVICE } from '../../common/agentHostCheckpointService.js'; import { IAgentHostChatContributions } from '../../common/agentHostChatContributionsService.js'; import { IAgentHostTerminalManager } from '../../node/agentHostTerminalManager.js'; @@ -120,6 +120,7 @@ suite('AgentSideEffects — turn tracker telemetry', () => { let telemetry: CapturingTelemetryService; let logService: NullLogService; let turnTracker: AgentHostTurnTracker; + let chatContributions: AgentHostChatContributions; const sessionUri = AgentSession.uri('mock', 'session-1'); const sessionKey = sessionUri.toString(); @@ -265,7 +266,7 @@ suite('AgentSideEffects — turn tracker telemetry', () => { }], ); const instantiationService = disposables.add(new InstantiationService(services, /*strict*/ true)); - const chatContributions = disposables.add(new AgentHostChatContributions(logService, instantiationService)); + chatContributions = disposables.add(new AgentHostChatContributions(logService, instantiationService)); services.set(IAgentHostChatContributions, chatContributions); services.set(IAgentHostTurnService, new AgentHostTurnService(stateManager, chatContributions, instantiationService)); services.set(IAgentHostSessionTitleController, disposables.add(new AgentHostSessionTitleController(stateManager, { sessionDataService }, logService))); @@ -425,9 +426,17 @@ suite('AgentSideEffects — turn tracker telemetry', () => { setSessionConfig({ autoApprove: 'autopilot', mode: 'interactive' }); startTurn('turn-original'); await new Promise(resolve => setTimeout(resolve, 0)); - fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-original', duration: 1000 }); + fireModelCallFinished('turn-original', 'call-before-steering', 100, 'success', true); + let previousTurnId = 'turn-original'; for (const turnId of ['turn-steering-1', 'turn-steering-2']) { + stateManager.dispatchClientAction(defaultChatUri, { + type: ActionType.ChatPendingMessageSet, + kind: PendingMessageKind.Steering, + id: `queued-${turnId}`, + message: { text: 'edit the file', origin: { kind: MessageKind.User } }, + }, { clientId: 'test', clientSeq: 2 }); + fire({ type: ActionType.ChatTurnComplete, turnId: previousTurnId, duration: 1000 }); fire({ type: ActionType.ChatTurnStarted, turnId, @@ -436,8 +445,9 @@ suite('AgentSideEffects — turn tracker telemetry', () => { queuedMessageId: `queued-${turnId}`, }); fireModelCallFinished(turnId, `call-${turnId}`, 250, 'success', true); - fire({ type: ActionType.ChatTurnComplete, turnId, duration: 1000 }); + previousTurnId = turnId; } + fire({ type: ActionType.ChatTurnComplete, turnId: previousTurnId, duration: 1000 }); await new Promise(resolve => setTimeout(resolve, 0)); assert.deepStrictEqual({ @@ -447,6 +457,9 @@ suite('AgentSideEffects — turn tracker telemetry', () => { return { turnId: data.turnId, timeToFirstEdit: data.timeToFirstEdit, + timeToFirstEditClassifierVersion: data.timeToFirstEditClassifierVersion, + startedWithSteering: data.startedWithSteering, + receivedSteering: data.receivedSteering, hostLaunchKind: data.hostLaunchKind, permissionLevel: data.permissionLevel, interactionMode: data.interactionMode, @@ -457,7 +470,10 @@ suite('AgentSideEffects — turn tracker telemetry', () => { sentPrompts: ['hello'], completed: ['turn-original', 'turn-steering-1', 'turn-steering-2'].map(turnId => ({ turnId, - timeToFirstEdit: turnId === 'turn-original' ? undefined : 250, + timeToFirstEdit: turnId === 'turn-original' ? 100 : 250, + timeToFirstEditClassifierVersion: 1, + startedWithSteering: turnId !== 'turn-original', + receivedSteering: turnId !== 'turn-steering-2', hostLaunchKind: undefined, permissionLevel: 'autopilot', interactionMode: 'interactive', @@ -466,6 +482,139 @@ suite('AgentSideEffects — turn tracker telemetry', () => { }); }); + for (const result of ['success', 'cancelled', 'error'] as const) { + test(`preserves steering submission and first edit on ${result} even if steering is removed`, () => { + setupSession(); + startTurn('turn-steered'); + fireModelCallFinished('turn-steered', 'edit', 150, 'success', true); + stateManager.dispatchClientAction(defaultChatUri, { + type: ActionType.ChatPendingMessageSet, + kind: PendingMessageKind.Steering, + id: 'steer', + message: { text: 'change direction', origin: { kind: MessageKind.User } }, + }, { clientId: 'test', clientSeq: 2 }); + stateManager.dispatchClientAction(defaultChatUri, { + type: ActionType.ChatPendingMessageRemoved, + kind: PendingMessageKind.Steering, + id: 'steer', + }, { clientId: 'test', clientSeq: 3 }); + if (result === 'error') { + fire({ type: ActionType.ChatError, turnId: 'turn-steered', duration: 1000, part: createErrorResponsePart({ errorType: 'test', message: 'failed' }) }); + } else { + fire({ type: result === 'success' ? ActionType.ChatTurnComplete : ActionType.ChatTurnCancelled, turnId: 'turn-steered', duration: 1000 }); + } + startTurn('turn-next'); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-next', duration: 1000 }); + + assert.deepStrictEqual(completedEvents().map(event => { + const data = event.data as Record; + return { + turnId: data.turnId, + result: data.result, + timeToFirstEdit: data.timeToFirstEdit, + startedWithSteering: data.startedWithSteering, + receivedSteering: data.receivedSteering, + }; + }), [ + { turnId: 'turn-steered', result, timeToFirstEdit: 150, startedWithSteering: false, receivedSteering: true }, + { turnId: 'turn-next', result: 'success', timeToFirstEdit: undefined, startedWithSteering: false, receivedSteering: false }, + ]); + }); + } + + test('does not classify queued follow-ups as steering', () => { + setupSession(); + startTurn('turn-original'); + const queued: ChatAction = { + type: ActionType.ChatPendingMessageSet, + kind: PendingMessageKind.Queued, + id: 'queued-follow-up', + message: { text: 'follow up', origin: { kind: MessageKind.User } }, + }; + stateManager.dispatchClientAction(defaultChatUri, queued, { clientId: 'test', clientSeq: 2 }); + sideEffects.handleAction(defaultChatUri, queued); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-original', duration: 1000 }); + const queuedTurnId = stateManager.getActiveTurnId(defaultChatUri); + assert.ok(queuedTurnId); + fireModelCallFinished(queuedTurnId, 'edit', 200, 'success', true); + fire({ type: ActionType.ChatTurnComplete, turnId: queuedTurnId, duration: 1000 }); + + assert.deepStrictEqual(completedEvents().map(event => { + const data = event.data as Record; + return { + timeToFirstEdit: data.timeToFirstEdit, + startedWithSteering: data.startedWithSteering, + receivedSteering: data.receivedSteering, + }; + }), [ + { timeToFirstEdit: undefined, startedWithSteering: false, receivedSteering: false }, + { timeToFirstEdit: 200, startedWithSteering: false, receivedSteering: false }, + ]); + }); + + test('ignores rejected steering actions and submissions made while idle', () => { + setupSession(); + const action: ChatAction = { + type: ActionType.ChatPendingMessageSet, + kind: PendingMessageKind.Steering, + id: 'steer-idle', + message: { text: 'change direction', origin: { kind: MessageKind.User } }, + }; + stateManager.dispatchClientAction(defaultChatUri, action, { clientId: 'test', clientSeq: 1 }); + startTurn('turn-after-idle'); + chatContributions.didDispatchAction({ channel: defaultChatUri, session: sessionKey, action, rejectionReason: 'rejected' }); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-after-idle', duration: 1000 }); + + const data = completedEvents()[0].data as Record; + assert.deepStrictEqual({ + startedWithSteering: data.startedWithSteering, + receivedSteering: data.receivedSteering, + }, { startedWithSteering: false, receivedSteering: false }); + }); + + test('attributes steering only to the targeted peer chat', () => { + setupSession(); + const peerChatUri = buildChatUri(sessionUri, 'peer'); + stateManager.addChat(sessionKey, peerChatUri); + startTurn('turn-default'); + startTurn('turn-peer', 'hello peer', undefined, peerChatUri); + stateManager.dispatchClientAction(peerChatUri, { + type: ActionType.ChatPendingMessageSet, + kind: PendingMessageKind.Steering, + id: 'steer-peer', + message: { text: 'change direction', origin: { kind: MessageKind.User } }, + }, { clientId: 'test', clientSeq: 2 }); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-peer', duration: 1000 }, peerChatUri); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-default', duration: 1000 }); + + assert.deepStrictEqual(completedEvents().map(event => { + const data = event.data as Record; + return { turnId: data.turnId, startedWithSteering: data.startedWithSteering, receivedSteering: data.receivedSteering }; + }), [ + { turnId: 'turn-peer', startedWithSteering: false, receivedSteering: true }, + { turnId: 'turn-default', startedWithSteering: false, receivedSteering: false }, + ]); + }); + + test('does not classify provider starts without a pending message as steering', () => { + setupSession(); + fire({ + type: ActionType.ChatTurnStarted, + turnId: 'turn-notification', + startedAt: new Date().toISOString(), + message: { text: 'background task done', origin: { kind: MessageKind.SystemNotification } }, + }); + fireModelCallFinished('turn-notification', 'edit', 300, 'success', true); + fire({ type: ActionType.ChatTurnComplete, turnId: 'turn-notification', duration: 1000 }); + + const data = completedEvents()[0].data as Record; + assert.deepStrictEqual({ + timeToFirstEdit: data.timeToFirstEdit, + startedWithSteering: data.startedWithSteering, + receivedSteering: data.receivedSteering, + }, { timeToFirstEdit: 300, startedWithSteering: false, receivedSteering: false }); + }); + test('deduplicates model-call attempts and leaves time to first edit absent when no edit is requested', () => { setupSession(); startTurn('turn-no-edit'); diff --git a/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts b/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts index fbe286966e07dd..c478bde1b261ec 100644 --- a/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts +++ b/src/vs/platform/agentHost/test/node/copilotAgentSession.test.ts @@ -6657,15 +6657,17 @@ Use the attached image as context. assert.strictEqual(steeringCompletions.length, 0, 'an aborted steering turn must not be completed'); }); - for (const { name, interactionId, expectedOriginalTurn } of [ - { name: 'new steering interaction before the next assistant.turn_start', interactionId: 'interaction-steer', expectedOriginalTurn: false }, - { name: 'stale original interaction after steering promotion', interactionId: 'interaction-original', expectedOriginalTurn: true }, - { name: 'missing interaction fallback after steering promotion', interactionId: undefined, expectedOriginalTurn: false }, + for (const { name, interactionId, expectedDropped } of [ + { name: 'new steering interaction before the next assistant.turn_start', interactionId: 'interaction-steer', expectedDropped: false }, + { name: 'stale original interaction after steering promotion', interactionId: 'interaction-original', expectedDropped: true }, + { name: 'missing interaction fallback after steering promotion', interactionId: undefined, expectedDropped: false }, ]) { test(`maps model-call lifecycle events with ${name}`, async () => { const { session, mockSession, signals } = await createAgentSession(disposables); session.resetTurnState('turn-original'); mockSession.fire('assistant.turn_start', { turnId: 'sdk-0', interactionId: 'interaction-original' }); + const originalTurn = session['_currentTurn'].value; + assert.ok(originalTurn); await session.sendSteering({ id: 'steer-1', message: { text: 'focus on tests', origin: { kind: MessageKind.User } } }); mockSession.fire('user.message', { @@ -6714,21 +6716,63 @@ Use the attached image as context. } assert.deepStrictEqual({ + originalCorrelations: [originalTurn.sdkTurnIds.size, originalTurn.interactionIds.size, originalTurn.activeSdkTurnId], modelCallTurnIds: signals.filter(signal => signal.kind === 'model_call_finished').map(signal => signal.turnId), completedTurns: telemetryService.events.filter(event => event.eventName === 'agentHost.turnCompleted').map(event => { const data = event.data as { turnId: string; timeToFirstEdit?: number }; return { turnId: data.turnId, timeToFirstEdit: data.timeToFirstEdit }; }), }, { - modelCallTurnIds: [expectedOriginalTurn ? 'turn-original' : steeringTurnId], + originalCorrelations: [0, 0, undefined], + modelCallTurnIds: expectedDropped ? [] : [steeringTurnId], completedTurns: [ { turnId: 'turn-original', timeToFirstEdit: undefined }, - { turnId: steeringTurnId, timeToFirstEdit: expectedOriginalTurn ? undefined : 250 }, + { turnId: steeringTurnId, timeToFirstEdit: expectedDropped ? undefined : 250 }, ], }); }); } + for (const ending of ['complete', 'abort', 'fail', 'discard', 'replace'] as const) { + test(`does not carry an old SDK turn into steering after ${ending}`, async () => { + const { session, mockSession, signals } = await createAgentSession(disposables); + session.resetTurnState('old-turn'); + mockSession.fire('assistant.turn_start', { turnId: 'old-sdk-turn', interactionId: 'old-interaction' }); + switch (ending) { + case 'complete': + mockSession.fire('session.idle', {}); + break; + case 'abort': + mockSession.fire('session.idle', { aborted: true }); + break; + case 'fail': + session.failActiveTurn({ errorType: 'test', message: 'failure' }); + break; + case 'discard': + session.discardActiveTurn(); + break; + case 'replace': + break; + } + session.resetTurnState('new-turn'); + await session.sendSteering({ id: 'steer', message: { text: 'follow up', origin: { kind: MessageKind.User } } }); + mockSession.fire('user.message', { content: 'follow up', interactionId: 'steering-interaction' }); + mockSession.fireRaw({ + type: 'model.call_finished', + ephemeral: true, + id: 'late-old-call', + data: { + turnId: 'old-sdk-turn', + dispatchDurationMs: 250, + outcome: 'success', + containsBuiltInFileEditRequest: true, + editClassifierVersion: 1, + }, + }); + assert.deepStrictEqual(signals.filter(signal => signal.kind === 'model_call_finished'), []); + }); + } + test('does not signal cleanup when send fails', async () => { const { session, mockSession, signals } = await createAgentSession(disposables); @@ -7627,6 +7671,97 @@ Use the attached image as context. ); }); + for (const ending of ['complete', 'abort', 'fail', 'discard', 'replace', 'dispose'] as const) { + test(`releases model-call correlations when a host turn ends via ${ending}`, async () => { + const { session, mockSession, signals } = await createAgentSession(disposables); + const counts: { sdkTurnIds: number; interactionIds: number; activeSdkTurnId: string | undefined }[] = []; + const iterations = ending === 'dispose' ? 1 : 25; + for (let i = 0; i < iterations; i++) { + session.resetTurnState(`host-turn-${i}`); + mockSession.fire('assistant.turn_start', { turnId: `sdk-turn-${i}`, interactionId: `interaction-${i}` }); + mockSession.fire('assistant.turn_start', { turnId: `sdk-turn-${i}-next`, interactionId: `interaction-${i}` }); + const turn = session['_currentTurn'].value; + assert.ok(turn); + assert.deepStrictEqual([turn.sdkTurnIds.size, turn.interactionIds.size, turn.activeSdkTurnId], [2, 1, `sdk-turn-${i}-next`]); + + switch (ending) { + case 'complete': + mockSession.fire('session.idle', {}); + break; + case 'abort': + mockSession.fire('session.idle', { aborted: true }); + break; + case 'fail': + session.failActiveTurn({ errorType: 'test', message: 'failure' }); + break; + case 'discard': + session.discardActiveTurn(); + break; + case 'replace': + session.resetTurnState('replacement'); + break; + case 'dispose': + session.dispose(); + break; + } + + counts.push({ sdkTurnIds: turn.sdkTurnIds.size, interactionIds: turn.interactionIds.size, activeSdkTurnId: turn.activeSdkTurnId }); + if (ending !== 'dispose') { + session.resetTurnState('replacement'); + mockSession.fire('assistant.turn_start', { turnId: `sdk-turn-${i}-next`, interactionId: 'replacement-interaction' }); + for (const interactionId of [`interaction-${i}`, undefined]) { + mockSession.fireRaw({ + type: 'model.call_finished', + ephemeral: true, + id: `late-call-${i}-${interactionId}`, + data: { + turnId: interactionId ? `sdk-turn-${i}-next` : `sdk-turn-${i}`, + interactionId, + dispatchDurationMs: 250, + outcome: 'success', + containsBuiltInFileEditRequest: true, + editClassifierVersion: 1, + }, + }); + } + } + } + + assert.deepStrictEqual({ + counts, + modelCalls: signals.filter(signal => signal.kind === 'model_call_finished'), + }, { + counts: Array.from({ length: iterations }, () => ({ sdkTurnIds: 0, interactionIds: 0, activeSdkTurnId: undefined })), + modelCalls: [], + }); + }); + } + + test('keeps model-call correlations through SDK turn end until the host turn ends', async () => { + const { session, mockSession, signals } = await createAgentSession(disposables); + session.resetTurnState('host-turn'); + for (const sdkTurnId of ['0', '1']) { + mockSession.fire('assistant.turn_start', { turnId: sdkTurnId, interactionId: 'interaction' }); + mockSession.fire('assistant.turn_end', { turnId: sdkTurnId }); + } + for (const interactionId of ['interaction', undefined]) { + mockSession.fireRaw({ + type: 'model.call_finished', + ephemeral: true, + id: `model-call-${interactionId}`, + data: { + turnId: '0', + interactionId, + dispatchDurationMs: 250, + outcome: 'success', + containsBuiltInFileEditRequest: true, + editClassifierVersion: 1, + }, + }); + } + assert.deepStrictEqual(signals.filter(signal => signal.kind === 'model_call_finished').map(signal => signal.turnId), ['host-turn', 'host-turn']); + }); + test('resumes a subagent on turn start before mapping model.call_finished', async () => { const { session, mockSession, signals } = await createAgentSession(disposables); session.resetTurnState('host-turn-1');