Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/vs/platform/agentHost/node/agentHostTelemetryReporter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,8 @@ export interface IAgentHostTurnCompletedEvent extends IAgentHostEventTelemetry {
timeToFirstProgress: number | undefined;
timeToFirstEdit: number | undefined;
timeToFirstEditClassifierVersion: number | undefined;
startedWithSteering: boolean;
receivedSteering: boolean;
totalTime: number;
result: AgentHostTurnResult;
model: string | TelemetryTrustedValue<string> | undefined;
Expand Down Expand Up @@ -251,6 +253,8 @@ export type IAgentHostTurnCompletedClassification = IAgentHostEventClassificatio
timeToFirstProgress: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Time in milliseconds from turn start to the first visible progress (text delta, response part, tool call start, or reasoning).' };
timeToFirstEdit: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Cumulative provider-dispatch time in milliseconds through the first accepted response that requests a built-in file edit. Excludes prompt construction, retry backoff, tool execution, confirmations, and post-response processing.' };
timeToFirstEditClassifierVersion: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Version of the built-in file-edit request classifier used for timeToFirstEdit.' };
startedWithSteering: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether the provider promoted a steering message into this turn. Time to first edit remains a per-turn measurement, not a cumulative measurement across the preceding turn.' };
receivedSteering: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Whether a steering message was submitted to this chat while this turn was active, regardless of whether the provider consumed it. Previously recorded time to first edit is preserved.' };
totalTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Total time in milliseconds from turn start to turn completion.' };
result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the turn completed successfully, with an error, or was cancelled.' };
model: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The trusted provider model identifier selected at turn start, or a generic value for BYOK and unknown models.' };
Expand Down Expand Up @@ -327,6 +331,8 @@ export interface IAgentHostTurnCompletedReport extends IAgentHostTurnAttributedR
timeToFirstProgress: number | undefined;
timeToFirstEditMs: number | undefined;
timeToFirstEditClassifierVersion: number | undefined;
startedWithSteering: boolean;
receivedSteering: boolean;
totalTime: number;
result: AgentHostTurnResult;
model: string | undefined;
Expand Down Expand Up @@ -1352,6 +1358,8 @@ export class AgentHostTelemetryReporter {
timeToFirstProgress: report.timeToFirstProgress,
timeToFirstEdit: report.timeToFirstEditMs,
timeToFirstEditClassifierVersion: report.timeToFirstEditClassifierVersion,
startedWithSteering: report.startedWithSteering,
receivedSteering: report.receivedSteering,
totalTime: report.totalTime,
result: report.result,
model,
Expand Down
17 changes: 17 additions & 0 deletions src/vs/platform/agentHost/node/agentHostTurnTracker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ interface ITurnTiming {
modelCallDispatchDurationMs: number;
timeToFirstEditMs: number | undefined;
timeToFirstEditClassifierVersion: number | undefined;
startedWithSteering: boolean;
receivedSteering: boolean;
firstProgressMs: number | undefined;
currentStage: AgentHostTurnFailureStage;

Expand Down Expand Up @@ -201,6 +203,8 @@ export class AgentHostTurnTracker extends Disposable {
modelCallDispatchDurationMs: 0,
timeToFirstEditMs: undefined,
timeToFirstEditClassifierVersion: undefined,
startedWithSteering: false,
receivedSteering: false,
firstProgressMs: undefined,
currentStage: 'validation',
quietStopWatch: StopWatch.create(false),
Expand Down Expand Up @@ -378,6 +382,17 @@ export class AgentHostTurnTracker extends Disposable {
this._turnTimings.get(this._key(session, turnId))?.completedModelCallIds.add(modelCallId);
}

markSteering(session: string, turnId: string, kind: 'started' | 'received'): void {
const timing = this._turnTimings.get(this._key(session, turnId));
if (timing) {
if (kind === 'started') {
timing.startedWithSteering = true;
} else {
timing.receivedSteering = true;
}
}
}

modelCallFinished(session: string, turnId: string, modelCallId: string, dispatchDurationMs: number, outcome: AgentModelCallFinishedOutcome, containsBuiltInFileEditRequest: boolean | undefined, editClassifierVersion: number): void {
const timing = this._turnTimings.get(this._key(session, turnId));
if (!timing || timing.finishedModelCallIds.has(modelCallId)) {
Expand Down Expand Up @@ -452,6 +467,8 @@ export class AgentHostTurnTracker extends Disposable {
timeToFirstProgress: timing.firstProgressMs,
timeToFirstEditMs: timing.timeToFirstEditMs,
timeToFirstEditClassifierVersion: timing.timeToFirstEditClassifierVersion,
startedWithSteering: timing.startedWithSteering,
receivedSteering: timing.receivedSteering,
totalTime,
result,
model: timing.model,
Expand Down
4 changes: 4 additions & 0 deletions src/vs/platform/agentHost/node/agentSideEffects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,10 @@ export class AgentSideEffects extends Disposable {
hostLaunchKind: this._options.hostLaunchKind ?? AgentHostLaunchKind.Unknown,
};
this._turnTracker.turnStarted(agent, sessionKey, action.turnId, model, modelTelemetryKind, modelSelectionKind, permissionLevel, interactionMode, clientContext, undefined, undefined, undefined, getMessageOriginTelemetryKind(action.message, this._stateManager.isEphemeralSession(sessionChannel)));
// Queue-drained starts are host-owned; a provider start consuming a pending message is steering.
if (action.queuedMessageId !== undefined) {
this._turnTracker.markSteering(sessionKey, action.turnId, 'started');
}
this._turnTracker.setCurrentStage(sessionKey, action.turnId, 'provider');
} else if (action.type === ActionType.ChatTurnComplete) {
this._runTurnCompleteSideEffects(sessionKey, undefined);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { SessionFlagsContribution } from './sessionFlags/sessionFlagsContributio
import { SessionInputNeededContribution } from './sessionInputNeeded/sessionInputNeededContribution.js';
import { SessionTitleContribution } from './sessionTitle/sessionTitleContribution.js';
import { SideChatContribution } from './sideChat/sideChatContribution.js';
import { SteeringTelemetryContribution } from './steeringTelemetry/steeringTelemetryContribution.js';
import { TurnAdmissionContribution } from './turnAdmission/turnAdmissionContribution.js';
import { TurnDelegationContribution } from './turnDelegation/turnDelegationContribution.js';
import { WorktreeAnnouncementContribution } from './worktreeAnnouncement/worktreeAnnouncementContribution.js';
Expand All @@ -35,6 +36,7 @@ export function registerBuiltInChatContributions(
registrations.add(contributions.registerContribution(PullRequestChatContribution));
registrations.add(contributions.registerContribution(TurnDelegationContribution));
registrations.add(contributions.registerContribution(PersistedTurnUsageContribution));
registrations.add(contributions.registerContribution(SteeringTelemetryContribution));
registrations.add(contributions.registerContribution(WorktreeAnnouncementContribution));
registrations.add(contributions.registerContribution(CheckpointAndChangesetContribution));
registrations.add(contributions.registerContribution(SessionWorkspaceConversionContribution));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { Disposable } from '../../../../../base/common/lifecycle.js';
import type { IAgentHostChatContribution, IAgentHostChatContributionContext, IDispatchedAction } from '../../../common/agentHostChatContributionsService.js';
import { ActionType } from '../../../common/state/sessionActions.js';
import { isAhpChatChannel, PendingMessageKind } from '../../../common/state/sessionState.js';
import { AgentHostStateManager, IAgentHostStateManager } from '../../agentHostStateManager.js';
import { AgentHostTurnTracker, IAgentHostTurnTracker } from '../../agentHostTurnTracker.js';

export class SteeringTelemetryContribution extends Disposable implements IAgentHostChatContribution {
static readonly id = 'steeringTelemetry';
readonly order = 150;

constructor(
_context: IAgentHostChatContributionContext,
@IAgentHostStateManager private readonly _stateManager: AgentHostStateManager,
@IAgentHostTurnTracker private readonly _turnTracker: AgentHostTurnTracker,
) {
super();
}

onDidDispatchAction({ channel, action, rejectionReason }: IDispatchedAction): void {
if (rejectionReason !== undefined || !isAhpChatChannel(channel)
|| action.type !== ActionType.ChatPendingMessageSet || action.kind !== PendingMessageKind.Steering) {
return;
}
if (this._stateManager.getChatState(channel)?.steeringMessage?.id !== action.id) {
return;
}
const turnId = this._stateManager.getActiveTurnId(channel);
if (turnId) {
this._turnTracker.markSteering(channel, turnId, 'received');
}
}
}
58 changes: 25 additions & 33 deletions src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts
Original file line number Diff line number Diff line change
Expand Up @@ -705,6 +705,10 @@ class CopilotTurn extends Disposable {
*/
readonly toolCounts = new Map<string, number>();
readonly mainModelCallIds = new Set<string>();
/** Root SDK correlations are valid only while their owning protocol turn is active. */
readonly sdkTurnIds = new Set<string>();
readonly interactionIds = new Set<string>();
activeSdkTurnId: string | undefined;
toolCallRounds = 0;
totalToolCalls = 0;
parallelToolCallRounds = 0;
Expand Down Expand Up @@ -783,6 +787,9 @@ class CopilotTurn extends Disposable {
* Rejects {@link eventId} before disposal so pending fork-boundary checks do not hang.
*/
override dispose(): void {
this.sdkTurnIds.clear();
this.interactionIds.clear();
this.activeSdkTurnId = undefined;
if (!this._eventId.isSettled) {
this._eventId.error(new Error(`Turn ${this.id} was disposed before its SDK event id was recorded`));
}
Expand Down Expand Up @@ -829,11 +836,6 @@ export class CopilotAgentSession extends Disposable {
* the same id, so mappings live until session teardown.
*/
private readonly _parentToolCallIdsByAgentId = new Map<string, string>();
/** Maps SDK root-agent turn ids to their owning host protocol turn ids. */
private readonly _hostTurnIdsBySdkTurnId = new Map<string, string>();
/** Maps runtime interactions to their owning host protocol turn ids. */
private readonly _hostTurnIdsByInteractionId = new Map<string, string>();
private _activeRootSdkTurnId: string | undefined;
private readonly _rootTurnIdBySubagentToolCallId = new Map<string, string>();
readonly modelCallTurnCorrelation = new ModelCallTurnCorrelation();
private readonly _subagentDirectUsageByToolCallId = new Map<string, DirectUsageAccumulator>();
Expand Down Expand Up @@ -1367,13 +1369,14 @@ export class CopilotAgentSession extends Disposable {
this._logService.trace(`[Copilot:${this.sessionId}] Ignoring unroutable subagent model.call_finished: agentId=${event.agentId}, sdkTurnId=${event.data.turnId}`);
return;
}
const turn = this._currentTurn.value;
let turnId: string | undefined;
if (event.agentId) {
turnId = this._turnId;
} else if (event.data.interactionId) {
turnId = this._hostTurnIdsByInteractionId.get(event.data.interactionId);
turnId = turn?.interactionIds.has(event.data.interactionId) ? turn.id : undefined;
} else {
turnId = this._hostTurnIdsBySdkTurnId.get(event.data.turnId);
turnId = turn?.sdkTurnIds.has(event.data.turnId) ? turn.id : undefined;
}
if (!turnId) {
this._logService.trace(`[Copilot:${this.sessionId}] Ignoring model.call_finished without a host turn mapping: sdkTurnId=${event.data.turnId}`);
Expand All @@ -1393,23 +1396,11 @@ export class CopilotAgentSession extends Disposable {
}

/**
* Promotes a pending steering message into its own protocol turn:
* closes the in-flight turn (so its responseParts settle into history)
* and dispatches {@link ActionType.ChatTurnStarted} for a fresh
* turn whose user message is the steering content. The action's
* `queuedMessageId` atomically clears the corresponding pending
* steering message from the session state.
*
* All subsequent SDK events (message deltas, tool calls, …) emitted
* by the agent now reference the new `_turnId`, so the steering
* response lands in the new turn rather than being folded into the
* original.
*
* Returns the new turn id so callers (notably the `user.message`
* handler) can associate the SDK event id with the steering turn for
* history.truncate / sessions.fork mapping.
* Closes the in-flight protocol turn and promotes a pending steering message into its own turn.
* Carries the active SDK turn association forward so subsequent events can target the steering turn.
*/
private _beginSteeringTurn(steering: PendingMessage): string {
private _beginSteeringTurn(steering: PendingMessage): void {
const activeSdkTurnId = this._currentTurn.value?.activeSdkTurnId;
this._completeActiveTurn();
const newTurnId = generateUuid();
this._emitAction({
Expand All @@ -1432,10 +1423,10 @@ export class CopilotAgentSession extends Disposable {
turn.messageCharLen = steering.message.text.length;
turn.markRunning();
}
if (this._activeRootSdkTurnId) {
this._hostTurnIdsBySdkTurnId.set(this._activeRootSdkTurnId, newTurnId);
if (activeSdkTurnId && turn) {
turn.activeSdkTurnId = activeSdkTurnId;
turn.sdkTurnIds.add(activeSdkTurnId);
}
return newTurnId;
}

/**
Expand Down Expand Up @@ -5025,9 +5016,9 @@ export class CopilotAgentSession extends Disposable {
this._currentTurn.value?.markRunning();
const steering = this._takeMatchingPendingSteering(e.data.content);
if (steering) {
const turnId = this._beginSteeringTurn(steering);
this._beginSteeringTurn(steering);
if (e.data.interactionId) {
this._hostTurnIdsByInteractionId.set(e.data.interactionId, turnId);
this._currentTurn.value?.interactionIds.add(e.data.interactionId);
}
}
if (this._turnId) {
Expand Down Expand Up @@ -6724,11 +6715,11 @@ export class CopilotAgentSession extends Disposable {
this._logService.trace(`[Copilot:${sessionId}] Turn started: ${e.data.turnId}`);
this._resumeSubagentForEvent(e);
if (!e.agentId) {
this._activeRootSdkTurnId = e.data.turnId;
if (this._currentTurn.value) {
this._hostTurnIdsBySdkTurnId.set(e.data.turnId, this._currentTurn.value.id);
this._currentTurn.value.activeSdkTurnId = e.data.turnId;
this._currentTurn.value.sdkTurnIds.add(e.data.turnId);
if (e.data.interactionId) {
this._hostTurnIdsByInteractionId.set(e.data.interactionId, this._currentTurn.value.id);
this._currentTurn.value.interactionIds.add(e.data.interactionId);
}
}
const telemetryMessageId = this._currentTurn.value?.id ?? e.data.turnId;
Expand Down Expand Up @@ -6771,8 +6762,9 @@ export class CopilotAgentSession extends Disposable {

this._register(wrapper.onTurnEnd(e => {
this._logService.trace(`[Copilot:${sessionId}] Turn ended: ${e.data.turnId}`);
if (!e.agentId && this._activeRootSdkTurnId === e.data.turnId) {
this._activeRootSdkTurnId = undefined;
const turn = this._currentTurn.value;
if (!e.agentId && turn?.activeSdkTurnId === e.data.turnId) {
turn.activeSdkTurnId = undefined;
}
}));

Expand Down
Loading
Loading