From 0dafa4d0b29942c334d1353868530c4794c4eb6f Mon Sep 17 00:00:00 2001 From: amunger <2019016+amunger@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:03:32 -0700 Subject: [PATCH] Add edit tracker characterization tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../helpers/documentWithAnnotatedEdits.ts | 3 +- .../browser/telemetry/arcTelemetryReporter.ts | 4 +- .../browser/telemetry/arcTelemetrySender.ts | 6 +- .../telemetry/editSourceTrackingImpl.ts | 54 ++-- .../browser/telemetry/scmAdapter.ts | 10 +- .../documentWithAnnotatedEdits.test.ts | 146 +++++++++++ .../test/browser/editSourceCategories.test.ts | 86 +++++++ .../browser/editSourceTrackingImpl.test.ts | 231 ++++++++++++++++++ .../test/browser/editTracker.test.ts | 140 +++++++++++ 9 files changed, 642 insertions(+), 38 deletions(-) create mode 100644 src/vs/workbench/contrib/editTelemetry/test/browser/documentWithAnnotatedEdits.test.ts create mode 100644 src/vs/workbench/contrib/editTelemetry/test/browser/editSourceCategories.test.ts create mode 100644 src/vs/workbench/contrib/editTelemetry/test/browser/editSourceTrackingImpl.test.ts create mode 100644 src/vs/workbench/contrib/editTelemetry/test/browser/editTracker.test.ts diff --git a/src/vs/workbench/contrib/editTelemetry/browser/helpers/documentWithAnnotatedEdits.ts b/src/vs/workbench/contrib/editTelemetry/browser/helpers/documentWithAnnotatedEdits.ts index c059121d53849..db0153cc1dc94 100644 --- a/src/vs/workbench/contrib/editTelemetry/browser/helpers/documentWithAnnotatedEdits.ts +++ b/src/vs/workbench/contrib/editTelemetry/browser/helpers/documentWithAnnotatedEdits.ts @@ -146,7 +146,7 @@ export class InlineSuggestEditSource extends EditSourceBase { public readonly type: 'word' | 'line' | undefined, ) { super(); } - override toString() { return `${this.category}/${this.feature}/${this.kind}/${this.extensionId}/${this.type}`; } + override toString() { return `${this.category}/${this.feature}/${this.kind}/${this.extensionId}/${this.providerId}/${this.type}`; } public getColor(): string { return '#00ff0033'; } } @@ -330,4 +330,3 @@ export function createDocWithJustReason(docWithAnnotatedEdits: IDocumentWithAnno }; return docWithJustReason; } - diff --git a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/arcTelemetryReporter.ts b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/arcTelemetryReporter.ts index 387563fbd4af2..eb5c36319f466 100644 --- a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/arcTelemetryReporter.ts +++ b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/arcTelemetryReporter.ts @@ -9,7 +9,7 @@ import { BaseStringEdit } from '../../../../../editor/common/core/edits/stringEd import { StringText } from '../../../../../editor/common/core/text/abstractText.js'; import { ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; import { ArcTracker } from '../../common/arcTracker.js'; -import type { ScmRepoAdapter } from './scmAdapter.js'; +import type { IScmRepoAdapter } from './scmAdapter.js'; export class ArcTelemetryReporter extends Disposable { private readonly _arcTracker; @@ -22,7 +22,7 @@ export class ArcTelemetryReporter extends Disposable { private readonly _documentValueBeforeTrackedEdit: StringText, private readonly _document: { value: IObservableWithChange }, // _markedEdits -> document.value - private readonly _gitRepo: IObservable, + private readonly _gitRepo: IObservable, private readonly _trackedEdit: BaseStringEdit, private readonly _sendTelemetryEvent: (res: ArcTelemetryReporterData) => void, private readonly _dispose: () => void, diff --git a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/arcTelemetrySender.ts b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/arcTelemetrySender.ts index a0abb90038473..263aad985d005 100644 --- a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/arcTelemetrySender.ts +++ b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/arcTelemetrySender.ts @@ -11,7 +11,7 @@ import { EditDeltaInfo, EditSuggestionId, ITextModelEditSourceMetadata } from '. import { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js'; import { EditSourceData, IDocumentWithAnnotatedEdits, createDocWithJustReason } from '../helpers/documentWithAnnotatedEdits.js'; import { IAiEditTelemetryService } from './aiEditTelemetry/aiEditTelemetryService.js'; -import type { ScmRepoAdapter } from './scmAdapter.js'; +import type { IScmRepoAdapter } from './scmAdapter.js'; import { forwardToChannelIf, isCopilotLikeExtension } from '../../../../../platform/dataChannel/browser/forwardingTelemetryService.js'; import { ProviderId } from '../../../../../editor/common/languages.js'; import { ArcTelemetryReporter } from './arcTelemetryReporter.js'; @@ -20,7 +20,7 @@ import { IRandomService } from '../randomService.js'; export class EditTelemetryReportInlineEditArcSender extends Disposable { constructor( docWithAnnotatedEdits: IDocumentWithAnnotatedEdits, - scmRepoBridge: IObservable, + scmRepoBridge: IObservable, @IInstantiationService private readonly _instantiationService: IInstantiationService ) { super(); @@ -154,7 +154,7 @@ export class CreateSuggestionIdForChatOrInlineChatCaller extends Disposable { export class EditTelemetryReportEditArcForChatOrInlineChatSender extends Disposable { constructor( docWithAnnotatedEdits: IDocumentWithAnnotatedEdits, - scmRepoBridge: IObservable, + scmRepoBridge: IObservable, @IInstantiationService private readonly _instantiationService: IInstantiationService, @IRandomService private readonly _randomService: IRandomService, ) { diff --git a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/editSourceTrackingImpl.ts b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/editSourceTrackingImpl.ts index 55fb2fdf16f96..ff534ab884a5e 100644 --- a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/editSourceTrackingImpl.ts +++ b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/editSourceTrackingImpl.ts @@ -15,12 +15,29 @@ import { CreateSuggestionIdForChatOrInlineChatCaller, EditTelemetryReportEditArc import { createDocWithJustReason, EditSource } from '../helpers/documentWithAnnotatedEdits.js'; import { DocumentEditSourceTracker, TrackedEdit } from './editTracker.js'; import { sumByCategory } from '../helpers/utils.js'; -import { ScmAdapter, ScmRepoAdapter } from './scmAdapter.js'; +import { IScmRepoAdapter, ScmAdapter } from './scmAdapter.js'; import { IRandomService } from '../randomService.js'; type EditTelemetryMode = 'longterm' | '10minFocusWindow' | '20minFocusWindow'; type EditTelemetryTrigger = '10hours' | 'hashChange' | 'branchChange' | 'closed' | 'time'; +export type EditTelemetryCategory = 'nes' | 'inlineCompletionsCopilot' | 'inlineCompletionsNES' | 'inlineCompletionsOther' | 'otherAI' | 'user' | 'ide' | 'external' | 'unknown'; + +export function getEditTelemetryCategory(source: EditSource): EditTelemetryCategory { + if (source.category === 'ai' && source.kind === 'nes') { return 'nes'; } + + if (source.category === 'ai' && source.kind === 'completion' && source.extensionId === 'github.copilot') { return 'inlineCompletionsCopilot'; } + if (source.category === 'ai' && source.kind === 'completion' && source.extensionId === 'github.copilot-chat' && source.providerId === 'nes') { return 'inlineCompletionsNES'; } + if (source.category === 'ai' && source.kind === 'completion' && source.extensionId === 'github.copilot-chat' && source.providerId === 'completions') { return 'inlineCompletionsCopilot'; } + if (source.category === 'ai' && source.kind === 'completion') { return 'inlineCompletionsOther'; } + + if (source.category === 'ai') { return 'otherAI'; } + if (source.category === 'user') { return 'user'; } + if (source.category === 'ide') { return 'ide'; } + if (source.category === 'external') { return 'external'; } + return 'unknown'; +} + export class EditSourceTrackingImpl extends Disposable { public readonly docsState; private readonly _states; @@ -47,7 +64,7 @@ class TrackedDocumentInfo extends Disposable { public readonly windowedTracker: IObservable | undefined>; public readonly windowedFocusTracker: IObservable | undefined>; - private readonly _repo: IObservable; + private readonly _repo: IObservable; constructor( private readonly _doc: AnnotatedDocument, @@ -182,21 +199,17 @@ class TrackedDocumentInfo extends Disposable { const statsUuid = this._randomService.generateUuid(); const sums = sumByCategory(ranges, r => r.range.length, r => r.sourceKey); - const entries = Object.entries(sums).filter(([key, value]) => value !== undefined); - entries.sort(reverseOrder(compareBy(([key, value]) => value!, numberComparator))); - entries.length = mode === 'longterm' ? 30 : 10; - for (const key of keys) { if (!sums[key]) { sums[key] = 0; } } + const entries = Object.entries(sums) + .filter((entry): entry is [string, number] => entry[1] !== undefined) + .sort(reverseOrder(compareBy(([, value]) => value, numberComparator))) + .slice(0, mode === 'longterm' ? 30 : 10); - for (const [key, value] of Object.entries(sums)) { - if (value === undefined) { - continue; - } - + for (const [key, value] of entries) { const repr = t.getRepresentative(key)!; const deltaModifiedCount = t.getTotalInsertedCharactersCount(key); @@ -321,24 +334,7 @@ class TrackedDocumentInfo extends Disposable { } getTelemetryData(ranges: readonly TrackedEdit[]) { - const getEditCategory = (source: EditSource) => { - if (source.category === 'ai' && source.kind === 'nes') { return 'nes'; } - - if (source.category === 'ai' && source.kind === 'completion' && source.extensionId === 'github.copilot') { return 'inlineCompletionsCopilot'; } - if (source.category === 'ai' && source.kind === 'completion' && source.extensionId === 'github.copilot-chat' && source.providerId === 'completions') { return 'inlineCompletionsCopilot'; } - if (source.category === 'ai' && source.kind === 'completion' && source.extensionId === 'github.copilot-chat' && source.providerId === 'nes') { return 'inlineCompletionsNES'; } - if (source.category === 'ai' && source.kind === 'completion') { return 'inlineCompletionsOther'; } - - if (source.category === 'ai') { return 'otherAI'; } - if (source.category === 'user') { return 'user'; } - if (source.category === 'ide') { return 'ide'; } - if (source.category === 'external') { return 'external'; } - if (source.category === 'unknown') { return 'unknown'; } - - return 'unknown'; - }; - - const sums = sumByCategory(ranges, r => r.range.length, r => getEditCategory(r.source)); + const sums = sumByCategory(ranges, r => r.range.length, r => getEditTelemetryCategory(r.source)); const totalModifiedCharactersInFinalState = sumBy(ranges, r => r.range.length); return { diff --git a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/scmAdapter.ts b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/scmAdapter.ts index 02b8d24513737..98738835158b9 100644 --- a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/scmAdapter.ts +++ b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/scmAdapter.ts @@ -20,7 +20,7 @@ export class ScmAdapter { this._reposChangedSignal = observableSignalFromEvent(this, Event.any(this._scmService.onDidAddRepository, this._scmService.onDidRemoveRepository)); } - public getRepo(uri: URI, reader: IReader | undefined): ScmRepoAdapter | undefined { + public getRepo(uri: URI, reader: IReader | undefined): IScmRepoAdapter | undefined { this._reposChangedSignal.read(reader); const repo = this._scmService.getRepository(uri); if (!repo) { @@ -30,7 +30,13 @@ export class ScmAdapter { } } -export class ScmRepoAdapter { +export interface IScmRepoAdapter { + readonly headBranchNameObs: IObservable; + readonly headCommitHashObs: IObservable; + isIgnored(uri: URI): Promise; +} + +export class ScmRepoAdapter implements IScmRepoAdapter { public readonly headBranchNameObs: IObservable = derived(reader => this._repo.provider.historyProvider.read(reader)?.historyItemRef.read(reader)?.name); public readonly headCommitHashObs: IObservable = derived(reader => this._repo.provider.historyProvider.read(reader)?.historyItemRef.read(reader)?.revision); diff --git a/src/vs/workbench/contrib/editTelemetry/test/browser/documentWithAnnotatedEdits.test.ts b/src/vs/workbench/contrib/editTelemetry/test/browser/documentWithAnnotatedEdits.test.ts new file mode 100644 index 0000000000000..c9ea85b97961c --- /dev/null +++ b/src/vs/workbench/contrib/editTelemetry/test/browser/documentWithAnnotatedEdits.test.ts @@ -0,0 +1,146 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { timeout } from '../../../../../base/common/async.js'; +import { Disposable, DisposableStore } from '../../../../../base/common/lifecycle.js'; +import { IObservableWithChange, ISettableObservable, observableValue, runOnChange } from '../../../../../base/common/observable.js'; +import { runWithFakedTimers } from '../../../../../base/test/common/timeTravelScheduler.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/test/common/utils.js'; +import { AnnotatedStringEdit, StringEdit } from '../../../../../editor/common/core/edits/stringEdit.js'; +import { OffsetRange } from '../../../../../editor/common/core/ranges/offsetRange.js'; +import { StringText } from '../../../../../editor/common/core/text/abstractText.js'; +import { computeStringDiff } from '../../../../../editor/common/services/editorWebWorker.js'; +import { EditSources, TextModelEditSource } from '../../../../../editor/common/textModelEditSource.js'; +import { ServiceCollection } from '../../../../../platform/instantiation/common/serviceCollection.js'; +import { TestInstantiationService } from '../../../../../platform/instantiation/test/common/instantiationServiceMock.js'; +import { CombineStreamedChanges, DiffService, EditSourceData, IDocumentWithAnnotatedEdits, MinimizeEditsProcessor } from '../../browser/helpers/documentWithAnnotatedEdits.js'; + +suite('Documents with Annotated Edits', () => { + ensureNoDisposablesAreLeakedInTestSuite(); + + test('collapses streamed chat edits into one diff', () => runWithFakedTimers({}, async () => { + const context = setup(''); + const source = chatEdit(); + await timeout(0); + + context.document.apply(StringEdit.insert(0, 'a'), source); + await timeout(500); + context.document.apply(StringEdit.insert(1, 'b'), source); + await timeout(1100); + + assert.deepStrictEqual(context.changes, [{ + value: 'ab', + source: 'ai/chat/sidebar', + replacements: [{ start: 0, endExclusive: 0, newText: 'ab' }], + }]); + context.disposables.dispose(); + })); + + test('preserves ordering when a user edit interrupts streamed chat edits', () => runWithFakedTimers({}, async () => { + const context = setup(''); + await timeout(0); + + context.document.apply(StringEdit.insert(0, 'a'), chatEdit()); + await timeout(500); + context.document.apply(StringEdit.insert(1, 'U'), EditSources.cursor({ kind: 'type' })); + await timeout(1100); + + assert.deepStrictEqual(context.changes, [ + { + value: 'a', + source: 'ai/chat/sidebar', + replacements: [{ start: 0, endExclusive: 0, newText: 'a' }], + }, + { + value: 'aU', + source: 'user', + replacements: [{ start: 1, endExclusive: 1, newText: 'U' }], + }, + ]); + context.disposables.dispose(); + })); + + test('minimizes common prefixes and suffixes', () => { + const disposables = new DisposableStore(); + const document = disposables.add(new TestSourceDocument('hello world')); + const minimized = disposables.add(new MinimizeEditsProcessor(document)); + const changes: Array<{ value: string; replacements: Array<{ start: number; endExclusive: number; newText: string }> }> = []; + disposables.add(runOnChange(minimized.value, (value, _previous, edits) => { + const edit = AnnotatedStringEdit.compose(edits.map(change => change.edit)); + changes.push({ + value: value.value, + replacements: edit.replacements.map(replacement => ({ + start: replacement.replaceRange.start, + endExclusive: replacement.replaceRange.endExclusive, + newText: replacement.newText, + })), + }); + })); + + document.apply(StringEdit.replace(OffsetRange.ofLength(11), 'hello brave world'), chatEdit()); + + assert.deepStrictEqual(changes, [{ + value: 'hello brave world', + replacements: [{ start: 5, endExclusive: 5, newText: ' brave' }], + }]); + disposables.dispose(); + }); +}); + +function setup(initialValue: string) { + const disposables = new DisposableStore(); + const instantiationService = disposables.add(new TestInstantiationService(new ServiceCollection(), false, undefined, true)); + instantiationService.stubInstance(DiffService, { + computeDiff: async (original, modified) => computeStringDiff(original, modified, { maxComputationTimeMs: 500 }, 'advanced'), + }); + const document = disposables.add(new TestSourceDocument(initialValue)); + const combined = disposables.add(instantiationService.createInstance(CombineStreamedChanges, document)); + const changes: Array<{ value: string; source: string; replacements: Array<{ start: number; endExclusive: number; newText: string }> }> = []; + disposables.add(runOnChange(combined.value, (value, _previous, edits) => { + const edit = AnnotatedStringEdit.compose(edits.map(change => change.edit)); + changes.push({ + value: value.value, + source: edit.replacements[0]?.data.source.toString(), + replacements: edit.replacements.map(replacement => ({ + start: replacement.replaceRange.start, + endExclusive: replacement.replaceRange.endExclusive, + newText: replacement.newText, + })), + }); + })); + return { disposables, document, changes }; +} + +class TestSourceDocument extends Disposable implements IDocumentWithAnnotatedEdits { + private readonly _value: ISettableObservable }>; + readonly value: IObservableWithChange }>; + + constructor(initialValue: string) { + super(); + this.value = this._value = observableValue(this, new StringText(initialValue)); + } + + apply(edit: StringEdit, source: TextModelEditSource): void { + const data = new EditSourceData(source); + this._value.set(edit.applyOnText(this._value.get()), undefined, { edit: edit.mapData(() => data) }); + } + + waitForQueue(): Promise { + return Promise.resolve(); + } +} + +function chatEdit(): TextModelEditSource { + return EditSources.chatApplyEdits({ + modelId: undefined, + sessionId: 'session-1', + requestId: 'request-1', + languageId: 'typescript', + mode: 'agent', + extensionId: undefined, + codeBlockSuggestionId: undefined, + }); +} diff --git a/src/vs/workbench/contrib/editTelemetry/test/browser/editSourceCategories.test.ts b/src/vs/workbench/contrib/editTelemetry/test/browser/editSourceCategories.test.ts new file mode 100644 index 0000000000000..e5673e3df2c4b --- /dev/null +++ b/src/vs/workbench/contrib/editTelemetry/test/browser/editSourceCategories.test.ts @@ -0,0 +1,86 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { ProviderId } from '../../../../../editor/common/languages.js'; +import { EditSources } from '../../../../../editor/common/textModelEditSource.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/test/common/utils.js'; +import { EditSourceBase } from '../../browser/helpers/documentWithAnnotatedEdits.js'; +import { getEditTelemetryCategory } from '../../browser/telemetry/editSourceTrackingImpl.js'; + +suite('Edit Telemetry Source Categories', () => { + ensureNoDisposablesAreLeakedInTestSuite(); + + test('maps every edit source category', () => { + const sources = { + chat: EditSources.chatApplyEdits({ + modelId: undefined, + sessionId: undefined, + requestId: undefined, + languageId: 'typescript', + mode: 'agent', + extensionId: undefined, + codeBlockSuggestionId: undefined, + }), + copilotCompletion: EditSources.inlineCompletionAccept({ + nes: false, + requestUuid: 'request-1', + languageId: 'typescript', + providerId: new ProviderId('github.copilot', '1.0.0', 'completions'), + correlationId: undefined, + }), + copilotChatCompletion: EditSources.inlineCompletionAccept({ + nes: false, + requestUuid: 'request-2', + languageId: 'typescript', + providerId: new ProviderId('github.copilot-chat', '1.0.0', 'completions'), + correlationId: undefined, + }), + nes: EditSources.inlineCompletionAccept({ + nes: true, + requestUuid: 'request-3', + languageId: 'typescript', + providerId: new ProviderId('github.copilot-chat', '1.0.0', 'nes'), + correlationId: undefined, + }), + inlineNesProvider: EditSources.inlineCompletionAccept({ + nes: false, + requestUuid: 'request-4', + languageId: 'typescript', + providerId: new ProviderId('github.copilot-chat', '1.0.0', 'nes'), + correlationId: undefined, + }), + otherCompletion: EditSources.inlineCompletionAccept({ + nes: false, + requestUuid: 'request-5', + languageId: 'typescript', + providerId: new ProviderId('other.extension', '1.0.0', 'other'), + correlationId: undefined, + }), + user: EditSources.cursor({ kind: 'type' }), + snippet: EditSources.snippet(), + format: EditSources.unknown({ name: 'formatEditsCommand' }), + external: EditSources.reloadFromDisk(), + unknown: EditSources.unknown({}), + }; + + assert.deepStrictEqual(Object.fromEntries(Object.entries(sources).map(([key, source]) => [ + key, + getEditTelemetryCategory(EditSourceBase.create(source)), + ])), { + chat: 'otherAI', + copilotCompletion: 'inlineCompletionsCopilot', + copilotChatCompletion: 'inlineCompletionsCopilot', + nes: 'nes', + inlineNesProvider: 'inlineCompletionsNES', + otherCompletion: 'inlineCompletionsOther', + user: 'user', + snippet: 'ide', + format: 'ide', + external: 'external', + unknown: 'unknown', + }); + }); +}); diff --git a/src/vs/workbench/contrib/editTelemetry/test/browser/editSourceTrackingImpl.test.ts b/src/vs/workbench/contrib/editTelemetry/test/browser/editSourceTrackingImpl.test.ts new file mode 100644 index 0000000000000..f84b57dab6410 --- /dev/null +++ b/src/vs/workbench/contrib/editTelemetry/test/browser/editSourceTrackingImpl.test.ts @@ -0,0 +1,231 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { timeout } from '../../../../../base/common/async.js'; +import { Disposable, DisposableStore } from '../../../../../base/common/lifecycle.js'; +import { constObservable, ISettableObservable, observableValue } from '../../../../../base/common/observable.js'; +import { URI } from '../../../../../base/common/uri.js'; +import { runWithFakedTimers } from '../../../../../base/test/common/timeTravelScheduler.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/test/common/utils.js'; +import { OffsetRange } from '../../../../../editor/common/core/ranges/offsetRange.js'; +import { computeStringDiff } from '../../../../../editor/common/services/editorWebWorker.js'; +import { EditSources, EditSuggestionId } from '../../../../../editor/common/textModelEditSource.js'; +import { ServiceCollection } from '../../../../../platform/instantiation/common/serviceCollection.js'; +import { TestInstantiationService } from '../../../../../platform/instantiation/test/common/instantiationServiceMock.js'; +import { ILogService, NullLogService } from '../../../../../platform/log/common/log.js'; +import { ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; +import { IUserAttentionService } from '../../../../services/userAttention/common/userAttentionService.js'; +import { AnnotatedDocuments, UriVisibilityProvider } from '../../browser/helpers/annotatedDocuments.js'; +import { DiffService } from '../../browser/helpers/documentWithAnnotatedEdits.js'; +import { StringEditWithReason } from '../../browser/helpers/observableWorkspace.js'; +import { IAiEditTelemetryService } from '../../browser/telemetry/aiEditTelemetry/aiEditTelemetryService.js'; +import { EditSourceTrackingImpl } from '../../browser/telemetry/editSourceTrackingImpl.js'; +import { IScmRepoAdapter, ScmAdapter } from '../../browser/telemetry/scmAdapter.js'; +import { IRandomService } from '../../browser/randomService.js'; +import { MutableObservableWorkspace } from './editTelemetry.test.js'; + +suite('Edit Source Tracking Windows', () => { + ensureNoDisposablesAreLeakedInTestSuite(); + + test('flushes and recreates the long-term tracker on hash and branch changes', () => runWithFakedTimers({}, async () => { + const context = setup(); + await timeout(10); + + context.document.applyEdit(StringEditWithReason.replace(context.document.findRange('hello'), 'alpha', chatEdit('request-1'))); + await timeout(1500); + context.headHash.set('hash-2', undefined); + + context.document.applyEdit(StringEditWithReason.replace(context.document.findRange('alpha'), 'beta', chatEdit('request-2'))); + await timeout(1500); + context.branch.set('feature', undefined); + + assert.deepStrictEqual(context.details.map(event => ({ + trigger: event.trigger, + requestId: event.requestId, + modifiedCount: event.modifiedCount, + deltaModifiedCount: event.deltaModifiedCount, + })), [ + { trigger: 'hashChange', requestId: 'request-1', modifiedCount: 5, deltaModifiedCount: 5 }, + { trigger: 'branchChange', requestId: 'request-2', modifiedCount: 3, deltaModifiedCount: 3 }, + ]); + + context.disposables.dispose(); + })); + + test('flushes the long-term tracker when the document closes', () => runWithFakedTimers({}, async () => { + const context = setup(); + await timeout(10); + + context.document.applyEdit(StringEditWithReason.replace(context.document.findRange('hello'), 'alpha', chatEdit('request-1'))); + await timeout(1500); + context.document.dispose(); + await timeout(0); + + assert.deepStrictEqual(context.details.map(event => ({ + trigger: event.trigger, + requestId: event.requestId, + })), [{ trigger: 'closed', requestId: 'request-1' }]); + + context.disposables.dispose(); + })); + + test('flushes and recreates the long-term tracker after ten hours', () => runWithFakedTimers({}, async () => { + const context = setup(); + await timeout(10); + + context.document.applyEdit(StringEditWithReason.replace(context.document.findRange('hello'), 'alpha', chatEdit('request-1'))); + await timeout(1500); + await timeout(10 * 60 * 60 * 1000); + + context.document.applyEdit(StringEditWithReason.replace(context.document.findRange('alpha'), 'beta', chatEdit('request-2'))); + await timeout(1500); + context.headHash.set('hash-2', undefined); + + assert.deepStrictEqual(context.details.map(event => ({ + trigger: event.trigger, + requestId: event.requestId, + })), [ + { trigger: '10hours', requestId: 'request-1' }, + { trigger: 'hashChange', requestId: 'request-2' }, + ]); + + context.disposables.dispose(); + })); + + test('emits only the top thirty long-term sources by retained count', () => runWithFakedTimers({}, async () => { + const context = setup(); + await timeout(10); + + for (let i = 1; i <= 31; i++) { + context.document.applyEdit(StringEditWithReason.replace( + OffsetRange.emptyAt(context.document.value.get().value.length), + 'x'.repeat(i), + EditSources.unknown({ name: `source-${i}` }), + )); + } + await timeout(10); + context.headHash.set('hash-2', undefined); + + assert.deepStrictEqual({ + count: context.details.length, + first: context.details[0].sourceKey, + last: context.details.at(-1)?.sourceKey, + containsSmallest: context.details.some(event => event.sourceKey === 'source:unknown-name:source-1'), + }, { + count: 30, + first: 'source:unknown-name:source-31', + last: 'source:unknown-name:source-2', + containsSmallest: false, + }); + + context.disposables.dispose(); + })); + + test('starts after first visibility and keeps only the long-term tracker while hidden', () => runWithFakedTimers({}, async () => { + const visible = observableValue('visible', false); + const context = setup(visible); + await timeout(10); + + assert.strictEqual(context.impl.docsState.get().size, 0); + + visible.set(true, undefined); + const visibleState = context.impl.docsState.get().get(context.document); + if (!visibleState) { + throw new Error('Expected visible document state'); + } + assert.ok(visibleState.longtermTracker.get()); + const firstWindowedTracker = visibleState.windowedTracker.get(); + assert.ok(firstWindowedTracker); + assert.ok(visibleState.windowedFocusTracker.get()); + + visible.set(false, undefined); + const hiddenState = context.impl.docsState.get().get(context.document); + if (!hiddenState) { + throw new Error('Expected hidden document state'); + } + assert.ok(hiddenState.longtermTracker.get()); + assert.strictEqual(hiddenState.windowedTracker.get(), undefined); + assert.strictEqual(hiddenState.windowedFocusTracker.get(), undefined); + + visible.set(true, undefined); + const visibleAgainState = context.impl.docsState.get().get(context.document); + if (!visibleAgainState) { + throw new Error('Expected visible document state after reopening'); + } + assert.ok(visibleAgainState.windowedTracker.get()); + assert.notStrictEqual(visibleAgainState.windowedTracker.get(), firstWindowedTracker); + + context.disposables.dispose(); + })); +}); + +function setup(visible: ISettableObservable = observableValue('visible', true)) { + const disposables = new DisposableStore(); + const headHash = observableValue('headHash', 'hash-1'); + const branch = observableValue('branch', 'main'); + const repo = { + headCommitHashObs: headHash, + headBranchNameObs: branch, + isIgnored: async () => false, + } satisfies IScmRepoAdapter; + const details: Array<{ sourceKey: string; trigger: string; requestId: string | undefined; modifiedCount: number; deltaModifiedCount: number }> = []; + let uuid = 0; + const instantiationService = disposables.add(new TestInstantiationService(new ServiceCollection(), false, undefined, true)); + instantiationService.stub(ITelemetryService, { + publicLog2(eventName, data) { + const eventData = data as { mode?: string } | undefined; + if (eventName === 'editTelemetry.editSources.details' && eventData?.mode === 'longterm') { + details.push(data as typeof details[number]); + } + }, + }); + instantiationService.stubInstance(DiffService, { computeDiff: async (original, modified) => computeStringDiff(original, modified, { maxComputationTimeMs: 500 }, 'advanced') }); + instantiationService.stubInstance(ScmAdapter, { getRepo: () => repo }); + instantiationService.stubInstance(UriVisibilityProvider, { isVisible: (_uri, reader) => visible.read(reader) }); + instantiationService.stub(IRandomService, { + _serviceBrand: undefined, + generateUuid: () => `stats-${++uuid}`, + generatePrefixedUuid: namespace => `${namespace}-${++uuid}`, + }); + instantiationService.stub(IUserAttentionService, { + _serviceBrand: undefined, + isVsCodeFocused: constObservable(true), + isUserActive: constObservable(true), + hasUserAttention: constObservable(true), + totalFocusTimeMs: 0, + fireAfterGivenFocusTimePassed: () => Disposable.None, + }); + instantiationService.stub(IAiEditTelemetryService, { + _serviceBrand: undefined, + createSuggestionId: () => EditSuggestionId.newId(() => 'sgt-test'), + handleCodeAccepted: () => { }, + handleCodeRejected: () => { }, + }); + instantiationService.stub(ILogService, new NullLogService()); + + const workspace = new MutableObservableWorkspace(); + const annotatedDocuments = disposables.add(new AnnotatedDocuments(workspace, instantiationService)); + const impl = disposables.add(new EditSourceTrackingImpl(constObservable(true), annotatedDocuments, instantiationService)); + const document = disposables.add(workspace.createDocument({ + uri: URI.file('C:\\repo\\file.ts'), + initialValue: 'hello', + languageId: 'typescript', + })); + + return { disposables, document, details, headHash, branch, impl }; +} + +function chatEdit(requestId: string) { + return EditSources.chatApplyEdits({ + modelId: undefined, + sessionId: 'session-1', + requestId, + languageId: 'typescript', + mode: 'agent', + extensionId: undefined, + codeBlockSuggestionId: undefined, + }); +} diff --git a/src/vs/workbench/contrib/editTelemetry/test/browser/editTracker.test.ts b/src/vs/workbench/contrib/editTelemetry/test/browser/editTracker.test.ts new file mode 100644 index 0000000000000..8ef814c7b4267 --- /dev/null +++ b/src/vs/workbench/contrib/editTelemetry/test/browser/editTracker.test.ts @@ -0,0 +1,140 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { Disposable } from '../../../../../base/common/lifecycle.js'; +import { IObservableWithChange, ISettableObservable, observableValue } from '../../../../../base/common/observable.js'; +import { OffsetRange } from '../../../../../editor/common/core/ranges/offsetRange.js'; +import { AnnotatedStringEdit, StringEdit } from '../../../../../editor/common/core/edits/stringEdit.js'; +import { StringText } from '../../../../../editor/common/core/text/abstractText.js'; +import { EditSources, TextModelEditSource } from '../../../../../editor/common/textModelEditSource.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/test/common/utils.js'; +import { EditKeySourceData, EditSourceData, IDocumentWithAnnotatedEdits } from '../../browser/helpers/documentWithAnnotatedEdits.js'; +import { DocumentEditSourceTracker } from '../../browser/telemetry/editTracker.js'; + +suite('DocumentEditSourceTracker', () => { + const disposables = ensureNoDisposablesAreLeakedInTestSuite(); + + test('ignores an initial external edit', () => { + const document = disposables.add(new TestAnnotatedDocument('initial')); + const tracker = disposables.add(new DocumentEditSourceTracker(document, undefined)); + + document.apply(StringEdit.replace(OffsetRange.ofLength(7), 'external'), EditSources.reloadFromDisk()); + + assert.deepStrictEqual(snapshot(tracker), []); + }); + + test('applies queued external edits before the next attributed edit', () => { + const document = disposables.add(new TestAnnotatedDocument('')); + const tracker = disposables.add(new DocumentEditSourceTracker(document, undefined)); + const ai = chatEditSource('gpt-5', 'request-1'); + const user = EditSources.cursor({ kind: 'type' }); + + document.apply(StringEdit.insert(0, 'abcdef'), ai); + document.apply(StringEdit.delete(new OffsetRange(2, 4)), EditSources.reloadFromDisk()); + + assert.deepStrictEqual(snapshot(tracker), [{ + key: ai.toKey(1), + delta: 6, + retained: 6, + requestId: 'request-1', + }]); + + document.apply(StringEdit.insert(4, 'X'), user); + + assert.deepStrictEqual(snapshot(tracker), [ + { + key: ai.toKey(1), + delta: 6, + retained: 4, + requestId: 'request-1', + }, + { + key: 'source:cursor-kind:type', + delta: 1, + retained: 1, + requestId: undefined, + }, + { + key: 'source:reloadFromDisk', + delta: 0, + retained: 0, + requestId: undefined, + }, + ]); + }); + + test('joins level-one keys but keeps distinct model ids separate', () => { + const document = disposables.add(new TestAnnotatedDocument('')); + const tracker = disposables.add(new DocumentEditSourceTracker(document, undefined)); + const gptFirst = chatEditSource('gpt-5', 'request-1'); + const gptSecond = chatEditSource('gpt-5', 'request-2'); + const claude = chatEditSource('claude-sonnet', 'request-3'); + + document.apply(StringEdit.insert(0, 'one'), gptFirst); + document.apply(StringEdit.insert(3, 'two'), gptSecond); + document.apply(StringEdit.insert(6, 'three'), claude); + + assert.deepStrictEqual(snapshot(tracker), [ + { + key: claude.toKey(1), + delta: 5, + retained: 5, + requestId: 'request-3', + }, + { + key: gptFirst.toKey(1), + delta: 6, + retained: 6, + requestId: 'request-1', + }, + ]); + assert.strictEqual(gptFirst.toKey(1), gptSecond.toKey(1)); + }); +}); + +class TestAnnotatedDocument extends Disposable implements IDocumentWithAnnotatedEdits { + private readonly _value: ISettableObservable }>; + readonly value: IObservableWithChange }>; + + constructor(initialValue: string) { + super(); + this.value = this._value = observableValue(this, new StringText(initialValue)); + } + + apply(edit: StringEdit, source: TextModelEditSource): void { + const data = new EditSourceData(source).toEditSourceData(); + this._value.set(edit.applyOnText(this._value.get()), undefined, { edit: edit.mapData(() => data) }); + } + + waitForQueue(): Promise { + return Promise.resolve(); + } +} + +function chatEditSource(modelId: string, requestId: string): TextModelEditSource { + return EditSources.chatApplyEdits({ + modelId, + sessionId: 'session-1', + requestId, + languageId: 'typescript', + mode: 'agent', + extensionId: undefined, + codeBlockSuggestionId: undefined, + }); +} + +function snapshot(tracker: DocumentEditSourceTracker): Array<{ key: string; delta: number; retained: number; requestId: string | undefined }> { + const retained = new Map(); + for (const range of tracker.getTrackedRanges()) { + retained.set(range.sourceKey, (retained.get(range.sourceKey) ?? 0) + range.range.length); + } + return tracker.getAllKeys().map(key => ({ + key, + delta: tracker.getTotalInsertedCharactersCount(key), + retained: retained.get(key) ?? 0, + requestId: tracker.getRepresentative(key)?.props.$$requestId, + })).sort((a, b) => a.key.localeCompare(b.key)); +}