From a92981b456b93198cc8ce15ecc45f9a82a06e269 Mon Sep 17 00:00:00 2001 From: Hani Nemati Date: Mon, 28 Sep 2026 16:22:50 -0700 Subject: [PATCH] Add PMU performance counters to LTTng execution events table Ports the performance counter support from 560cd2f onto develop: - Capture integer perf context fields on each sched_switch - Compute per-execution-event counter deltas between switch-in and switch-out - Add Instruction Count, LLC Misses, CPU Cycle and Cache Misses columns Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../DataOutputTypes/IExecutionEvent.cs | 2 ++ .../SourceDataCookers/Thread/ContextSwitch.cs | 18 ++++++++++ .../Thread/ExecutionEvent.cs | 19 +++++++++++ .../Thread/LTTngThreadDataCooker.cs | 34 +++++++++++-------- .../Tables/ExecutionEventTable.cs | 29 ++++++++++++++++ 5 files changed, 87 insertions(+), 15 deletions(-) diff --git a/LTTngDataExtensions/DataOutputTypes/IExecutionEvent.cs b/LTTngDataExtensions/DataOutputTypes/IExecutionEvent.cs index 1507aa5..45a54c8 100644 --- a/LTTngDataExtensions/DataOutputTypes/IExecutionEvent.cs +++ b/LTTngDataExtensions/DataOutputTypes/IExecutionEvent.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System.Collections.Generic; using Microsoft.Performance.SDK; namespace LTTngDataExtensions.SourceDataCookers.Thread @@ -23,5 +24,6 @@ public interface IExecutionEvent Timestamp SwitchInTime { get; } Timestamp SwitchOutTime { get; } Timestamp NextThreadPreviousSwitchOutTime { get; } + IReadOnlyDictionary PerformanceCountersDiffByName { get; } } } diff --git a/LTTngDataExtensions/SourceDataCookers/Thread/ContextSwitch.cs b/LTTngDataExtensions/SourceDataCookers/Thread/ContextSwitch.cs index a324cbf..5d36751 100644 --- a/LTTngDataExtensions/SourceDataCookers/Thread/ContextSwitch.cs +++ b/LTTngDataExtensions/SourceDataCookers/Thread/ContextSwitch.cs @@ -1,8 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System.Collections.Generic; using Microsoft.Performance.SDK; using LTTngCds.CookerData; +using CtfPlayback.Metadata; namespace LTTngDataExtensions.SourceDataCookers.Thread { @@ -23,6 +25,7 @@ public class ContextSwitch private TimestampDelta waitTime; private Timestamp switchInTime; private Timestamp nextThreadPreviousSwitchOutTime; + private Dictionary performanceCountersByName; public ContextSwitch(LTTngEvent data, ThreadInfo nextThread, ThreadInfo previousThread, uint cpu) { @@ -61,6 +64,20 @@ public ContextSwitch(LTTngEvent data, ThreadInfo nextThread, ThreadInfo previous this.previousCommand = data.Payload.ReadFieldAsArray("_prev_comm").GetValueAsString(); this.switchInTime = data.Timestamp; this.nextThreadPreviousSwitchOutTime = nextThread.previousSwitchOutTime; + + // PMU counters (e.g. _perf_cpu_instructions) are added to the stream event context by "lttng add-context -t perf:cpu:..." + this.performanceCountersByName = new Dictionary(); + var eventContext = data.StreamDefinedEventContext; + if (eventContext != null) + { + foreach (var field in eventContext.FieldsByName) + { + if (field.Value.FieldType == CtfTypes.Integer && field.Key.Contains("perf")) + { + this.performanceCountersByName[field.Key] = eventContext.ReadFieldAsInt64(field.Key); + } + } + } } public uint Cpu => this.cpu; @@ -78,5 +95,6 @@ public ContextSwitch(LTTngEvent data, ThreadInfo nextThread, ThreadInfo previous public TimestampDelta WaitTime => this.waitTime; public Timestamp SwitchInTime => this.switchInTime; public Timestamp NextThreadPreviousSwitchOutTime => this.nextThreadPreviousSwitchOutTime; + public IReadOnlyDictionary PerformanceCountersByName => this.performanceCountersByName; } } diff --git a/LTTngDataExtensions/SourceDataCookers/Thread/ExecutionEvent.cs b/LTTngDataExtensions/SourceDataCookers/Thread/ExecutionEvent.cs index 35ac96c..ee89dc3 100644 --- a/LTTngDataExtensions/SourceDataCookers/Thread/ExecutionEvent.cs +++ b/LTTngDataExtensions/SourceDataCookers/Thread/ExecutionEvent.cs @@ -27,8 +27,14 @@ public class ExecutionEvent private Timestamp switchInTime; private Timestamp switchOutTime; private Timestamp nextThreadPreviousSwitchOutTime; + private Dictionary performanceCountersDiffByName; public ExecutionEvent(ContextSwitch contextSwitch, Timestamp switchOutTime) + : this(contextSwitch, null, switchOutTime) + { + } + + public ExecutionEvent(ContextSwitch contextSwitch, ContextSwitch switchOutContextSwitch, Timestamp switchOutTime) { this.cpu = contextSwitch.Cpu; this.nextPid = contextSwitch.NextPid; @@ -46,6 +52,18 @@ public ExecutionEvent(ContextSwitch contextSwitch, Timestamp switchOutTime) this.switchInTime = contextSwitch.SwitchInTime; this.switchOutTime = switchOutTime; this.nextThreadPreviousSwitchOutTime = contextSwitch.NextThreadPreviousSwitchOutTime; + + this.performanceCountersDiffByName = new Dictionary(); + if (switchOutContextSwitch != null) + { + foreach (var counter in contextSwitch.PerformanceCountersByName) + { + if (switchOutContextSwitch.PerformanceCountersByName.TryGetValue(counter.Key, out long switchOutValue)) + { + this.performanceCountersDiffByName[counter.Key] = switchOutValue - counter.Value; + } + } + } } public void RecoverPids(Dictionary recoveredPids) @@ -90,5 +108,6 @@ private int reconstructPid(string pid) public Timestamp SwitchInTime => this.switchInTime; public Timestamp SwitchOutTime => this.switchOutTime; public Timestamp NextThreadPreviousSwitchOutTime => this.nextThreadPreviousSwitchOutTime; + public IReadOnlyDictionary PerformanceCountersDiffByName => this.performanceCountersDiffByName; } } diff --git a/LTTngDataExtensions/SourceDataCookers/Thread/LTTngThreadDataCooker.cs b/LTTngDataExtensions/SourceDataCookers/Thread/LTTngThreadDataCooker.cs index ab9272b..25890f3 100644 --- a/LTTngDataExtensions/SourceDataCookers/Thread/LTTngThreadDataCooker.cs +++ b/LTTngDataExtensions/SourceDataCookers/Thread/LTTngThreadDataCooker.cs @@ -577,19 +577,6 @@ void ProcessThreadMetadata(LTTngEvent data, ThreadInfo thread) public void ProcessContextSwitch(LTTngEvent data, LTTngContext context) { int prevTid = data.Payload.ReadFieldAsInt32("_prev_tid"); - if (lastContextSwitch.TryGetValue(context.CurrentCpu, out ContextSwitch previousContextSwitch)) - { - if (prevTid == previousContextSwitch.NextTid) - { - processedExecutionEvents.Add(new ExecutionEvent(previousContextSwitch, data.Timestamp)); - } - else - { - ///If we missed context switch events, - processedExecutionEvents.Add(new ExecutionEvent(previousContextSwitch, previousContextSwitch.SwitchInTime)); - } - - } int nextTid = data.Payload.ReadFieldAsInt32("_next_tid"); ThreadInfo.ThreadState switchOutState; var prevStateValue = data.Payload.FieldsByName["_prev_state"]; @@ -620,17 +607,34 @@ public void ProcessContextSwitch(LTTngEvent data, LTTngContext context) } ThreadInfo nextThread; + ContextSwitch currentContextSwitch; if (runningThreads.TryGetValue(nextTid, out nextThread)) { - lastContextSwitch[context.CurrentCpu] = new ContextSwitch(data, nextThread, prevThread, context.CurrentCpu); + currentContextSwitch = new ContextSwitch(data, nextThread, prevThread, context.CurrentCpu); nextThread.SwitchIn(data.Timestamp); } else { nextThread = new ThreadInfo(nextTid, data.Timestamp, ThreadInfo.ThreadState.TASK_RUNNING); - lastContextSwitch[context.CurrentCpu] = new ContextSwitch(data, nextThread, prevThread, context.CurrentCpu); + currentContextSwitch = new ContextSwitch(data, nextThread, prevThread, context.CurrentCpu); this.AddNewThread(nextThread); } + + // The execution event for the previous context switch is completed here so the PMU counter deltas can be computed against this switch-out. + if (lastContextSwitch.TryGetValue(context.CurrentCpu, out ContextSwitch previousContextSwitch)) + { + if (prevTid == previousContextSwitch.NextTid) + { + processedExecutionEvents.Add(new ExecutionEvent(previousContextSwitch, currentContextSwitch, data.Timestamp)); + } + else + { + ///If we missed context switch events, + processedExecutionEvents.Add(new ExecutionEvent(previousContextSwitch, previousContextSwitch.SwitchInTime)); + } + } + + lastContextSwitch[context.CurrentCpu] = currentContextSwitch; } public void ProcessThreadExit(LTTngEvent data) diff --git a/LTTngDataExtensions/Tables/ExecutionEventTable.cs b/LTTngDataExtensions/Tables/ExecutionEventTable.cs index 03ca340..0a141c7 100644 --- a/LTTngDataExtensions/Tables/ExecutionEventTable.cs +++ b/LTTngDataExtensions/Tables/ExecutionEventTable.cs @@ -108,6 +108,26 @@ public class ExecutionEventTable new ColumnMetadata(new Guid("{5AB96EE5-FE5C-45AE-9BA2-965EFF4833D8}"), "Last Switch-Out Time"), new UIHints { Width = 80, }); + private static readonly ColumnConfiguration instructionCountColumn = + new ColumnConfiguration( + new ColumnMetadata(new Guid("{63447522-514D-4F7D-81E0-544E0670BB1D}"), "Instruction Count"), + new UIHints { Width = 80, }); + + private static readonly ColumnConfiguration lastLevelCacheMissesColumn = + new ColumnConfiguration( + new ColumnMetadata(new Guid("{7EB3DBFA-B1CA-4CB7-9D70-66E0BEFA0F32}"), "LLC Misses"), + new UIHints { Width = 80, }); + + private static readonly ColumnConfiguration cpuCycleColumn = + new ColumnConfiguration( + new ColumnMetadata(new Guid("{AC255EA6-BA80-4A7D-BCF5-710B6B3618A9}"), "CPU Cycle"), + new UIHints { Width = 80, }); + + private static readonly ColumnConfiguration cacheMissesColumn = + new ColumnConfiguration( + new ColumnMetadata(new Guid("{6BBB2F84-4215-46CF-9C92-FA79DB8DE55C}"), "Cache Misses"), + new UIHints { Width = 80, }); + private static readonly ColumnConfiguration cpuUsageInViewportPreset = new ColumnConfiguration( new ColumnMetadata(new Guid("{0cf6ffa6-2f41-4460-a201-718c37cbf413}"), "CPU Usage (in view)"), new UIHints @@ -271,6 +291,10 @@ public static void BuildTable(ITableBuilder tableBuilder, IDataExtensionRetrieva table.AddColumn(switchInTimeColumn, switchInTime); table.AddColumn(switchOutTimeColumn, switchOutTime); table.AddColumn(previousPidColumn, Projection.CreateUsingFuncAdaptor((i) => threads[i].PreviousPid)); + table.AddColumn(instructionCountColumn, Projection.CreateUsingFuncAdaptor((i) => GetPerformanceCounterDiff(threads[i], "_perf_cpu_instructions"))); + table.AddColumn(lastLevelCacheMissesColumn, Projection.CreateUsingFuncAdaptor((i) => GetPerformanceCounterDiff(threads[i], "_perf_cpu_LLC_load_misses"))); + table.AddColumn(cpuCycleColumn, Projection.CreateUsingFuncAdaptor((i) => GetPerformanceCounterDiff(threads[i], "_perf_cpu_cpu_cycles"))); + table.AddColumn(cacheMissesColumn, Projection.CreateUsingFuncAdaptor((i) => GetPerformanceCounterDiff(threads[i], "_perf_cpu_cache_misses"))); table.AddColumn(nextCommandColumn, Projection.CreateUsingFuncAdaptor((i) => threads[i].NextImage)); table.AddColumn(previousCommandColumn, Projection.CreateUsingFuncAdaptor((i) => threads[i].PreviousImage)); @@ -294,6 +318,11 @@ public static void BuildTable(ITableBuilder tableBuilder, IDataExtensionRetrieva table.AddColumn(cpuUsagePreset, cpuUsageColumn); table.AddColumn(percentCpuUsagePreset, percentCpuUsageColumn); } + + private static long GetPerformanceCounterDiff(IExecutionEvent executionEvent, string counterName) + { + return executionEvent.PerformanceCountersDiffByName.TryGetValue(counterName, out long value) ? value : 0; + } } struct ReduceTimeSinceLastDiff