OBSINTA-1006: Add UI performance benchmarks for Incidents page

docs/incident_detection/tests/performance/03.endurance_test_source.md

@@ -0,0 +1,152 @@
+# 03. Endurance Test — Shelved Source
+
+Shelved due to Cypress DOM snapshot accumulation causing ~10x degradation over 100 cycles regardless of application performance. See [overview.md](./overview.md) for details.
+
+**To re-enable:** save the code block below as `web/cypress/e2e/incidents/performance/03.performance_endurance.cy.ts`.
+
+```typescript
+import { incidentsPage } from '../../../views/incidents-page';
+
+const MCP = {
+  namespace: Cypress.env('COO_NAMESPACE'),
+  packageName: 'cluster-observability-operator',
+  operatorName: 'Cluster Observability Operator',
+  config: {
+    kind: 'UIPlugin',
+    name: 'monitoring',
+  },
+};
+
+const MP = {
+  namespace: 'openshift-monitoring',
+  operatorName: 'Cluster Monitoring Operator',
+};
+
+const INCIDENT_COUNT = 20;
+const CYCLES_PER_INCIDENT = 5;
+const TOTAL_CYCLES = INCIDENT_COUNT * CYCLES_PER_INCIDENT;
+const DEGRADATION_FACTOR = 5;
+
+const INCIDENT_IDS = Array.from({ length: INCIDENT_COUNT }, (_, i) =>
+  `bench-${String(i + 1).padStart(2, '0')}`,
+);
+
+const TRACKED_AVERAGE = 'bench-01';
+const TRACKED_HEAVIEST = 'bench-03';
+
+describe('Performance: Endurance - Select/Deselect Cycling', { tags: ['@incidents', '@performance', '@endurance'] }, () => {
+
+  before(() => {
+    cy.beforeBlockCOO(MCP, MP, { dashboards: false, troubleshootingPanel: false });
+  });
+
+  it('8.1 Endurance: Incident select/deselect cycling across 20 incidents', () => {
+    cy.mockIncidentFixture('incident-scenarios/22-benchmark-20-incidents.yaml');
+    incidentsPage.clearAllFilters();
+    incidentsPage.setDays('1 day');
+    incidentsPage.elements.incidentsChartBarsGroups()
+      .should('have.length', 20);
+
+    const cycleTimes: number[] = [];
+    const trackedTimes: Record<string, number[]> = {
+      [TRACKED_AVERAGE]: [],
+      [TRACKED_HEAVIEST]: [],
+    };
+
+    const sum = (arr: number[]) => arr.reduce((a, b) => a + b, 0);
+    const avg = (arr: number[]) => Math.round(sum(arr) / arr.length);
+
+    const logTrackedSnapshot = (label: string) => {
+      const lines: string[] = [label];
+      for (const [id, times] of Object.entries(trackedTimes)) {
+        if (times.length === 0) continue;
+        lines.push(
+          `  ${id} (${times.length} cycles): avg ${avg(times)}ms, last ${times[times.length - 1]}ms`,
+        );
+      }
+      lines.forEach((line) => {
+        cy.log(line);
+        cy.task('log', line);
+      });
+    };
+
+    cy.window({ log: false }).then((win) => {
+      win.performance.clearMarks('endurance-start');
+      win.performance.mark('endurance-start');
+    });
+
+    const runCycle = (cycleIndex: number) => {
+      if (cycleIndex >= TOTAL_CYCLES) return;
+
+      const incidentId = INCIDENT_IDS[cycleIndex % INCIDENT_IDS.length];
+      const cycleMark = `cycle-${cycleIndex}`;
+
+      cy.window({ log: false }).then((win) => {
+        win.performance.mark(cycleMark);
+      });
+
+      incidentsPage.selectIncidentById(incidentId);
+      incidentsPage.elements.alertsChartCard().should('be.visible');
+
+      incidentsPage.deselectIncidentById(incidentId);
+
+      cy.window({ log: false }).then((win) => {
+        const measure = win.performance.measure(`${cycleMark}-measure`, cycleMark);
+        const elapsed = Math.round(measure.duration);
+        cycleTimes.push(elapsed);
+
+        if (incidentId in trackedTimes) {
+          trackedTimes[incidentId].push(elapsed);
+        }
+
+        if ((cycleIndex + 1) % 20 === 0 || cycleIndex === 0) {
+          const totalMeasure = win.performance.measure('endurance-progress', 'endurance-start');
+          const totalElapsed = Math.round(totalMeasure.duration / 1000);
+          win.performance.clearMeasures('endurance-progress');
+          const msg = `Cycle ${cycleIndex + 1}/${TOTAL_CYCLES} [${incidentId}]: ${elapsed}ms (${totalElapsed}s elapsed)`;
+          cy.log(msg);
+          cy.task('log', msg);
+          logTrackedSnapshot('--- Tracked incidents snapshot ---');
+        }
+      });
+
+      cy.then(() => { runCycle(cycleIndex + 1); });
+    };
+
+    runCycle(0);
+
+    cy.window({ log: false }).then((win) => {
+      const totalMeasure = win.performance.measure('endurance-total', 'endurance-start');
+      const totalElapsed = totalMeasure.duration;
+
+      const min = Math.min(...cycleTimes);
+      const max = Math.max(...cycleTimes);
+      const mean = avg(cycleTimes);
+      const first10Avg = avg(cycleTimes.slice(0, 10));
+      const last10Avg = avg(cycleTimes.slice(-10));
+
+      const summary = [
+        `=== ENDURANCE TEST SUMMARY ===`,
+        `Total cycles: ${cycleTimes.length}`,
+        `Total time: ${Math.round(totalElapsed / 1000)}s`,
+        `Cycle times - min: ${min}ms, max: ${max}ms, avg: ${mean}ms`,
+        `First 10 avg: ${first10Avg}ms`,
+        `Last 10 avg: ${last10Avg}ms`,
+        `Degradation ratio: ${(last10Avg / first10Avg).toFixed(2)}x (threshold: ${DEGRADATION_FACTOR}x)`,
+      ];
+
+      summary.forEach((line) => {
+        cy.log(line);
+        cy.task('log', line);
+      });
+
+      logTrackedSnapshot('=== TRACKED INCIDENTS FINAL ===');
+
+      expect(
+        last10Avg,
+        `Last 10 cycles avg (${last10Avg}ms) should not exceed ${DEGRADATION_FACTOR}x first 10 avg (${first10Avg}ms)`,
+      ).to.be.lessThan(first10Avg * DEGRADATION_FACTOR);
+    });
+  });
+});
+```

docs/incident_detection/tests/performance/overview.md

@@ -0,0 +1,93 @@
+# Performance Testing - Incidents Page
+
+Location: `web/cypress/e2e/incidents/performance/`
+Verifies: OBSINTA-1006
+
+## Test Suite
+
+### 01. Performance Benchmark (`01.performance_benchmark.cy.ts`)
+
+Measures wall-clock render time for chart operations under escalating data loads. Uses `performance.mark()`/`performance.measure()` for timing.
+
+**What it tests:**
+- Incidents chart render: 100, 200, 500 alerts (single incident)
+- Alerts detail chart render after incident selection: 100, 200, 500 alerts
+- Multi-incident chart: 20 uniform incidents, 12 mixed-size incidents (67 alerts)
+
+**Thresholds** are regression indicators, not absolute targets. They include Cypress overhead (~3-5s per navigation cycle). Calibrate by running 3-5 times on a clean build, then set at ~2x median.
+
+**Known limitation:** 1000-alert tests are disabled — mocking that volume triggers a maximum call stack error in the mock generator, though equivalent non-mocked simulated data renders without issue. To re-enable, apply:
+
+```diff
+--- a/web/cypress/e2e/incidents/performance/01.performance_benchmark.cy.ts
++++ b/web/cypress/e2e/incidents/performance/01.performance_benchmark.cy.ts
+@@ end of it('6.1 ...')
++    cy.log('6.1.4 Incidents chart with 1000 alerts (single incident)');
++    benchmarkIncidentsChart(
++      '18-stress-test-1000-alerts-single.yaml',
++      1,
++      THRESHOLDS.INCIDENTS_CHART_1000_ALERTS,
++      'Incidents chart - 1000 alerts',
++      '7 days',
++    );
+
+@@ end of it('6.2 ...')
++    cy.log('6.2.4 Alerts chart after selecting incident with 1000 alerts');
++    benchmarkAlertsChart(
++      '18-stress-test-1000-alerts-single.yaml',
++      'cluster-wide-failure-1000-alerts',
++      THRESHOLDS.ALERTS_CHART_1000_ALERTS,
++      'Alerts chart - 1000 alerts',
++      '7 days',
++    );
+```
+
+### 02. Interactive Walkthrough (`02.performance_walkthrough.cy.ts`)
+
+Measures incremental re-render cost during a realistic user session (as opposed to 01 which measures initial render).
+
+**What it tests:**
+- Filter apply/clear cycle times with 20 incidents loaded
+- Time range switching (1d → 3d → 7d → 15d → 1d)
+- Table row expansion with 100 and 500 alerts
+
+### 03. Endurance Cycling — Shelved
+
+Rapidly cycles select/deselect across 20 incidents to detect progressive degradation or memory leaks. Compares first-10 vs last-10 cycle averages against a degradation threshold.
+
+**Status:** Shelved — results are unreliable due to Cypress overhead (see below). The full test source is preserved in [`03.endurance_test_source.md`](./03.endurance_test_source.md). To re-enable, copy it to `web/cypress/e2e/incidents/performance/`.
+
+## Cypress Overhead
+
+All timings include Cypress command processing time. This adds a constant baseline to each operation and, more importantly, a **progressive overhead** that grows with test length.
+
+### DOM snapshot accumulation
+
+Cypress captures a full DOM snapshot for every logged command (to enable time-travel debugging in the runner). The OpenShift Console DOM is large (thousands of nodes), so each snapshot is expensive. Over many cycles, these snapshots accumulate in memory, causing:
+
+- Increasing GC pressure → micro-jank on interactions
+- Cypress command queue growth → longer per-command scheduling
+- Linear slowdown: cycle times grow ~10x over 100 cycles regardless of application performance
+
+### Mitigation attempts and findings
+
+| Approach | Result |
+|----------|--------|
+| `{ log: false }` on commands | ~20% total time reduction, ~34% faster by cycle 100. Gap widens over time, confirming snapshot accumulation contributes. But `.should()` assertions have no `log` option and still snapshot. |
+| `numTestsKeptInMemory: 0` | Only purges between `it()` blocks, not within a single long-running test. |
+| Split into multiple `it()` blocks | `testIsolation: false` preserves page state, but fixture/mock setup between blocks adds complexity and the shared state management is fragile. |
+| Override `Cypress.log` with no-op | Cypress internally chains `.snapshot()`, `.end()`, `.set()` on the log return value. Stubbing all methods is brittle across Cypress versions. |
+
+### Implication for the endurance test
+
+The endurance test (03) cannot reliably distinguish between "the UI is getting slower" and "Cypress is getting slower." Both logged and unlogged runs show ~9-10x degradation over 100 cycles, dominated by Cypress overhead. The test would only catch catastrophic leaks that add degradation far above the Cypress baseline.
+
+## Potential: In-App Performance Measurement
+
+A more accurate approach would instrument the React components directly:
+
+1. Add `performance.mark()` calls inside the incidents component's render/effect lifecycle
+2. Trigger interactions from Cypress but measure only the React work via `cy.window()`
+3. The endurance loop would still run in Cypress, but measured duration would exclude Cypress command queue and snapshot overhead
+
+This is not implemented because it requires modifying production source code (or adding test-only hooks), and the benchmark tests (01, 02) already provide sufficient regression signal for practical purposes — their shorter duration keeps Cypress overhead in the flat range (~2-3x) where real application regressions are visible.

web/cypress.config.ts

@@ -3,6 +3,7 @@ import * as fs from 'fs-extra';
 import * as console from 'console';
 import * as path from 'path';
 import registerCypressGrep from '@cypress/grep/src/plugin';
+import { writeBenchmarkReport, injectBenchmarksIntoMochawesome } from './cypress/plugins/benchmark-reporter';
 
 const getLoginCredentials = (index: number): { username: string; password: string } => {
   const users = (process.env.CYPRESS_LOGIN_USERS || '').split(',').filter(Boolean);
@@ -150,18 +151,24 @@ export default defineConfig({
           return files;
         },
 
+        writeBenchmarkReport,
+
       });
       on('after:spec', (spec: Cypress.Spec, results: CypressCommandLine.RunResult) => {
         if (results && results.video) {
-          // Do we have failures for any retry attempts?
           const failures = results.tests.some((test) =>
             test.attempts.some((attempt) => attempt.state === 'failed'),
           );
           if (!failures && fs.existsSync(results.video)) {
-            // Delete the video if the spec passed and no tests retried
             fs.unlinkSync(results.video);
           }
         }
+
+        try {
+          injectBenchmarksIntoMochawesome(spec.relative);
+        } catch (e) {
+          console.log(`Benchmark injection skipped: ${(e as Error).message}`);
+        }
       });
       return config;
     },