From e65fe806edefd7d3886cb41365c38395d136fa41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Mon, 27 Apr 2026 17:45:22 +0200 Subject: [PATCH 1/5] feat: performance tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- operator-framework-core/pom.xml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/operator-framework-core/pom.xml b/operator-framework-core/pom.xml index fef44f614a..8c6c2d1554 100644 --- a/operator-framework-core/pom.xml +++ b/operator-framework-core/pom.xml @@ -121,6 +121,17 @@ org.apache.maven.plugins maven-surefire-plugin + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + io.github.git-commit-id From 7d108d925147b092d35ab640128228cb18452f98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Mon, 27 Apr 2026 17:54:02 +0200 Subject: [PATCH 2/5] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .github/workflows/performance-tests.yml | 61 ++++++ performance-tests/pom.xml | 177 ++++++++++++++++++ .../event/EventProcessorBenchmark.java | 88 +++++++++ .../PrimaryToSecondaryIndexBenchmark.java | 88 +++++++++ .../event/EventProcessorThroughputTest.java | 105 +++++++++++ .../src/test/resources/log4j2-test.xml | 32 ++++ 6 files changed, 551 insertions(+) create mode 100644 .github/workflows/performance-tests.yml create mode 100644 performance-tests/pom.xml create mode 100644 performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java create mode 100644 performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java create mode 100644 performance-tests/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java create mode 100644 performance-tests/src/test/resources/log4j2-test.xml diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml new file mode 100644 index 0000000000..95bd3dd27b --- /dev/null +++ b/.github/workflows/performance-tests.yml @@ -0,0 +1,61 @@ +name: Performance Tests + +env: + MAVEN_ARGS: -V -ntp -e + +on: + push: + branches: + - main + paths: + - 'operator-framework-core/src/main/**' + - 'performance-tests/**' + workflow_dispatch: + +jobs: + jmh-benchmarks: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - name: Set up Java and Maven + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: 17 + cache: 'maven' + - name: Build project + run: ./mvnw ${MAVEN_ARGS} install -DskipTests --file pom.xml + - name: Run JMH benchmarks + run: java -jar performance-tests/target/benchmarks.jar -rf json -rff performance-tests/target/jmh-result.json + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + with: + name: jmh-result + path: performance-tests/target/jmh-result.json + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + if: github.ref == 'refs/heads/main' + with: + tool: 'jmh' + output-file-path: performance-tests/target/jmh-result.json + auto-push: true + alert-threshold: '130%' + comment-on-alert: true + fail-on-alert: false + gh-pages-branch: gh-pages + benchmark-data-dir-path: dev/bench + + integration-perf-tests: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - name: Set up Java and Maven + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: 17 + cache: 'maven' + - name: Run integration performance tests + run: ./mvnw ${MAVEN_ARGS} verify -P performance-tests -pl performance-tests -am --file pom.xml diff --git a/performance-tests/pom.xml b/performance-tests/pom.xml new file mode 100644 index 0000000000..88b3cbf678 --- /dev/null +++ b/performance-tests/pom.xml @@ -0,0 +1,177 @@ + + + + 4.0.0 + + io.javaoperatorsdk + java-operator-sdk + 5.6.2-SNAPSHOT + ../pom.xml + + + performance-tests + jar + Operator SDK - Performance Tests + JMH benchmarks and performance tests for the Operator SDK + + + true + true + true + + + + + + io.javaoperatorsdk + operator-framework-core + + + io.javaoperatorsdk + operator-framework-core + test-jar + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + org.mockito + mockito-core + + + + + io.fabric8 + kubernetes-client + + + io.fabric8 + kubernetes-httpclient-okhttp + + + + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.assertj + assertj-core + test + + + org.awaitility + awaitility + test + + + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-slf4j2-impl + test + + + org.apache.logging.log4j + log4j-core + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.0 + + + + shade + + package + + benchmarks + + + org.openjdk.jmh.Main + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${skipPerformanceTests} + + + + + diff --git a/performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java b/performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java new file mode 100644 index 0000000000..9e63c55d1d --- /dev/null +++ b/performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java @@ -0,0 +1,88 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.processing.event; + +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import io.javaoperatorsdk.operator.TestUtils; +import io.javaoperatorsdk.operator.api.config.BaseConfigurationService; +import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; +import io.javaoperatorsdk.operator.processing.event.rate.LinearRateLimiter; +import io.javaoperatorsdk.operator.processing.event.source.ResourceAction; +import io.javaoperatorsdk.operator.processing.event.source.controller.ControllerEventSource; +import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceEvent; +import io.javaoperatorsdk.operator.processing.event.source.timer.TimerEventSource; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork(2) +@State(Scope.Thread) +@SuppressWarnings({"rawtypes", "unchecked"}) +public class EventProcessorBenchmark { + + private EventProcessor eventProcessor; + + @Setup(Level.Iteration) + public void setup() { + ReconciliationDispatcher reconciliationDispatcher = mock(ReconciliationDispatcher.class); + when(reconciliationDispatcher.handleExecution(any())) + .thenReturn(PostExecutionControl.defaultDispatch()); + + EventSourceManager eventSourceManager = mock(EventSourceManager.class); + ControllerEventSource controllerEventSource = mock(ControllerEventSource.class); + when(eventSourceManager.getControllerEventSource()).thenReturn(controllerEventSource); + + TimerEventSource retryTimerEventSource = mock(TimerEventSource.class); + + ControllerConfiguration config = mock(ControllerConfiguration.class); + when(config.getName()).thenReturn("benchmark"); + when(config.getRetry()).thenReturn(null); + when(config.getRateLimiter()).thenReturn(LinearRateLimiter.deactivatedRateLimiter()); + when(config.maxReconciliationInterval()).thenReturn(Optional.of(Duration.ofHours(1))); + when(config.getConfigurationService()).thenReturn(new BaseConfigurationService()); + when(config.triggerReconcilerOnAllEvents()).thenReturn(false); + + eventProcessor = + spy(new EventProcessor(config, reconciliationDispatcher, eventSourceManager, null)); + when(eventProcessor.retryEventSource()).thenReturn(retryTimerEventSource); + eventProcessor.start(); + } + + @TearDown(Level.Iteration) + public void tearDown() { + eventProcessor.stop(); + } + + @Benchmark + public void handleSingleEvent(Blackhole bh) { + var cr = TestUtils.testCustomResource(); + ResourceID resourceID = ResourceID.fromResource(cr); + ResourceEvent event = new ResourceEvent(ResourceAction.UPDATED, resourceID, cr); + eventProcessor.handleEvent(event); + } +} diff --git a/performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java b/performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java new file mode 100644 index 0000000000..9e60d59626 --- /dev/null +++ b/performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java @@ -0,0 +1,88 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.processing.event.source.informer; + +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.processing.event.ResourceID; + +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +@Fork(2) +@State(Scope.Benchmark) +public class PrimaryToSecondaryIndexBenchmark { + + @Param({"10", "100", "1000"}) + private int indexSize; + + private DefaultPrimaryToSecondaryIndex index; + private ConfigMap sampleResource; + private ResourceID samplePrimaryId; + + @Setup(Level.Iteration) + public void setup() { + ResourceID primaryId = new ResourceID("primary-0", "default"); + index = new DefaultPrimaryToSecondaryIndex<>(resource -> Set.of(primaryId)); + + for (int i = 0; i < indexSize; i++) { + ConfigMap cm = createConfigMap("secondary-" + i); + index.onAddOrUpdate(cm); + } + + sampleResource = createConfigMap("new-secondary"); + samplePrimaryId = primaryId; + } + + @Benchmark + public void onAddOrUpdate() { + index.onAddOrUpdate(sampleResource); + } + + @Benchmark + public void getSecondaryResources(Blackhole bh) { + bh.consume(index.getSecondaryResources(samplePrimaryId)); + } + + @Benchmark + public void onDelete() { + index.onDelete(sampleResource); + } + + @Benchmark + public void addThenDelete() { + index.onAddOrUpdate(sampleResource); + index.onDelete(sampleResource); + } + + private static ConfigMap createConfigMap(String name) { + ConfigMap cm = new ConfigMap(); + cm.setMetadata( + new ObjectMetaBuilder() + .withName(name) + .withNamespace("default") + .withResourceVersion("1") + .build()); + return cm; + } +} diff --git a/performance-tests/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java b/performance-tests/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java new file mode 100644 index 0000000000..6e5f59940a --- /dev/null +++ b/performance-tests/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java @@ -0,0 +1,105 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.processing.event; + +import java.time.Duration; +import java.util.Optional; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.javaoperatorsdk.operator.TestUtils; +import io.javaoperatorsdk.operator.api.config.BaseConfigurationService; +import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; +import io.javaoperatorsdk.operator.processing.event.rate.LinearRateLimiter; +import io.javaoperatorsdk.operator.processing.event.source.ResourceAction; +import io.javaoperatorsdk.operator.processing.event.source.controller.ControllerEventSource; +import io.javaoperatorsdk.operator.processing.event.source.controller.ResourceEvent; +import io.javaoperatorsdk.operator.processing.event.source.timer.TimerEventSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@SuppressWarnings({"rawtypes", "unchecked"}) +class EventProcessorThroughputTest { + + private static final Logger log = LoggerFactory.getLogger(EventProcessorThroughputTest.class); + private static final int EVENT_COUNT = 1000; + + private EventProcessor eventProcessor; + + @BeforeEach + void setup() { + ReconciliationDispatcher reconciliationDispatcher = mock(ReconciliationDispatcher.class); + when(reconciliationDispatcher.handleExecution(any())) + .thenReturn(PostExecutionControl.defaultDispatch()); + + EventSourceManager eventSourceManager = mock(EventSourceManager.class); + ControllerEventSource controllerEventSource = mock(ControllerEventSource.class); + when(eventSourceManager.getControllerEventSource()).thenReturn(controllerEventSource); + + TimerEventSource retryTimerEventSource = mock(TimerEventSource.class); + + ControllerConfiguration config = mock(ControllerConfiguration.class); + when(config.getName()).thenReturn("throughput-test"); + when(config.getRetry()).thenReturn(null); + when(config.getRateLimiter()).thenReturn(LinearRateLimiter.deactivatedRateLimiter()); + when(config.maxReconciliationInterval()).thenReturn(Optional.of(Duration.ofHours(1))); + when(config.getConfigurationService()).thenReturn(new BaseConfigurationService()); + when(config.triggerReconcilerOnAllEvents()).thenReturn(false); + + eventProcessor = + spy(new EventProcessor(config, reconciliationDispatcher, eventSourceManager, null)); + when(eventProcessor.retryEventSource()).thenReturn(retryTimerEventSource); + eventProcessor.start(); + } + + @AfterEach + void tearDown() { + eventProcessor.stop(); + } + + @Test + void shouldProcessEventsWithAcceptableThroughput() { + long startTime = System.nanoTime(); + + for (int i = 0; i < EVENT_COUNT; i++) { + var cr = TestUtils.testCustomResource(); + ResourceID resourceID = ResourceID.fromResource(cr); + ResourceEvent event = new ResourceEvent(ResourceAction.UPDATED, resourceID, cr); + eventProcessor.handleEvent(event); + } + + long elapsedMs = (System.nanoTime() - startTime) / 1_000_000; + double eventsPerSecond = EVENT_COUNT * 1000.0 / elapsedMs; + + log.info( + "EventProcessor throughput: {} events in {} ms ({} events/sec)", + EVENT_COUNT, + elapsedMs, + String.format("%.1f", eventsPerSecond)); + + assertThat(eventsPerSecond) + .as("Event processing throughput should exceed 100 events/sec") + .isGreaterThan(100.0); + } +} diff --git a/performance-tests/src/test/resources/log4j2-test.xml b/performance-tests/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000000..651b49c24a --- /dev/null +++ b/performance-tests/src/test/resources/log4j2-test.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + From 6f908ca19c7ab23cc3f4bbd25e63f757c8d698d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 25 Aug 2026 14:51:41 +0200 Subject: [PATCH 3/5] feat: performance test modules and result history Split performance-tests into submodules and record the results of every run so they can be compared across commits. - jmh: the existing benchmarks and in-process throughput tests - e2e: reconciliation throughput of an operator against a real cluster - reporting: writes the measurements of both as JSON Results are keyed by commit, ordered by commit timestamp and indexed in index.json. The workflow runs on main and next, merges the results of all jobs and commits them to the performance-test-results branch. Also fixes PrimaryToSecondaryIndexBenchmark, which did not compile against the current onAddOrUpdate signature, and excludes the JMH generated classes from spotbugs so that the module passes verify. --- .github/workflows/performance-tests.yml | 125 +++++++++++- performance-tests/README.md | 100 ++++++++++ performance-tests/e2e/pom.xml | 124 ++++++++++++ .../PerformanceCustomResource.java | 31 +++ .../PerformanceCustomResourceSpec.java | 34 ++++ .../PerformanceCustomResourceStatus.java | 49 +++++ .../performance/PerformanceReconciler.java | 58 ++++++ .../ReconciliationThroughputE2E.java | 142 ++++++++++++++ .../e2e/src/test/resources/log4j2-test.xml | 32 +++ performance-tests/jmh/pom.xml | 183 ++++++++++++++++++ .../performance/jmh/BenchmarkRunner.java | 105 ++++++++++ .../event/EventProcessorBenchmark.java | 0 .../PrimaryToSecondaryIndexBenchmark.java | 6 +- .../event/EventProcessorThroughputTest.java | 13 +- .../src/test/resources/log4j2-test.xml | 0 performance-tests/pom.xml | 153 +-------------- performance-tests/reporting/pom.xml | 121 ++++++++++++ .../operator/performance/results/Json.java | 49 +++++ .../performance/results/Measurement.java | 38 ++++ .../results/PerformanceResultsExtension.java | 102 ++++++++++ .../performance/results/PerformanceTest.java | 53 +++++ .../results/PerformanceTestResults.java | 64 ++++++ .../performance/results/ResultsIndex.java | 45 +++++ .../performance/results/ResultsIndexer.java | 98 ++++++++++ .../performance/results/ResultsWriter.java | 85 ++++++++ .../performance/results/RunMetadata.java | 122 ++++++++++++ .../performance/results/TestResult.java | 56 ++++++ .../results/ResultsWriterTest.java | 91 +++++++++ spotbugs-exclude.xml | 5 + 29 files changed, 1930 insertions(+), 154 deletions(-) create mode 100644 performance-tests/README.md create mode 100644 performance-tests/e2e/pom.xml create mode 100644 performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResource.java create mode 100644 performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResourceSpec.java create mode 100644 performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResourceStatus.java create mode 100644 performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceReconciler.java create mode 100644 performance-tests/e2e/src/test/java/io/javaoperatorsdk/operator/performance/ReconciliationThroughputE2E.java create mode 100644 performance-tests/e2e/src/test/resources/log4j2-test.xml create mode 100644 performance-tests/jmh/pom.xml create mode 100644 performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/performance/jmh/BenchmarkRunner.java rename performance-tests/{ => jmh}/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java (100%) rename performance-tests/{ => jmh}/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java (94%) rename performance-tests/{ => jmh}/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java (87%) rename performance-tests/{ => jmh}/src/test/resources/log4j2-test.xml (100%) create mode 100644 performance-tests/reporting/pom.xml create mode 100644 performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Json.java create mode 100644 performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Measurement.java create mode 100644 performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceResultsExtension.java create mode 100644 performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTest.java create mode 100644 performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTestResults.java create mode 100644 performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndex.java create mode 100644 performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndexer.java create mode 100644 performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsWriter.java create mode 100644 performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/RunMetadata.java create mode 100644 performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/TestResult.java create mode 100644 performance-tests/reporting/src/test/java/io/javaoperatorsdk/operator/performance/results/ResultsWriterTest.java diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml index 95bd3dd27b..60e0774c4a 100644 --- a/.github/workflows/performance-tests.yml +++ b/.github/workflows/performance-tests.yml @@ -2,11 +2,15 @@ name: Performance Tests env: MAVEN_ARGS: -V -ntp -e + # branch the results of every run are stored on, created on the first run + RESULTS_BRANCH: performance-test-results + RESULTS_PATH: performance-tests/results on: push: branches: - main + - next paths: - 'operator-framework-core/src/main/**' - 'performance-tests/**' @@ -27,18 +31,27 @@ jobs: - name: Build project run: ./mvnw ${MAVEN_ARGS} install -DskipTests --file pom.xml - name: Run JMH benchmarks - run: java -jar performance-tests/target/benchmarks.jar -rf json -rff performance-tests/target/jmh-result.json + run: | + java -Dperformance.results.dir=performance-tests/jmh/target/performance-results \ + -Dperformance.results.branch=${{ github.ref_name }} \ + -jar performance-tests/jmh/target/benchmarks.jar \ + -rf json -rff performance-tests/jmh/target/jmh-result.json - name: Upload benchmark results uses: actions/upload-artifact@v4 with: name: jmh-result - path: performance-tests/target/jmh-result.json + path: performance-tests/jmh/target/jmh-result.json + - name: Upload performance results + uses: actions/upload-artifact@v4 + with: + name: performance-results-jmh + path: performance-tests/jmh/target/performance-results - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 if: github.ref == 'refs/heads/main' with: tool: 'jmh' - output-file-path: performance-tests/target/jmh-result.json + output-file-path: performance-tests/jmh/target/jmh-result.json auto-push: true alert-threshold: '130%' comment-on-alert: true @@ -46,9 +59,67 @@ jobs: gh-pages-branch: gh-pages benchmark-data-dir-path: dev/bench - integration-perf-tests: + in-process-tests: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - name: Set up Java and Maven + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: 17 + cache: 'maven' + - name: Run in-process performance tests + run: | + ./mvnw ${MAVEN_ARGS} install -DskipTests --file pom.xml + ./mvnw ${MAVEN_ARGS} verify -P performance-tests \ + -Dperformance.results.branch=${{ github.ref_name }} \ + -pl performance-tests/jmh --file pom.xml + - name: Upload performance results + uses: actions/upload-artifact@v4 + with: + name: performance-results-in-process + path: performance-tests/jmh/target/performance-results + + e2e-performance-tests: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - name: Set up Java and Maven + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: 17 + cache: 'maven' + - name: Set up Minikube + uses: manusa/actions-setup-minikube@v2.18.0 + with: + minikube version: 'v1.38.1' + kubernetes version: 'v1.35.2' + github token: ${{ github.token }} + - name: Run performance tests against the cluster + run: | + ./mvnw ${MAVEN_ARGS} install -DskipTests --file pom.xml + ./mvnw ${MAVEN_ARGS} verify -P performance-tests \ + -Dperformance.results.branch=${{ github.ref_name }} \ + -pl performance-tests/e2e --file pom.xml + - name: Upload performance results + uses: actions/upload-artifact@v4 + with: + name: performance-results-e2e + path: performance-tests/e2e/target/performance-results + + publish-results: + needs: [jmh-benchmarks, in-process-tests, e2e-performance-tests] + # results of a partially failed run would be misleading when comparing commits, and only the + # long living branches are worth keeping a history for + if: github.ref_name == 'main' || github.ref_name == 'next' runs-on: ubuntu-latest timeout-minutes: 15 + permissions: + contents: write steps: - uses: actions/checkout@v6 - name: Set up Java and Maven @@ -57,5 +128,47 @@ jobs: distribution: temurin java-version: 17 cache: 'maven' - - name: Run integration performance tests - run: ./mvnw ${MAVEN_ARGS} verify -P performance-tests -pl performance-tests -am --file pom.xml + - name: Build the results tooling + run: | + ./mvnw ${MAVEN_ARGS} package -Dmaven.test.skip=true \ + -pl performance-tests/reporting -am --file pom.xml + - name: Download the results of all performance test jobs + uses: actions/download-artifact@v4 + with: + pattern: performance-results-* + # all jobs of a run write into the same directory, named after the commit + merge-multiple: true + path: collected-results + - name: Check out the results branch + env: + REPOSITORY_URL: https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git + run: | + set -euo pipefail + if git ls-remote --exit-code --heads "$REPOSITORY_URL" "$RESULTS_BRANCH" > /dev/null; then + git clone --depth 1 --branch "$RESULTS_BRANCH" "$REPOSITORY_URL" results-branch + else + # first run: the branch holds only results, so it is started without any history + echo "$RESULTS_BRANCH does not exist yet, creating it" + git init results-branch + git -C results-branch remote add origin "$REPOSITORY_URL" + git -C results-branch checkout -b "$RESULTS_BRANCH" + fi + - name: Add the results of this commit + run: | + mkdir -p "results-branch/$RESULTS_PATH" + cp -R collected-results/. "results-branch/$RESULTS_PATH/" + java -cp performance-tests/reporting/target/performance-results-tools.jar \ + io.javaoperatorsdk.operator.performance.results.ResultsIndexer \ + "results-branch/$RESULTS_PATH" + - name: Push the results + working-directory: results-branch + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add "$RESULTS_PATH" + if git diff --cached --quiet; then + echo "No new results to store" + exit 0 + fi + git commit -m "perf: results of ${{ github.ref_name }} at ${GITHUB_SHA:0:7}" + git push origin "HEAD:$RESULTS_BRANCH" diff --git a/performance-tests/README.md b/performance-tests/README.md new file mode 100644 index 0000000000..871077892a --- /dev/null +++ b/performance-tests/README.md @@ -0,0 +1,100 @@ +# Performance Tests + +Performance related tests of the SDK. All of them are **skipped by default**; activate them with the +`performance-tests` profile. + +| module | what it measures | needs a cluster | +|-------------|---------------------------------------------------------------|-----------------| +| `jmh` | JMH micro-benchmarks and in-process throughput tests | no | +| `e2e` | end-to-end reconciliation of a real operator on a real cluster | yes | +| `reporting` | records the results of the above so they can be compared | - | + +## `jmh` + +JMH micro-benchmarks (`src/main/java`) and in-process throughput tests (`src/test/java`) for hot +paths of the core. + +Run the throughput tests: + +```shell +./mvnw verify -P performance-tests -pl performance-tests/jmh -am +``` + +Run the benchmarks (the shaded jar is built by a plain `install`): + +```shell +./mvnw install -DskipTests +java -jar performance-tests/jmh/target/benchmarks.jar +``` + +The jar takes the same arguments as JMH itself, it only wraps `org.openjdk.jmh.Main` to also record +the scores as results, see `BenchmarkRunner`. + +## `e2e` + +Tests running a real operator against a real Kubernetes cluster, measuring end-to-end reconciliation +throughput. They use the current kube context, so a cluster (minikube, kind, ...) has to be +available. + +```shell +./mvnw verify -P performance-tests -pl performance-tests/e2e -am +``` + +Scenario size can be tuned per test, see the javadoc of the individual tests, e.g.: + +```shell +./mvnw verify -P performance-tests -pl performance-tests/e2e -am \ + -Dperformance.resourceCount=1000 -Dperformance.timeoutSeconds=600 +``` + +## `reporting` + +Records the measurements of the other two modules as JSON, so that the numbers of different commits +can be compared and visualized. A test opts in by being annotated with `@PerformanceTest`, and +receives a `PerformanceTestResults` parameter to add measurements to. The duration of the test method +itself is always recorded, results of failed tests are not recorded at all. + +```java +@PerformanceTest(type = PerformanceTest.END_TO_END) +class SomethingE2E { + + @Test + void measuresSomething(PerformanceTestResults results) { + results.param("resourceCount", 100).recordElapsed("create", start, end); + } +} +``` + +Results are written below `target/performance-results` of the module, override with +`-Dperformance.results.dir=...`. + +### How results are stored + +The `Performance Tests` workflow runs on every push to `main` and `next`, collects the results of all +jobs and commits them to the **`performance-test-results`** branch, which holds nothing but results: + +``` +performance-tests/results/ +├── index.json all runs, ordered oldest to newest commit +└── 20260825T070249Z-b2bace4/ - + ├── run.json commit, branch and timestamps of the run + ├── jmh/ + │ └── ..json + ├── in-process/ + │ └── ..json + └── e2e/ + └── ..json +``` + +A run directory is named after the commit, prefixed with the **commit** timestamp (not the time the +tests ran) so that runs sort chronologically without consulting the git history, and so that a re-run +of a commit updates its results in place. `index.json` additionally lists the runs in order with +their commit, branch and the categories of tests that produced results, which is what tooling should +read instead of listing directories. It is regenerated from the run directories after every push: + +```shell +java -cp performance-tests/reporting/target/performance-results-tools.jar \ + io.javaoperatorsdk.operator.performance.results.ResultsIndexer performance-tests/results +``` + +The branch is created by the workflow on its first run and needs no manual setup. diff --git a/performance-tests/e2e/pom.xml b/performance-tests/e2e/pom.xml new file mode 100644 index 0000000000..28c931b81d --- /dev/null +++ b/performance-tests/e2e/pom.xml @@ -0,0 +1,124 @@ + + + + 4.0.0 + + io.javaoperatorsdk + performance-tests + 5.6.2-SNAPSHOT + ../pom.xml + + + performance-tests-e2e + jar + Operator SDK - Performance Tests - End to End + Performance tests running a real operator against a real Kubernetes cluster + + + + io.javaoperatorsdk + operator-framework + + + io.fabric8 + kubernetes-httpclient-${fabric8-httpclient-impl.name} + + + org.slf4j + slf4j-api + + + + + io.javaoperatorsdk + performance-tests-reporting + ${project.version} + test + + + io.javaoperatorsdk + operator-framework-junit + ${project.version} + test + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.assertj + assertj-core + test + + + org.awaitility + awaitility + test + + + + + org.apache.logging.log4j + log4j-slf4j2-impl + test + + + org.apache.logging.log4j + log4j-core + test + + + + + + + io.fabric8 + crd-generator-maven-plugin + ${fabric8-client.version} + + + + generate + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${skipPerformanceTests} + + **/*E2E.java + + + + ${project.build.directory}/performance-results + + + + + + diff --git a/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResource.java b/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResource.java new file mode 100644 index 0000000000..9e38957b33 --- /dev/null +++ b/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResource.java @@ -0,0 +1,31 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.Kind; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +@Group("perf.sample.javaoperatorsdk") +@Version("v1") +@Kind("PerformanceResource") +@ShortNames("perf") +public class PerformanceCustomResource + extends CustomResource + implements Namespaced {} diff --git a/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResourceSpec.java b/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResourceSpec.java new file mode 100644 index 0000000000..36b80a3a0a --- /dev/null +++ b/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResourceSpec.java @@ -0,0 +1,34 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance; + +public class PerformanceCustomResourceSpec { + + private String value; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + @Override + public String toString() { + return "PerformanceCustomResourceSpec{value='" + value + "'}"; + } +} diff --git a/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResourceStatus.java b/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResourceStatus.java new file mode 100644 index 0000000000..154b99ea7e --- /dev/null +++ b/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceCustomResourceStatus.java @@ -0,0 +1,49 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance; + +public class PerformanceCustomResourceStatus { + + /** Generation of the spec the reconciler has last seen. */ + private Long observedGeneration; + + private String observedValue; + + public Long getObservedGeneration() { + return observedGeneration; + } + + public void setObservedGeneration(Long observedGeneration) { + this.observedGeneration = observedGeneration; + } + + public String getObservedValue() { + return observedValue; + } + + public void setObservedValue(String observedValue) { + this.observedValue = observedValue; + } + + @Override + public String toString() { + return "PerformanceCustomResourceStatus{observedGeneration=" + + observedGeneration + + ", observedValue='" + + observedValue + + "'}"; + } +} diff --git a/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceReconciler.java b/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceReconciler.java new file mode 100644 index 0000000000..d120d3e049 --- /dev/null +++ b/performance-tests/e2e/src/main/java/io/javaoperatorsdk/operator/performance/PerformanceReconciler.java @@ -0,0 +1,58 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance; + +import java.util.concurrent.atomic.AtomicInteger; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; + +/** + * Minimal reconciler: it only records what it has seen in the status subresource. Keeping the + * reconciliation itself trivial means the measured throughput reflects the cost of the SDK event + * processing and the API server round trips, not the cost of the business logic. + */ +@ControllerConfiguration +public class PerformanceReconciler implements Reconciler { + + private final AtomicInteger numberOfExecutions = new AtomicInteger(0); + + @Override + public UpdateControl reconcile( + PerformanceCustomResource resource, Context context) { + numberOfExecutions.incrementAndGet(); + + var statusPatch = new PerformanceCustomResource(); + statusPatch.setMetadata( + new ObjectMetaBuilder() + .withName(resource.getMetadata().getName()) + .withNamespace(resource.getMetadata().getNamespace()) + .build()); + var status = new PerformanceCustomResourceStatus(); + status.setObservedGeneration(resource.getMetadata().getGeneration()); + status.setObservedValue(resource.getSpec().getValue()); + statusPatch.setStatus(status); + + return UpdateControl.patchStatus(statusPatch); + } + + public int getNumberOfExecutions() { + return numberOfExecutions.get(); + } +} diff --git a/performance-tests/e2e/src/test/java/io/javaoperatorsdk/operator/performance/ReconciliationThroughputE2E.java b/performance-tests/e2e/src/test/java/io/javaoperatorsdk/operator/performance/ReconciliationThroughputE2E.java new file mode 100644 index 0000000000..12ed931b1d --- /dev/null +++ b/performance-tests/e2e/src/test/java/io/javaoperatorsdk/operator/performance/ReconciliationThroughputE2E.java @@ -0,0 +1,142 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance; + +import java.time.Duration; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension; +import io.javaoperatorsdk.operator.performance.results.PerformanceTest; +import io.javaoperatorsdk.operator.performance.results.PerformanceTestResults; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +/** + * Measures how fast a locally run operator reconciles a batch of custom resources against a real + * cluster. The reconciler itself does close to nothing, so what is measured is the SDK event + * processing plus the API server round trips. + * + *

The scenario is tunable with system properties: + * + *

    + *
  • {@code performance.resourceCount} - number of custom resources to create (default 100) + *
  • {@code performance.timeoutSeconds} - how long to wait for all reconciliations (default 120) + *
+ */ +@PerformanceTest(type = PerformanceTest.END_TO_END) +class ReconciliationThroughputE2E { + + private static final Logger log = LoggerFactory.getLogger(ReconciliationThroughputE2E.class); + + private static final String RESOURCE_NAME_PREFIX = "perf-resource-"; + private static final String INITIAL_VALUE = "initial"; + private static final String UPDATED_VALUE = "updated"; + + private static final int RESOURCE_COUNT = Integer.getInteger("performance.resourceCount", 100); + private static final Duration TIMEOUT = + Duration.ofSeconds(Integer.getInteger("performance.timeoutSeconds", 120)); + + @RegisterExtension + LocallyRunOperatorExtension operator = + LocallyRunOperatorExtension.builder().withReconciler(new PerformanceReconciler()).build(); + + @Test + void reconcilesBatchOfResourcesOnCreateAndUpdate(PerformanceTestResults results) { + results.param("resourceCount", RESOURCE_COUNT); + + log.info("Creating {} resources", RESOURCE_COUNT); + var start = System.nanoTime(); + for (int i = 0; i < RESOURCE_COUNT; i++) { + operator.create(resource(i, INITIAL_VALUE)); + } + var createdAt = System.nanoTime(); + awaitAllObserved(INITIAL_VALUE); + report(results, "create", start, createdAt, System.nanoTime()); + + log.info("Updating {} resources", RESOURCE_COUNT); + start = System.nanoTime(); + for (var resource : operator.resources(PerformanceCustomResource.class).list().getItems()) { + resource.getSpec().setValue(UPDATED_VALUE); + operator.update(resource); + } + var updatedAt = System.nanoTime(); + awaitAllObserved(UPDATED_VALUE); + report(results, "update", start, updatedAt, System.nanoTime()); + + var executions = + operator.getReconcilerOfType(PerformanceReconciler.class).getNumberOfExecutions(); + log.info( + "Total reconciliations: {} for {} resources ({} per resource)", + executions, + RESOURCE_COUNT, + String.format("%.2f", executions / (double) RESOURCE_COUNT)); + results.record("reconciliationsPerResource", executions / (double) RESOURCE_COUNT, "count"); + + // one reconciliation for the create, one for the update; more means redundant work + assertThat(executions).isGreaterThanOrEqualTo(2 * RESOURCE_COUNT); + } + + private void awaitAllObserved(String expectedValue) { + await() + .atMost(TIMEOUT) + .pollInterval(Duration.ofMillis(100)) + .untilAsserted(() -> assertThat(numberOfObserved(expectedValue)).isEqualTo(RESOURCE_COUNT)); + } + + private long numberOfObserved(String expectedValue) { + return operator.resources(PerformanceCustomResource.class).list().getItems().stream() + .filter(r -> r.getStatus() != null) + .filter(r -> expectedValue.equals(r.getStatus().getObservedValue())) + .filter(r -> r.getMetadata().getGeneration().equals(r.getStatus().getObservedGeneration())) + .count(); + } + + private void report( + PerformanceTestResults results, String phase, long start, long submitted, long end) { + var submitMs = (submitted - start) / 1_000_000; + var totalMs = (end - start) / 1_000_000; + log.info( + "{}: submitted {} resources in {} ms, all reconciled after {} ms ({} reconciliations/sec)", + phase, + RESOURCE_COUNT, + submitMs, + totalMs, + String.format("%.1f", RESOURCE_COUNT * 1000.0 / Math.max(totalMs, 1))); + + results + .recordElapsed(phase + ".submit", start, submitted) + .recordElapsed(phase, start, end) + .record( + phase + ".throughput", + RESOURCE_COUNT * 1_000_000_000.0 / (end - start), + "reconciliations/s"); + } + + private PerformanceCustomResource resource(int index, String value) { + var resource = new PerformanceCustomResource(); + resource.setMetadata(new ObjectMetaBuilder().withName(RESOURCE_NAME_PREFIX + index).build()); + var spec = new PerformanceCustomResourceSpec(); + spec.setValue(value); + resource.setSpec(spec); + return resource; + } +} diff --git a/performance-tests/e2e/src/test/resources/log4j2-test.xml b/performance-tests/e2e/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000000..06faebd0ea --- /dev/null +++ b/performance-tests/e2e/src/test/resources/log4j2-test.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + diff --git a/performance-tests/jmh/pom.xml b/performance-tests/jmh/pom.xml new file mode 100644 index 0000000000..c7a53802fe --- /dev/null +++ b/performance-tests/jmh/pom.xml @@ -0,0 +1,183 @@ + + + + 4.0.0 + + io.javaoperatorsdk + performance-tests + 5.6.2-SNAPSHOT + ../pom.xml + + + performance-tests-jmh + jar + Operator SDK - Performance Tests - JMH + JMH micro-benchmarks and in-process throughput tests for the Operator SDK + + + + + io.javaoperatorsdk + operator-framework-core + + + io.javaoperatorsdk + operator-framework-core + test-jar + + + + + io.javaoperatorsdk + performance-tests-reporting + ${project.version} + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + org.mockito + mockito-core + + + + + io.fabric8 + kubernetes-client + + + io.fabric8 + kubernetes-httpclient-okhttp + + + + + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.assertj + assertj-core + test + + + org.awaitility + awaitility + test + + + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-slf4j2-impl + test + + + org.apache.logging.log4j + log4j-core + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + + shade + + package + + benchmarks + false + + + + io.javaoperatorsdk.operator.performance.jmh.BenchmarkRunner + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${skipPerformanceTests} + + ${project.build.directory}/performance-results + + + + + + diff --git a/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/performance/jmh/BenchmarkRunner.java b/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/performance/jmh/BenchmarkRunner.java new file mode 100644 index 0000000000..c365bfcda5 --- /dev/null +++ b/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/performance/jmh/BenchmarkRunner.java @@ -0,0 +1,105 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.jmh; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.CommandLineOptions; + +import io.javaoperatorsdk.operator.performance.results.Measurement; +import io.javaoperatorsdk.operator.performance.results.PerformanceTest; +import io.javaoperatorsdk.operator.performance.results.ResultsWriter; +import io.javaoperatorsdk.operator.performance.results.TestResult; + +/** + * Main class of the benchmark jar. Takes the same arguments as {@link org.openjdk.jmh.Main} and in + * addition records the scores as performance test results, so that they can be stored and compared + * per commit. + * + *

The scores are read from the objects the runner returns, not from the JMH result file, so + * whatever {@code -rf} / {@code -rff} produce stays exactly what JMH itself writes. + */ +public class BenchmarkRunner { + + private static final String NO_PARAMS_MEASUREMENT = "score"; + + public static void main(String[] args) throws Exception { + var options = new CommandLineOptions(args); + if (options.shouldHelp() + || options.shouldList() + || options.shouldListWithParams() + || options.shouldListResultFormats() + || options.shouldListProfilers()) { + org.openjdk.jmh.Main.main(args); + return; + } + record(new Runner(options).run()); + } + + private static void record(Collection runResults) { + var writer = new ResultsWriter(); + // a benchmark method has one result per parameter combination, all of them end up as + // measurements in the result file of that method + Map> measurementsByBenchmark = new LinkedHashMap<>(); + for (RunResult runResult : runResults) { + var primary = runResult.getPrimaryResult(); + var params = parameters(runResult); + measurementsByBenchmark + .computeIfAbsent(runResult.getParams().getBenchmark(), benchmark -> new ArrayList<>()) + .add( + new Measurement( + measurementName(params), primary.getScore(), primary.getScoreUnit(), params)); + } + measurementsByBenchmark.forEach( + (benchmark, measurements) -> { + var lastDot = benchmark.lastIndexOf('.'); + writer.write( + TestResult.of( + PerformanceTest.JMH, + benchmark.substring(0, lastDot), + benchmark.substring(lastDot + 1), + writer.run(), + measurements)); + }); + } + + private static Map parameters(RunResult runResult) { + var benchmarkParams = runResult.getParams(); + Map params = new TreeMap<>(); + for (Object key : benchmarkParams.getParamsKeys()) { + var name = String.valueOf(key); + params.put(name, benchmarkParams.getParam(name)); + } + return params; + } + + private static String measurementName(Map params) { + if (params.isEmpty()) { + return NO_PARAMS_MEASUREMENT; + } + return params.entrySet().stream() + .map(param -> param.getKey() + "=" + param.getValue()) + .collect(Collectors.joining(",")); + } +} diff --git a/performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java b/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java similarity index 100% rename from performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java rename to performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/processing/event/EventProcessorBenchmark.java diff --git a/performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java b/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java similarity index 94% rename from performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java rename to performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java index 9e60d59626..72e2029aca 100644 --- a/performance-tests/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java +++ b/performance-tests/jmh/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/PrimaryToSecondaryIndexBenchmark.java @@ -47,7 +47,7 @@ public void setup() { for (int i = 0; i < indexSize; i++) { ConfigMap cm = createConfigMap("secondary-" + i); - index.onAddOrUpdate(cm); + index.onAddOrUpdate(cm, null); } sampleResource = createConfigMap("new-secondary"); @@ -56,7 +56,7 @@ public void setup() { @Benchmark public void onAddOrUpdate() { - index.onAddOrUpdate(sampleResource); + index.onAddOrUpdate(sampleResource, null); } @Benchmark @@ -71,7 +71,7 @@ public void onDelete() { @Benchmark public void addThenDelete() { - index.onAddOrUpdate(sampleResource); + index.onAddOrUpdate(sampleResource, null); index.onDelete(sampleResource); } diff --git a/performance-tests/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java b/performance-tests/jmh/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java similarity index 87% rename from performance-tests/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java rename to performance-tests/jmh/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java index 6e5f59940a..89fd873f8f 100644 --- a/performance-tests/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java +++ b/performance-tests/jmh/src/test/java/io/javaoperatorsdk/operator/processing/event/EventProcessorThroughputTest.java @@ -27,6 +27,8 @@ import io.javaoperatorsdk.operator.TestUtils; import io.javaoperatorsdk.operator.api.config.BaseConfigurationService; import io.javaoperatorsdk.operator.api.config.ControllerConfiguration; +import io.javaoperatorsdk.operator.performance.results.PerformanceTest; +import io.javaoperatorsdk.operator.performance.results.PerformanceTestResults; import io.javaoperatorsdk.operator.processing.event.rate.LinearRateLimiter; import io.javaoperatorsdk.operator.processing.event.source.ResourceAction; import io.javaoperatorsdk.operator.processing.event.source.controller.ControllerEventSource; @@ -39,6 +41,7 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +@PerformanceTest(type = PerformanceTest.IN_PROCESS) @SuppressWarnings({"rawtypes", "unchecked"}) class EventProcessorThroughputTest { @@ -79,7 +82,7 @@ void tearDown() { } @Test - void shouldProcessEventsWithAcceptableThroughput() { + void shouldProcessEventsWithAcceptableThroughput(PerformanceTestResults results) { long startTime = System.nanoTime(); for (int i = 0; i < EVENT_COUNT; i++) { @@ -89,7 +92,8 @@ void shouldProcessEventsWithAcceptableThroughput() { eventProcessor.handleEvent(event); } - long elapsedMs = (System.nanoTime() - startTime) / 1_000_000; + long elapsedNanos = System.nanoTime() - startTime; + long elapsedMs = elapsedNanos / 1_000_000; double eventsPerSecond = EVENT_COUNT * 1000.0 / elapsedMs; log.info( @@ -98,6 +102,11 @@ void shouldProcessEventsWithAcceptableThroughput() { elapsedMs, String.format("%.1f", eventsPerSecond)); + results + .param("eventCount", EVENT_COUNT) + .recordMillis("handleEvent", elapsedNanos / 1_000_000.0) + .record("handleEvent.throughput", eventsPerSecond, "events/s"); + assertThat(eventsPerSecond) .as("Event processing throughput should exceed 100 events/sec") .isGreaterThan(100.0); diff --git a/performance-tests/src/test/resources/log4j2-test.xml b/performance-tests/jmh/src/test/resources/log4j2-test.xml similarity index 100% rename from performance-tests/src/test/resources/log4j2-test.xml rename to performance-tests/jmh/src/test/resources/log4j2-test.xml diff --git a/performance-tests/pom.xml b/performance-tests/pom.xml index 88b3cbf678..80aea5332f 100644 --- a/performance-tests/pom.xml +++ b/performance-tests/pom.xml @@ -26,152 +26,19 @@ performance-tests - jar + pom Operator SDK - Performance Tests - JMH benchmarks and performance tests for the Operator SDK + Aggregator for the performance test modules of the Operator SDK + + + reporting + jmh + e2e + - true + true - true - - - - - io.javaoperatorsdk - operator-framework-core - - - io.javaoperatorsdk - operator-framework-core - test-jar - - - - - org.openjdk.jmh - jmh-core - ${jmh.version} - - - org.openjdk.jmh - jmh-generator-annprocess - ${jmh.version} - provided - - - - - org.mockito - mockito-core - - - - - io.fabric8 - kubernetes-client - - - io.fabric8 - kubernetes-httpclient-okhttp - - - - - - - org.junit.jupiter - junit-jupiter-api - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - org.assertj - assertj-core - test - - - org.awaitility - awaitility - test - - - - - org.slf4j - slf4j-api - - - org.apache.logging.log4j - log4j-slf4j2-impl - test - - - org.apache.logging.log4j - log4j-core - test - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - org.openjdk.jmh - jmh-generator-annprocess - ${jmh.version} - - - - - - org.apache.maven.plugins - maven-shade-plugin - 3.6.0 - - - - shade - - package - - benchmarks - - - org.openjdk.jmh.Main - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - ${skipPerformanceTests} - - - - diff --git a/performance-tests/reporting/pom.xml b/performance-tests/reporting/pom.xml new file mode 100644 index 0000000000..b97ecc0dae --- /dev/null +++ b/performance-tests/reporting/pom.xml @@ -0,0 +1,121 @@ + + + + 4.0.0 + + io.javaoperatorsdk + performance-tests + 5.6.2-SNAPSHOT + ../pom.xml + + + performance-tests-reporting + jar + Operator SDK - Performance Tests - Reporting + Records performance test results as JSON so they can be stored, compared and visualized + + + + com.fasterxml.jackson.core + jackson-databind + + + + org.junit.jupiter + junit-jupiter-api + + + org.slf4j + slf4j-api + + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.assertj + assertj-core + test + + + + + + + + io.github.git-commit-id + git-commit-id-maven-plugin + ${git-commit-id-maven-plugin.version} + + true + ${project.build.outputDirectory}/performance-tests-git.properties + + ^git.commit.id.(abbrev|full)$ + ^git.commit.time$ + ^git.branch$ + + full + yyyy-MM-dd'T'HH:mm:ss'Z' + UTC + + false + + + + get-the-git-infos + + revision + + initialize + + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + + shade + + package + + performance-results-tools + false + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Json.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Json.java new file mode 100644 index 0000000000..d5e948e28c --- /dev/null +++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Json.java @@ -0,0 +1,49 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.results; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +/** Single place configuring how result files are read and written. */ +final class Json { + + private static final ObjectMapper MAPPER = + new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT) + // keeps diffs on the results branch stable + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + + private Json() {} + + static void write(Path file, Object value) throws IOException { + var directory = file.getParent(); + if (directory != null) { + Files.createDirectories(directory); + } + Files.writeString( + file, MAPPER.writeValueAsString(value) + System.lineSeparator(), StandardCharsets.UTF_8); + } + + static T read(Path file, Class type) throws IOException { + return MAPPER.readValue(Files.readString(file, StandardCharsets.UTF_8), type); + } +} diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Measurement.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Measurement.java new file mode 100644 index 0000000000..dd2e8ccc35 --- /dev/null +++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/Measurement.java @@ -0,0 +1,38 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.results; + +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * A single measured value of a performance test. + * + * @param name identifies the measurement within a test, e.g. {@code create} or {@code total} + * @param value the measured value + * @param unit unit of the value, e.g. {@code ms} or {@code ops/s} + * @param params what the measurement was parameterized with, e.g. {@code resourceCount=100} + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record Measurement(String name, double value, String unit, Map params) { + + public static final String MILLISECONDS = "ms"; + + public Measurement(String name, double value, String unit) { + this(name, value, unit, Map.of()); + } +} diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceResultsExtension.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceResultsExtension.java new file mode 100644 index 0000000000..0a8495238a --- /dev/null +++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceResultsExtension.java @@ -0,0 +1,102 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.results; + +import java.util.ArrayList; +import java.util.Objects; + +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolver; + +/** + * Records the measurements of tests annotated with {@link PerformanceTest}. Registered by that + * annotation, so it is not meant to be used with {@code @ExtendWith} directly. + * + *

Results of failed tests are not recorded, a partial measurement would only distort the + * comparison with other runs. + */ +public class PerformanceResultsExtension + implements BeforeEachCallback, AfterEachCallback, ParameterResolver { + + private static final ExtensionContext.Namespace NAMESPACE = + ExtensionContext.Namespace.create(PerformanceResultsExtension.class); + private static final String RESULTS = "results"; + private static final String START_TIME = "startTime"; + private static final String TOTAL_MEASUREMENT = "total"; + + private final ResultsWriter writer; + + public PerformanceResultsExtension() { + this(new ResultsWriter()); + } + + PerformanceResultsExtension(ResultsWriter writer) { + this.writer = writer; + } + + @Override + public void beforeEach(ExtensionContext context) { + var store = context.getStore(NAMESPACE); + store.put(RESULTS, new PerformanceTestResults()); + store.put(START_TIME, System.nanoTime()); + } + + @Override + public void afterEach(ExtensionContext context) { + var endTime = System.nanoTime(); + if (context.getExecutionException().isPresent()) { + return; + } + var store = context.getStore(NAMESPACE); + var results = Objects.requireNonNull(store.get(RESULTS, PerformanceTestResults.class)); + var startTime = Objects.requireNonNull(store.get(START_TIME, Long.class)); + var measurements = new ArrayList<>(results.measurements()); + measurements.add( + new Measurement( + TOTAL_MEASUREMENT, (endTime - startTime) / 1_000_000.0, Measurement.MILLISECONDS)); + + writer.write( + TestResult.of( + typeOf(context), + context.getRequiredTestClass().getName(), + context.getRequiredTestMethod().getName(), + writer.run(), + measurements)); + } + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext context) { + return PerformanceTestResults.class.equals(parameterContext.getParameter().getType()); + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext context) { + return context.getStore(NAMESPACE).get(RESULTS, PerformanceTestResults.class); + } + + private static String typeOf(ExtensionContext context) { + var testClass = context.getRequiredTestClass(); + var annotation = testClass.getAnnotation(PerformanceTest.class); + if (annotation == null) { + throw new IllegalStateException( + testClass.getName() + " is not annotated with @" + PerformanceTest.class.getSimpleName()); + } + return annotation.type(); + } +} diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTest.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTest.java new file mode 100644 index 0000000000..1040f3f455 --- /dev/null +++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTest.java @@ -0,0 +1,53 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.results; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Marks a test class as a performance test, its measurements are recorded by {@link + * PerformanceResultsExtension}. Test methods can take a {@link PerformanceTestResults} parameter to + * add measurements, the duration of the test method itself is always recorded. + */ +@Documented +@Inherited +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +@ExtendWith(PerformanceResultsExtension.class) +public @interface PerformanceTest { + + /** JMH benchmarks. */ + String JMH = "jmh"; + + /** Tests measuring the SDK in process, without a cluster. */ + String IN_PROCESS = "in-process"; + + /** Tests measuring an operator running against a real cluster. */ + String END_TO_END = "e2e"; + + /** + * Category of the performance test, becomes the directory the results are stored in. Use one of + * the constants of this annotation unless a new category is added. + */ + String type(); +} diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTestResults.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTestResults.java new file mode 100644 index 0000000000..d5aca1c5ed --- /dev/null +++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/PerformanceTestResults.java @@ -0,0 +1,64 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.results; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Collects the measurements of a single test method. Injected as a test method parameter by {@link + * PerformanceResultsExtension}. + */ +public class PerformanceTestResults { + + private final List measurements = new ArrayList<>(); + private final Map params = new LinkedHashMap<>(); + + /** + * Adds a parameter describing the scenario, for example the number of resources. It is attached + * to every measurement recorded afterwards, so that results are only compared with results of the + * same scenario. + */ + public PerformanceTestResults param(String name, Object value) { + params.put(name, String.valueOf(value)); + return this; + } + + public PerformanceTestResults record(String name, double value, String unit) { + measurements.add(new Measurement(name, value, unit, Map.copyOf(params))); + return this; + } + + public PerformanceTestResults recordDuration(String name, Duration duration) { + return recordMillis(name, duration.toNanos() / 1_000_000.0); + } + + /** Records the duration between two {@link System#nanoTime()} readings. */ + public PerformanceTestResults recordElapsed(String name, long startNanos, long endNanos) { + return recordMillis(name, (endNanos - startNanos) / 1_000_000.0); + } + + public PerformanceTestResults recordMillis(String name, double millis) { + return record(name, millis, Measurement.MILLISECONDS); + } + + public List measurements() { + return List.copyOf(measurements); + } +} diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndex.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndex.java new file mode 100644 index 0000000000..b94031f990 --- /dev/null +++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndex.java @@ -0,0 +1,45 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.results; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * Content of {@value ResultsIndexer#INDEX_FILE}: all runs stored on the results branch, ordered + * from oldest to newest commit. Lets tooling read the results in order without looking at the git + * history of the branch they are stored on. + * + *

Holds nothing that changes between two regenerations of the same runs, so that re-indexing + * without new results does not produce a commit. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record ResultsIndex(List runs) { + + /** + * @param directory name of the directory the results of the run are in + * @param types the categories of performance tests that produced results in this run + */ + @JsonIgnoreProperties(ignoreUnknown = true) + public record Run( + String directory, + String commit, + String commitAbbrev, + String commitTimestamp, + String branch, + List types) {} +} diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndexer.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndexer.java new file mode 100644 index 0000000000..5f1418f932 --- /dev/null +++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsIndexer.java @@ -0,0 +1,98 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.results; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; + +/** + * Regenerates {@value #INDEX_FILE} for a results directory by scanning the run directories in it. + * Rebuilding instead of appending keeps the index correct no matter in which order runs are added, + * and avoids conflicts when several jobs contribute results for the same commit. + * + *

Invoked by CI after the results of a commit have been merged onto the results branch: + * + * 

+ * java -cp performance-results-tools.jar \
+ *   io.javaoperatorsdk.operator.performance.results.ResultsIndexer performance-tests/results
+ *
+ */ +public final class ResultsIndexer { + + public static final String INDEX_FILE = "index.json"; + + private ResultsIndexer() {} + + public static void main(String[] args) throws IOException { + if (args.length != 1) { + System.err.println("Usage: " + ResultsIndexer.class.getName() + " "); + System.exit(1); + return; + } + var resultsDirectory = Path.of(args[0]); + var index = index(resultsDirectory); + var indexFile = resultsDirectory.resolve(INDEX_FILE); + Json.write(indexFile, index); + System.out.println("Indexed " + index.runs().size() + " run(s) into " + indexFile); + } + + /** Reads all runs of a results directory, ordered from the oldest to the newest commit. */ + public static ResultsIndex index(Path resultsDirectory) throws IOException { + var runs = new ArrayList(); + try (var directories = Files.list(resultsDirectory)) { + for (var directory : directories.filter(Files::isDirectory).toList()) { + var run = read(directory); + if (run != null) { + runs.add(run); + } + } + } + runs.sort( + Comparator.comparing(ResultsIndex.Run::commitTimestamp) + .thenComparing(ResultsIndex.Run::directory)); + return new ResultsIndex(runs); + } + + private static ResultsIndex.Run read(Path directory) throws IOException { + var runFile = directory.resolve(ResultsWriter.RUN_FILE); + if (!Files.isRegularFile(runFile)) { + return null; + } + var metadata = Json.read(runFile, RunMetadata.class); + return new ResultsIndex.Run( + String.valueOf(directory.getFileName()), + metadata.commit(), + metadata.commitAbbrev(), + metadata.commitTimestamp(), + metadata.branch(), + types(directory)); + } + + private static List types(Path directory) throws IOException { + try (Stream content = Files.list(directory)) { + return content + .filter(Files::isDirectory) + .map(path -> String.valueOf(path.getFileName())) + .sorted() + .toList(); + } + } +} diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsWriter.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsWriter.java new file mode 100644 index 0000000000..8368e39a1e --- /dev/null +++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/ResultsWriter.java @@ -0,0 +1,85 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.results; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Writes performance test results into the directory of the current run: + * + * 
+ *   <results dir>/<commit timestamp>-<short commit>/run.json
+ *   <results dir>/<commit timestamp>-<short commit>/<type>/<test class>.<test method>.json
+ *
+ * + *

The results directory defaults to {@code target/performance-results} and can be set with the + * {@value RunMetadata#RESULTS_DIR_PROPERTY} system property. CI collects those directories from all + * performance test jobs and merges them onto the results branch, which is why the layout below the + * results directory is the same as the one on that branch. + */ +public class ResultsWriter { + + public static final String RUN_FILE = "run.json"; + private static final String DEFAULT_RESULTS_DIR = "target/performance-results"; + + private static final Logger log = LoggerFactory.getLogger(ResultsWriter.class); + + private final RunMetadata run; + private final Path runDirectory; + + public ResultsWriter() { + this(defaultResultsDirectory(), RunMetadata.current()); + } + + public ResultsWriter(Path resultsDirectory, RunMetadata run) { + this.run = run; + this.runDirectory = resultsDirectory.resolve(run.directoryName()); + } + + public static Path defaultResultsDirectory() { + return Path.of(System.getProperty(RunMetadata.RESULTS_DIR_PROPERTY, DEFAULT_RESULTS_DIR)); + } + + public RunMetadata run() { + return run; + } + + public Path runDirectory() { + return runDirectory; + } + + /** Writes a result file, creating {@value #RUN_FILE} for the run if it is not there yet. */ + public Path write(TestResult result) { + try { + var runFile = runDirectory.resolve(RUN_FILE); + if (!Files.exists(runFile)) { + Json.write(runFile, run); + } + var resultFile = runDirectory.resolve(result.fileName()); + Json.write(resultFile, result); + log.info("Recorded {} result of {} to {}", result.type(), result.testMethod(), resultFile); + return resultFile; + } catch (IOException e) { + throw new UncheckedIOException("Cannot write results to " + runDirectory, e); + } + } +} diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/RunMetadata.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/RunMetadata.java new file mode 100644 index 0000000000..adb67a9ddc --- /dev/null +++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/RunMetadata.java @@ -0,0 +1,122 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.results; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Properties; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * Identifies the commit a set of results belongs to. Written as {@code run.json} next to the + * results of a run. + * + *

The values are taken from {@code performance-tests-git.properties}, generated at build time by + * the git-commit-id plugin, so they are also available when the results are recorded outside of + * Maven, for example from the shaded JMH benchmark jar. Every value can be overridden with a system + * property, which is what CI does for the branch, since a GitHub Actions checkout has a detached + * HEAD. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record RunMetadata( + String commit, + String commitAbbrev, + String commitTimestamp, + String branch, + String runTimestamp) { + + public static final String RESULTS_DIR_PROPERTY = "performance.results.dir"; + public static final String COMMIT_PROPERTY = "performance.results.commit"; + public static final String COMMIT_TIMESTAMP_PROPERTY = "performance.results.commitTimestamp"; + public static final String BRANCH_PROPERTY = "performance.results.branch"; + + private static final Logger log = LoggerFactory.getLogger(RunMetadata.class); + + private static final String PROPERTIES_RESOURCE = "/performance-tests-git.properties"; + private static final String UNKNOWN = "unknown"; + private static final int ABBREV_LENGTH = 7; + + /** + * Name of the directory the results of this run are stored in. Prefixed with the commit timestamp + * so that runs sort chronologically without consulting the git history. + */ + @JsonIgnore + public String directoryName() { + return compactTimestamp(commitTimestamp) + "-" + commitAbbrev; + } + + public static RunMetadata current() { + var gitProperties = loadGitProperties(); + + var commit = value(COMMIT_PROPERTY, gitProperties.getProperty("git.commit.id.full"), UNKNOWN); + var runTimestamp = Instant.now().truncatedTo(ChronoUnit.SECONDS).toString(); + + return new RunMetadata( + commit, + abbreviate(gitProperties.getProperty("git.commit.id.abbrev"), commit), + value( + COMMIT_TIMESTAMP_PROPERTY, gitProperties.getProperty("git.commit.time"), runTimestamp), + value(BRANCH_PROPERTY, gitProperties.getProperty("git.branch"), UNKNOWN), + runTimestamp); + } + + private static Properties loadGitProperties() { + var properties = new Properties(); + try (InputStream is = RunMetadata.class.getResourceAsStream(PROPERTIES_RESOURCE)) { + if (is == null) { + log.warn( + "{} not found on the classpath, results will not be associated with a commit", + PROPERTIES_RESOURCE); + } else { + properties.load(is); + } + } catch (IOException e) { + log.warn("Cannot read {}", PROPERTIES_RESOURCE, e); + } + return properties; + } + + private static String value(String systemProperty, String fromGit, String fallback) { + var override = System.getProperty(systemProperty); + if (isSet(override)) { + return override; + } + return isSet(fromGit) ? fromGit : fallback; + } + + private static String abbreviate(String fromGit, String commit) { + if (isSet(fromGit) && !isSet(System.getProperty(COMMIT_PROPERTY))) { + return fromGit; + } + return commit.length() > ABBREV_LENGTH ? commit.substring(0, ABBREV_LENGTH) : commit; + } + + private static boolean isSet(String value) { + // the git plugin leaves unresolved placeholders in place when not run in a git checkout + return value != null && !value.isBlank() && !value.startsWith("$"); + } + + private static String compactTimestamp(String timestamp) { + return timestamp.replace("-", "").replace(":", ""); + } +} diff --git a/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/TestResult.java b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/TestResult.java new file mode 100644 index 0000000000..f8f1d621e4 --- /dev/null +++ b/performance-tests/reporting/src/main/java/io/javaoperatorsdk/operator/performance/results/TestResult.java @@ -0,0 +1,56 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.results; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * All measurements of a single performance test, the content of one result file. + * + * @param type category of the performance test, see {@link PerformanceTest#type()} + * @param testClass fully qualified name of the test, for JMH the benchmark class + * @param testMethod test or benchmark method the measurements belong to + * @param commitAbbrev repeated from {@code run.json} so a single file is self describing + * @param commitTimestamp repeated from {@code run.json} so a single file is self describing + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record TestResult( + String type, + String testClass, + String testMethod, + String commitAbbrev, + String commitTimestamp, + List measurements) { + + public static TestResult of( + String type, + String testClass, + String testMethod, + RunMetadata run, + List measurements) { + return new TestResult( + type, testClass, testMethod, run.commitAbbrev(), run.commitTimestamp(), measurements); + } + + /** Name of the result file, relative to the directory of the run. */ + @JsonIgnore + public String fileName() { + return type + "/" + testClass + "." + testMethod + ".json"; + } +} diff --git a/performance-tests/reporting/src/test/java/io/javaoperatorsdk/operator/performance/results/ResultsWriterTest.java b/performance-tests/reporting/src/test/java/io/javaoperatorsdk/operator/performance/results/ResultsWriterTest.java new file mode 100644 index 0000000000..08f63cdf24 --- /dev/null +++ b/performance-tests/reporting/src/test/java/io/javaoperatorsdk/operator/performance/results/ResultsWriterTest.java @@ -0,0 +1,91 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.performance.results; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.assertj.core.api.Assertions.assertThat; + +class ResultsWriterTest { + + private static final RunMetadata RUN = + new RunMetadata( + "abc123def456", "abc123d", "2026-08-25T12:00:00Z", "main", "2026-08-25T13:00:00Z"); + + @Test + void storesResultsUnderADirectoryNamedAfterTheCommit(@TempDir Path resultsDirectory) { + var writer = new ResultsWriter(resultsDirectory, RUN); + + writer.write( + TestResult.of( + PerformanceTest.END_TO_END, + "com.acme.ThroughputE2E", + "measuresThroughput", + RUN, + List.of(new Measurement("create", 12.5, "ms", Map.of("resourceCount", "100"))))); + + var runDirectory = resultsDirectory.resolve("20260825T120000Z-abc123d"); + assertThat(runDirectory.resolve(ResultsWriter.RUN_FILE)).exists(); + assertThat(runDirectory.resolve("e2e/com.acme.ThroughputE2E.measuresThroughput.json")).exists(); + } + + @Test + void indexListsRunsOldestFirstWithTheirTypes(@TempDir Path resultsDirectory) throws IOException { + var older = + new RunMetadata("1", "aaaaaaa", "2026-08-24T10:00:00Z", "main", "2026-08-26T10:00:00Z"); + var newer = + new RunMetadata("2", "bbbbbbb", "2026-08-25T10:00:00Z", "next", "2026-08-25T10:00:00Z"); + // written newest first on purpose, the index has to order by commit, not by insertion + write(resultsDirectory, newer, PerformanceTest.JMH); + write(resultsDirectory, older, PerformanceTest.END_TO_END); + write(resultsDirectory, older, PerformanceTest.IN_PROCESS); + + var index = ResultsIndexer.index(resultsDirectory); + + assertThat(index.runs()) + .map(ResultsIndex.Run::commitAbbrev) + .containsExactly("aaaaaaa", "bbbbbbb"); + assertThat(index.runs().get(0).types()).containsExactly("e2e", "in-process"); + assertThat(index.runs().get(0).branch()).isEqualTo("main"); + assertThat(index.runs().get(1).types()).containsExactly("jmh"); + } + + @Test + void indexIgnoresDirectoriesWithoutRunMetadata(@TempDir Path resultsDirectory) + throws IOException { + write(resultsDirectory, RUN, PerformanceTest.JMH); + java.nio.file.Files.createDirectory(resultsDirectory.resolve("not-a-run")); + + assertThat(ResultsIndexer.index(resultsDirectory).runs()).hasSize(1); + } + + private static void write(Path resultsDirectory, RunMetadata run, String type) { + new ResultsWriter(resultsDirectory, run) + .write( + TestResult.of( + type, + "com.acme.SomeTest", + "someMeasurement", + run, + List.of(new Measurement("total", 1.0, "ms")))); + } +} diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml index f6107ac4ea..fd8f6e22c4 100644 --- a/spotbugs-exclude.xml +++ b/spotbugs-exclude.xml @@ -17,6 +17,11 @@ --> + + + + + true + true + 1.37 + 2.21.4 + 3.6.2 + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + io.javaoperatorsdk + operator-framework-core + ${project.version} + test-jar + + +