From db9bea71f643046f4312d966b64da23c4ac18525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jordi=20G=C3=B3mez?= Date: Wed, 13 May 2026 14:57:27 +0200 Subject: [PATCH 1/3] fix(perf): make Gatling simulations runnable again The simulations were broken by a Netty version conflict, spring boot was pining a lower version of netty so gatling was not working. Ideally we may want to consider extracting these tests to their own package. Within this patch I'm also consolidating the scripts for running perf tests as gradle tasks. Assisted-By: github:claude-opus-4.6, anthropic:claude-opus-4-7[1m] --- server/.gitignore | 2 + server/build.gradle | 95 ++++++++++++++++--- server/gradle/libs.versions.toml | 2 +- server/src/gatling/README.md | 37 +++++--- .../src/gatling/resources/access-tokens.csv | 1 + .../gatling/resources/application.properties | 2 +- .../scala/org/eclipse/openvsx/Scenarios.scala | 38 ++++---- server/src/gatling/scripts/fill-database.sh | 6 -- .../src/gatling/scripts/test-registry-api.sh | 18 ---- .../gatling/scripts/test-vscode-adapter.sh | 9 -- 10 files changed, 132 insertions(+), 78 deletions(-) delete mode 100644 server/src/gatling/scripts/fill-database.sh delete mode 100644 server/src/gatling/scripts/test-registry-api.sh delete mode 100644 server/src/gatling/scripts/test-vscode-adapter.sh diff --git a/server/.gitignore b/server/.gitignore index 22fe684e2..508484672 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -6,6 +6,8 @@ .settings/ .classpath .project +.metals +.bloop DEPENDENCIES /src/dev/resources/application-ovsx.properties diff --git a/server/build.gradle b/server/build.gradle index 12f770a14..37db316ac 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -48,27 +48,20 @@ configurations { devImplementation.extendsFrom implementation devRuntimeOnly.extendsFrom runtimeOnly - gatling.exclude group: "io.gatling.highcharts", module: "gatling-charts-highcharts" - // exclude commons-logging to avoid runtime warnings like these // Standard Commons Logging discovery in action with spring-jcl: // please remove commons-logging.jar from classpath in order to avoid potential conflicts configureEach { exclude group: "commons-logging", module: "commons-logging" } -} -dependencyManagement { - gatling { - dependencies { - dependencySet(group: 'io.netty', version: '4.2.13.Final') { - entry 'netty-codec-http' - entry 'netty-codec' - entry 'netty-handler' - entry 'netty-buffer' - entry 'netty-transport' - entry 'netty-common' - entry 'netty-transport-native-epoll' + // Gatling 3.15 requires Netty 4.2.x but Spring Boot's dependency management forces 4.1.x. + // Override all io.netty modules (except tcnative which has its own versioning) on the + // gatling configuration to keep its classpath internally consistent. + matching { it.name.startsWith('gatling') }.configureEach { + resolutionStrategy.eachDependency { details -> + if (details.requested.group == 'io.netty' && !details.requested.name.startsWith('netty-tcnative')) { + details.useVersion '4.2.10.Final' } } } @@ -273,5 +266,79 @@ tasks.withType(ScalaCompile).configureEach { } } +import io.gatling.gradle.GatlingRunTask + +// Per-simulation GatlingRunTask instances. Aggregator tasks below pull them +// in via dependsOn. The plugin's includes/excludes filter isn't exposed +// per-task, so we register one task per simulation explicitly. +def gatlingSimulationGroups = [ + fillDatabase: [ + 'org.eclipse.openvsx.RegistryAPICreateNamespaceSimulation', + 'org.eclipse.openvsx.RegistryAPIPublishExtensionSimulation', + ], + registryApi: [ + 'org.eclipse.openvsx.RegistryAPIGetNamespaceSimulation', + 'org.eclipse.openvsx.RegistryAPIGetNamespaceDetailsSimulation', + 'org.eclipse.openvsx.RegistryAPIGetExtensionSimulation', + 'org.eclipse.openvsx.RegistryAPIGetExtensionTargetPlatformSimulation', + 'org.eclipse.openvsx.RegistryAPIGetExtensionVersionSimulation', + 'org.eclipse.openvsx.RegistryAPIGetExtensionVersionTargetPlatformSimulation', + 'org.eclipse.openvsx.RegistryAPIGetVersionReferencesSimulation', + 'org.eclipse.openvsx.RegistryAPIGetVersionReferencesTargetPlatformSimulation', + 'org.eclipse.openvsx.RegistryAPIGetFileSimulation', + 'org.eclipse.openvsx.RegistryAPIGetFileTargetPlatformSimulation', + 'org.eclipse.openvsx.RegistryAPIGetQuerySimulation', + 'org.eclipse.openvsx.RegistryAPIGetQueryV2Simulation', + 'org.eclipse.openvsx.RegistryAPISearchSimulation', + 'org.eclipse.openvsx.RegistryAPIVerifyTokenSimulation', + ], + vscodeAdapter: [ + 'org.eclipse.openvsx.adapter.VSCodeAdapterExtensionQuerySimulation', + 'org.eclipse.openvsx.adapter.VSCodeAdapterGetAssetSimulation', + 'org.eclipse.openvsx.adapter.VSCodeAdapterGetWebResourceSimulation', + 'org.eclipse.openvsx.adapter.VSCodeAdapterItemSimulation', + 'org.eclipse.openvsx.adapter.VSCodeAdapterUnpkgSimulation', + 'org.eclipse.openvsx.adapter.VSCodeAdapterVspackageSimulation', + ], +] + +def simTaskName = { String fqcn -> "gatling${fqcn.tokenize('.').last()}" as String } + +gatlingSimulationGroups.values().flatten().each { String fqcn -> + tasks.register(simTaskName(fqcn), GatlingRunTask) { + simulationClassName = fqcn + nonInteractive = true + } +} + +// Within fillDatabase, namespaces must be created before extensions can be published. +tasks.named(simTaskName('org.eclipse.openvsx.RegistryAPIPublishExtensionSimulation')).configure { + mustRunAfter simTaskName('org.eclipse.openvsx.RegistryAPICreateNamespaceSimulation') +} + +tasks.register('perfFillDatabase') { + description = 'Populates the DB: creates namespaces then publishes extensions' + group = 'Performance testing' + dependsOn gatlingSimulationGroups.fillDatabase.collect(simTaskName) +} + +tasks.register('perfTestRegistryApi') { + description = 'Runs read-only Gatling simulations against the registry API' + group = 'Performance testing' + dependsOn gatlingSimulationGroups.registryApi.collect(simTaskName) +} + +tasks.register('perfTestVscodeAdapter') { + description = 'Runs read-only Gatling simulations against the VS Code adapter' + group = 'Performance testing' + dependsOn gatlingSimulationGroups.vscodeAdapter.collect(simTaskName) +} + +tasks.register('perfTestAll') { + description = 'Runs all read-only Gatling simulations (registry API + VS Code adapter)' + group = 'Performance testing' + dependsOn 'perfTestRegistryApi', 'perfTestVscodeAdapter' +} + apply from: 'dependencies.gradle' apply from: 'test-extensions.gradle' diff --git a/server/gradle/libs.versions.toml b/server/gradle/libs.versions.toml index ebcc94418..76213c50e 100644 --- a/server/gradle/libs.versions.toml +++ b/server/gradle/libs.versions.toml @@ -6,7 +6,7 @@ bucket4j-spring = "0.12.10" bucket4j = "8.10.1" commons-lang3 = "3.20.0" flyway = "11.20.3" -gatling = "3.14.9" +gatling = "3.15.0" gcloud = "2.62.1" hibernate = "6.6.42.Final" ipaddress = "5.5.1" diff --git a/server/src/gatling/README.md b/server/src/gatling/README.md index a3780b185..726c3c7e0 100644 --- a/server/src/gatling/README.md +++ b/server/src/gatling/README.md @@ -14,23 +14,36 @@ Simulations use the `auth` value to set the Authorization request header. ### resources/access-tokens.csv: -- contains API access tokens. Create a couple of tokens using the web UI and add them to this file. +- contains API access tokens. `super_token` is pre-seeded for the dev profile; create more via the web UI for testing against a remote server. - make sure to keep the `access_token` header at the top of the file. ### scala/org/eclipse/openvsx/RegistryAPIPublishExtensionSimulation.scala: -- Change `extensionDir` property in `resources/application.properties` to a directory that **only** contains extensions (*.vsix files). The simulation uses those files to upload them to the server. +- Defaults to `build/test-extensions/`. Either populate it with `.vsix` files (e.g. `./gradlew downloadTestExtensions`) or change `extensionDir` in `resources/application.properties` to a directory that **only** contains extensions. # Running Gatling -**The Gatling 'post' simulations need to be run to fill the database:** -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPICreateNamespaceSimulation` -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIPublishExtensionSimulation` - -**After running those simulations all other simulations can be run in no particular order:** -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetNamespaceSimulation` -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetExtensionSimulation` -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetExtensionVersionSimulation` -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetQuerySimulation` -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.adapter.VSCodeAdapterExtensionQuerySimulation` + +Gradle tasks (group: `Performance testing`): + +| Task | What it does | +|-------------------------|-----------------------------------------------------------------------------------------------| +| `perfFillDatabase` | Seeds the DB: creates namespaces, then publishes extensions from `extensionDir` | +| `perfTestRegistryApi` | Runs all read-only registry API simulations | +| `perfTestVscodeAdapter` | Runs all read-only VS Code adapter simulations | +| `perfTestAll` | Runs `perfTestRegistryApi` + `perfTestVscodeAdapter` | +| `gatlingRun` | Built-in plugin task; pass `--simulation=` to run a single simulation | + +Typical run against a freshly started server: + +```sh +./gradlew perfFillDatabase +./gradlew perfTestAll +``` + +To run a single simulation: + +```sh +./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPISearchSimulation +``` ## Empty the database If you wish to empty the database after running the Gatling simulations, you can run: diff --git a/server/src/gatling/resources/access-tokens.csv b/server/src/gatling/resources/access-tokens.csv index f40bd5ed0..c6d18db68 100644 --- a/server/src/gatling/resources/access-tokens.csv +++ b/server/src/gatling/resources/access-tokens.csv @@ -1 +1,2 @@ access_token +super_token diff --git a/server/src/gatling/resources/application.properties b/server/src/gatling/resources/application.properties index 2fd32a25c..d766c648f 100644 --- a/server/src/gatling/resources/application.properties +++ b/server/src/gatling/resources/application.properties @@ -1,3 +1,3 @@ baseUrl=http://localhost:8080 -extensionDir= +extensionDir=build/test-extensions #auth= \ No newline at end of file diff --git a/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala b/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala index 71b6a6f22..eae60b983 100644 --- a/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala +++ b/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala @@ -15,7 +15,8 @@ import io.gatling.core.session.Expression import io.gatling.core.structure.ScenarioBuilder import io.gatling.http.Predef._ -import java.nio.file.Files +import java.nio.file.{Files, Paths} +import java.util.UUID import scala.collection.mutable.ListBuffer import scala.concurrent.duration.DurationInt import scala.reflect.io.File @@ -42,11 +43,13 @@ object Scenarios { } def createNamespaceScenario(): ScenarioBuilder = { - val namespacesCount = 780 - val repeatTimes = namespacesCount / users + // Generate unique names per request so the sim works against any DB state. + val totalRequests = 780 + val repeatTimes = totalRequests / users scenario("RegistryAPI: Create Namespace") .repeat(repeatTimes) { - feed(csv(NamespaceFeed)) + exec(session => + session.set("namespace", "perfns-" + UUID.randomUUID().toString.replace("-", "").take(20))) .feed(csv(AccessTokenFeed).circular) .exec(http("RegistryAPI.createNamespace") .post(s"/api/-/namespace/create") @@ -55,20 +58,17 @@ object Scenarios { .body(StringBody("""{"name":"#{namespace}"}""")).asJson .requestTimeout(3.minutes) .check(status.is(201))) - // useful for debugging responses - // .check(bodyString.saveAs("BODY"))) - // .exec(session => { - // val response = session("BODY").as[String] - // println(s"Response body: \n$response") - // session - // }) } } private def extensionFilesFeeder(extensionDir: String): Array[Map[String,String]] = { - val extensionFiles = new java.io.File(extensionDir).list() - val feeder = new Array[Map[String, String]](extensionFiles.length) + // File.list() returns null when the directory does not exist. + val extensionFiles = Option(new java.io.File(extensionDir).list()) + .getOrElse(Array.empty[String]) + .filter(_.endsWith(".vsix")) + if (extensionFiles.isEmpty) return Array.empty[Map[String, String]] + val feeder = new Array[Map[String, String]](extensionFiles.length) var mapIndex = 0 var feederIndex = 0 // make sure that versions of same extension are not right after one another @@ -91,8 +91,11 @@ object Scenarios { def publishScenario(users: Int): ScenarioBuilder = { val extensionDir = conf.getString("extensionDir") val feeder = this.extensionFilesFeeder(extensionDir) + if (feeder.isEmpty) { + return scenario("RegistryAPI: Publish Extension (no extensions found at " + extensionDir + ")") + } - val repeatTimes = feeder.length / users + val repeatTimes = Math.max(1, feeder.length / users) scenario("RegistryAPI: Publish Extension") .repeat(repeatTimes) { feed(feeder) @@ -102,11 +105,12 @@ object Scenarios { .headers(headers()) .queryParam("token", """#{access_token}""") .body(ByteArrayBody(session => { - val path = extensionDir + "\\" + session("extension_file").as[String] + val path = Paths.get(extensionDir, session("extension_file").as[String]).toString File(path).toByteArray() })) .requestTimeout(3.minutes) - .check(status.is(201))) + // 201 = new publish, 400 = already published (idempotent on re-runs). + .check(status.in(201, 400))) } } @@ -288,7 +292,7 @@ object Scenarios { .repeat(1000) { feed(csv(ExtensionVersionFeed).circular) .feed(Array( - Map("file" -> """#{namespace}.#{name}-#{version}.vsix"""), + Map("file" -> ""), Map("file" -> "package.json"), Map("file" -> "extension.vsixmanifest"), Map("file" -> "CHANGELOG.md"), diff --git a/server/src/gatling/scripts/fill-database.sh b/server/src/gatling/scripts/fill-database.sh deleted file mode 100644 index 3e3d90684..000000000 --- a/server/src/gatling/scripts/fill-database.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -cd ../../.. -./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPICreateNamespaceSimulation -./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIPublishExtensionSimulation -cd src/gatling/scripts || exit diff --git a/server/src/gatling/scripts/test-registry-api.sh b/server/src/gatling/scripts/test-registry-api.sh deleted file mode 100644 index 12cb2d856..000000000 --- a/server/src/gatling/scripts/test-registry-api.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -cd ../../.. -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetNamespaceSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetNamespaceDetailsSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetExtensionSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetExtensionTargetPlatformSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetExtensionVersionSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetExtensionVersionTargetPlatformSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetVersionReferencesSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetVersionReferencesTargetPlatformSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetFileSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetFileTargetPlatformSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetQuerySimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetQueryV2Simulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPISearchSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIVerifyTokenSimulation -cd src/gatling/scripts || exit diff --git a/server/src/gatling/scripts/test-vscode-adapter.sh b/server/src/gatling/scripts/test-vscode-adapter.sh deleted file mode 100644 index 366692c57..000000000 --- a/server/src/gatling/scripts/test-vscode-adapter.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -cd ../../.. -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.adapter.VSCodeAdapterExtensionQuerySimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.adapter.VSCodeAdapterGetAssetSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.adapter.VSCodeAdapterItemSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.adapter.VSCodeAdapterUnpkgSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.adapter.VSCodeAdapterVspackageSimulation -cd src/gatling/scripts || exit From 07bcf1138fb332ce3c68e426d156c5c7f0408ba9 Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Fri, 15 May 2026 13:05:27 +0200 Subject: [PATCH 2/3] revert unqiue namespace names as this breaks the publish extension scenario --- .../scala/org/eclipse/openvsx/Scenarios.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala b/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala index eae60b983..c47050133 100644 --- a/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala +++ b/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala @@ -43,13 +43,11 @@ object Scenarios { } def createNamespaceScenario(): ScenarioBuilder = { - // Generate unique names per request so the sim works against any DB state. val totalRequests = 780 val repeatTimes = totalRequests / users scenario("RegistryAPI: Create Namespace") .repeat(repeatTimes) { - exec(session => - session.set("namespace", "perfns-" + UUID.randomUUID().toString.replace("-", "").take(20))) + feed(csv(NamespaceFeed)) .feed(csv(AccessTokenFeed).circular) .exec(http("RegistryAPI.createNamespace") .post(s"/api/-/namespace/create") @@ -58,6 +56,13 @@ object Scenarios { .body(StringBody("""{"name":"#{namespace}"}""")).asJson .requestTimeout(3.minutes) .check(status.is(201))) + // useful for debugging responses + // .check(bodyString.saveAs("BODY"))) + // .exec(session => { + // val response = session("BODY").as[String] + // println(s"Response body: \n$response") + // session + // }) } } @@ -292,7 +297,7 @@ object Scenarios { .repeat(1000) { feed(csv(ExtensionVersionFeed).circular) .feed(Array( - Map("file" -> ""), + Map("file" -> """#{namespace}.#{name}-#{version}.vsix"""), Map("file" -> "package.json"), Map("file" -> "extension.vsixmanifest"), Map("file" -> "CHANGELOG.md"), From 2205539cb0344fdd0cb70f7f5b34d18c95cabdc9 Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Fri, 15 May 2026 13:42:30 +0200 Subject: [PATCH 3/3] bump to latest 4.2 netty version, remove --rerun-tasks from gradle invocation as gatlingRun task has caching disabled by default which, running without --rerun-tasks does not trigger an app reload --- server/build.gradle | 2 +- server/src/gatling/README.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/build.gradle b/server/build.gradle index 37db316ac..14e9c26f0 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -61,7 +61,7 @@ configurations { matching { it.name.startsWith('gatling') }.configureEach { resolutionStrategy.eachDependency { details -> if (details.requested.group == 'io.netty' && !details.requested.name.startsWith('netty-tcnative')) { - details.useVersion '4.2.10.Final' + details.useVersion '4.2.13.Final' } } } diff --git a/server/src/gatling/README.md b/server/src/gatling/README.md index 726c3c7e0..cf923b95d 100644 --- a/server/src/gatling/README.md +++ b/server/src/gatling/README.md @@ -1,7 +1,6 @@ # Setup ## build.gradle ### Running on dev environment: -- comment out `devRuntimeOnly "org.springframework.boot:spring-boot-devtools"` to prevent spring boot restart when you compile a gatling simulation. - add `jvmArgs = ['-Xverify:none']` to runServer task if you want to attach VisualVM to the server. ### Running against remote server: @@ -42,7 +41,7 @@ Typical run against a freshly started server: To run a single simulation: ```sh -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPISearchSimulation +./gradlew gatlingRun --simulation=org.eclipse.openvsx.RegistryAPISearchSimulation ``` ## Empty the database