diff --git a/server/.gitignore b/server/.gitignore index 22fe684e2..508484672 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -6,6 +6,8 @@ .settings/ .classpath .project +.metals +.bloop DEPENDENCIES /src/dev/resources/application-ovsx.properties diff --git a/server/build.gradle b/server/build.gradle index 12f770a14..14e9c26f0 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -48,27 +48,20 @@ configurations { devImplementation.extendsFrom implementation devRuntimeOnly.extendsFrom runtimeOnly - gatling.exclude group: "io.gatling.highcharts", module: "gatling-charts-highcharts" - // exclude commons-logging to avoid runtime warnings like these // Standard Commons Logging discovery in action with spring-jcl: // please remove commons-logging.jar from classpath in order to avoid potential conflicts configureEach { exclude group: "commons-logging", module: "commons-logging" } -} -dependencyManagement { - gatling { - dependencies { - dependencySet(group: 'io.netty', version: '4.2.13.Final') { - entry 'netty-codec-http' - entry 'netty-codec' - entry 'netty-handler' - entry 'netty-buffer' - entry 'netty-transport' - entry 'netty-common' - entry 'netty-transport-native-epoll' + // Gatling 3.15 requires Netty 4.2.x but Spring Boot's dependency management forces 4.1.x. + // Override all io.netty modules (except tcnative which has its own versioning) on the + // gatling configuration to keep its classpath internally consistent. + matching { it.name.startsWith('gatling') }.configureEach { + resolutionStrategy.eachDependency { details -> + if (details.requested.group == 'io.netty' && !details.requested.name.startsWith('netty-tcnative')) { + details.useVersion '4.2.13.Final' } } } @@ -273,5 +266,79 @@ tasks.withType(ScalaCompile).configureEach { } } +import io.gatling.gradle.GatlingRunTask + +// Per-simulation GatlingRunTask instances. Aggregator tasks below pull them +// in via dependsOn. The plugin's includes/excludes filter isn't exposed +// per-task, so we register one task per simulation explicitly. +def gatlingSimulationGroups = [ + fillDatabase: [ + 'org.eclipse.openvsx.RegistryAPICreateNamespaceSimulation', + 'org.eclipse.openvsx.RegistryAPIPublishExtensionSimulation', + ], + registryApi: [ + 'org.eclipse.openvsx.RegistryAPIGetNamespaceSimulation', + 'org.eclipse.openvsx.RegistryAPIGetNamespaceDetailsSimulation', + 'org.eclipse.openvsx.RegistryAPIGetExtensionSimulation', + 'org.eclipse.openvsx.RegistryAPIGetExtensionTargetPlatformSimulation', + 'org.eclipse.openvsx.RegistryAPIGetExtensionVersionSimulation', + 'org.eclipse.openvsx.RegistryAPIGetExtensionVersionTargetPlatformSimulation', + 'org.eclipse.openvsx.RegistryAPIGetVersionReferencesSimulation', + 'org.eclipse.openvsx.RegistryAPIGetVersionReferencesTargetPlatformSimulation', + 'org.eclipse.openvsx.RegistryAPIGetFileSimulation', + 'org.eclipse.openvsx.RegistryAPIGetFileTargetPlatformSimulation', + 'org.eclipse.openvsx.RegistryAPIGetQuerySimulation', + 'org.eclipse.openvsx.RegistryAPIGetQueryV2Simulation', + 'org.eclipse.openvsx.RegistryAPISearchSimulation', + 'org.eclipse.openvsx.RegistryAPIVerifyTokenSimulation', + ], + vscodeAdapter: [ + 'org.eclipse.openvsx.adapter.VSCodeAdapterExtensionQuerySimulation', + 'org.eclipse.openvsx.adapter.VSCodeAdapterGetAssetSimulation', + 'org.eclipse.openvsx.adapter.VSCodeAdapterGetWebResourceSimulation', + 'org.eclipse.openvsx.adapter.VSCodeAdapterItemSimulation', + 'org.eclipse.openvsx.adapter.VSCodeAdapterUnpkgSimulation', + 'org.eclipse.openvsx.adapter.VSCodeAdapterVspackageSimulation', + ], +] + +def simTaskName = { String fqcn -> "gatling${fqcn.tokenize('.').last()}" as String } + +gatlingSimulationGroups.values().flatten().each { String fqcn -> + tasks.register(simTaskName(fqcn), GatlingRunTask) { + simulationClassName = fqcn + nonInteractive = true + } +} + +// Within fillDatabase, namespaces must be created before extensions can be published. +tasks.named(simTaskName('org.eclipse.openvsx.RegistryAPIPublishExtensionSimulation')).configure { + mustRunAfter simTaskName('org.eclipse.openvsx.RegistryAPICreateNamespaceSimulation') +} + +tasks.register('perfFillDatabase') { + description = 'Populates the DB: creates namespaces then publishes extensions' + group = 'Performance testing' + dependsOn gatlingSimulationGroups.fillDatabase.collect(simTaskName) +} + +tasks.register('perfTestRegistryApi') { + description = 'Runs read-only Gatling simulations against the registry API' + group = 'Performance testing' + dependsOn gatlingSimulationGroups.registryApi.collect(simTaskName) +} + +tasks.register('perfTestVscodeAdapter') { + description = 'Runs read-only Gatling simulations against the VS Code adapter' + group = 'Performance testing' + dependsOn gatlingSimulationGroups.vscodeAdapter.collect(simTaskName) +} + +tasks.register('perfTestAll') { + description = 'Runs all read-only Gatling simulations (registry API + VS Code adapter)' + group = 'Performance testing' + dependsOn 'perfTestRegistryApi', 'perfTestVscodeAdapter' +} + apply from: 'dependencies.gradle' apply from: 'test-extensions.gradle' diff --git a/server/gradle/libs.versions.toml b/server/gradle/libs.versions.toml index ebcc94418..76213c50e 100644 --- a/server/gradle/libs.versions.toml +++ b/server/gradle/libs.versions.toml @@ -6,7 +6,7 @@ bucket4j-spring = "0.12.10" bucket4j = "8.10.1" commons-lang3 = "3.20.0" flyway = "11.20.3" -gatling = "3.14.9" +gatling = "3.15.0" gcloud = "2.62.1" hibernate = "6.6.42.Final" ipaddress = "5.5.1" diff --git a/server/src/gatling/README.md b/server/src/gatling/README.md index a3780b185..cf923b95d 100644 --- a/server/src/gatling/README.md +++ b/server/src/gatling/README.md @@ -1,7 +1,6 @@ # Setup ## build.gradle ### Running on dev environment: -- comment out `devRuntimeOnly "org.springframework.boot:spring-boot-devtools"` to prevent spring boot restart when you compile a gatling simulation. - add `jvmArgs = ['-Xverify:none']` to runServer task if you want to attach VisualVM to the server. ### Running against remote server: @@ -14,23 +13,36 @@ Simulations use the `auth` value to set the Authorization request header. ### resources/access-tokens.csv: -- contains API access tokens. Create a couple of tokens using the web UI and add them to this file. +- contains API access tokens. `super_token` is pre-seeded for the dev profile; create more via the web UI for testing against a remote server. - make sure to keep the `access_token` header at the top of the file. ### scala/org/eclipse/openvsx/RegistryAPIPublishExtensionSimulation.scala: -- Change `extensionDir` property in `resources/application.properties` to a directory that **only** contains extensions (*.vsix files). The simulation uses those files to upload them to the server. +- Defaults to `build/test-extensions/`. Either populate it with `.vsix` files (e.g. `./gradlew downloadTestExtensions`) or change `extensionDir` in `resources/application.properties` to a directory that **only** contains extensions. # Running Gatling -**The Gatling 'post' simulations need to be run to fill the database:** -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPICreateNamespaceSimulation` -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIPublishExtensionSimulation` - -**After running those simulations all other simulations can be run in no particular order:** -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetNamespaceSimulation` -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetExtensionSimulation` -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetExtensionVersionSimulation` -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIGetQuerySimulation` -- `./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.adapter.VSCodeAdapterExtensionQuerySimulation` + +Gradle tasks (group: `Performance testing`): + +| Task | What it does | +|-------------------------|-----------------------------------------------------------------------------------------------| +| `perfFillDatabase` | Seeds the DB: creates namespaces, then publishes extensions from `extensionDir` | +| `perfTestRegistryApi` | Runs all read-only registry API simulations | +| `perfTestVscodeAdapter` | Runs all read-only VS Code adapter simulations | +| `perfTestAll` | Runs `perfTestRegistryApi` + `perfTestVscodeAdapter` | +| `gatlingRun` | Built-in plugin task; pass `--simulation=` to run a single simulation | + +Typical run against a freshly started server: + +```sh +./gradlew perfFillDatabase +./gradlew perfTestAll +``` + +To run a single simulation: + +```sh +./gradlew gatlingRun --simulation=org.eclipse.openvsx.RegistryAPISearchSimulation +``` ## Empty the database If you wish to empty the database after running the Gatling simulations, you can run: diff --git a/server/src/gatling/resources/access-tokens.csv b/server/src/gatling/resources/access-tokens.csv index f40bd5ed0..c6d18db68 100644 --- a/server/src/gatling/resources/access-tokens.csv +++ b/server/src/gatling/resources/access-tokens.csv @@ -1 +1,2 @@ access_token +super_token diff --git a/server/src/gatling/resources/application.properties b/server/src/gatling/resources/application.properties index 2fd32a25c..d766c648f 100644 --- a/server/src/gatling/resources/application.properties +++ b/server/src/gatling/resources/application.properties @@ -1,3 +1,3 @@ baseUrl=http://localhost:8080 -extensionDir= +extensionDir=build/test-extensions #auth= \ No newline at end of file diff --git a/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala b/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala index 71b6a6f22..c47050133 100644 --- a/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala +++ b/server/src/gatling/scala/org/eclipse/openvsx/Scenarios.scala @@ -15,7 +15,8 @@ import io.gatling.core.session.Expression import io.gatling.core.structure.ScenarioBuilder import io.gatling.http.Predef._ -import java.nio.file.Files +import java.nio.file.{Files, Paths} +import java.util.UUID import scala.collection.mutable.ListBuffer import scala.concurrent.duration.DurationInt import scala.reflect.io.File @@ -42,8 +43,8 @@ object Scenarios { } def createNamespaceScenario(): ScenarioBuilder = { - val namespacesCount = 780 - val repeatTimes = namespacesCount / users + val totalRequests = 780 + val repeatTimes = totalRequests / users scenario("RegistryAPI: Create Namespace") .repeat(repeatTimes) { feed(csv(NamespaceFeed)) @@ -66,9 +67,13 @@ object Scenarios { } private def extensionFilesFeeder(extensionDir: String): Array[Map[String,String]] = { - val extensionFiles = new java.io.File(extensionDir).list() - val feeder = new Array[Map[String, String]](extensionFiles.length) + // File.list() returns null when the directory does not exist. + val extensionFiles = Option(new java.io.File(extensionDir).list()) + .getOrElse(Array.empty[String]) + .filter(_.endsWith(".vsix")) + if (extensionFiles.isEmpty) return Array.empty[Map[String, String]] + val feeder = new Array[Map[String, String]](extensionFiles.length) var mapIndex = 0 var feederIndex = 0 // make sure that versions of same extension are not right after one another @@ -91,8 +96,11 @@ object Scenarios { def publishScenario(users: Int): ScenarioBuilder = { val extensionDir = conf.getString("extensionDir") val feeder = this.extensionFilesFeeder(extensionDir) + if (feeder.isEmpty) { + return scenario("RegistryAPI: Publish Extension (no extensions found at " + extensionDir + ")") + } - val repeatTimes = feeder.length / users + val repeatTimes = Math.max(1, feeder.length / users) scenario("RegistryAPI: Publish Extension") .repeat(repeatTimes) { feed(feeder) @@ -102,11 +110,12 @@ object Scenarios { .headers(headers()) .queryParam("token", """#{access_token}""") .body(ByteArrayBody(session => { - val path = extensionDir + "\\" + session("extension_file").as[String] + val path = Paths.get(extensionDir, session("extension_file").as[String]).toString File(path).toByteArray() })) .requestTimeout(3.minutes) - .check(status.is(201))) + // 201 = new publish, 400 = already published (idempotent on re-runs). + .check(status.in(201, 400))) } } diff --git a/server/src/gatling/scripts/fill-database.sh b/server/src/gatling/scripts/fill-database.sh deleted file mode 100644 index 3e3d90684..000000000 --- a/server/src/gatling/scripts/fill-database.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -cd ../../.. -./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPICreateNamespaceSimulation -./gradlew --rerun-tasks gatlingRun-org.eclipse.openvsx.RegistryAPIPublishExtensionSimulation -cd src/gatling/scripts || exit diff --git a/server/src/gatling/scripts/test-registry-api.sh b/server/src/gatling/scripts/test-registry-api.sh deleted file mode 100644 index 12cb2d856..000000000 --- a/server/src/gatling/scripts/test-registry-api.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -cd ../../.. -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetNamespaceSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetNamespaceDetailsSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetExtensionSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetExtensionTargetPlatformSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetExtensionVersionSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetExtensionVersionTargetPlatformSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetVersionReferencesSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetVersionReferencesTargetPlatformSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetFileSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetFileTargetPlatformSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetQuerySimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIGetQueryV2Simulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPISearchSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.RegistryAPIVerifyTokenSimulation -cd src/gatling/scripts || exit diff --git a/server/src/gatling/scripts/test-vscode-adapter.sh b/server/src/gatling/scripts/test-vscode-adapter.sh deleted file mode 100644 index 366692c57..000000000 --- a/server/src/gatling/scripts/test-vscode-adapter.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -cd ../../.. -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.adapter.VSCodeAdapterExtensionQuerySimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.adapter.VSCodeAdapterGetAssetSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.adapter.VSCodeAdapterItemSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.adapter.VSCodeAdapterUnpkgSimulation -./gradlew --rerun-tasks gatlingRun --simulation=org.eclipse.openvsx.adapter.VSCodeAdapterVspackageSimulation -cd src/gatling/scripts || exit