diff --git a/MODULE.bazel b/MODULE.bazel index efcac0b..abc5654 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -58,10 +58,38 @@ maven.install( "org.apache.commons:commons-pool2:2.11.1", "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.5.2", "org.mockito.kotlin:mockito-kotlin:5.4.0", + "software.amazon.awssdk:url-connection-client:2.46.7", ], fail_if_repin_required = True, lock_file = "//:maven_install.json", ) + +# AWS SDK for the `serve` S3 hash-cache backend. The Netty/Apache HTTP clients are +# excluded (the service explicitly uses the lightweight url-connection-client above) +# to keep the deploy jar lean. `sts` is runtime-only: DefaultCredentialsProvider needs +# it to assume the pod's IAM role via web identity (IRSA) when running on EKS. +maven.artifact( + name = "bazel_diff_maven", + artifact = "s3", + exclusions = [ + "software.amazon.awssdk:netty-nio-client", + "software.amazon.awssdk:apache-client", + "software.amazon.awssdk:apache5-client", + ], + group = "software.amazon.awssdk", + version = "2.46.7", +) +maven.artifact( + name = "bazel_diff_maven", + artifact = "sts", + exclusions = [ + "software.amazon.awssdk:netty-nio-client", + "software.amazon.awssdk:apache-client", + "software.amazon.awssdk:apache5-client", + ], + group = "software.amazon.awssdk", + version = "2.46.7", +) use_repo( maven, bazel_diff_maven = "bazel_diff_maven", diff --git a/README.md b/README.md index 5ae0f57..dfac9aa 100644 --- a/README.md +++ b/README.md @@ -206,11 +206,41 @@ curl 'http://localhost:8080/metrics' "ready": true, "gitEngine": "subprocess", "trackDeps": false, - "cache": {"directory": "/var/cache/bazel-diff", "entries": 128, "sizeBytes": 4823913, "sizeHuman": "4.6 MB"}, + "cache": {"directory": "/var/cache/bazel-diff", "remote": "s3://my-bucket/bazel-diff/", "entries": 128, "sizeBytes": 4823913, "sizeHuman": "4.6 MB"}, "jvm": {"usedBytes": 123456789, "maxBytes": 2147483648} } ``` +### Shared S3 cache for multi-instance deployments + +A single instance caches hashes on local disk only. When you run several replicas behind a load +balancer (e.g. a Kubernetes Deployment behind a Service, with the readiness probe on `/health`), +give them a shared S3 cache tier so a revision is cold-hashed once fleet-wide instead of once per +pod: + +```bash +bazel-diff serve \ + --workspacePath /path/to/workspace-clone \ + --cacheDir /var/cache/bazel-diff \ + --s3Bucket my-hash-cache-bucket \ + --s3Prefix bazel-diff/my-repo +``` + +With `--s3Bucket` set the cache becomes two-tiered: reads check local disk first and fall back to +the bucket (backfilling local disk on a hit), and every generated entry is published to both, so +any replica can serve a revision another replica already hashed. Credentials and region resolve +through the standard AWS default provider chains (environment variables, profile, IRSA web +identity on EKS, IMDS), or pin the region with `--s3Region`. `--s3Endpoint` plus +`--s3ForcePathStyle` point the client at an S3-compatible store (MinIO, LocalStack) for local +testing. + +S3 errors never fail a request: a failed read is treated as a cache miss (the revision is +regenerated — slower, but correct) and a failed write leaves the entry local-only, so an S3 outage +degrades throughput rather than availability. Concurrent replicas racing to hash the same +revision are also harmless — entries are deterministic per key, so last-write-wins over identical +content. The `--cacheMax*` pruning flags bound the *local* tier only; bound the bucket with an S3 +lifecycle policy instead. + Notes and current limitations: * Distance metrics (`/impacted_targets_with_distances`) require the dependency-edge graph, which is @@ -234,10 +264,8 @@ Notes and current limitations: sweeper enforces the limits once at startup and then every `--cachePruneInterval` (default `1h`), evicting least-recently-used entries first — a cache hit refreshes an entry's recency, so revisions under active query are not expired out from under live traffic. With no `--cacheMax*` flag set the - cache is never pruned (the previous behavior). The cache layer is pluggable behind a byte-oriented - interface so a remote backend (e.g. S3) can be added without touching callers; such a backend - manages its own retention (e.g. a bucket lifecycle policy), and the in-process `--cacheMax*` flags - do not apply to it. + cache is never pruned (the previous behavior). The `--cacheMax*` flags always bound the local-disk + tier only; the shared S3 tier (see above) manages its own retention via a bucket lifecycle policy. * Query-affecting flags (`--useCquery`, `--fineGrainedHashExternalRepos`, etc.) mirror `generate-hashes`, and are folded into the cache key so a server started with different flags never serves another configuration's cached hashes. @@ -248,8 +276,8 @@ Notes and current limitations: shared-base full-hash cache is not reused (each distinct changed-set re-hashes the base), but each such hash is cheaper because it skips reading unchanged files. The extra entries are bounded by the same LRU `--cacheMax*` pruning as everything else. -* Containerization, multi-instance deployment manifests, and remote cache backends are not yet - included. +* Containerization and multi-instance deployment manifests are not yet included; the shared S3 + cache tier above is the building block for running replicas behind a load balancer. ## CLI Interface @@ -508,9 +536,9 @@ Command-line utility to analyze the state of the bazel build graph ```terminal Usage: bazel-diff serve [-hkvV] [--[no-]excludeExternalTargets] - [--no-initial-fetch] [--[no-]trackDeps] [--[no-] - useCquery] [-b=] --cacheDir= - [--cacheMaxAge=] + [--no-initial-fetch] [--[no-]s3ForcePathStyle] [--[no-] + trackDeps] [--[no-]useCquery] [-b=] + --cacheDir= [--cacheMaxAge=] [--cacheMaxEntries=] [--cacheMaxSize=] [--cachePruneInterval=] @@ -519,7 +547,9 @@ Usage: bazel-diff serve [-hkvV] [--[no-]excludeExternalTargets] [--fineGrainedHashExternalReposFile=] [--gitPath=] [--port=] [--requestTimeout=] - [-s=] -w= + [-s=] [--s3Bucket=] + [--s3Endpoint=] [--s3Prefix=] + [--s3Region=] -w= [-co=]... [--cqueryCommandOptions=]... [--fineGrainedHashExternalRepos= A text file with a newline separated list of filepaths used as a SHA256 seed for all targets. + --s3Bucket= S3 bucket used as a shared hash cache behind the + local --cacheDir tier. Generated hashes are + published to the bucket and local cache misses + fall back to it, so replicas behind a load + balancer share one cache. Credentials and region + come from the AWS default provider chains (env + vars, profile, IRSA, IMDS). Unset (the default) + means local-disk caching only. + --s3Endpoint= + Custom S3 endpoint URL for S3-compatible stores + (MinIO, LocalStack). Usually combined with + --s3ForcePathStyle. + --[no-]s3ForcePathStyle + Use path-style S3 addressing (bucket in the URL + path, not the hostname), required by most + S3-compatible stores. Defaults to false. + --s3Prefix= Key prefix for cache objects in --s3Bucket, e.g. + 'bazel-diff/my-repo'. Defaults to no prefix + (objects at the bucket root). + --s3Region= AWS region of --s3Bucket. Defaults to the SDK's + default region chain (env vars, profile, IMDS). -so, --bazelStartupOptions= Additional space separated Bazel client startup options used when invoking Bazel diff --git a/cli/BUILD b/cli/BUILD index 2f74d43..fe4dab0 100644 --- a/cli/BUILD +++ b/cli/BUILD @@ -30,6 +30,11 @@ kt_jvm_library( name = "cli-lib", srcs = glob(["src/main/kotlin/**/*.kt"]), resources = [":version_file"], + # sts is runtime-only: the SDK's DefaultCredentialsProvider loads it reflectively to + # assume a pod IAM role via web identity (IRSA) for the serve S3 cache backend. + runtime_deps = [ + "@bazel_diff_maven//:software_amazon_awssdk_sts", + ], deps = [ "//proto:analysis_v2_java_proto", "//proto:build_java_proto", @@ -39,6 +44,10 @@ kt_jvm_library( "@bazel_diff_maven//:io_insert_koin_koin_core_jvm", "@bazel_diff_maven//:org_apache_commons_commons_pool2", "@bazel_diff_maven//:org_jetbrains_kotlinx_kotlinx_coroutines_core_jvm", + "@bazel_diff_maven//:software_amazon_awssdk_regions", + "@bazel_diff_maven//:software_amazon_awssdk_s3", + "@bazel_diff_maven//:software_amazon_awssdk_sdk_core", + "@bazel_diff_maven//:software_amazon_awssdk_url_connection_client", ], ) @@ -244,6 +253,21 @@ kt_jvm_test( runtime_deps = [":cli-test-lib"], ) +kt_jvm_test( + name = "S3HashCacheStorageTest", + jvm_flags = [ + "-Dnet.bytebuddy.experimental=true", + ], + test_class = "com.bazel_diff.server.S3HashCacheStorageTest", + runtime_deps = [":cli-test-lib"], +) + +kt_jvm_test( + name = "TieredHashCacheStorageTest", + test_class = "com.bazel_diff.server.TieredHashCacheStorageTest", + runtime_deps = [":cli-test-lib"], +) + kt_jvm_test( name = "CachePrunerTest", test_class = "com.bazel_diff.server.CachePrunerTest", @@ -304,6 +328,8 @@ kt_jvm_library( "@bazel_diff_maven//:io_insert_koin_koin_test_jvm", "@bazel_diff_maven//:junit_junit", "@bazel_diff_maven//:org_mockito_kotlin_mockito_kotlin", + "@bazel_diff_maven//:software_amazon_awssdk_s3", + "@bazel_diff_maven//:software_amazon_awssdk_sdk_core", ], ) diff --git a/cli/src/main/kotlin/com/bazel_diff/cli/ServeCommand.kt b/cli/src/main/kotlin/com/bazel_diff/cli/ServeCommand.kt index ab72521..08401a0 100644 --- a/cli/src/main/kotlin/com/bazel_diff/cli/ServeCommand.kt +++ b/cli/src/main/kotlin/com/bazel_diff/cli/ServeCommand.kt @@ -22,7 +22,10 @@ import com.bazel_diff.server.LocalDiskHashCacheStorage import com.bazel_diff.server.MetricsService import com.bazel_diff.server.ProcessGitClient import com.bazel_diff.server.PrunableHashCacheStorage +import com.bazel_diff.server.S3HashCacheStorage +import com.bazel_diff.server.TieredHashCacheStorage import java.io.File +import java.net.URI import java.nio.file.Path import java.time.Duration import java.util.concurrent.Callable @@ -141,6 +144,51 @@ class ServeCommand : Callable { defaultValue = "1h") var cachePruneInterval: Duration = Duration.ofHours(1) + @CommandLine.Option( + names = ["--s3Bucket"], + description = + [ + "S3 bucket used as a shared hash cache behind the local --cacheDir tier. Generated " + + "hashes are published to the bucket and local cache misses fall back to it, so " + + "replicas behind a load balancer share one cache. Credentials and region come " + + "from the AWS default provider chains (env vars, profile, IRSA, IMDS). Unset " + + "(the default) means local-disk caching only."]) + var s3Bucket: String? = null + + @CommandLine.Option( + names = ["--s3Prefix"], + description = + [ + "Key prefix for cache objects in --s3Bucket, e.g. 'bazel-diff/my-repo'. Defaults to " + + "no prefix (objects at the bucket root)."], + defaultValue = "") + var s3Prefix: String = "" + + @CommandLine.Option( + names = ["--s3Region"], + description = + [ + "AWS region of --s3Bucket. Defaults to the SDK's default region chain (env vars, " + + "profile, IMDS)."]) + var s3Region: String? = null + + @CommandLine.Option( + names = ["--s3Endpoint"], + description = + [ + "Custom S3 endpoint URL for S3-compatible stores (MinIO, LocalStack). Usually " + + "combined with --s3ForcePathStyle."]) + var s3Endpoint: URI? = null + + @CommandLine.Option( + names = ["--s3ForcePathStyle"], + negatable = true, + description = + [ + "Use path-style S3 addressing (bucket in the URL path, not the hostname), required " + + "by most S3-compatible stores. Defaults to false."]) + var s3ForcePathStyle = false + @CommandLine.Option( names = ["--warmupRevision"], description = @@ -284,7 +332,7 @@ class ServeCommand : Callable { } return try { - val server = buildAndStartServer(createGitClient(), LocalDiskHashCacheStorage(cacheDir)) + val server = buildAndStartServer(createGitClient(), createStorage()) awaitShutdown(server) CommandLine.ExitCode.OK } finally { @@ -292,6 +340,29 @@ class ServeCommand : Callable { } } + /** + * Builds the cache storage stack: the local `--cacheDir` tier, backed by the shared [remote] tier + * when one is configured. [remote] is injectable for tests; the default builds the S3 tier from + * the `--s3*` flags (null -- local-only -- when `--s3Bucket` is unset). + */ + fun createStorage(remote: HashCacheStorage? = createS3Storage()): HashCacheStorage { + val local = LocalDiskHashCacheStorage(cacheDir) + return if (remote == null) local else TieredHashCacheStorage(local, remote) + } + + /** The S3 remote cache tier from the `--s3*` flags, or null when `--s3Bucket` is unset. */ + fun createS3Storage(): S3HashCacheStorage? = + s3Bucket?.let { bucket -> + S3HashCacheStorage( + S3HashCacheStorage.buildClient(s3Region, s3Endpoint, s3ForcePathStyle), + bucket, + s3Prefix) + } + + /** Human-readable location of the shared remote cache tier for `/metrics`; null when unset. */ + fun remoteCacheLocation(): String? = + s3Bucket?.let { "s3://$it/${S3HashCacheStorage.normalizePrefix(s3Prefix)}" } + /** * Builds the [GitClient]. Git fetch/checkout operations shell out to the `git` binary at * [gitPath], so a `git` binary must be available on the host. @@ -324,6 +395,7 @@ class ServeCommand : Callable { gitEngine = "subprocess", trackDeps = trackDeps, cacheDir = cacheDir.toString(), + remoteCache = remoteCacheLocation(), storage = storage, readiness = { ready.get() }, ) diff --git a/cli/src/main/kotlin/com/bazel_diff/server/MetricsService.kt b/cli/src/main/kotlin/com/bazel_diff/server/MetricsService.kt index 8fc06c5..d07f716 100644 --- a/cli/src/main/kotlin/com/bazel_diff/server/MetricsService.kt +++ b/cli/src/main/kotlin/com/bazel_diff/server/MetricsService.kt @@ -18,10 +18,13 @@ data class ServerMetrics( /** * Footprint of the hash cache. [entries]/[sizeBytes]/[sizeHuman] are null when the backend does not - * cheaply report its size (e.g. a remote store) -- see [MeasurableHashCacheStorage]. + * cheaply report its size (e.g. a remote store) -- see [MeasurableHashCacheStorage]. [remote] is + * the shared remote tier's location (e.g. `s3://bucket/prefix/`) when one is configured, null for a + * local-only cache; the size fields always describe the local tier only. */ data class CacheMetrics( val directory: String?, + val remote: String?, val entries: Long?, val sizeBytes: Long?, val sizeHuman: String?, @@ -57,6 +60,7 @@ class MetricsService( private val trackDeps: Boolean, private val cacheDir: String, private val storage: HashCacheStorage, + private val remoteCache: String? = null, private val readiness: () -> Boolean, private val clock: () -> Long = System::currentTimeMillis, ) : MetricsProvider { @@ -72,6 +76,7 @@ class MetricsService( cache = CacheMetrics( directory = cacheDir, + remote = remoteCache, entries = cacheStats?.entryCount, sizeBytes = cacheStats?.totalBytes, sizeHuman = cacheStats?.totalBytes?.let(::humanReadableBytes), diff --git a/cli/src/main/kotlin/com/bazel_diff/server/S3HashCacheStorage.kt b/cli/src/main/kotlin/com/bazel_diff/server/S3HashCacheStorage.kt new file mode 100644 index 0000000..89870ca --- /dev/null +++ b/cli/src/main/kotlin/com/bazel_diff/server/S3HashCacheStorage.kt @@ -0,0 +1,122 @@ +package com.bazel_diff.server + +import java.net.URI +import software.amazon.awssdk.core.exception.SdkException +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.GetObjectRequest +import software.amazon.awssdk.services.s3.model.HeadObjectRequest +import software.amazon.awssdk.services.s3.model.NoSuchKeyException +import software.amazon.awssdk.services.s3.model.PutObjectRequest +import software.amazon.awssdk.services.s3.model.S3Exception + +/** + * [HashCacheStorage] backed by an S3 bucket -- the shared-cache backend the RFC (issue #29) calls + * out so `serve` replicas behind a load balancer share one hash cache instead of each cold-hashing + * every revision. Typically wrapped in a [TieredHashCacheStorage] with a + * [LocalDiskHashCacheStorage] in front. + * + * Objects are stored as `.json`, mirroring [LocalDiskHashCacheStorage]'s `.json` + * layout so entries are recognizable across backends. Cache keys (see `HashService.cacheKey`) are + * `.[.]`, already S3-key-safe. + * + * S3 failures are contained here rather than propagated: a read failure degrades to a cache miss + * (the caller regenerates -- slower but correct, since an entry's content is deterministic for its + * key) and a write failure is logged and dropped (whichever instance next generates the revision + * re-publishes it). An S3 outage therefore slows the fleet but never fails requests. Concurrent + * writers racing the same key are equally harmless: last-write-wins over identical content. + * + * Deliberately neither [MeasurableHashCacheStorage] (sizing a bucket in-process would be a paged + * LIST per `/metrics` scrape) nor [PrunableHashCacheStorage] (retention belongs to a bucket + * lifecycle policy) -- the local tier keeps both roles. + */ +class S3HashCacheStorage( + private val s3: S3Client, + private val bucket: String, + prefix: String = "", + private val warn: (String) -> Unit = { System.err.println("[Warn] $it") }, +) : HashCacheStorage { + private val prefix = normalizePrefix(prefix) + + /** Human-readable location (`s3://bucket/prefix/`) for logs and `/metrics`. */ + val location: String + get() = "s3://$bucket/$prefix" + + private fun objectKey(key: String) = "$prefix$key.json" + + override fun get(key: String): ByteArray? = + try { + s3.getObjectAsBytes(GetObjectRequest.builder().bucket(bucket).key(objectKey(key)).build()) + .asByteArray() + } catch (e: NoSuchKeyException) { + null + } catch (e: SdkException) { + warn("S3 cache read of ${objectKey(key)} failed (treating as a miss): ${e.message}") + null + } + + override fun put(key: String, data: ByteArray) { + try { + s3.putObject( + PutObjectRequest.builder() + .bucket(bucket) + .key(objectKey(key)) + .contentType("application/json") + .build(), + RequestBody.fromBytes(data)) + } catch (e: SdkException) { + warn("S3 cache write of ${objectKey(key)} failed (entry not shared): ${e.message}") + } + } + + override fun contains(key: String): Boolean = + try { + s3.headObject(HeadObjectRequest.builder().bucket(bucket).key(objectKey(key)).build()) + true + } catch (e: NoSuchKeyException) { + false + } catch (e: S3Exception) { + // HeadObject reports a missing key as a bare 404 with no error body, which the SDK does + // not always map to NoSuchKeyException. + if (e.statusCode() != 404) { + warn("S3 cache check of ${objectKey(key)} failed (treating as a miss): ${e.message}") + } + false + } catch (e: SdkException) { + warn("S3 cache check of ${objectKey(key)} failed (treating as a miss): ${e.message}") + false + } + + companion object { + /** + * Normalizes a user-supplied key prefix to either `""` or a slash-terminated path with no + * leading slash, so `team/repo`, `/team/repo/` etc. all address the same objects. + */ + fun normalizePrefix(prefix: String): String { + val trimmed = prefix.trim('/') + return if (trimmed.isEmpty()) "" else "$trimmed/" + } + + /** + * Builds the [S3Client] the `serve` wiring uses, explicitly on the JDK's UrlConnection HTTP + * client (the SDK's Netty/Apache clients are excluded from the dependency tree to keep the + * deploy jar lean). Region and credentials fall back to the SDK default provider chains (env + * vars, profile, IRSA web identity, IMDS) when not given, so on EKS the pod's IAM role just + * works. [endpoint] plus [forcePathStyle] point the client at an S3-compatible store (MinIO, + * LocalStack) for testing. + */ + fun buildClient( + region: String? = null, + endpoint: URI? = null, + forcePathStyle: Boolean = false, + ): S3Client { + val builder = S3Client.builder().httpClientBuilder(UrlConnectionHttpClient.builder()) + region?.let { builder.region(Region.of(it)) } + endpoint?.let { builder.endpointOverride(it) } + if (forcePathStyle) builder.forcePathStyle(true) + return builder.build() + } + } +} diff --git a/cli/src/main/kotlin/com/bazel_diff/server/TieredHashCacheStorage.kt b/cli/src/main/kotlin/com/bazel_diff/server/TieredHashCacheStorage.kt new file mode 100644 index 0000000..16871a3 --- /dev/null +++ b/cli/src/main/kotlin/com/bazel_diff/server/TieredHashCacheStorage.kt @@ -0,0 +1,40 @@ +package com.bazel_diff.server + +/** + * Two-tier [HashCacheStorage]: a fast local tier backed by a shared remote tier (the RFC issue #29 + * S3 backend), so `serve` replicas behind a load balancer share generated hashes while hot + * revisions are still served from local disk. + * + * Reads hit [local] first; a remote hit is backfilled into [local] so subsequent reads (and the + * local LRU recency that pruning relies on) stay local. Writes go to both tiers, publishing every + * generated revision to the fleet. + * + * The measurable/prunable surface is the *local* tier only: `--cacheMax*` pruning bounds local disk + * usage and `/metrics` reports the local footprint. Remote retention belongs to the backend (e.g. a + * bucket lifecycle policy) -- see [S3HashCacheStorage]. A pruned-locally entry that still exists + * remotely is simply re-backfilled on next use. + */ +class TieredHashCacheStorage( + private val local: L, + private val remote: HashCacheStorage, +) : MeasurableHashCacheStorage, PrunableHashCacheStorage where +L : MeasurableHashCacheStorage, +L : PrunableHashCacheStorage { + override fun get(key: String): ByteArray? { + local.get(key)?.let { + return it + } + return remote.get(key)?.also { local.put(key, it) } + } + + override fun put(key: String, data: ByteArray) { + local.put(key, data) + remote.put(key, data) + } + + override fun contains(key: String): Boolean = local.contains(key) || remote.contains(key) + + override fun stats(): CacheStorageStats = local.stats() + + override fun prune(limits: CachePruneLimits): CachePruneResult = local.prune(limits) +} diff --git a/cli/src/test/kotlin/com/bazel_diff/cli/ServeCommandTest.kt b/cli/src/test/kotlin/com/bazel_diff/cli/ServeCommandTest.kt index 7529b69..8671413 100644 --- a/cli/src/test/kotlin/com/bazel_diff/cli/ServeCommandTest.kt +++ b/cli/src/test/kotlin/com/bazel_diff/cli/ServeCommandTest.kt @@ -16,6 +16,7 @@ import com.bazel_diff.server.HashCacheStorage import com.bazel_diff.server.LocalDiskHashCacheStorage import com.bazel_diff.server.ProcessGitClient import com.bazel_diff.server.ServerMetrics +import com.bazel_diff.server.TieredHashCacheStorage import com.google.gson.Gson import com.google.gson.GsonBuilder import java.net.HttpURLConnection @@ -303,6 +304,113 @@ class ServeCommandTest : KoinTest { assertThat(cmd.buildCachePruner(storage)).isNotNull() } + @Test + fun parsesS3Flags() { + val cmd = ServeCommand() + picocli + .CommandLine(cmd) + .parseArgs( + "--workspacePath", + "/tmp/ws", + "--cacheDir", + "/tmp/cache", + "--s3Bucket", + "my-bucket", + "--s3Prefix", + "bazel-diff/repo", + "--s3Region", + "us-east-1", + "--s3Endpoint", + "http://localhost:9000", + "--s3ForcePathStyle") + + assertThat(cmd.s3Bucket).isEqualTo("my-bucket") + assertThat(cmd.s3Prefix).isEqualTo("bazel-diff/repo") + assertThat(cmd.s3Region).isEqualTo("us-east-1") + assertThat(cmd.s3Endpoint).isEqualTo(java.net.URI("http://localhost:9000")) + assertThat(cmd.s3ForcePathStyle).isEqualTo(true) + } + + @Test + fun createStorageIsLocalOnlyWithoutABucket() { + val cmd = command() + // --s3Bucket unset: no remote tier is built and the stack stays a bare local-disk cache. + assertThat(cmd.createS3Storage()).isNull() + assertThat(cmd.createStorage()).isInstanceOf(LocalDiskHashCacheStorage::class) + } + + @Test + fun createStorageIsTieredWhenARemoteIsConfigured() { + val cmd = command() + val remote = InMemoryStorage() + + val storage = cmd.createStorage(remote = remote) + + assertThat(storage).isInstanceOf(TieredHashCacheStorage::class) + // A write must land in the shared tier (that is what publishes hashes to the fleet), and a + // local miss must fall back to it. + storage.put("k", byteArrayOf(1)) + assertThat(remote.get("k")).isNotNull() + remote.put("only-remote", byteArrayOf(2)) + assertThat(storage.get("only-remote")).isNotNull() + } + + @Test + fun createS3StorageBuildsTheTierFromFlags() { + // Building the client is offline (no credential/network resolution), so the full flag-driven + // construction path -- including the S3-compatible-store shape -- is exercised directly. + val cmd = + command().apply { + s3Bucket = "my-bucket" + s3Prefix = "/bazel-diff/repo/" + s3Region = "us-east-1" + s3Endpoint = java.net.URI("http://localhost:9000") + s3ForcePathStyle = true + } + + val s3 = cmd.createS3Storage() + + assertThat(s3).isNotNull() + assertThat(s3!!.location).isEqualTo("s3://my-bucket/bazel-diff/repo/") + } + + @Test + fun remoteCacheLocationReflectsFlags() { + assertThat(command().remoteCacheLocation()).isNull() + val cmd = command().apply { s3Bucket = "b" } + assertThat(cmd.remoteCacheLocation()).isEqualTo("s3://b/") + cmd.s3Prefix = "team/repo" + assertThat(cmd.remoteCacheLocation()).isEqualTo("s3://b/team/repo/") + } + + @Test + fun buildCachePrunerPrunesTheLocalTierOfATieredStorage() { + // --cacheMax* limits must keep working when the S3 tier is enabled: they bound the local disk + // tier, while the remote's retention is the bucket lifecycle policy's job. + val cmd = command().apply { cacheMaxEntries = 10 } + val tiered = + TieredHashCacheStorage( + LocalDiskHashCacheStorage(temp.newFolder().toPath()), InMemoryStorage()) + assertThat(cmd.buildCachePruner(tiered)).isNotNull() + } + + @Test + fun metricsReportTheRemoteCacheLocationWhenConfigured() { + val cmd = command().apply { s3Bucket = "shared-bucket" } + val server = + cmd.buildAndStartServer(FakeGitClient(), InMemoryStorage()).also { startedServers += it } + + val conn = + URL("http://localhost:${server.boundPort()}/metrics").openConnection() as HttpURLConnection + try { + val parsed = + Gson().fromJson(conn.inputStream.bufferedReader().readText(), ServerMetrics::class.java) + assertThat(parsed.cache.remote).isEqualTo("s3://shared-bucket/") + } finally { + conn.disconnect() + } + } + @Test fun buildAndStartServerLameDucksOnFetchFailure() { val cmd = command() diff --git a/cli/src/test/kotlin/com/bazel_diff/server/BazelDiffServerTest.kt b/cli/src/test/kotlin/com/bazel_diff/server/BazelDiffServerTest.kt index 346f68e..fb087c7 100644 --- a/cli/src/test/kotlin/com/bazel_diff/server/BazelDiffServerTest.kt +++ b/cli/src/test/kotlin/com/bazel_diff/server/BazelDiffServerTest.kt @@ -55,7 +55,7 @@ class BazelDiffServerTest : KoinTest { ready = true, gitEngine = "subprocess", trackDeps = false, - cache = CacheMetrics("/cache", 2, 2048, "2.0 KB"), + cache = CacheMetrics("/cache", null, 2, 2048, "2.0 KB"), jvm = JvmMetrics(1000, 2000), ) } diff --git a/cli/src/test/kotlin/com/bazel_diff/server/MetricsServiceTest.kt b/cli/src/test/kotlin/com/bazel_diff/server/MetricsServiceTest.kt index ca6a216..3e0b578 100644 --- a/cli/src/test/kotlin/com/bazel_diff/server/MetricsServiceTest.kt +++ b/cli/src/test/kotlin/com/bazel_diff/server/MetricsServiceTest.kt @@ -27,6 +27,7 @@ class MetricsServiceTest { ready: Boolean = true, nowMillis: Long = 10_000, startedAtMillis: Long = 4_000, + remoteCache: String? = null, ) = MetricsService( version = "1.2.3", @@ -35,6 +36,7 @@ class MetricsServiceTest { trackDeps = true, cacheDir = "/var/cache/bazel-diff", storage = storage, + remoteCache = remoteCache, readiness = { ready }, clock = { nowMillis }, ) @@ -68,6 +70,13 @@ class MetricsServiceTest { assertThat(snap.cache.sizeHuman).isNull() } + @Test + fun remoteCacheLocationIsReportedWhenConfigured() { + assertThat(service(FakePlain()).snapshot().cache.remote).isNull() + assertThat(service(FakePlain(), remoteCache = "s3://bucket/prefix/").snapshot().cache.remote) + .isEqualTo("s3://bucket/prefix/") + } + @Test fun readinessIsReflectedLive() { assertThat(service(FakePlain(), ready = false).snapshot().ready).isFalse() diff --git a/cli/src/test/kotlin/com/bazel_diff/server/S3HashCacheStorageTest.kt b/cli/src/test/kotlin/com/bazel_diff/server/S3HashCacheStorageTest.kt new file mode 100644 index 0000000..74269b5 --- /dev/null +++ b/cli/src/test/kotlin/com/bazel_diff/server/S3HashCacheStorageTest.kt @@ -0,0 +1,181 @@ +package com.bazel_diff.server + +import assertk.assertThat +import assertk.assertions.contains +import assertk.assertions.hasSize +import assertk.assertions.isEmpty +import assertk.assertions.isEqualTo +import assertk.assertions.isFalse +import assertk.assertions.isNull +import assertk.assertions.isTrue +import java.net.URI +import java.nio.charset.StandardCharsets +import org.junit.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.doAnswer +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.doThrow +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import software.amazon.awssdk.core.ResponseBytes +import software.amazon.awssdk.core.exception.SdkClientException +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.GetObjectRequest +import software.amazon.awssdk.services.s3.model.GetObjectResponse +import software.amazon.awssdk.services.s3.model.HeadObjectRequest +import software.amazon.awssdk.services.s3.model.HeadObjectResponse +import software.amazon.awssdk.services.s3.model.NoSuchKeyException +import software.amazon.awssdk.services.s3.model.PutObjectRequest +import software.amazon.awssdk.services.s3.model.PutObjectResponse +import software.amazon.awssdk.services.s3.model.S3Exception + +class S3HashCacheStorageTest { + private val s3: S3Client = mock() + private val warnings = mutableListOf() + + private fun storage(prefix: String = "") = + S3HashCacheStorage(s3, "bucket", prefix, warn = { warnings += it }) + + private fun bytes(s: String) = s.toByteArray(StandardCharsets.UTF_8) + + private fun stubGetReturning(data: ByteArray) { + whenever(s3.getObjectAsBytes(any())) + .doReturn(ResponseBytes.fromByteArray(GetObjectResponse.builder().build(), data)) + } + + @Test + fun getReturnsObjectBytesUnderThePrefixedKey() { + stubGetReturning(bytes("hello")) + + val result = storage(prefix = "team/repo").get("sha.fp") + + assertThat(String(result!!, StandardCharsets.UTF_8)).isEqualTo("hello") + val captor = argumentCaptor() + org.mockito.kotlin.verify(s3).getObjectAsBytes(captor.capture()) + assertThat(captor.firstValue.bucket()).isEqualTo("bucket") + assertThat(captor.firstValue.key()).isEqualTo("team/repo/sha.fp.json") + } + + @Test + fun getMissReturnsNullWithoutWarning() { + whenever(s3.getObjectAsBytes(any())) + .doThrow(NoSuchKeyException.builder().build()) + + assertThat(storage().get("absent")).isNull() + assertThat(warnings).isEmpty() + } + + @Test + fun getErrorDegradesToMissWithWarning() { + // An S3 outage must slow the service (regenerate), never fail the request. + whenever(s3.getObjectAsBytes(any())) + .doThrow(SdkClientException.create("connection refused")) + + assertThat(storage().get("sha.fp")).isNull() + assertThat(warnings).hasSize(1) + assertThat(warnings.first()).contains("treating as a miss") + } + + @Test + fun putUploadsJsonObjectUnderThePrefixedKey() { + val requestCaptor = argumentCaptor() + val bodyCaptor = argumentCaptor() + whenever(s3.putObject(requestCaptor.capture(), bodyCaptor.capture())) + .doReturn(PutObjectResponse.builder().build()) + + storage(prefix = "p").put("sha.fp", bytes("data")) + + assertThat(requestCaptor.firstValue.bucket()).isEqualTo("bucket") + assertThat(requestCaptor.firstValue.key()).isEqualTo("p/sha.fp.json") + assertThat(requestCaptor.firstValue.contentType()).isEqualTo("application/json") + val uploaded = bodyCaptor.firstValue.contentStreamProvider().newStream().readBytes() + assertThat(String(uploaded, StandardCharsets.UTF_8)).isEqualTo("data") + } + + @Test + fun putErrorIsSwallowedWithWarning() { + // A failed publish leaves the entry local-only; the next instance to generate re-publishes. + whenever(s3.putObject(any(), any())) + .doThrow(SdkClientException.create("boom")) + + storage().put("sha.fp", bytes("data")) + + assertThat(warnings).hasSize(1) + assertThat(warnings.first()).contains("not shared") + } + + @Test + fun containsIsTrueWhenHeadSucceeds() { + whenever(s3.headObject(any())).doReturn(HeadObjectResponse.builder().build()) + + assertThat(storage().contains("sha.fp")).isTrue() + } + + @Test + fun containsIsFalseOnNoSuchKey() { + whenever(s3.headObject(any())).doThrow(NoSuchKeyException.builder().build()) + + assertThat(storage().contains("absent")).isFalse() + assertThat(warnings).isEmpty() + } + + @Test + fun containsIsFalseOnBare404WithoutWarning() { + // HeadObject reports a missing key as a bare 404 with no error body, which the SDK does not + // always map to NoSuchKeyException. + whenever(s3.headObject(any())).doAnswer { + throw S3Exception.builder().statusCode(404).build() + } + + assertThat(storage().contains("absent")).isFalse() + assertThat(warnings).isEmpty() + } + + @Test + fun containsIsFalseOnServiceErrorWithWarning() { + whenever(s3.headObject(any())).doAnswer { + throw S3Exception.builder().statusCode(500).message("oops").build() + } + + assertThat(storage().contains("sha.fp")).isFalse() + assertThat(warnings).hasSize(1) + } + + @Test + fun containsIsFalseOnClientErrorWithWarning() { + whenever(s3.headObject(any())) + .doThrow(SdkClientException.create("no route to host")) + + assertThat(storage().contains("sha.fp")).isFalse() + assertThat(warnings).hasSize(1) + } + + @Test + fun prefixIsNormalizedToSlashTerminatedWithNoLeadingSlash() { + assertThat(S3HashCacheStorage.normalizePrefix("")).isEqualTo("") + assertThat(S3HashCacheStorage.normalizePrefix("/")).isEqualTo("") + assertThat(S3HashCacheStorage.normalizePrefix("a")).isEqualTo("a/") + assertThat(S3HashCacheStorage.normalizePrefix("/a/b/")).isEqualTo("a/b/") + assertThat(S3HashCacheStorage.normalizePrefix("a/b")).isEqualTo("a/b/") + } + + @Test + fun locationDescribesBucketAndNormalizedPrefix() { + assertThat(storage(prefix = "/team/repo/").location).isEqualTo("s3://bucket/team/repo/") + assertThat(storage().location).isEqualTo("s3://bucket/") + } + + @Test + fun buildClientAcceptsRegionEndpointAndPathStyle() { + // Building a client performs no network I/O and resolves no credentials, so the factory the + // serve wiring uses can be exercised directly, including the S3-compatible-store shape. + S3HashCacheStorage.buildClient( + region = "us-east-1", endpoint = URI("http://localhost:9000"), forcePathStyle = true) + .use { client -> assertThat(client.serviceName()).isEqualTo("s3") } + S3HashCacheStorage.buildClient(region = "us-west-2").use { client -> + assertThat(client.serviceName()).isEqualTo("s3") + } + } +} diff --git a/cli/src/test/kotlin/com/bazel_diff/server/TieredHashCacheStorageTest.kt b/cli/src/test/kotlin/com/bazel_diff/server/TieredHashCacheStorageTest.kt new file mode 100644 index 0000000..d037b34 --- /dev/null +++ b/cli/src/test/kotlin/com/bazel_diff/server/TieredHashCacheStorageTest.kt @@ -0,0 +1,124 @@ +package com.bazel_diff.server + +import assertk.assertThat +import assertk.assertions.isEqualTo +import assertk.assertions.isFalse +import assertk.assertions.isNull +import assertk.assertions.isTrue +import java.nio.charset.StandardCharsets +import org.junit.Test + +class TieredHashCacheStorageTest { + /** In-memory local tier that records prune calls and reports canned stats. */ + private class FakeLocal : MeasurableHashCacheStorage, PrunableHashCacheStorage { + val entries = mutableMapOf() + var gets = 0 + var prunedWith: CachePruneLimits? = null + + override fun get(key: String): ByteArray? { + gets++ + return entries[key] + } + + override fun put(key: String, data: ByteArray) { + entries[key] = data + } + + override fun stats() = CacheStorageStats(entryCount = entries.size.toLong(), totalBytes = 42) + + override fun prune(limits: CachePruneLimits): CachePruneResult { + prunedWith = limits + return CachePruneResult(scanned = entries.size, evicted = 1, bytesFreed = 10) + } + } + + /** In-memory remote tier that counts reads and writes. */ + private class FakeRemote : HashCacheStorage { + val entries = mutableMapOf() + var gets = 0 + var puts = 0 + + override fun get(key: String): ByteArray? { + gets++ + return entries[key] + } + + override fun put(key: String, data: ByteArray) { + puts++ + entries[key] = data + } + } + + private val local = FakeLocal() + private val remote = FakeRemote() + private val tiered = TieredHashCacheStorage(local, remote) + + private fun bytes(s: String) = s.toByteArray(StandardCharsets.UTF_8) + + @Test + fun localHitNeverTouchesRemote() { + local.entries["k"] = bytes("local") + + assertThat(String(tiered.get("k")!!, StandardCharsets.UTF_8)).isEqualTo("local") + assertThat(remote.gets).isEqualTo(0) + } + + @Test + fun remoteHitIsBackfilledIntoLocal() { + remote.entries["k"] = bytes("shared") + + assertThat(String(tiered.get("k")!!, StandardCharsets.UTF_8)).isEqualTo("shared") + // The next read must be served locally: the whole point of the backfill. + assertThat(String(local.entries["k"]!!, StandardCharsets.UTF_8)).isEqualTo("shared") + assertThat(String(tiered.get("k")!!, StandardCharsets.UTF_8)).isEqualTo("shared") + assertThat(remote.gets).isEqualTo(1) + } + + @Test + fun missInBothTiersReturnsNull() { + assertThat(tiered.get("absent")).isNull() + assertThat(remote.gets).isEqualTo(1) + } + + @Test + fun putWritesBothTiers() { + tiered.put("k", bytes("v")) + + assertThat(String(local.entries["k"]!!, StandardCharsets.UTF_8)).isEqualTo("v") + assertThat(String(remote.entries["k"]!!, StandardCharsets.UTF_8)).isEqualTo("v") + assertThat(remote.puts).isEqualTo(1) + } + + @Test + fun containsChecksBothTiers() { + assertThat(tiered.contains("absent")).isFalse() + + local.entries["l"] = bytes("x") + assertThat(tiered.contains("l")).isTrue() + + remote.entries["r"] = bytes("y") + assertThat(tiered.contains("r")).isTrue() + } + + @Test + fun statsReportTheLocalTierOnly() { + local.entries["a"] = bytes("x") + remote.entries["b"] = bytes("y") + + // Remote footprint is the bucket's business (lifecycle policy), not /metrics'. + assertThat(tiered.stats()).isEqualTo(CacheStorageStats(entryCount = 1, totalBytes = 42)) + } + + @Test + fun pruneBoundsTheLocalTierOnly() { + remote.entries["b"] = bytes("y") + val limits = CachePruneLimits(maxEntries = 5) + + val result = tiered.prune(limits) + + assertThat(local.prunedWith).isEqualTo(limits) + assertThat(result.evicted).isEqualTo(1) + // The remote entry is untouched by local pruning. + assertThat(remote.entries.containsKey("b")).isTrue() + } +} diff --git a/maven_install.json b/maven_install.json index 30da09a..f80d443 100644 --- a/maven_install.json +++ b/maven_install.json @@ -11,7 +11,10 @@ "org.apache.commons:commons-pool2": -501602100, "org.jetbrains.kotlinx:kotlinx-coroutines-core": -542524036, "org.mockito.kotlin:mockito-kotlin": 1836434344, - "repositories": -1949687017 + "repositories": -1949687017, + "software.amazon.awssdk:s3": -357938842, + "software.amazon.awssdk:sts": -252807166, + "software.amazon.awssdk:url-connection-client": -1774081775 }, "__RESOLVED_ARTIFACTS_HASH": { "co.touchlab:stately-concurrency-jvm": 840813890, @@ -48,7 +51,40 @@ "org.mockito.kotlin:mockito-kotlin": 2123507963, "org.mockito:mockito-core": -960687581, "org.objenesis:objenesis": 1798216877, - "org.opentest4j:opentest4j": -1584531193 + "org.opentest4j:opentest4j": -1584531193, + "org.reactivestreams:reactive-streams": 1281941191, + "org.slf4j:slf4j-api": -844459515, + "software.amazon.awssdk:annotations": -1907982020, + "software.amazon.awssdk:arns": -331969567, + "software.amazon.awssdk:auth": -1304499746, + "software.amazon.awssdk:aws-core": 1022743112, + "software.amazon.awssdk:aws-query-protocol": 1541096812, + "software.amazon.awssdk:aws-xml-protocol": 1222571574, + "software.amazon.awssdk:checksums": -849815427, + "software.amazon.awssdk:checksums-spi": -1573645255, + "software.amazon.awssdk:crt-core": 863172458, + "software.amazon.awssdk:endpoints-spi": -331509201, + "software.amazon.awssdk:http-auth": -751577575, + "software.amazon.awssdk:http-auth-aws": -929270720, + "software.amazon.awssdk:http-auth-aws-eventstream": 548779349, + "software.amazon.awssdk:http-auth-spi": 1109320635, + "software.amazon.awssdk:http-client-spi": 350288267, + "software.amazon.awssdk:identity-spi": 1339412679, + "software.amazon.awssdk:json-utils": -1894065695, + "software.amazon.awssdk:metrics-spi": -1682165638, + "software.amazon.awssdk:profiles": 1666666417, + "software.amazon.awssdk:protocol-core": -403970318, + "software.amazon.awssdk:regions": -1068450009, + "software.amazon.awssdk:retries": 496346634, + "software.amazon.awssdk:retries-spi": 249510908, + "software.amazon.awssdk:s3": -102542933, + "software.amazon.awssdk:sdk-core": -680189205, + "software.amazon.awssdk:sts": -1882991067, + "software.amazon.awssdk:third-party-jackson-core": 498836391, + "software.amazon.awssdk:url-connection-client": 1912663129, + "software.amazon.awssdk:utils": 1520148684, + "software.amazon.awssdk:utils-lite": 992114661, + "software.amazon.eventstream:eventstream": 666815114 }, "conflict_resolution": { "io.insert-koin:koin-core-jvm:3.1.6": "io.insert-koin:koin-core-jvm:4.0.0" @@ -263,6 +299,204 @@ "jar": "58812de60898d976fb81ef3b62da05c6604c18fd4a249f5044282479fc286af2" }, "version": "1.2.0" + }, + "org.reactivestreams:reactive-streams": { + "shasums": { + "jar": "f75ca597789b3dac58f61857b9ac2e1034a68fa672db35055a8fb4509e325f28" + }, + "version": "1.0.4" + }, + "org.slf4j:slf4j-api": { + "shasums": { + "jar": "d3ef575e3e4979678dc01bf1dcce51021493b4d11fb7f1be8ad982877c16a1c0" + }, + "version": "1.7.36" + }, + "software.amazon.awssdk:annotations": { + "shasums": { + "jar": "0e66de12a3ed381780f586fff21a3d8cbbee34f7a750d591e6808f0eb7ff11d7" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:arns": { + "shasums": { + "jar": "43bcdc92354d04bfbc7c0146a2142e7441da765b5a94f53cb042dcc639b41746" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:auth": { + "shasums": { + "jar": "d243b7d35e9f4359379327893615493f29951e65a0f9b2877dc57ef802cb5d43" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:aws-core": { + "shasums": { + "jar": "3a11acee17918f0748902e8ce3361cecaba3cdd7c8bd938fcf3f215f0af72b8e" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:aws-query-protocol": { + "shasums": { + "jar": "13bea147a77a7ca1b401e57f1a957821fbdef21366854bfff028922272ef66b6" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:aws-xml-protocol": { + "shasums": { + "jar": "17c45c621eb8d09bb1d3d8300ff71ab47285247d7be7f0e7f42ed1db5b59e9eb" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:checksums": { + "shasums": { + "jar": "6ff01216cb086fe5d49e8e65130dacdb85ab45d0f75236b902f3c22f3b855dde" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:checksums-spi": { + "shasums": { + "jar": "d3b0e2f03b42a2f243bb3b98fc5947fd91df6e4c3613eb144da9be7296c51d07" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:crt-core": { + "shasums": { + "jar": "f30033d6d5e70d05baeb23bd6e02136a48a82c0bca43809345bda8cca5de1eba" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:endpoints-spi": { + "shasums": { + "jar": "0a182c0c57711a25894489b41a64cb6db39d9e3e238d2612028fe8e8215a0260" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:http-auth": { + "shasums": { + "jar": "cfb65870787413dc20585c12be24988fdf3860940e29ed4a9abbf2429c75ba70" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:http-auth-aws": { + "shasums": { + "jar": "7db13526c33499bbfff15aa4e322140cf4506b338c74443debf8b0b6f0a4e49a" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:http-auth-aws-eventstream": { + "shasums": { + "jar": "d44351439fbd6cab242f3668a717fee893432bee2e7fe847361bf101eab07290" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:http-auth-spi": { + "shasums": { + "jar": "db8ce223e673f25560feae07239164521b407aae6e7813f46f0f9184611b20a5" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:http-client-spi": { + "shasums": { + "jar": "6ac595a06a587c3aa5043337622913c267b6fb6d8703ee9941b33ba956d6d9b0" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:identity-spi": { + "shasums": { + "jar": "04037b0af50aa081b7af15cbe6436cc5a13de38918a195447f48d9e9ff69ef6f" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:json-utils": { + "shasums": { + "jar": "87cc4f9e28b6f25cb17560dab8b880dd6434697fd14eedce1b75b616c34e67a3" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:metrics-spi": { + "shasums": { + "jar": "11cb8851d2606b49b247f99bced6ee1badbf0bade142a716d4b40dfb0535c437" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:profiles": { + "shasums": { + "jar": "7fc1e03b890ac2196f2a9c597f8b0074710ed62b67f9e753a62897893002ffd3" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:protocol-core": { + "shasums": { + "jar": "b83955e778376391157ffc653baacd8527694bb15c8e542e7e49333bd5cef98b" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:regions": { + "shasums": { + "jar": "b5d24f9cc4c7c8b251df9c211a9b9207f23dc93d4615a5d84edfff53149a0fae" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:retries": { + "shasums": { + "jar": "40bd2c5dfeb7ec2107de09d9732d100fded1cef86501604a7b200ce4a878d7b6" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:retries-spi": { + "shasums": { + "jar": "3b4d05cc05436413266b555b9a8682c2cca79ac7456912d27a37ad2123141724" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:s3": { + "shasums": { + "jar": "8621845c19a7b5f47d1c2c7c80a7c2a9385f79173930a3540c2d1b9599668cd0" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:sdk-core": { + "shasums": { + "jar": "9ce1df0c7e0c088fb5d32167a8240b5b7b7eb324128f0bedb6ad5878243d05e9" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:sts": { + "shasums": { + "jar": "04add8e379aa38012708fbec017e356f257be670fb2115be3179f354a45a6f5e" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:third-party-jackson-core": { + "shasums": { + "jar": "d53b00bec0af215d25c57fa05b19737fe7a6fb841ad895ac700fd998eab10e78" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:url-connection-client": { + "shasums": { + "jar": "b72393dadd12698c94a00bc93bcbcbe79d1f1525a0bc7c54584916447a0631f7" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:utils": { + "shasums": { + "jar": "b6403e4ee0640678bb228dc6340499750eea3e48918dc4c8a76ff8c26d22690e" + }, + "version": "2.46.7" + }, + "software.amazon.awssdk:utils-lite": { + "shasums": { + "jar": "b7b3ebb6a902f6f89edd2ba12b3145b8f000245b891d5b2ea21f4c005b842dd0" + }, + "version": "2.46.7" + }, + "software.amazon.eventstream:eventstream": { + "shasums": { + "jar": "0c37d8e696117f02c302191b8110b0d0eb20fa412fce34c3a269ec73c16ce822" + }, + "version": "1.0.1" } }, "dependencies": { @@ -345,6 +579,224 @@ "net.bytebuddy:byte-buddy", "net.bytebuddy:byte-buddy-agent", "org.objenesis:objenesis" + ], + "software.amazon.awssdk:arns": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:auth": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:checksums-spi", + "software.amazon.awssdk:http-auth", + "software.amazon.awssdk:http-auth-aws", + "software.amazon.awssdk:http-auth-aws-eventstream", + "software.amazon.awssdk:http-auth-spi", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:identity-spi", + "software.amazon.awssdk:json-utils", + "software.amazon.awssdk:profiles", + "software.amazon.awssdk:regions", + "software.amazon.awssdk:sdk-core", + "software.amazon.awssdk:utils", + "software.amazon.eventstream:eventstream" + ], + "software.amazon.awssdk:aws-core": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:auth", + "software.amazon.awssdk:endpoints-spi", + "software.amazon.awssdk:http-auth", + "software.amazon.awssdk:http-auth-spi", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:identity-spi", + "software.amazon.awssdk:metrics-spi", + "software.amazon.awssdk:profiles", + "software.amazon.awssdk:regions", + "software.amazon.awssdk:retries", + "software.amazon.awssdk:retries-spi", + "software.amazon.awssdk:sdk-core", + "software.amazon.awssdk:utils", + "software.amazon.awssdk:utils-lite", + "software.amazon.eventstream:eventstream" + ], + "software.amazon.awssdk:aws-query-protocol": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:aws-core", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:protocol-core", + "software.amazon.awssdk:sdk-core", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:aws-xml-protocol": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:aws-core", + "software.amazon.awssdk:aws-query-protocol", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:protocol-core", + "software.amazon.awssdk:sdk-core", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:checksums": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:checksums-spi", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:checksums-spi": [ + "software.amazon.awssdk:annotations" + ], + "software.amazon.awssdk:crt-core": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:endpoints-spi": [ + "software.amazon.awssdk:annotations" + ], + "software.amazon.awssdk:http-auth": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:http-auth-spi", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:identity-spi", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:http-auth-aws": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:checksums", + "software.amazon.awssdk:checksums-spi", + "software.amazon.awssdk:http-auth-spi", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:identity-spi", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:http-auth-aws-eventstream": [ + "software.amazon.awssdk:annotations", + "software.amazon.eventstream:eventstream" + ], + "software.amazon.awssdk:http-auth-spi": [ + "org.reactivestreams:reactive-streams", + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:checksums-spi", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:identity-spi", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:http-client-spi": [ + "org.reactivestreams:reactive-streams", + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:metrics-spi", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:identity-spi": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:json-utils": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:third-party-jackson-core", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:metrics-spi": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:profiles": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:protocol-core": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:sdk-core", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:regions": [ + "org.slf4j:slf4j-api", + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:json-utils", + "software.amazon.awssdk:profiles", + "software.amazon.awssdk:sdk-core", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:retries": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:retries-spi", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:retries-spi": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:s3": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:arns", + "software.amazon.awssdk:auth", + "software.amazon.awssdk:aws-core", + "software.amazon.awssdk:aws-xml-protocol", + "software.amazon.awssdk:checksums", + "software.amazon.awssdk:checksums-spi", + "software.amazon.awssdk:crt-core", + "software.amazon.awssdk:endpoints-spi", + "software.amazon.awssdk:http-auth", + "software.amazon.awssdk:http-auth-aws", + "software.amazon.awssdk:http-auth-spi", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:identity-spi", + "software.amazon.awssdk:json-utils", + "software.amazon.awssdk:metrics-spi", + "software.amazon.awssdk:profiles", + "software.amazon.awssdk:protocol-core", + "software.amazon.awssdk:regions", + "software.amazon.awssdk:retries-spi", + "software.amazon.awssdk:sdk-core", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:sdk-core": [ + "org.reactivestreams:reactive-streams", + "org.slf4j:slf4j-api", + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:checksums", + "software.amazon.awssdk:checksums-spi", + "software.amazon.awssdk:endpoints-spi", + "software.amazon.awssdk:http-auth-aws", + "software.amazon.awssdk:http-auth-spi", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:identity-spi", + "software.amazon.awssdk:metrics-spi", + "software.amazon.awssdk:profiles", + "software.amazon.awssdk:retries", + "software.amazon.awssdk:retries-spi", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:sts": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:auth", + "software.amazon.awssdk:aws-core", + "software.amazon.awssdk:aws-query-protocol", + "software.amazon.awssdk:endpoints-spi", + "software.amazon.awssdk:http-auth", + "software.amazon.awssdk:http-auth-aws", + "software.amazon.awssdk:http-auth-spi", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:identity-spi", + "software.amazon.awssdk:json-utils", + "software.amazon.awssdk:metrics-spi", + "software.amazon.awssdk:profiles", + "software.amazon.awssdk:protocol-core", + "software.amazon.awssdk:regions", + "software.amazon.awssdk:retries-spi", + "software.amazon.awssdk:sdk-core", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:url-connection-client": [ + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:utils" + ], + "software.amazon.awssdk:utils": [ + "org.reactivestreams:reactive-streams", + "org.slf4j:slf4j-api", + "software.amazon.awssdk:annotations" + ], + "software.amazon.awssdk:utils-lite": [ + "software.amazon.awssdk:annotations" ] }, "packages": { @@ -796,6 +1248,309 @@ ], "org.opentest4j:opentest4j": [ "org.opentest4j" + ], + "org.reactivestreams:reactive-streams": [ + "org.reactivestreams" + ], + "org.slf4j:slf4j-api": [ + "org.slf4j", + "org.slf4j.event", + "org.slf4j.helpers", + "org.slf4j.spi" + ], + "software.amazon.awssdk:annotations": [ + "software.amazon.awssdk.annotations" + ], + "software.amazon.awssdk:arns": [ + "software.amazon.awssdk.arns" + ], + "software.amazon.awssdk:auth": [ + "software.amazon.awssdk.auth.credentials", + "software.amazon.awssdk.auth.credentials.internal", + "software.amazon.awssdk.auth.signer", + "software.amazon.awssdk.auth.signer.internal", + "software.amazon.awssdk.auth.signer.internal.chunkedencoding", + "software.amazon.awssdk.auth.signer.internal.util", + "software.amazon.awssdk.auth.signer.params", + "software.amazon.awssdk.auth.token.credentials", + "software.amazon.awssdk.auth.token.credentials.aws", + "software.amazon.awssdk.auth.token.internal", + "software.amazon.awssdk.auth.token.signer", + "software.amazon.awssdk.auth.token.signer.aws" + ], + "software.amazon.awssdk:aws-core": [ + "software.amazon.awssdk.awscore", + "software.amazon.awssdk.awscore.auth", + "software.amazon.awssdk.awscore.client.builder", + "software.amazon.awssdk.awscore.client.config", + "software.amazon.awssdk.awscore.client.handler", + "software.amazon.awssdk.awscore.defaultsmode", + "software.amazon.awssdk.awscore.endpoint", + "software.amazon.awssdk.awscore.endpoints", + "software.amazon.awssdk.awscore.endpoints.authscheme", + "software.amazon.awssdk.awscore.eventstream", + "software.amazon.awssdk.awscore.exception", + "software.amazon.awssdk.awscore.interceptor", + "software.amazon.awssdk.awscore.internal", + "software.amazon.awssdk.awscore.internal.auth", + "software.amazon.awssdk.awscore.internal.authcontext", + "software.amazon.awssdk.awscore.internal.client.config", + "software.amazon.awssdk.awscore.internal.defaultsmode", + "software.amazon.awssdk.awscore.internal.interceptor", + "software.amazon.awssdk.awscore.internal.token", + "software.amazon.awssdk.awscore.internal.useragent", + "software.amazon.awssdk.awscore.presigner", + "software.amazon.awssdk.awscore.retry", + "software.amazon.awssdk.awscore.retry.conditions", + "software.amazon.awssdk.awscore.util" + ], + "software.amazon.awssdk:aws-query-protocol": [ + "software.amazon.awssdk.protocols.query", + "software.amazon.awssdk.protocols.query.internal.marshall", + "software.amazon.awssdk.protocols.query.internal.unmarshall", + "software.amazon.awssdk.protocols.query.unmarshall" + ], + "software.amazon.awssdk:aws-xml-protocol": [ + "software.amazon.awssdk.protocols.xml", + "software.amazon.awssdk.protocols.xml.internal.marshall", + "software.amazon.awssdk.protocols.xml.internal.unmarshall" + ], + "software.amazon.awssdk:checksums": [ + "software.amazon.awssdk.checksums", + "software.amazon.awssdk.checksums.internal" + ], + "software.amazon.awssdk:checksums-spi": [ + "software.amazon.awssdk.checksums.spi" + ], + "software.amazon.awssdk:crt-core": [ + "software.amazon.awssdk.crtcore" + ], + "software.amazon.awssdk:endpoints-spi": [ + "software.amazon.awssdk.endpoints" + ], + "software.amazon.awssdk:http-auth": [ + "software.amazon.awssdk.http.auth.internal.scheme", + "software.amazon.awssdk.http.auth.internal.signer", + "software.amazon.awssdk.http.auth.scheme", + "software.amazon.awssdk.http.auth.signer" + ], + "software.amazon.awssdk:http-auth-aws": [ + "software.amazon.awssdk.http.auth.aws.crt.internal.io", + "software.amazon.awssdk.http.auth.aws.crt.internal.signer", + "software.amazon.awssdk.http.auth.aws.crt.internal.util", + "software.amazon.awssdk.http.auth.aws.eventstream.internal.io", + "software.amazon.awssdk.http.auth.aws.eventstream.internal.signer", + "software.amazon.awssdk.http.auth.aws.internal.scheme", + "software.amazon.awssdk.http.auth.aws.internal.signer", + "software.amazon.awssdk.http.auth.aws.internal.signer.checksums", + "software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding", + "software.amazon.awssdk.http.auth.aws.internal.signer.io", + "software.amazon.awssdk.http.auth.aws.internal.signer.util", + "software.amazon.awssdk.http.auth.aws.scheme", + "software.amazon.awssdk.http.auth.aws.signer" + ], + "software.amazon.awssdk:http-auth-aws-eventstream": [ + "software.amazon.awssdk.http.auth.aws.eventstream" + ], + "software.amazon.awssdk:http-auth-spi": [ + "software.amazon.awssdk.http.auth.spi.internal.scheme", + "software.amazon.awssdk.http.auth.spi.internal.signer", + "software.amazon.awssdk.http.auth.spi.scheme", + "software.amazon.awssdk.http.auth.spi.signer" + ], + "software.amazon.awssdk:http-client-spi": [ + "software.amazon.awssdk.http", + "software.amazon.awssdk.http.async", + "software.amazon.awssdk.internal.http" + ], + "software.amazon.awssdk:identity-spi": [ + "software.amazon.awssdk.identity.spi", + "software.amazon.awssdk.identity.spi.internal" + ], + "software.amazon.awssdk:json-utils": [ + "software.amazon.awssdk.protocols.jsoncore", + "software.amazon.awssdk.protocols.jsoncore.internal" + ], + "software.amazon.awssdk:metrics-spi": [ + "software.amazon.awssdk.metrics", + "software.amazon.awssdk.metrics.internal" + ], + "software.amazon.awssdk:profiles": [ + "software.amazon.awssdk.profiles", + "software.amazon.awssdk.profiles.internal" + ], + "software.amazon.awssdk:protocol-core": [ + "software.amazon.awssdk.protocols.core" + ], + "software.amazon.awssdk:regions": [ + "software.amazon.awssdk.regions", + "software.amazon.awssdk.regions.internal", + "software.amazon.awssdk.regions.internal.util", + "software.amazon.awssdk.regions.partitionmetadata", + "software.amazon.awssdk.regions.providers", + "software.amazon.awssdk.regions.regionmetadata", + "software.amazon.awssdk.regions.servicemetadata", + "software.amazon.awssdk.regions.util" + ], + "software.amazon.awssdk:retries": [ + "software.amazon.awssdk.retries", + "software.amazon.awssdk.retries.internal", + "software.amazon.awssdk.retries.internal.circuitbreaker", + "software.amazon.awssdk.retries.internal.ratelimiter" + ], + "software.amazon.awssdk:retries-spi": [ + "software.amazon.awssdk.retries.api", + "software.amazon.awssdk.retries.api.internal", + "software.amazon.awssdk.retries.api.internal.backoff" + ], + "software.amazon.awssdk:s3": [ + "software.amazon.awssdk.services.s3", + "software.amazon.awssdk.services.s3.auth.scheme", + "software.amazon.awssdk.services.s3.auth.scheme.internal", + "software.amazon.awssdk.services.s3.crt", + "software.amazon.awssdk.services.s3.endpoints", + "software.amazon.awssdk.services.s3.endpoints.authscheme", + "software.amazon.awssdk.services.s3.endpoints.internal", + "software.amazon.awssdk.services.s3.internal", + "software.amazon.awssdk.services.s3.internal.checksums", + "software.amazon.awssdk.services.s3.internal.client", + "software.amazon.awssdk.services.s3.internal.crossregion", + "software.amazon.awssdk.services.s3.internal.crossregion.endpointprovider", + "software.amazon.awssdk.services.s3.internal.crossregion.utils", + "software.amazon.awssdk.services.s3.internal.crt", + "software.amazon.awssdk.services.s3.internal.endpoints", + "software.amazon.awssdk.services.s3.internal.handlers", + "software.amazon.awssdk.services.s3.internal.multipart", + "software.amazon.awssdk.services.s3.internal.plugins", + "software.amazon.awssdk.services.s3.internal.resource", + "software.amazon.awssdk.services.s3.internal.s3express", + "software.amazon.awssdk.services.s3.internal.settingproviders", + "software.amazon.awssdk.services.s3.internal.signing", + "software.amazon.awssdk.services.s3.jmespath.internal", + "software.amazon.awssdk.services.s3.model", + "software.amazon.awssdk.services.s3.model.selectobjectcontenteventstream", + "software.amazon.awssdk.services.s3.multipart", + "software.amazon.awssdk.services.s3.paginators", + "software.amazon.awssdk.services.s3.presigner", + "software.amazon.awssdk.services.s3.presigner.model", + "software.amazon.awssdk.services.s3.s3express", + "software.amazon.awssdk.services.s3.transform", + "software.amazon.awssdk.services.s3.waiters", + "software.amazon.awssdk.services.s3.waiters.internal" + ], + "software.amazon.awssdk:sdk-core": [ + "software.amazon.awssdk.core", + "software.amazon.awssdk.core.adapter", + "software.amazon.awssdk.core.async", + "software.amazon.awssdk.core.async.listener", + "software.amazon.awssdk.core.checksums", + "software.amazon.awssdk.core.client.builder", + "software.amazon.awssdk.core.client.config", + "software.amazon.awssdk.core.client.handler", + "software.amazon.awssdk.core.document", + "software.amazon.awssdk.core.document.internal", + "software.amazon.awssdk.core.endpointdiscovery", + "software.amazon.awssdk.core.endpointdiscovery.providers", + "software.amazon.awssdk.core.exception", + "software.amazon.awssdk.core.http", + "software.amazon.awssdk.core.identity", + "software.amazon.awssdk.core.interceptor", + "software.amazon.awssdk.core.interceptor.trait", + "software.amazon.awssdk.core.internal", + "software.amazon.awssdk.core.internal.async", + "software.amazon.awssdk.core.internal.capacity", + "software.amazon.awssdk.core.internal.checksums", + "software.amazon.awssdk.core.internal.chunked", + "software.amazon.awssdk.core.internal.compression", + "software.amazon.awssdk.core.internal.handler", + "software.amazon.awssdk.core.internal.http", + "software.amazon.awssdk.core.internal.http.async", + "software.amazon.awssdk.core.internal.http.loader", + "software.amazon.awssdk.core.internal.http.pipeline", + "software.amazon.awssdk.core.internal.http.pipeline.stages", + "software.amazon.awssdk.core.internal.http.pipeline.stages.utils", + "software.amazon.awssdk.core.internal.http.timers", + "software.amazon.awssdk.core.internal.interceptor", + "software.amazon.awssdk.core.internal.interceptor.trait", + "software.amazon.awssdk.core.internal.io", + "software.amazon.awssdk.core.internal.metrics", + "software.amazon.awssdk.core.internal.pagination.async", + "software.amazon.awssdk.core.internal.retry", + "software.amazon.awssdk.core.internal.signer", + "software.amazon.awssdk.core.internal.sync", + "software.amazon.awssdk.core.internal.transform", + "software.amazon.awssdk.core.internal.useragent", + "software.amazon.awssdk.core.internal.util", + "software.amazon.awssdk.core.internal.waiters", + "software.amazon.awssdk.core.io", + "software.amazon.awssdk.core.metrics", + "software.amazon.awssdk.core.pagination.async", + "software.amazon.awssdk.core.pagination.sync", + "software.amazon.awssdk.core.protocol", + "software.amazon.awssdk.core.retry", + "software.amazon.awssdk.core.retry.backoff", + "software.amazon.awssdk.core.retry.conditions", + "software.amazon.awssdk.core.runtime", + "software.amazon.awssdk.core.runtime.transform", + "software.amazon.awssdk.core.signer", + "software.amazon.awssdk.core.sync", + "software.amazon.awssdk.core.traits", + "software.amazon.awssdk.core.useragent", + "software.amazon.awssdk.core.util", + "software.amazon.awssdk.core.waiters" + ], + "software.amazon.awssdk:sts": [ + "software.amazon.awssdk.services.sts", + "software.amazon.awssdk.services.sts.auth", + "software.amazon.awssdk.services.sts.auth.scheme", + "software.amazon.awssdk.services.sts.auth.scheme.internal", + "software.amazon.awssdk.services.sts.endpoints", + "software.amazon.awssdk.services.sts.endpoints.internal", + "software.amazon.awssdk.services.sts.internal", + "software.amazon.awssdk.services.sts.model", + "software.amazon.awssdk.services.sts.transform" + ], + "software.amazon.awssdk:third-party-jackson-core": [ + "software.amazon.awssdk.thirdparty.jackson.core", + "software.amazon.awssdk.thirdparty.jackson.core.async", + "software.amazon.awssdk.thirdparty.jackson.core.base", + "software.amazon.awssdk.thirdparty.jackson.core.exc", + "software.amazon.awssdk.thirdparty.jackson.core.filter", + "software.amazon.awssdk.thirdparty.jackson.core.format", + "software.amazon.awssdk.thirdparty.jackson.core.internal.shaded.fdp.v2_21_3", + "software.amazon.awssdk.thirdparty.jackson.core.internal.shaded.fdp.v2_21_3.bte", + "software.amazon.awssdk.thirdparty.jackson.core.internal.shaded.fdp.v2_21_3.chr", + "software.amazon.awssdk.thirdparty.jackson.core.io", + "software.amazon.awssdk.thirdparty.jackson.core.io.schubfach", + "software.amazon.awssdk.thirdparty.jackson.core.json", + "software.amazon.awssdk.thirdparty.jackson.core.json.async", + "software.amazon.awssdk.thirdparty.jackson.core.sym", + "software.amazon.awssdk.thirdparty.jackson.core.type", + "software.amazon.awssdk.thirdparty.jackson.core.util" + ], + "software.amazon.awssdk:url-connection-client": [ + "software.amazon.awssdk.http.urlconnection" + ], + "software.amazon.awssdk:utils": [ + "software.amazon.awssdk.utils", + "software.amazon.awssdk.utils.async", + "software.amazon.awssdk.utils.builder", + "software.amazon.awssdk.utils.cache", + "software.amazon.awssdk.utils.cache.bounded", + "software.amazon.awssdk.utils.cache.lru", + "software.amazon.awssdk.utils.http", + "software.amazon.awssdk.utils.internal", + "software.amazon.awssdk.utils.internal.async", + "software.amazon.awssdk.utils.internal.proxy", + "software.amazon.awssdk.utils.io", + "software.amazon.awssdk.utils.uri", + "software.amazon.awssdk.utils.uri.internal" + ], + "software.amazon.awssdk:utils-lite": [ + "software.amazon.awssdk.utilslite" + ], + "software.amazon.eventstream:eventstream": [ + "software.amazon.eventstream" ] }, "repositories": { @@ -834,7 +1589,40 @@ "org.mockito.kotlin:mockito-kotlin", "org.mockito:mockito-core", "org.objenesis:objenesis", - "org.opentest4j:opentest4j" + "org.opentest4j:opentest4j", + "org.reactivestreams:reactive-streams", + "org.slf4j:slf4j-api", + "software.amazon.awssdk:annotations", + "software.amazon.awssdk:arns", + "software.amazon.awssdk:auth", + "software.amazon.awssdk:aws-core", + "software.amazon.awssdk:aws-query-protocol", + "software.amazon.awssdk:aws-xml-protocol", + "software.amazon.awssdk:checksums", + "software.amazon.awssdk:checksums-spi", + "software.amazon.awssdk:crt-core", + "software.amazon.awssdk:endpoints-spi", + "software.amazon.awssdk:http-auth", + "software.amazon.awssdk:http-auth-aws", + "software.amazon.awssdk:http-auth-aws-eventstream", + "software.amazon.awssdk:http-auth-spi", + "software.amazon.awssdk:http-client-spi", + "software.amazon.awssdk:identity-spi", + "software.amazon.awssdk:json-utils", + "software.amazon.awssdk:metrics-spi", + "software.amazon.awssdk:profiles", + "software.amazon.awssdk:protocol-core", + "software.amazon.awssdk:regions", + "software.amazon.awssdk:retries", + "software.amazon.awssdk:retries-spi", + "software.amazon.awssdk:s3", + "software.amazon.awssdk:sdk-core", + "software.amazon.awssdk:sts", + "software.amazon.awssdk:third-party-jackson-core", + "software.amazon.awssdk:url-connection-client", + "software.amazon.awssdk:utils", + "software.amazon.awssdk:utils-lite", + "software.amazon.eventstream:eventstream" ] }, "services": { @@ -847,6 +1635,16 @@ "kotlin.reflect.jvm.internal.impl.load.java.FieldOverridabilityCondition", "kotlin.reflect.jvm.internal.impl.load.java.JavaIncompatibilityRulesOverridabilityCondition" ] + }, + "software.amazon.awssdk:third-party-jackson-core": { + "software.amazon.awssdk.thirdparty.jackson.core.JsonFactory": [ + "software.amazon.awssdk.thirdparty.jackson.core.JsonFactory" + ] + }, + "software.amazon.awssdk:url-connection-client": { + "software.amazon.awssdk.http.SdkHttpService": [ + "software.amazon.awssdk.http.urlconnection.UrlConnectionSdkHttpService" + ] } }, "version": "3" diff --git a/tools/readme_template.md b/tools/readme_template.md index bbda5d9..b32ab9b 100644 --- a/tools/readme_template.md +++ b/tools/readme_template.md @@ -206,11 +206,41 @@ curl 'http://localhost:8080/metrics' "ready": true, "gitEngine": "subprocess", "trackDeps": false, - "cache": {"directory": "/var/cache/bazel-diff", "entries": 128, "sizeBytes": 4823913, "sizeHuman": "4.6 MB"}, + "cache": {"directory": "/var/cache/bazel-diff", "remote": "s3://my-bucket/bazel-diff/", "entries": 128, "sizeBytes": 4823913, "sizeHuman": "4.6 MB"}, "jvm": {"usedBytes": 123456789, "maxBytes": 2147483648} } ``` +### Shared S3 cache for multi-instance deployments + +A single instance caches hashes on local disk only. When you run several replicas behind a load +balancer (e.g. a Kubernetes Deployment behind a Service, with the readiness probe on `/health`), +give them a shared S3 cache tier so a revision is cold-hashed once fleet-wide instead of once per +pod: + +```bash +bazel-diff serve \ + --workspacePath /path/to/workspace-clone \ + --cacheDir /var/cache/bazel-diff \ + --s3Bucket my-hash-cache-bucket \ + --s3Prefix bazel-diff/my-repo +``` + +With `--s3Bucket` set the cache becomes two-tiered: reads check local disk first and fall back to +the bucket (backfilling local disk on a hit), and every generated entry is published to both, so +any replica can serve a revision another replica already hashed. Credentials and region resolve +through the standard AWS default provider chains (environment variables, profile, IRSA web +identity on EKS, IMDS), or pin the region with `--s3Region`. `--s3Endpoint` plus +`--s3ForcePathStyle` point the client at an S3-compatible store (MinIO, LocalStack) for local +testing. + +S3 errors never fail a request: a failed read is treated as a cache miss (the revision is +regenerated — slower, but correct) and a failed write leaves the entry local-only, so an S3 outage +degrades throughput rather than availability. Concurrent replicas racing to hash the same +revision are also harmless — entries are deterministic per key, so last-write-wins over identical +content. The `--cacheMax*` pruning flags bound the *local* tier only; bound the bucket with an S3 +lifecycle policy instead. + Notes and current limitations: * Distance metrics (`/impacted_targets_with_distances`) require the dependency-edge graph, which is @@ -234,10 +264,8 @@ Notes and current limitations: sweeper enforces the limits once at startup and then every `--cachePruneInterval` (default `1h`), evicting least-recently-used entries first — a cache hit refreshes an entry's recency, so revisions under active query are not expired out from under live traffic. With no `--cacheMax*` flag set the - cache is never pruned (the previous behavior). The cache layer is pluggable behind a byte-oriented - interface so a remote backend (e.g. S3) can be added without touching callers; such a backend - manages its own retention (e.g. a bucket lifecycle policy), and the in-process `--cacheMax*` flags - do not apply to it. + cache is never pruned (the previous behavior). The `--cacheMax*` flags always bound the local-disk + tier only; the shared S3 tier (see above) manages its own retention via a bucket lifecycle policy. * Query-affecting flags (`--useCquery`, `--fineGrainedHashExternalRepos`, etc.) mirror `generate-hashes`, and are folded into the cache key so a server started with different flags never serves another configuration's cached hashes. @@ -248,8 +276,8 @@ Notes and current limitations: shared-base full-hash cache is not reused (each distinct changed-set re-hashes the base), but each such hash is cheaper because it skips reading unchanged files. The extra entries are bounded by the same LRU `--cacheMax*` pruning as everything else. -* Containerization, multi-instance deployment manifests, and remote cache backends are not yet - included. +* Containerization and multi-instance deployment manifests are not yet included; the shared S3 + cache tier above is the building block for running replicas behind a load balancer.