From 16bede3fbca1a6248727eb60ad3e3fbb249c0142 Mon Sep 17 00:00:00 2001 From: Elliot Otchet Date: Wed, 29 Jul 2026 18:16:38 +0000 Subject: [PATCH 01/10] Move tests to resource/service so they can be exercised to confirm requirements in the running ES cluster. --- DEVNOTES.md | 54 +- docs/plans/es-security-capability-record.md | 364 +++++ .../consent/http/ConsentApplication.java | 2 + .../consent/http/ConsentModule.java | 45 +- .../elastic_search/CapabilityVerdict.java | 43 + .../ElasticSearchCapability.java | 12 + .../ElasticSearchCapabilityReport.java | 52 + .../ElasticSearchCapabilityResource.java | 80 + .../ElasticSearchCapabilityService.java | 1421 +++++++++++++++++ src/main/resources/assets/api-docs.yaml | 2 + .../paths/elasticSearchCapabilities.yaml | 75 + .../ElasticSearchCapabilityReport.yaml | 222 +++ .../consent/http/ConsentModuleTest.java | 63 +- .../ElasticSearchCapabilityResourceTest.java | 108 ++ .../ElasticSearchCapabilityServiceTest.java | 1036 ++++++++++++ 15 files changed, 3543 insertions(+), 36 deletions(-) create mode 100644 docs/plans/es-security-capability-record.md create mode 100644 src/main/java/org/broadinstitute/consent/http/models/elastic_search/CapabilityVerdict.java create mode 100644 src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapability.java create mode 100644 src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapabilityReport.java create mode 100644 src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java create mode 100644 src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java create mode 100644 src/main/resources/assets/paths/elasticSearchCapabilities.yaml create mode 100644 src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml create mode 100644 src/test/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResourceTest.java create mode 100644 src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java diff --git a/DEVNOTES.md b/DEVNOTES.md index dd69beee4..69601adac 100644 --- a/DEVNOTES.md +++ b/DEVNOTES.md @@ -71,41 +71,41 @@ and the defaults should be correct. ### Developing with a local Elastic Search instance: -Update the compose file to include a new section for an ES instance: +`config/docker-compose.yaml` already ships an `elastic` service, and `config/consent.yaml` already +points at it (`servers: [elastic]`), so a normal `docker-compose up` gives you a local cluster with +no edits. I suggest changing the default bucket location so uploaded ontology files do not +interfere with other dev environments. -``` -es: - image: docker.elastic.co/elasticsearch/elasticsearch:5.5.0 - ports: - - "9200:9200" - volumes: - - ../data:/usr/share/elasticsearch/data - environment: - transport.host: 127.0.0.1 - xpack.security.enabled: "false" - http.host: 0.0.0.0 -``` +#### Running with X-Pack Security enabled -Add a line to the `app` section to link to that: +Security is **on by default**, which is what the security work (`GET /api/elasticSearch/capabilities` +and native DLS/FLS) needs. Two things make that work, and they have to agree: -``` - links: - - es:es -``` +* `ELASTIC_PASSWORD` in the compose `elastic` service bootstraps the `elastic` superuser. +* `authUser` / `authPassword` in `config/consent.yaml` are the credentials Consent sends. They are + harmless when security is off — the ES client only sends credentials in response to a 401 + challenge, which a security-disabled cluster never issues. -Finally, update the servers in consent.conf to point to this instance: +DLS and FLS are Platinum features, so the compose file self-generates a 30-day **trial** license via +`xpack.license.self_generated.type`. That setting only applies the first time a cluster forms. If +your `elastic` data volume predates it, the cluster keeps its `basic` license and DLS/FLS come back +`LICENSE_BLOCKED`; activate the trial once, by hand: +```bash +curl -u elastic:devpassword -XPOST 'localhost:9200/_license/start_trial?acknowledge=true' +curl -s -u elastic:devpassword localhost:9200/_license # expect "type": "trial" ``` -elasticSearch: - servers: - - es - indexName: local-ontology - datasetIndexName: datasetIName + +Note that transport SSL stays disabled. ES logs a bootstrap warning about it, which is expected and +correct here — transport SSL is only required for multi-node clusters. + +To get the old security-disabled cluster back for a run, without editing the committed file: + +```bash +ES_SECURITY_ENABLED=false docker-compose -p consent -f config/docker-compose.yaml up ``` -Consent will now point to a local ES instance. -I also suggest changing the default bucket location so uploaded -ontology files do not interfere with other dev environments. +Work that lives entirely at the application layer needs no security and is unaffected either way. ## How To... diff --git a/docs/plans/es-security-capability-record.md b/docs/plans/es-security-capability-record.md new file mode 100644 index 000000000..da1bb3b74 --- /dev/null +++ b/docs/plans/es-security-capability-record.md @@ -0,0 +1,364 @@ +# Elasticsearch Security Capability Record — Ticket A-1 + +Written record of the Elasticsearch security feature inventory for the three Consent +environments, plus the Epic D / Epic E decision that depends on it. + +Companion to +[`elasticsearch-service-duos-ui-usage.md`](elasticsearch-service-duos-ui-usage.md) (Ticket A-1). + +## How this record is produced + +One tool: [`GET /api/elasticSearch/capabilities`](../../src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java), +which reports the full inventory for whichever cluster a deployment is pointed at — inferred, or with +`writeProbes=true` proven. All it needs is an Admin token for that environment. + +Each environment already runs its own Consent deployment holding its own cluster credential, so that +token yields the per-environment record without anyone obtaining cluster network access or a copy of a +secret. Nothing is read from a secret store; the endpoint uses the credential its own deployment is +already configured with. + +Two earlier tools did the same job from outside the application and have been removed: a +`scripts/es-security-audit.sh` that reimplemented the whole verdict matrix in untested bash, and an +`ElasticSearchSecurityProbeTest` that drove the security APIs through the production +`ElasticSearchSupport.createRestClient` path but was inert unless `ES_PROBE_URL` was set, so it never +ran in CI and never guarded anything. `ElasticSearchCapabilityService` supersedes both: it makes the +same calls from inside the application, its verdict logic is unit-tested, and a successful response +from it in any environment is itself the client-compatibility evidence the probe test was written to +supply. Two implementations that can disagree about something as consequential as "is DLS enforced +here" are worse than one that is tested. + +To capture a report file, redirect the endpoint's JSON: + +```shell +curl -s -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + 'https:///api/elasticSearch/capabilities?writeProbes=true' \ + | tee "es-capability--$(date +%F).json" | jq +``` + +To measure a cluster no deployment points at — a new environment, or a throwaway container used as a +control — point a local Consent deployment's `elasticSearch` configuration block at it and call the +endpoint against that. + +### Running the capability endpoint + +```shell +# Read-only. Safe anywhere, but DLS/FLS/API-key verdicts are inferred from the license tier. +curl -s -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + https:///api/elasticSearch/capabilities | jq + +# Proven instead of inferred: creates and tears down a short-lived key and role. +curl -s -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + 'https:///api/elasticSearch/capabilities?writeProbes=true' | jq + +# Optionally probe run_as against a specific username rather than the credential's own principal +curl -s -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + 'https:///api/elasticSearch/capabilities?runAsUser=some-user' | jq +``` + +**Read-only mode** creates, modifies, and deletes nothing. That safety is what costs certainty: DLS, +FLS, and API-key support cannot be proven without writing, so they come back as `INFERRED_SUPPORTED` +/ `LICENSE_BLOCKED` reasoned from the license tier. Only `run_as` (a header on a read-only request) +and X-Pack Security itself are observed. + +**`writeProbes=true`** mints a short-lived API key and authenticates as it, creates a role carrying +both a DLS query and an FLS grant, then uses keys whose `role_descriptors` carry those filters +against the real dataset index to check the cluster *enforces* them: a `match_none` DLS key must +return zero of the documents the shared credential can see, and a key granting one field must return +only that field. That distinction is the whole point — a Basic-licensed cluster accepts a key +carrying a DLS descriptor at creation and fails only later at search time, which no license +inference can tell you and which this probe reproduces exactly. Everything created is namespaced +`duos-capability-probe-*` / `duos_dlsfls_probe_*`, expires within 10 minutes regardless, and is torn +down before the response returns; a teardown that fails is reported in `notes` rather than left for +you to find in the logs. + +Three fields carry most of the interpretive weight: + +- **`write_probes_run`** — read this first. It tells you whether the DLS/FLS/API-key verdicts below + are observations or inferences. +- **`cluster_privileges`** — what the deployment's *own* shared credential may do, which is the + constraint Epic D has to work within. If it holds neither `manage_security` nor `manage_api_key`, + the write probes cannot run and the report says so explicitly rather than reading their refusal as + a verdict against the native path (see the decision table below). +- **`security_settings`** — filtered to the dozen or so values that gate a capability, out of the + ~50 defaults a cluster reports. + +## Environment inventory + +### Local (`config/docker-compose.yaml`) — measured 2026-07-29 with `writeProbes=true` + +Ticket A-0 is closed: the compose file now sets `xpack.security.enabled` to **true** by default +(overridable per-run with `ES_SECURITY_ENABLED=false`) and self-generates a **trial** license, so the +security features are exercisable locally as shipped. The endpoint has now been run against the +running local cluster in write-probe mode, so every row below is an observation rather than an +inference — this is the first environment where all five capabilities came back `SUPPORTED`: + +| Capability | Verdict | Evidence | +| --- | --- | --- | +| Elasticsearch version | 9.4.4 | `GET /` → `version.number` | +| Distribution | elasticsearch (not OpenSearch) | `GET /` → `version.distribution` | +| Edition / license | Trial (Platinum-equivalent), `status: active`, expires 2026-08-28 | `GET /_license` → `type: trial` | +| X-Pack Security enabled | **`SUPPORTED`** | `GET /_xpack` 200; `GET /_security/_authenticate` 200 | +| DLS | **`SUPPORTED` — enforced, not merely accepted** | a `match_none` DLS key returned **0 of 1158** documents from `GET /dataset/_search` | +| FLS | **`SUPPORTED` — enforced** | a key granting only `datasetIdentifier` returned documents carrying only that field | +| API keys | **`SUPPORTED`** | key created (`POST /_security/api_key` 200), authenticated as `elastic`, invalidated | +| `run_as` | **`SUPPORTED`** | `es-security-runas-user: elastic` honoured, request resolved to `elastic` | +| Credential privileges | all six probed privileges true: `manage_security`, `manage_api_key`, `grant_api_key`, `manage_own_api_key`, `read_security`, `monitor` | `POST /_security/user/_has_privileges` as `elastic` (`superuser`) | +| Relevant cluster settings | `dls_fls.enabled=true`, `authc.api_key.enabled=true`, `authc.run_as.enabled=true`; `audit.enabled=false`, `authc.token.enabled=false`, both SSL layers off | `security_settings` in the report | +| `dataset` index | 1158 docs | non-empty, so the `match_none` DLS result means enforcement rather than an empty index | +| `elasticsearch-rest-client` (POM) | 9.4.4 | `rest_client_compatibility`: matches cluster major 9 | +| Recommendation | Epic D viable here, **observed** | probe role and keys carrying DLS/FLS descriptors accepted *and* enforced | + +Teardown behaved as documented: three short-lived keys and one probe role were created under the +`duos-capability-probe` / `duos_dlsfls_probe` names and removed again, with no teardown failure +reported in `notes`. + +One caveat survives the measurement: the trial license is 30 days. After expiry the cluster silently +drops to `basic` and DLS/FLS revert to `LICENSE_BLOCKED` — worth recognising as a license expiry +rather than reading as a regression in the feature work. + +Unlike the deployed environments, the local credential is the `elastic` superuser, so it holds the +`manage_security` / `manage_api_key` grants the write probes need. That makes local the one place the +probes are guaranteed *not* to be inconclusive — useful for exercising the probe path itself, and a +reminder that this clean local run says nothing about whether the shared credential in dev, staging, +or production can do the same. Read it as evidence that the *probe path and the cluster features* +work, not as a preview of the deployed rows. + +#### Notice: developers must update their own local configuration + +A local cluster does **not** pick these settings up on its own. `config/docker-compose.yaml` is +committed, but most people carry local edits to it (the bucket location, ports, memory limits) or run +a copy of their own, so a pull of this branch will not necessarily put these settings into the file +you actually start ES with. Each developer has to enable them in their own compose file before the +endpoint will report anything like the table above — and a local cluster that lags behind produces +`UNAVAILABLE` / `LICENSE_BLOCKED` verdicts that read like findings when they are only local drift. + +What has to be true in your `config/docker-compose.yaml` (and any personal copy or override file you +run instead of it): + +- `xpack.security.enabled=${ES_SECURITY_ENABLED:-true}` — without this the `/_security` API is absent + and every security verdict follows from that one fact. +- `xpack.license.self_generated.type=${ES_LICENSE_TYPE:-trial}` — a `basic` license leaves DLS and + FLS `LICENSE_BLOCKED`, so Epic D cannot be developed against. +- `ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-devpassword}`, matching `authUser` / `authPassword` in + `config/consent.yaml` — otherwise the deployment cannot authenticate at all. +- `xpack.security.transport.ssl.enabled=false` and `xpack.security.http.ssl.enabled=false` — keeps the + HTTP layer on plain `http` so consent's `protocol: http` client keeps working with security on. +- The image at `docker.elastic.co/elasticsearch/elasticsearch:9.4.4`, which is the version measured + above and the version of the pinned REST client. + +Two things that trip people up, both consequences of state that outlives a compose edit: + +- The self-generated license type only takes effect **the first time a cluster forms**. On an existing + `elastic` volume that already registered a `basic` license, editing the compose file changes + nothing; activate the trial once by hand: + + ```shell + curl -u elastic:devpassword -XPOST 'localhost:9200/_license/start_trial?acknowledge=true' + ``` + + Or discard the volume and let the cluster form fresh. +- `api_key.enabled`, `run_as.enabled`, and `dls_fls.enabled` are cluster defaults and need no compose + entry; if the report shows any of them false, something in your local setup has explicitly disabled + it. `audit.enabled=false` and `authc.token.enabled=false` are expected and gate nothing this work + needs. + +DEVNOTES.md ("Developing with a local Elastic Search instance") carries the full workflow, including +getting the old security-disabled cluster back for a run with `ES_SECURITY_ENABLED=false` — which +remains fine for Epics A–C and E, since none of them need security. + +### Control clusters (ES 9.3.3 and 9.4.4, security enabled) — measured 2026-07-28, 9.4.4 added 2026-07-29 + +Not a Consent environment. A throwaway container was run under both license tiers to +establish what each tier permits, so the deployed-environment results below can be read +against a known baseline — and so the verdict logic is validated in both directions rather +than only against a security-disabled cluster. + +The exercise was run twice, on **9.3.3** and again on **9.4.4** — the latter being both the local +cluster's version and the version of the REST client in `pom.xml`, so the client is now known to work +against a same-version cluster and not only across a minor-version gap. Every verdict below was +identical on the two versions, in both license tiers and in both endpoint modes; the table therefore +records one set of results rather than two. + +| Capability | Basic license | Trial (Platinum-equivalent) license | +| --- | --- | --- | +| X-Pack Security enabled | true | true | +| API keys | **supported** — created, authenticated, invalidated | **supported** | +| `run_as` | **supported** — header honoured, resolved to target user | **supported** | +| DLS | **blocked by license** — role with `indices[].query` rejected 403 | **proven end-to-end** — `match_none` API key returned 0 of 2 docs | +| FLS | **blocked by license** — role with `field_security` rejected 403 | **supported** — role accepted | + +The operative finding: **API keys and `run_as` are Basic-tier features; DLS and FLS are not.** +Epic D therefore has a license dependency that Epic E does not, and a Basic-licensed cluster +will accept an API key carrying a DLS role descriptor at creation time and only fail at +search time with a 403 — a failure mode worth designing around. + +#### The endpoint was validated against these same clusters + +The endpoint was run against the control clusters in both modes and under both license tiers, on both +9.3.3 and 9.4.4. Its read-only inferences agree with the tier-by-tier measurements above, and its own +write probes independently reproduce them — so the verdicts have been checked rather than trusted: + +| Capability | Basic, read-only | Basic, `writeProbes` | Trial, read-only | Trial, `writeProbes` | +| --- | --- | --- | --- | --- | +| X-Pack Security | `SUPPORTED` | `SUPPORTED` | `SUPPORTED` | `SUPPORTED` | +| API keys | `INFERRED_SUPPORTED` | `SUPPORTED` — created, authenticated, invalidated | `INFERRED_SUPPORTED` | `SUPPORTED` | +| DLS | `LICENSE_BLOCKED` | `LICENSE_BLOCKED` — key accepted, 403 at search | `INFERRED_SUPPORTED` | `SUPPORTED` — 0 of 2 docs through a `match_none` key | +| FLS | `LICENSE_BLOCKED` | `LICENSE_BLOCKED` — as above | `INFERRED_SUPPORTED` | `SUPPORTED` — only the granted field returned | +| `run_as` | `SUPPORTED` | `SUPPORTED` | `SUPPORTED` | `SUPPORTED` | +| Recommendation | Epic E | Epic E | Epic D | Epic D, observed | + +Two things worth recording from that exercise. First, the Basic write-probe run reproduced the exact +failure mode this document warns about: the API key carrying a DLS role descriptor was **accepted at +creation** and failed only at search time with `current license is non-compliant`. Read-only +inference and write probes reach the same verdict there by different routes, which is the strongest +form of agreement available. Second, teardown was verified from the cluster side, not just trusted: +after the runs, `GET /_security/role/duos_dlsfls_probe_*` returned `{}` and every +`duos-capability-probe-*` key showed `invalidated: true`. Both held on 9.4.4 as well, including the +Basic-tier accepted-then-403-at-search behaviour — so that failure mode is not an artefact of one +minor version. + +#### Running the probes as a least-privilege credential + +Because the deployed environments' shared credential is unlikely to hold `manage_security`, the +probes were also run as a purpose-built user with only `monitor` plus `read` on the indices — the +shape the real service credential is expected to have. All six probed cluster privileges came back +false except `monitor`, every write was refused, and the report's conclusion is the important part: + +> Inconclusive from the write probes: this deployment's credential is not permitted to create a role +> or an API key, so the DLS and FLS verdicts describe the credential rather than the cluster (see +> `cluster_privileges`). Re-run with a credential holding `manage_security` and `manage_api_key` to +> settle it. On the license alone: Epic D … is viable on this cluster … + +That distinction matters more than it looks: a privilege refusal tells you nothing about whether the +cluster licenses DLS, so it must not be recorded as a verdict against the native path. The endpoint +falls back to the license reading and says which of the two you are looking at. + +### `dev` — not yet measured + +> Call `GET /api/elasticSearch/capabilities?writeProbes=true` against dev with an Admin token and +> paste the findings here. Dev is the right place to run the write probes first in a *deployed* +> environment — teardown has been confirmed on the control clusters and locally, so what dev adds is +> confirmation under a real shared credential rather than a superuser one. + +| Capability | Verdict | Evidence | +| --- | --- | --- | +| Elasticsearch version | | | +| Distribution | | | +| Edition / license | | | +| X-Pack Security enabled | | | +| DLS | | | +| FLS | | | +| API keys | | | +| `run_as` | | | + +### `staging` — not yet measured + +| Capability | Verdict | Evidence | +| --- | --- | --- | +| Elasticsearch version | | | +| Distribution | | | +| Edition / license | | | +| X-Pack Security enabled | | | +| DLS | | | +| FLS | | | +| API keys | | | +| `run_as` | | | + +### `production` — not yet measured + +Call the endpoint read-only first — in that mode it creates nothing, so it cannot leave anything +behind on the production cluster. Only add `writeProbes=true` after the same call has been run in dev +and staging and the teardown has been confirmed there; the probes are designed to be safe in +production (namespaced, 10-minute expiry, torn down in a `finally`, failures reported in `notes`), +but production is not the place to find out. + +If production's shared credential turns out to lack `manage_security`, the write probes there will be +inconclusive by construction. That is a finding to record rather than a problem to work around: it +means Epic D's per-request key minting needs a privilege grant before it can work in production at +all. + +| Capability | Verdict | Evidence | +| --- | --- | --- | +| Elasticsearch version | | | +| Distribution | | | +| Edition / license | | | +| X-Pack Security enabled | | | +| DLS | | | +| FLS | | | +| API keys | | | +| `run_as` | | | + +## REST client compatibility — resolved + +`org.elasticsearch.client:elasticsearch-rest-client` **9.4.4** (`pom.xml:865-869`) is +compatible with every security API call the plan requires. Verified, not assumed: against live +security-enabled **9.3.3 and 9.4.4** clusters, a client built through the production +`ElasticSearchSupport.createRestClient` path successfully: + +1. issued `GET /_security/_authenticate` and `GET /_xpack`; +2. issued `POST /_security/api_key` carrying a `role_descriptors` block with both a DLS + `query` and an `field_security` grant; +3. authenticated a second `RestClient` as that API key; +4. invalidated the key via `DELETE /_security/api_key`. + +The 9.4.4 run is the more direct evidence of the two, since it pairs the pinned client with a cluster +of its own version; the 9.3.3 run additionally shows the transport tolerates a minor-version gap +between client and cluster, which is what a deployed environment on an older minor would present. + +No dependency change is needed. The low-level `RestClient` is a version-agnostic HTTP +transport with no typed request model, so security endpoints are reached with +`RestClient.performRequest(Request)` and a JSON entity — neither the high-level REST client +(removed in 8.x) nor the new typed Java API client is required. The one caveat is that this +holds for Elasticsearch; against OpenSearch there is no `POST /_security/api_key` at all, and +the endpoint flags that case explicitly. + +`ElasticSearchCapabilityService` is the standing demonstration of that conclusion, which is why no +separate feasibility test is kept: it drives the same security APIs from inside the application +through the injected `RestClient`, so a successful response from the endpoint in any environment is +itself evidence that the transport reaches `/_security` there. The report says as much in its +`rest_client_compatibility` field. + +## Decision + +**Pending** — blocked on the `dev`, `staging`, and `production` rows above. The decision rule, +fixed in advance so the measurement determines the outcome: + +| Measured state of the deployed clusters | Decision | +| --- | --- | +| Security enabled and DLS/FLS licensed in all three | **Epic D** (native DLS/FLS). Epic E only if a rollout-safety fallback is wanted. | +| Security enabled, license lacks DLS/FLS | **Epic E**, and raise the Platinum/Enterprise upgrade as a separate infra decision before committing to Epic D. | +| Security disabled anywhere | **Epic E** now; Epic D stays blocked on infra enabling X-Pack Security in that environment. | +| Environments disagree | **Both** — Epic E as the portable path, Epic D where licensed. The access contract from Ticket A-2 must be identical either way, so the enforcement layer stays swappable. | +| Write probes refused for lack of privileges | **Not a decision.** The probes measured the credential, not the cluster; fall back to the license reading and treat the missing `manage_security` / `manage_api_key` grant as its own prerequisite for Epic D. | +| DLS/FLS accepted but **not enforced** | **Epic E**, and treat it as a defect report to infra: a filter that is accepted and silently ignored is worse than one that is refused, and Epic D cannot be built on it. | + +Known so far: the local environment now falls in the **first** row — security enabled, DLS and FLS +licensed *and* observed enforced — which closes Ticket A-0 but decides nothing on its own. The rule +above is about the deployed clusters, and local's superuser credential is not the shared credential +any of them use. + +## Notes for whoever runs this against the deployed clusters + +- The shared `authUser` almost certainly does **not** hold `manage_security` or `manage_api_key`. + The report's `cluster_privileges` block says exactly which it has, via + `POST /_security/user/_has_privileges`. If it lacks them, that is itself a finding: Epic D's + per-request key minting needs at minimum `grant_api_key` (preferred, since it mints keys on + behalf of a user without full `manage_api_key`). +- Because the endpoint authenticates as the deployment's own configured credential, that block *is* + the shared credential's privileges — there is no way to accidentally record an admin's instead, + which is what Epic D actually has to work with at runtime. When the credential holds neither + `manage_api_key` nor `grant_api_key`, API keys come back `NOT_PERMITTED` rather than supported: + the distinction between "the cluster can" and "we can." +- The end-to-end DLS check needs a non-empty index. The endpoint uses the configured + `datasetIndexName` automatically, and says so explicitly when that index is empty or unreadable + rather than reporting a false pass — an empty index makes a `match_none` key return zero documents + for the wrong reason. +- Everything the write probes create is namespaced `duos-capability-probe-*` / + `duos_dlsfls_probe_*` and expires in 10 minutes. Teardown is in a `finally` block, and any + teardown failure is reported in the response `notes` rather than left in the server log. +- Every probe key carries a `role_descriptors` block, including the plain round-trip key, whose + descriptor grants nothing at all. A key created without one would instead inherit a snapshot of + the deployment credential's own permissions. +- Both DLS and FLS are checked for *enforcement*, not just acceptance: a `match_none` key must return + zero documents, and a key granting one field must return only that field. Acceptance alone would + pass on a cluster that stores the descriptor and ignores it. diff --git a/src/main/java/org/broadinstitute/consent/http/ConsentApplication.java b/src/main/java/org/broadinstitute/consent/http/ConsentApplication.java index da746d269..899c4b02c 100644 --- a/src/main/java/org/broadinstitute/consent/http/ConsentApplication.java +++ b/src/main/java/org/broadinstitute/consent/http/ConsentApplication.java @@ -61,6 +61,7 @@ import org.broadinstitute.consent.http.resources.DatasetResource; import org.broadinstitute.consent.http.resources.DocumentResource; import org.broadinstitute.consent.http.resources.DraftResource; +import org.broadinstitute.consent.http.resources.ElasticSearchCapabilityResource; import org.broadinstitute.consent.http.resources.EmailNotifierResource; import org.broadinstitute.consent.http.resources.FeatureFlagResource; import org.broadinstitute.consent.http.resources.InstitutionResource; @@ -170,6 +171,7 @@ public void run(ConsentConfiguration config, Environment env) { env.jersey().register(injector.getInstance(DatasetResource.class)); env.jersey().register(injector.getInstance(DocumentResource.class)); env.jersey().register(injector.getInstance(DraftResource.class)); + env.jersey().register(injector.getInstance(ElasticSearchCapabilityResource.class)); env.jersey().register(injector.getInstance(EmailNotifierResource.class)); env.jersey().register(injector.getInstance(FeatureFlagResource.class)); env.jersey().register(injector.getInstance(PublicFeatureFlagResource.class)); diff --git a/src/main/java/org/broadinstitute/consent/http/ConsentModule.java b/src/main/java/org/broadinstitute/consent/http/ConsentModule.java index 09fdd684c..ecec9d5d9 100644 --- a/src/main/java/org/broadinstitute/consent/http/ConsentModule.java +++ b/src/main/java/org/broadinstitute/consent/http/ConsentModule.java @@ -10,6 +10,7 @@ import io.dropwizard.jdbi3.JdbiFactory; import io.dropwizard.lifecycle.Managed; import jakarta.ws.rs.client.Client; +import java.io.IOException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -55,6 +56,7 @@ import org.broadinstitute.consent.http.service.DataAccessRequestService; import org.broadinstitute.consent.http.service.DatasetRegistrationService; import org.broadinstitute.consent.http.service.DatasetService; +import org.broadinstitute.consent.http.service.ElasticSearchCapabilityService; import org.broadinstitute.consent.http.service.ElasticSearchService; import org.broadinstitute.consent.http.service.ElectionService; import org.broadinstitute.consent.http.service.EmailService; @@ -90,6 +92,7 @@ import org.broadinstitute.consent.http.util.HttpClientUtil; import org.broadinstitute.consent.http.util.InstitutionUtil; import org.broadinstitute.consent.http.util.gson.GsonUtil; +import org.elasticsearch.client.RestClient; import org.jdbi.v3.core.Jdbi; import org.jdbi.v3.gson2.Gson2Config; import org.jdbi.v3.gson2.Gson2Plugin; @@ -361,19 +364,51 @@ private DataUseMatcherV4 providesDataUseMatcherV4(DataUseUtil dataUseUtil) { return new DataUseMatcherV4(dataUseUtil); } + /** + * The application's Elasticsearch client. A singleton because each {@link RestClient} owns its + * own connection pool and background threads: building one per consumer multiplies pools against + * the same cluster for no benefit. Closed on shutdown, since nothing else releases those + * connections. + */ + @Provides + @Singleton + private RestClient providesElasticSearchRestClient( + ElasticSearchConfiguration elasticSearchConfiguration) { + RestClient restClient = ElasticSearchSupport.createRestClient(elasticSearchConfiguration); + environment.lifecycle().manage(new ElasticSearchClientShutdown(restClient)); + return restClient; + } + + /** + * Releases the shared Elasticsearch client's connection pool and background threads on shutdown. + * A named type rather than an anonymous {@link Managed} so that it can be told apart from the + * module's other lifecycle registrations. + */ + record ElasticSearchClientShutdown(RestClient restClient) implements Managed { + + @Override + public void stop() throws IOException { + restClient.close(); + } + } + @Provides @Singleton private ElasticSearchService providesElasticSearchService( Jdbi jdbi, DatasetServiceDAO datasetServiceDAO, + RestClient esClient, ElasticSearchConfiguration elasticSearchConfiguration, OntologyService ontologyService) { return new ElasticSearchService( - jdbi, - datasetServiceDAO, - ElasticSearchSupport.createRestClient(elasticSearchConfiguration), - elasticSearchConfiguration, - ontologyService); + jdbi, datasetServiceDAO, esClient, elasticSearchConfiguration, ontologyService); + } + + @Provides + @Singleton + private ElasticSearchCapabilityService providesElasticSearchCapabilityService( + RestClient esClient, ElasticSearchConfiguration elasticSearchConfiguration) { + return new ElasticSearchCapabilityService(esClient, elasticSearchConfiguration); } @Provides diff --git a/src/main/java/org/broadinstitute/consent/http/models/elastic_search/CapabilityVerdict.java b/src/main/java/org/broadinstitute/consent/http/models/elastic_search/CapabilityVerdict.java new file mode 100644 index 000000000..d2a29937c --- /dev/null +++ b/src/main/java/org/broadinstitute/consent/http/models/elastic_search/CapabilityVerdict.java @@ -0,0 +1,43 @@ +package org.broadinstitute.consent.http.models.elastic_search; + +/** + * Outcome of a single Elasticsearch security capability probe. + * + *

The distinction between the {@code INFERRED_*} values and the rest matters: a report produced + * without write probes cannot prove any feature that requires creating something on the cluster (a + * role, an API key), so those are reported as inferred from the license and cluster settings rather + * than as observed fact. A write-probe run replaces them with observations. + * + *

The other distinction that matters is between the ways a feature can fail to be usable: {@link + * #LICENSE_BLOCKED} is the cluster's answer, {@link #NOT_PERMITTED} is this credential's, and + * {@link #UNKNOWN} means the probe reached no conclusion at all. Only the first is a fact about the + * cluster, so only the first should carry weight in an architectural decision. + */ +public enum CapabilityVerdict { + + /** Observed to work. */ + SUPPORTED, + + /** + * Observed not to work. Either the feature is absent — typically because security is disabled on + * the cluster — or it was accepted and then silently not applied, which an enforcement probe + * caught. The second case is the more serious: a filter that is accepted and ignored is worse + * than one that is refused, because nothing can be built on it. + */ + UNAVAILABLE, + + /** Present in the distribution but not included in the cluster's current license tier. */ + LICENSE_BLOCKED, + + /** Present and licensed, but the credential the application authenticates with may not use it. */ + NOT_PERMITTED, + + /** Expected to work based on license tier and cluster settings, but not proven by a probe. */ + INFERRED_SUPPORTED, + + /** Expected not to work based on license tier and cluster settings, but not proven by a probe. */ + INFERRED_UNAVAILABLE, + + /** The probe could not reach a conclusion. */ + UNKNOWN +} diff --git a/src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapability.java b/src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapability.java new file mode 100644 index 000000000..a59f263ad --- /dev/null +++ b/src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapability.java @@ -0,0 +1,12 @@ +package org.broadinstitute.consent.http.models.elastic_search; + +/** + * A single line of the Elasticsearch security capability inventory. + * + * @param name the capability, e.g. {@code Document-level security (DLS)} + * @param verdict whether the capability is available + * @param detail human-readable expansion of the verdict + * @param evidence the probe the verdict is drawn from, so a reader can re-run it by hand + */ +public record ElasticSearchCapability( + String name, CapabilityVerdict verdict, String detail, String evidence) {} diff --git a/src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapabilityReport.java b/src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapabilityReport.java new file mode 100644 index 000000000..5d60584a5 --- /dev/null +++ b/src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapabilityReport.java @@ -0,0 +1,52 @@ +package org.broadinstitute.consent.http.models.elastic_search; + +import com.google.gson.annotations.SerializedName; +import java.util.List; +import java.util.Map; + +/** + * The Elasticsearch security feature inventory for the cluster this deployment is configured + * against. + * + *

Because each environment runs its own Consent deployment pointed at its own cluster, calling + * the endpoint that produces this report in dev, staging, and production yields the per-environment + * record that Ticket A-1 requires. + * + * @param clusterName the cluster's own name, to confirm which cluster was reached + * @param version Elasticsearch version, e.g. {@code 9.3.3} + * @param distribution {@code elasticsearch} or {@code opensearch}; the security APIs differ + * @param edition OSS, Basic, Enterprise, Elastic Cloud, or OpenSearch + * @param licenseType license tier reported by the cluster + * @param licenseStatus whether that license is active + * @param elasticCloud whether the deployment is configured with a cloud ID + * @param securityEnabled whether X-Pack Security is on + * @param authenticatedUser the principal the shared credential resolves to + * @param authenticatedUserRoles that principal's roles + * @param clusterPrivileges cluster privileges the shared credential holds, from a read-only check + * @param securitySettings the cluster's {@code xpack.security.*} settings + * @param writeProbesRun whether write probes ran; when false the DLS, FLS, and API-key verdicts are + * inferred from the license tier rather than observed, which is the first thing a reader of + * this report needs to know + * @param capabilities the capability inventory itself + * @param restClientCompatibility whether the bundled REST client can drive the security APIs + * @param recommendation which implementation path the findings point to + * @param notes caveats a reader needs in order to interpret the report correctly + */ +public record ElasticSearchCapabilityReport( + @SerializedName("cluster_name") String clusterName, + String version, + String distribution, + String edition, + @SerializedName("license_type") String licenseType, + @SerializedName("license_status") String licenseStatus, + @SerializedName("elastic_cloud") Boolean elasticCloud, + @SerializedName("security_enabled") Boolean securityEnabled, + @SerializedName("authenticated_user") String authenticatedUser, + @SerializedName("authenticated_user_roles") List authenticatedUserRoles, + @SerializedName("cluster_privileges") Map clusterPrivileges, + @SerializedName("security_settings") Map securitySettings, + @SerializedName("write_probes_run") Boolean writeProbesRun, + List capabilities, + @SerializedName("rest_client_compatibility") String restClientCompatibility, + String recommendation, + List notes) {} diff --git a/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java b/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java new file mode 100644 index 000000000..07e414e04 --- /dev/null +++ b/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java @@ -0,0 +1,80 @@ +package org.broadinstitute.consent.http.resources; + +import com.google.inject.Inject; +import io.dropwizard.auth.Auth; +import jakarta.annotation.security.RolesAllowed; +import jakarta.ws.rs.DefaultValue; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import org.broadinstitute.consent.http.models.DuosUser; +import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchCapabilityReport; +import org.broadinstitute.consent.http.service.ElasticSearchCapabilityService; + +/** + * Admin-only view onto the Elasticsearch security feature inventory of whichever cluster this + * deployment is configured against. + * + *

Each environment runs its own Consent deployment against its own cluster, so calling this in + * dev, staging, and production produces the per-environment record without anyone needing direct + * network access to the clusters or a copy of their credentials — the application already holds + * them. + * + *

The probes behind this endpoint are read-only by default. Pass {@code writeProbes=true} to + * additionally create and tear down a short-lived API key and role, which is the only way to + * observe DLS, FLS, and API-key support rather than infer it from the license tier — see {@link + * ElasticSearchCapabilityService} for what each mode establishes. + */ +@Path("api/elasticSearch") +public class ElasticSearchCapabilityResource extends Resource { + + private final ElasticSearchCapabilityService capabilityService; + + @Inject + public ElasticSearchCapabilityResource(ElasticSearchCapabilityService capabilityService) { + this.capabilityService = capabilityService; + } + + /** + * Report the cluster's security capabilities: version, edition, X-Pack Security, DLS, FLS, API + * keys, and run_as. + * + * @param duosUser the authenticated admin + * @param runAsUser optional username to attempt the run_as probe against; defaults to the + * credential's own principal, which still establishes whether the feature is licensed + * @param writeProbes when true, create and tear down a short-lived API key and role so DLS, FLS, + * and API-key support are observed rather than inferred. Off by default: the caller has to + * ask for writes against the cluster their environment depends on. + * @return the capability report + */ + @GET + @Path("/capabilities") + @Produces(MediaType.APPLICATION_JSON) + @RolesAllowed({ADMIN}) + public Response getCapabilities( + @Auth DuosUser duosUser, + @QueryParam("runAsUser") String runAsUser, + @QueryParam("writeProbes") @DefaultValue("false") boolean writeProbes) { + try { + // Worth an audit trail either way: this reports on the cluster's security posture, and with + // write probes it also creates and removes credentials on that cluster. + Integer userId = duosUser.getUser().getUserId(); + if (writeProbes) { + logWarn( + "Elasticsearch capability report with WRITE PROBES requested by user %d — a short-lived " + .formatted(userId) + + "API key and role will be created and torn down"); + } else { + logInfo("Elasticsearch capability report requested by user %d".formatted(userId)); + } + ElasticSearchCapabilityReport report = + capabilityService.getCapabilityReport(runAsUser, writeProbes); + return Response.ok(report).build(); + } catch (Exception e) { + return createExceptionResponse(e); + } + } +} diff --git a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java new file mode 100644 index 000000000..9fe9a6239 --- /dev/null +++ b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java @@ -0,0 +1,1421 @@ +package org.broadinstitute.consent.http.service; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.inject.Inject; +import jakarta.ws.rs.HttpMethod; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.UUID; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; +import org.broadinstitute.consent.http.configurations.ElasticSearchConfiguration; +import org.broadinstitute.consent.http.models.elastic_search.CapabilityVerdict; +import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchCapability; +import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchCapabilityReport; +import org.broadinstitute.consent.http.util.ConsentLogger; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; + +/** + * Inventories the security features of the Elasticsearch cluster this deployment is configured + * against: version, edition, X-Pack Security, DLS, FLS, API keys, and {@code run_as}. + * + *

The default pass is non-destructive. Nothing is created, updated, or deleted, so it is + * safe to run anywhere — but DLS, FLS, and API keys cannot be *proven* without creating a role or a + * key, so a read-only pass reports those with an {@code INFERRED_} verdict reasoned from the + * license tier. Only {@code run_as} — whose probe is a header on a read-only request — and X-Pack + * Security itself come back as observed fact. + * + *

The single exception to "read-only" in the strict HTTP sense is {@code POST + * /_security/user/_has_privileges}, which is a POST only because it takes a request body; it + * evaluates the caller's privileges and changes nothing. + * + *

Write probes turn those inferences into observations, and must be asked for explicitly. + * They mint a short-lived API key, create a role carrying a DLS query and an FLS grant, and then + * use a key whose {@code role_descriptors} carry those same filters to check that the cluster + * actually enforces them against the real dataset index — which is Epic D's exact mechanism, and + * the one thing a license inference cannot tell you, since a Basic cluster accepts a key carrying a + * DLS descriptor and only fails later at search time. Everything created is namespaced, expires + * within {@value #PROBE_KEY_EXPIRATION}, and is torn down in a {@code finally} block. + * + *

Because each environment's deployment already holds its own cluster credential, running this + * with write probes in each environment produces the measured per-environment record without anyone + * needing cluster network access or a copy of a secret. + * + *

All calls go through the low-level {@link RestClient} using {@link Request}/{@link Response}, + * which is what {@code ElasticSearchSupport.createRestClient} builds. That this class works at all + * is itself the answer to whether the bundled {@code elasticsearch-rest-client} can drive the + * security APIs. + */ +public class ElasticSearchCapabilityService implements ConsentLogger { + + /** License tiers that include document- and field-level security. */ + private static final Set DLS_FLS_LICENSES = Set.of("platinum", "enterprise", "trial"); + + /** License tiers that include security, but not DLS/FLS. */ + private static final Set SECURITY_ONLY_LICENSES = Set.of("basic", "standard", "gold"); + + /** Cluster privileges worth knowing about before designing per-request credential minting. */ + private static final List PROBED_CLUSTER_PRIVILEGES = + List.of( + "manage_security", + "manage_api_key", + "grant_api_key", + "manage_own_api_key", + "read_security", + "monitor"); + + /** + * The cluster-default security settings worth reporting: each one either gates a capability this + * report covers, or describes the authentication posture a reader needs in order to interpret the + * verdicts. Everything else in the {@code xpack.security.*} default namespace is tuning. + */ + private static final Set REPORTED_SECURITY_DEFAULTS = + Set.of( + "xpack.security.enabled", + "xpack.security.dls_fls.enabled", + "xpack.security.authc.api_key.enabled", + "xpack.security.authc.run_as.enabled", + "xpack.security.authc.token.enabled", + "xpack.security.authc.anonymous.username", + "xpack.security.authc.anonymous.roles", + "xpack.security.authc.reserved_realm.enabled", + "xpack.security.audit.enabled", + "xpack.security.operator_privileges.enabled", + "xpack.security.fips_mode.enabled", + "xpack.security.http.ssl.enabled", + "xpack.security.transport.ssl.enabled"); + + private static final String AUTHENTICATE_PATH = "/_security/_authenticate"; + private static final String API_KEY_PATH = "/_security/api_key"; + private static final String RUN_AS_HEADER = "es-security-runas-user"; + + /** + * A caller-supplied username goes into an HTTP header, and the transport does not validate header + * values, so a value carrying CR or LF could append arbitrary requests to the one this service + * sends on the deployment's own credential. Restricted to the characters an Elasticsearch + * username actually uses; anything else is refused before it reaches the cluster. + */ + private static final Pattern RUN_AS_USERNAME = Pattern.compile("[A-Za-z0-9._@+\\-]{1,255}"); + + private static final String API_KEYS = "API keys"; + private static final String DLS = "Document-level security (DLS)"; + private static final String FLS = "Field-level security (FLS)"; + private static final String RUN_AS = "run_as impersonation"; + + /** How long a probe API key lives even if teardown never runs. */ + private static final String PROBE_KEY_EXPIRATION = "10m"; + + /** The field a probe FLS grant is scoped to; a real field of the dataset index. */ + private static final String FLS_GRANT_FIELD = "datasetIdentifier"; + + /** + * The role descriptor every probe key that does not need privileges is created with. Omitting + * {@code role_descriptors} would instead give the key a snapshot of the deployment credential's + * own permissions — on a cluster where that credential is broadly privileged, a live copy of it. + * An empty descriptor grants nothing, and {@code GET /_security/_authenticate} needs nothing, so + * the round trip is proven just as well by a key that can do nothing else. + */ + private static final String PRIVILEGE_FREE_DESCRIPTOR = + """ + {"probe":{"cluster":[],"indices":[]}}"""; + + /** + * Builds a client that authenticates as an API key instead of the deployment's shared credential. + * A seam rather than an inline {@code RestClient.builder} call so the write probes can be tested + * without a live cluster. + */ + public interface ApiKeyClientFactory { + + RestClient create(String encodedApiKey); + } + + private final RestClient esClient; + private final ElasticSearchConfiguration esConfig; + private final ApiKeyClientFactory apiKeyClientFactory; + + @Inject + public ElasticSearchCapabilityService(RestClient esClient, ElasticSearchConfiguration esConfig) { + this(esClient, esConfig, defaultApiKeyClientFactory(esClient)); + } + + ElasticSearchCapabilityService( + RestClient esClient, + ElasticSearchConfiguration esConfig, + ApiKeyClientFactory apiKeyClientFactory) { + this.esClient = esClient; + this.esConfig = esConfig; + this.apiKeyClientFactory = apiKeyClientFactory; + } + + /** + * Points a second client at the same host the injected client uses, so an API key is exercised + * against the same cluster — including when the deployment is configured by cloud ID, which the + * injected client has already resolved to a host. + */ + private static ApiKeyClientFactory defaultApiKeyClientFactory(RestClient esClient) { + return encodedApiKey -> + RestClient.builder(esClient.getNodes().get(0).getHost()) + .setDefaultHeaders( + new Header[] {new BasicHeader("Authorization", "ApiKey " + encodedApiKey)}) + .build(); + } + + /** + * Builds the capability report. + * + * @param runAsUser username to attempt impersonation against; when null the probe targets the + * authenticated user itself, which still proves whether the feature is licensed and enabled + * @param writeProbes when true, additionally create and tear down a short-lived API key and role + * to convert the inferred DLS, FLS, and API-key verdicts into observed ones + * @return the inventory for this deployment's cluster + */ + public ElasticSearchCapabilityReport getCapabilityReport(String runAsUser, boolean writeProbes) { + List notes = new ArrayList<>(); + if (!writeProbes) { + notes.add( + "All probes were non-destructive. Nothing was created, modified, or deleted on the " + + "cluster."); + } + + ProbeResult root = probe(HttpMethod.GET, "/"); + if (root.status() == 0) { + return unreachableReport(notes); + } + + String version = string(root.body(), "version", "number"); + String distribution = stringOrDefault(root.body(), "elasticsearch", "version", "distribution"); + boolean isOpenSearch = "opensearch".equalsIgnoreCase(distribution); + + ProbeResult xpack = probe(HttpMethod.GET, "/_xpack"); + ProbeResult license = probe(HttpMethod.GET, "/_license"); + String licenseType = string(license.body(), "license", "type"); + String licenseStatus = string(license.body(), "license", "status"); + + Map securitySettings = securitySettings(); + Boolean securityEnabled = securityEnabled(xpack, securitySettings); + + ProbeResult authenticate = probe(HttpMethod.GET, AUTHENTICATE_PATH); + boolean securityApiPresent = securityApiPresent(authenticate.status()); + String authenticatedUser = string(authenticate.body(), "username"); + List roles = stringList(authenticate.body(), "roles"); + + Map clusterPrivileges = securityApiPresent ? clusterPrivileges() : Map.of(); + + boolean elasticCloud = esConfig.getCloudId() != null && !esConfig.getCloudId().trim().isEmpty(); + if (elasticCloud) { + notes.add( + "This deployment is configured with a cloud ID, so the cluster is Elastic Cloud and " + + "X-Pack Security is always present."); + } + if (isOpenSearch) { + notes.add( + "This cluster is OpenSearch, not Elasticsearch. The X-Pack security APIs do not exist " + + "here: DLS/FLS come from the OpenSearch security plugin under /_plugins/_security, " + + "and there is no POST /_security/api_key."); + } + boolean writeProbesRan = writeProbes && securityApiPresent && !isOpenSearch; + if (!securityApiPresent) { + notes.add( + "The /_security API is not available on this cluster, so no security feature can be " + + "exercised. Every security verdict below follows from that one fact."); + } else if (!writeProbesRan) { + notes.add( + "DLS, FLS, and API-key verdicts are inferred from the license tier and cluster " + + "settings. Re-run with writeProbes=true to create and tear down a short-lived key " + + "and role and observe them instead."); + } + if (writeProbes && !writeProbesRan) { + notes.add( + "Write probes were requested but not run: they need the /_security API on an " + + "Elasticsearch distribution."); + } + + WriteProbeOutcome writeProbeOutcome = writeProbesRan ? runWriteProbes(notes) : null; + + List capabilities = new ArrayList<>(); + capabilities.add(securityCapability(securityEnabled, securityApiPresent, xpack)); + capabilities.add( + writeProbeOutcome != null + ? writeProbeOutcome.apiKeys() + : apiKeyCapability( + securityApiPresent, isOpenSearch, securitySettings, clusterPrivileges)); + capabilities.add( + writeProbeOutcome != null + ? writeProbeOutcome.dls() + : dlsFlsCapability( + DLS, securityApiPresent, isOpenSearch, licenseType, securitySettings)); + capabilities.add( + writeProbeOutcome != null + ? writeProbeOutcome.fls() + : dlsFlsCapability( + FLS, securityApiPresent, isOpenSearch, licenseType, securitySettings)); + capabilities.add(runAsCapability(securityApiPresent, authenticatedUser, runAsUser)); + + return new ElasticSearchCapabilityReport( + string(root.body(), "cluster_name"), + version, + distribution, + edition( + elasticCloud, + isOpenSearch, + string(root.body(), "version", "build_flavor"), + xpack.status(), + licenseType), + licenseType, + licenseStatus, + elasticCloud, + securityEnabled, + authenticatedUser, + roles, + clusterPrivileges, + securitySettings, + writeProbesRan, + capabilities, + restClientCompatibility(version, isOpenSearch), + recommendation(securityApiPresent, isOpenSearch, licenseType, writeProbeOutcome), + notes); + } + + // --------------------------------------------------------------------------- + // Capability evaluation + // --------------------------------------------------------------------------- + + private ElasticSearchCapability securityCapability( + Boolean securityEnabled, boolean securityApiPresent, ProbeResult xpack) { + boolean enabled = Boolean.TRUE.equals(securityEnabled); + CapabilityVerdict verdict = + enabled && securityApiPresent ? CapabilityVerdict.SUPPORTED : CapabilityVerdict.UNAVAILABLE; + // The two signals can disagree — a cluster can report the feature as enabled while the + // /_security API is unreachable, say behind a proxy that strips it — and when they do, the + // detail has to say which, rather than repeating the verdict's own conclusion. + String detail; + if (enabled && securityApiPresent) { + detail = "X-Pack Security is enabled and the /_security API responds."; + } else if (enabled) { + detail = + "X-Pack Security reports itself enabled, but the /_security API did not respond as a " + + "security-enabled cluster would, so no security feature could be exercised."; + } else { + detail = "X-Pack Security is not enabled on this cluster."; + } + return new ElasticSearchCapability( + "X-Pack Security", + verdict, + detail, + "GET /_xpack (status %d), GET %s".formatted(xpack.status(), AUTHENTICATE_PATH)); + } + + private ElasticSearchCapability apiKeyCapability( + boolean securityApiPresent, + boolean isOpenSearch, + Map settings, + Map privileges) { + if (isOpenSearch) { + return new ElasticSearchCapability( + API_KEYS, + CapabilityVerdict.UNAVAILABLE, + "OpenSearch has no POST /_security/api_key endpoint.", + "GET / -> version.distribution"); + } + if (!securityApiPresent) { + return new ElasticSearchCapability( + API_KEYS, + CapabilityVerdict.UNAVAILABLE, + "Security is disabled, so API keys cannot be issued.", + "GET " + AUTHENTICATE_PATH); + } + if ("false".equals(settings.get("xpack.security.authc.api_key.enabled"))) { + return new ElasticSearchCapability( + API_KEYS, + CapabilityVerdict.UNAVAILABLE, + "API keys are explicitly disabled by cluster setting.", + "xpack.security.authc.api_key.enabled=false"); + } + // API keys are a Basic-tier feature, so a security-enabled cluster is expected to support + // them regardless of license. What actually gates Consent is whether the shared credential + // may mint them. + boolean canMint = + Boolean.TRUE.equals(privileges.get("manage_api_key")) + || Boolean.TRUE.equals(privileges.get("grant_api_key")); + if (!privileges.isEmpty() && !canMint) { + return new ElasticSearchCapability( + API_KEYS, + CapabilityVerdict.NOT_PERMITTED, + "The cluster supports API keys, but the credential this deployment authenticates with " + + "holds neither manage_api_key nor grant_api_key, so it cannot mint per-request keys.", + "POST /_security/user/_has_privileges"); + } + return new ElasticSearchCapability( + API_KEYS, + CapabilityVerdict.INFERRED_SUPPORTED, + "Security is enabled and API keys are a Basic-tier feature, so key creation is expected " + + "to work. Not proven: creating a key is a write.", + "xpack.security.authc.api_key.enabled=%s; POST /_security/user/_has_privileges" + .formatted(settings.getOrDefault("xpack.security.authc.api_key.enabled", "not-set"))); + } + + private ElasticSearchCapability dlsFlsCapability( + String name, + boolean securityApiPresent, + boolean isOpenSearch, + String licenseType, + Map settings) { + if (isOpenSearch) { + return new ElasticSearchCapability( + name, + CapabilityVerdict.UNKNOWN, + "OpenSearch provides DLS/FLS through its security plugin, which this probe does not " + + "inspect. Check /_plugins/_security instead.", + "GET / -> version.distribution"); + } + if (!securityApiPresent) { + return new ElasticSearchCapability( + name, + CapabilityVerdict.UNAVAILABLE, + "Security is disabled, so no role can carry a DLS query or an FLS grant.", + "GET " + AUTHENTICATE_PATH); + } + String license = licenseType == null ? "" : licenseType.toLowerCase(); + String dlsFlsSetting = settings.getOrDefault("xpack.security.dls_fls.enabled", "not-set"); + if (DLS_FLS_LICENSES.contains(license)) { + return new ElasticSearchCapability( + name, + CapabilityVerdict.INFERRED_SUPPORTED, + "The '%s' license includes DLS/FLS and xpack.security.dls_fls.enabled=%s." + .formatted(licenseType, dlsFlsSetting), + "GET /_license; GET /_cluster/settings"); + } + if (SECURITY_ONLY_LICENSES.contains(license)) { + return new ElasticSearchCapability( + name, + CapabilityVerdict.LICENSE_BLOCKED, + "The '%s' license does not include DLS/FLS. A role or API key carrying a DLS query is " + .formatted(licenseType) + + "rejected, and note that a key with a DLS role descriptor is accepted at creation " + + "and only fails at search time.", + "GET /_license"); + } + return new ElasticSearchCapability( + name, + CapabilityVerdict.UNKNOWN, + "License tier '%s' could not be mapped to a DLS/FLS entitlement.".formatted(licenseType), + "GET /_license"); + } + + /** + * The one security feature that can be proven without a write: {@code run_as} is requested with a + * header on an otherwise read-only call. + */ + private ElasticSearchCapability runAsCapability( + boolean securityApiPresent, String authenticatedUser, String requestedUser) { + if (!securityApiPresent) { + return new ElasticSearchCapability( + RUN_AS, + CapabilityVerdict.UNAVAILABLE, + "Security is disabled, so impersonation is not available.", + "GET " + AUTHENTICATE_PATH); + } + String target = + requestedUser != null && !requestedUser.isBlank() ? requestedUser : authenticatedUser; + if (target == null || target.isBlank()) { + return new ElasticSearchCapability( + RUN_AS, + CapabilityVerdict.UNKNOWN, + "No target user was available to attempt impersonation against.", + "GET " + AUTHENTICATE_PATH); + } + if (!RUN_AS_USERNAME.matcher(target).matches()) { + // The target ends up in a request header, so it is checked before it is sent rather than + // relying on the transport to reject it. + return new ElasticSearchCapability( + RUN_AS, + CapabilityVerdict.UNKNOWN, + "The requested run_as username is not a valid Elasticsearch username, so no request was " + + "sent. Expected only letters, digits, and the characters . _ @ + -", + "no request issued"); + } + + ProbeResult result = + probe(HttpMethod.GET, AUTHENTICATE_PATH, null, Map.of(RUN_AS_HEADER, target)); + String evidence = + "GET %s with %s: %s -> %d" + .formatted(AUTHENTICATE_PATH, RUN_AS_HEADER, target, result.status()); + + if (result.status() == 200) { + String resolved = string(result.body(), "username"); + if (target.equals(resolved)) { + return new ElasticSearchCapability( + RUN_AS, + CapabilityVerdict.SUPPORTED, + "The cluster honoured the run_as header and resolved the request to '%s'." + .formatted(resolved), + evidence); + } + return new ElasticSearchCapability( + RUN_AS, + CapabilityVerdict.UNKNOWN, + "The run_as header was accepted but the request still resolved to '%s'." + .formatted(resolved), + evidence); + } + if (result.status() == 403) { + return new ElasticSearchCapability( + RUN_AS, + refusalVerdict(result), + "Impersonation was refused: " + reason(result.body()), + evidence); + } + return new ElasticSearchCapability( + RUN_AS, + CapabilityVerdict.UNKNOWN, + "Impersonation probe returned an unexpected status. " + reason(result.body()), + evidence); + } + + // --------------------------------------------------------------------------- + // Write probes + // --------------------------------------------------------------------------- + + /** What a write-probe pass established, replacing the inferred verdicts for those features. */ + private record WriteProbeOutcome( + ElasticSearchCapability apiKeys, ElasticSearchCapability dls, ElasticSearchCapability fls) { + + /** Whether DLS came back usable, which is the pivot the Epic D / Epic E decision turns on. */ + boolean dlsUsable() { + return dls.verdict() == CapabilityVerdict.SUPPORTED; + } + + /** + * Whether the probes were stopped by the credential's privileges rather than by the cluster's + * capability. In that case the DLS and FLS verdicts say nothing about what the cluster can do, + * so nothing about Epic D can be concluded from them — a distinction that matters because the + * shared service credential is unlikely to hold {@code manage_security}. + */ + boolean credentialBlocked() { + return dls.verdict() == CapabilityVerdict.NOT_PERMITTED; + } + } + + /** + * Creates and then removes a short-lived API key and role in order to observe what the read-only + * probes can only infer. Every resource is namespaced {@code duos-capability-probe-*} / {@code + * duos_dlsfls_probe_*}, keys expire in {@value #PROBE_KEY_EXPIRATION} even if teardown never + * runs, and teardown is in a {@code finally} so it also fires when a probe throws. + * + *

Ordered cheapest-first so a credential that cannot mint keys at all fails on the first step + * rather than part-way through. + */ + private WriteProbeOutcome runWriteProbes(List notes) { + // Unique per run rather than per millisecond: two admins probing at once must not generate the + // same role name, or the second overwrites the first's role and both try to delete it. + String stamp = + "%d-%s".formatted(System.currentTimeMillis(), UUID.randomUUID().toString().substring(0, 8)); + String index = probeIndex(); + List createdKeyIds = new ArrayList<>(); + String roleName = "duos_dlsfls_probe_" + stamp; + boolean roleCreated = false; + + logInfo("Elasticsearch capability write probes starting against index " + index); + + try { + ElasticSearchCapability apiKeys = apiKeyRoundTripProbe(stamp, createdKeyIds); + // Run regardless of the key probe's outcome: role acceptance answers the license question on + // its own, and is the only DLS/FLS signal available to a credential that cannot mint keys. + RoleAcceptance acceptance = dlsFlsRoleProbe(roleName, index); + roleCreated = acceptance.created(); + + ElasticSearchCapability dls = acceptance.capability(DLS); + ElasticSearchCapability fls = acceptance.capability(FLS); + + // Enforcement needs a key to search with and documents to compare against; without either, + // the verdicts stand at role acceptance and the report has to say so rather than imply more. + if (apiKeys.verdict() != CapabilityVerdict.SUPPORTED) { + notes.add( + "No usable probe key, so DLS and FLS enforcement could not be observed end to end; " + + "those verdicts reflect whether the cluster accepted a role carrying the filters, " + + "not whether it applies them."); + } else { + long baseline = documentCount(index); + if (baseline < 0) { + notes.add( + "Index '%s' was not readable, so DLS and FLS enforcement could not be observed " + .formatted(index) + + "end to end; the verdicts reflect role acceptance only."); + } else if (baseline == 0) { + notes.add( + "Index '%s' is empty, so DLS and FLS enforcement could not be observed end to end; " + .formatted(index) + + "the verdicts reflect role acceptance only."); + } else { + Optional dlsObserved = + dlsEnforcementProbe(stamp, index, baseline, createdKeyIds); + if (dlsObserved.isEmpty()) { + notes.add( + "The DLS enforcement check could not be run to a conclusion, so the DLS verdict " + + "reflects role acceptance only."); + } + dls = dlsObserved.orElse(dls); + + Optional flsObserved = + flsProjectionProbe(stamp, index, createdKeyIds); + if (flsObserved.isEmpty()) { + notes.add( + "The FLS projection check returned no document fields to inspect (the index may " + + "not store _source, or its documents may not carry '%s'), so the FLS verdict " + .formatted(FLS_GRANT_FIELD) + + "reflects role acceptance only."); + } + fls = flsObserved.orElse(fls); + } + } + notes.add(writeProbeNote(createdKeyIds.size(), roleCreated)); + return new WriteProbeOutcome(apiKeys, dls, fls); + } finally { + tearDownProbeResources(createdKeyIds, roleCreated ? roleName : null, notes); + } + } + + /** + * Describes what the pass actually did. Worth getting exactly right: a run in which the cluster + * refused everything created nothing, and a note claiming otherwise would misrepresent both what + * happened to the cluster and what the verdicts mean. + */ + private String writeProbeNote(int keysCreated, boolean roleCreated) { + if (keysCreated == 0 && !roleCreated) { + return "Write probes were attempted but the cluster accepted nothing, so nothing was created " + + "or removed. Where the refusals were on privilege grounds, the verdicts below describe " + + "this deployment's credential rather than the cluster's capability — cluster_privileges " + + "shows what it is missing."; + } + return "Write probes ran: %d short-lived API key(s)%s were created under the " + .formatted(keysCreated, roleCreated ? " and one probe role" : "") + + "duos-capability-probe / duos_dlsfls_probe names and removed again. The DLS, FLS, and " + + "API-key verdicts below are observed rather than inferred."; + } + + /** + * The plain API-key lifecycle: mint a key, authenticate as it, and let teardown invalidate it. + * Authenticating is the part that matters — a key that cannot authenticate is no use as a + * per-request credential. + */ + private ElasticSearchCapability apiKeyRoundTripProbe(String stamp, List createdKeyIds) { + KeyCreation key = + createProbeKey("duos-capability-probe-" + stamp, PRIVILEGE_FREE_DESCRIPTOR, createdKeyIds); + String evidence = "POST %s -> %d".formatted(API_KEY_PATH, key.status()); + + if (!created(key.response())) { + return new ElasticSearchCapability( + API_KEYS, + refusalVerdict(key.response()), + "The cluster refused to create an API key: " + reason(key.response().body()), + evidence); + } + if (key.encoded() == null) { + return new ElasticSearchCapability( + API_KEYS, + CapabilityVerdict.UNKNOWN, + "The key was created but the response carried neither an encoded form nor an id and " + + "secret to build one from.", + evidence); + } + + ProbeResult asKey = probeAsApiKey(key.encoded(), HttpMethod.GET, AUTHENTICATE_PATH); + if (asKey.status() == 200) { + return new ElasticSearchCapability( + API_KEYS, + CapabilityVerdict.SUPPORTED, + "Observed: a key was created, authenticated as '%s', and invalidated." + .formatted(string(asKey.body(), "username")), + evidence + "; GET %s as the key -> 200".formatted(AUTHENTICATE_PATH)); + } + return new ElasticSearchCapability( + API_KEYS, + CapabilityVerdict.UNKNOWN, + "The key was created but could not authenticate: " + reason(asKey.body()), + evidence + "; GET %s as the key -> %d".formatted(AUTHENTICATE_PATH, asKey.status())); + } + + /** Whether the cluster accepted a role carrying a DLS query and an FLS grant, and why not. */ + private record RoleAcceptance( + boolean created, CapabilityVerdict verdict, String detail, String evidence) { + + ElasticSearchCapability capability(String name) { + return new ElasticSearchCapability(name, verdict, detail, evidence); + } + } + + /** + * Creating a role that carries both {@code query} and {@code field_security} is the cleanest + * license check there is: a cluster whose license excludes DLS/FLS rejects it outright with an + * explicit message, where an API key carrying the same descriptors would be accepted and only + * fail later at search time. + */ + private RoleAcceptance dlsFlsRoleProbe(String roleName, String index) { + String path = "/_security/role/" + roleName; + String body = + """ + {"indices":[{"names":["%s"],"privileges":["read"],\ + "query":{"term":{"accessPolicy.probe":"capability-probe"}},\ + "field_security":{"grant":["%s"],"except":[]}}]}""" + .formatted(index, FLS_GRANT_FIELD); + + ProbeResult result = probe(HttpMethod.PUT, path, body, Map.of()); + String evidence = + "PUT %s carrying query and field_security -> %d".formatted(path, result.status()); + + if (created(result)) { + return new RoleAcceptance( + true, + CapabilityVerdict.SUPPORTED, + "Observed: the cluster accepted a role carrying both a DLS query and an FLS grant.", + evidence); + } + CapabilityVerdict verdict = refusalVerdict(result); + String detail = + verdict == CapabilityVerdict.LICENSE_BLOCKED + ? "The license does not permit it: " + reason(result.body()) + : "The role was rejected: " + reason(result.body()); + return new RoleAcceptance(false, verdict, detail, evidence); + } + + /** + * The end-to-end check, and the one that exercises Epic D's exact mechanism: a per-request API + * key whose {@code role_descriptors} carry the DLS filter. A {@code match_none} filter must + * return zero of the {@code baseline} documents the shared credential can see. Anything else + * means the filter was accepted but not enforced — a far worse outcome than an honest refusal, + * and the whole reason this probe exists. + */ + private Optional dlsEnforcementProbe( + String stamp, String index, long baseline, List createdKeyIds) { + String descriptor = + """ + {"dls_probe":{"indices":[{"names":["%s"],"privileges":["read"],\ + "query":{"match_none":{}}}]}}""" + .formatted(index); + EnforcementAttempt attempt = + attemptEnforcement( + DLS, + "DLS", + "duos-capability-probe-dls-" + stamp, + descriptor, + "a DLS role_descriptor", + "a match_none DLS key", + index, + createdKeyIds); + if (attempt.settled() != null) { + return Optional.of(attempt.settled()); + } + if (attempt.search() == null) { + return Optional.empty(); + } + + long visible = hitCount(attempt.search()); + String evidence = + "%s; %d of %d documents visible".formatted(attempt.evidence(), visible, baseline); + if (visible < 0) { + // Do not read an unrecognised response shape as a failure to enforce: "not enforced" is the + // most serious verdict this report can return, and it has to mean what it says. + return Optional.of( + new ElasticSearchCapability( + DLS, + CapabilityVerdict.UNKNOWN, + "The search succeeded but its response carried no hit total, so whether the filter " + + "was enforced could not be established.", + evidence)); + } + if (visible == 0) { + return Optional.of( + new ElasticSearchCapability( + DLS, + CapabilityVerdict.SUPPORTED, + "Proven end to end: a match_none DLS key returned 0 of %d documents." + .formatted(baseline), + evidence)); + } + return Optional.of( + new ElasticSearchCapability( + DLS, + CapabilityVerdict.UNAVAILABLE, + "Not enforced: a match_none DLS key still returned %d of %d documents. The filter was " + .formatted(visible, baseline) + + "accepted but had no effect, so DLS cannot be relied on here.", + evidence)); + } + + /** + * The FLS counterpart: a key granting exactly one field must return documents carrying only that + * field. Role acceptance alone does not establish that the projection is applied. + */ + private Optional flsProjectionProbe( + String stamp, String index, List createdKeyIds) { + String descriptor = + """ + {"fls_probe":{"indices":[{"names":["%s"],"privileges":["read"],\ + "field_security":{"grant":["%s"]}}]}}""" + .formatted(index, FLS_GRANT_FIELD); + EnforcementAttempt attempt = + attemptEnforcement( + FLS, + "FLS", + "duos-capability-probe-fls-" + stamp, + descriptor, + "an FLS role_descriptor", + "a key granting only '%s'".formatted(FLS_GRANT_FIELD), + index, + createdKeyIds); + if (attempt.settled() != null) { + return Optional.of(attempt.settled()); + } + if (attempt.search() == null) { + return Optional.empty(); + } + + Set fields = firstHitSourceFields(attempt.search()); + if (fields.isEmpty()) { + return Optional.empty(); + } + if (fields.equals(Set.of(FLS_GRANT_FIELD))) { + return Optional.of( + new ElasticSearchCapability( + FLS, + CapabilityVerdict.SUPPORTED, + "Proven end to end: documents came back carrying only the granted '%s' field." + .formatted(FLS_GRANT_FIELD), + attempt.evidence())); + } + return Optional.of( + new ElasticSearchCapability( + FLS, + CapabilityVerdict.UNAVAILABLE, + "Not enforced: a key granting only '%s' returned documents carrying %s. The grant was " + .formatted(FLS_GRANT_FIELD, fields) + + "accepted but had no effect, so FLS cannot be relied on here.", + attempt.evidence())); + } + + /** + * What an enforcement attempt reached before interpretation: either a verdict the attempt itself + * settled — a refusal at key creation, or a failure at search time — or a successful search + * response for the caller to read the filter's effect out of, or neither, when the attempt could + * not be run to a conclusion and the caller must fall back to role acceptance. + */ + private record EnforcementAttempt( + ElasticSearchCapability settled, JsonObject search, String evidence) { + + static EnforcementAttempt settled(ElasticSearchCapability capability) { + return new EnforcementAttempt(capability, null, null); + } + + static EnforcementAttempt inconclusive() { + return new EnforcementAttempt(null, null, null); + } + + static EnforcementAttempt searched(JsonObject body, String evidence) { + return new EnforcementAttempt(null, body, evidence); + } + } + + /** + * The half the DLS and FLS enforcement checks share: mint a key carrying the filter under test, + * then search the real index through it. Only the reading of a successful response differs + * between the two, so only that is left to the callers. + * + * @param descriptorLabel how the {@code role_descriptors} block is described in evidence + * @param keyLabel how the key is described in evidence, e.g. {@code a match_none DLS key} + */ + private EnforcementAttempt attemptEnforcement( + String name, + String shortName, + String keyName, + String descriptor, + String descriptorLabel, + String keyLabel, + String index, + List createdKeyIds) { + KeyCreation key = createProbeKey(keyName, descriptor, createdKeyIds); + if (!created(key.response())) { + // On a license-blocked cluster the key may be refused here rather than at search time. + return EnforcementAttempt.settled( + new ElasticSearchCapability( + name, + refusalVerdict(key.response()), + "A key carrying %s was refused: ".formatted(descriptorLabel) + + reason(key.response().body()), + "POST %s with %s -> %d".formatted(API_KEY_PATH, descriptorLabel, key.status()))); + } + if (key.encoded() == null) { + return EnforcementAttempt.inconclusive(); + } + + String searchPath = "/%s/_search?size=1".formatted(index); + ProbeResult search = probeAsApiKey(key.encoded(), HttpMethod.GET, searchPath); + String evidence = "GET %s through %s -> %d".formatted(searchPath, keyLabel, search.status()); + return search.status() == 200 + ? EnforcementAttempt.searched(search.body(), evidence) + : EnforcementAttempt.settled(searchFailure(name, shortName, search, evidence)); + } + + /** + * A probe key's creation response paired with its encoded form, which is null when the cluster + * refused the key or returned nothing to build one from. + */ + private record KeyCreation(ProbeResult response, String encoded) { + + int status() { + return response.status(); + } + } + + /** + * Creates a short-lived probe key and records its id so teardown removes it. Every probe key goes + * through here, so the namespaced name, the expiry, and the teardown registration cannot be + * forgotten on one path and not another. + */ + private KeyCreation createProbeKey( + String keyName, String roleDescriptors, List createdKeyIds) { + String body = + """ + {"name":"%s","expiration":"%s","role_descriptors":%s}""" + .formatted(keyName, PROBE_KEY_EXPIRATION, roleDescriptors); + ProbeResult created = probe(HttpMethod.POST, API_KEY_PATH, body, Map.of()); + if (!created(created)) { + return new KeyCreation(created, null); + } + String keyId = string(created.body(), "id"); + if (keyId != null) { + createdKeyIds.add(keyId); + } + return new KeyCreation(created, encodedApiKey(created.body())); + } + + /** + * Removes everything the write probes created. Best effort by design: the keys expire on their + * own within {@value #PROBE_KEY_EXPIRATION}, so a failure here leaves nothing durable behind — + * but it is reported in the notes rather than swallowed, because an operator should not have to + * read server logs to find out something was left on the cluster. + */ + private void tearDownProbeResources( + List createdKeyIds, String roleName, List notes) { + for (String keyId : createdKeyIds) { + ProbeResult result = + probe( + HttpMethod.DELETE, + API_KEY_PATH, + """ + {"ids":["%s"]}""" + .formatted(keyId), + Map.of()); + if (result.status() != 200) { + logWarn( + "Failed to invalidate probe API key %s (status %d)".formatted(keyId, result.status())); + notes.add( + "Probe API key %s could not be invalidated (status %d); it expires on its own within %s." + .formatted(keyId, result.status(), PROBE_KEY_EXPIRATION)); + } + } + if (roleName == null) { + return; + } + ProbeResult result = probe(HttpMethod.DELETE, "/_security/role/" + roleName, null, Map.of()); + if (result.status() != 200) { + logWarn("Failed to delete probe role %s (status %d)".formatted(roleName, result.status())); + notes.add( + "Probe role %s could not be deleted (status %d) and must be removed by hand." + .formatted(roleName, result.status())); + } + } + + /** + * The single index every probe scopes itself to: the real dataset index, so that a DLS or FLS + * check measures the index Epic D would actually filter. {@code datasetIndexName} is + * {@code @NotNull} in the configuration, so the fallback is only reached by a hand-built + * configuration — and it is deliberately a name no real index uses, so a probe can never widen + * its own scope to a live index it was not pointed at. + */ + private String probeIndex() { + String index = esConfig.getDatasetIndexName(); + return index == null || index.isBlank() ? "duos-capability-probe-index" : index; + } + + /** + * Classifies a failed search through a probe key. The distinction is the whole value of the + * probe: a licence refusal means the feature is unavailable on this cluster, whereas a privilege + * refusal or a transport failure says nothing whatever about the feature — and an API key is + * limited by its owning credential's privileges, so a key that reads a restricted index is + * refused for reasons that have nothing to do with DLS or FLS. + */ + private ElasticSearchCapability searchFailure( + String name, String shortName, ProbeResult search, String evidence) { + if (search.status() == 0) { + return new ElasticSearchCapability( + name, + CapabilityVerdict.UNKNOWN, + "The key was accepted but the search could not be completed, so enforcement was not " + + "observed. This is a transport failure, not a finding about the cluster.", + evidence); + } + String reason = reason(search.body()); + if (licenseRefusal(reason)) { + return new ElasticSearchCapability( + name, + CapabilityVerdict.LICENSE_BLOCKED, + "Refused at search time on licensing grounds, which is exactly how a license that " + + "excludes %s fails — the key is accepted at creation and only rejected when used: " + .formatted(shortName) + + reason, + evidence); + } + if (search.status() == 401 || search.status() == 403) { + return new ElasticSearchCapability( + name, + CapabilityVerdict.NOT_PERMITTED, + "The key was accepted but the search was refused on privilege rather than licensing " + + "grounds, so %s itself was not established: ".formatted(shortName) + + reason, + evidence); + } + return new ElasticSearchCapability( + name, + CapabilityVerdict.UNKNOWN, + "The search through the probe key returned an unexpected status, so enforcement was not " + + "observed: " + + reason, + evidence); + } + + /** Distinguishes "the license forbids this" from "this credential may not do it". */ + private CapabilityVerdict refusalVerdict(ProbeResult result) { + if (licenseRefusal(reason(result.body()))) { + return CapabilityVerdict.LICENSE_BLOCKED; + } + if (result.status() == 401 || result.status() == 403) { + return CapabilityVerdict.NOT_PERMITTED; + } + return CapabilityVerdict.UNKNOWN; + } + + /** + * Whether a refusal was the cluster's licence talking rather than the credential's privileges. + * The single place that decision is made, so every probe classifies a refusal the same way — the + * distinction the whole report turns on, since only a licence refusal says anything about the + * cluster. + */ + private static boolean licenseRefusal(String reason) { + String lowered = reason.toLowerCase(); + return lowered.contains("non-compliant") || lowered.contains("license"); + } + + /** Whether the cluster created what was asked of it; PUT role returns 200 or 201. */ + private static boolean created(ProbeResult result) { + return result.status() == 200 || result.status() == 201; + } + + /** The {@code encoded} form of a created key, or one built from its id and secret. */ + private static String encodedApiKey(JsonObject created) { + String encoded = string(created, "encoded"); + if (encoded != null) { + return encoded; + } + String id = string(created, "id"); + String secret = string(created, "api_key"); + if (id == null || secret == null) { + return null; + } + return Base64.getEncoder().encodeToString((id + ":" + secret).getBytes(StandardCharsets.UTF_8)); + } + + /** The shared credential's view of how many documents the index holds, or -1 if unreadable. */ + private long documentCount(String index) { + ProbeResult result = probe(HttpMethod.GET, "/%s/_count".formatted(index)); + if (result.status() != 200 || !result.body().has("count")) { + return -1; + } + return result.body().get("count").getAsLong(); + } + + private static long hitCount(JsonObject searchBody) { + JsonObject hits = object(searchBody, "hits"); + if (hits == null || !hits.has("total")) { + return -1; + } + JsonElement total = hits.get("total"); + if (total.isJsonObject() && total.getAsJsonObject().has("value")) { + return total.getAsJsonObject().get("value").getAsLong(); + } + return total.isJsonPrimitive() ? total.getAsLong() : -1; + } + + private static Set firstHitSourceFields(JsonObject searchBody) { + JsonObject hits = object(searchBody, "hits"); + if (hits == null || !hits.has("hits") || !hits.get("hits").isJsonArray()) { + return Set.of(); + } + var array = hits.getAsJsonArray("hits"); + if (array.isEmpty() || !array.get(0).isJsonObject()) { + return Set.of(); + } + JsonObject source = object(array.get(0).getAsJsonObject(), "_source"); + return source == null ? Set.of() : source.keySet(); + } + + /** Issues a request authenticated as an API key rather than the deployment's own credential. */ + private ProbeResult probeAsApiKey(String encodedApiKey, String method, String path) { + try (RestClient keyClient = apiKeyClientFactory.create(encodedApiKey)) { + return toProbeResult(keyClient.performRequest(new Request(method, path))); + } catch (ResponseException e) { + return toProbeResult(e.getResponse()); + } catch (IOException | RuntimeException e) { + logWarn( + "Elasticsearch API-key probe failed for %s %s: %s" + .formatted(method, path, e.getMessage())); + return new ProbeResult(0, new JsonObject()); + } + } + + // --------------------------------------------------------------------------- + // Derived summary values + // --------------------------------------------------------------------------- + + /** + * X-Pack has shipped in every default Elasticsearch distribution since 6.3, so a missing {@code + * /_xpack} endpoint means an OSS build rather than merely a cluster with security switched off — + * a distinction worth keeping, because a security-disabled default distribution can have security + * enabled by configuration whereas an OSS build cannot. {@code build_flavor} says which directly + * when the cluster reports it. + */ + private String edition( + boolean elasticCloud, + boolean isOpenSearch, + String buildFlavor, + int xpackStatus, + String licenseType) { + if (elasticCloud) { + return "Elastic Cloud (X-Pack always present)"; + } + if (isOpenSearch) { + return "OpenSearch (Apache 2.0 / security plugin)"; + } + if ("oss".equalsIgnoreCase(buildFlavor) || xpackStatus == 400 || xpackStatus == 404) { + return "OSS (no X-Pack endpoint)"; + } + return licenseType == null ? "unknown" : licenseType; + } + + /** + * Reports whether the REST client this application ships can drive the security APIs. The + * low-level client is a version-agnostic HTTP transport with no typed request model, so the only + * real compatibility axis is major-version skew. + */ + private String restClientCompatibility(String clusterVersion, boolean isOpenSearch) { + String clientVersion = RestClient.class.getPackage().getImplementationVersion(); + if (isOpenSearch) { + return "Incompatible path: OpenSearch has no POST /_security/api_key. The low-level client " + + "can still reach /_plugins/_security, but the X-Pack API-key design does not apply."; + } + if (clientVersion == null || clusterVersion == null) { + return "Could not determine the client or cluster version at runtime. This report was itself " + + "produced through RestClient.performRequest, so the transport reaches /_security."; + } + String clientMajor = clientVersion.split("\\.")[0]; + String clusterMajor = clusterVersion.split("\\.")[0]; + if (clientMajor.equals(clusterMajor)) { + return "Compatible: elasticsearch-rest-client %s matches cluster major %s. Security calls go " + .formatted(clientVersion, clusterMajor) + + "through RestClient.performRequest(Request) with no typed-client dependency."; + } + return "Major-version skew: client %s vs cluster %s. The low-level client is version agnostic " + .formatted(clientVersion, clusterVersion) + + "over HTTP, but confirm the skew is within the supported range."; + } + + private String recommendation( + boolean securityApiPresent, + boolean isOpenSearch, + String licenseType, + WriteProbeOutcome writeProbeOutcome) { + // An observation outranks the license inference in either direction: a cluster that actually + // enforced DLS settles the question, and one that refused on licensing grounds settles it just + // as firmly. A refusal on *privilege* grounds settles nothing about the cluster, so that case + // falls through to the license reading rather than being read as a verdict against Epic D. + if (writeProbeOutcome != null && !isOpenSearch && securityApiPresent) { + if (writeProbeOutcome.dlsUsable()) { + return "Epic D (native DLS/FLS) is viable on this cluster, and this was observed rather " + + "than inferred: a probe role and API key carrying DLS/FLS descriptors were accepted " + + "and enforced. Keep Epic E in scope only if another environment cannot support " + + "DLS/FLS."; + } + if (writeProbeOutcome.credentialBlocked()) { + return "Inconclusive from the write probes: this deployment's credential is not permitted " + + "to create a role or an API key, so the DLS and FLS verdicts describe the credential " + + "rather than the cluster (see cluster_privileges). Re-run with a credential holding " + + "manage_security and manage_api_key to settle it. On the license alone: " + + licenseBasedRecommendation(licenseType); + } + return "Epic E (compatibility fallback). Write probes were run against this cluster and DLS " + + "was not usable: %s Epic D would need that resolved first." + .formatted(writeProbeOutcome.dls().detail()); + } + if (isOpenSearch) { + return "Epic E (compatibility fallback). This cluster is OpenSearch, where the X-Pack " + + "API-key and role-descriptor design behind Epic D does not exist."; + } + if (!securityApiPresent) { + return "Epic E (compatibility fallback) is the only path available on this cluster. " + + "Security is not enabled, so DLS, FLS, API keys, and run_as cannot be used at all. " + + "Epic D stays blocked until X-Pack Security is enabled here."; + } + return licenseBasedRecommendation(licenseType); + } + + /** What the license tier alone implies, used when no write probe settled the question. */ + private String licenseBasedRecommendation(String licenseType) { + String license = licenseType == null ? "" : licenseType.toLowerCase(); + if (DLS_FLS_LICENSES.contains(license)) { + return "Epic D (native DLS/FLS) is viable on this cluster: security is enabled and the '%s' " + .formatted(licenseType) + + "license includes DLS/FLS. Confirm with a write probe in a non-production environment " + + "before committing, and keep Epic E in scope only if another environment cannot " + + "support DLS/FLS."; + } + if (SECURITY_ONLY_LICENSES.contains(license)) { + return "Epic E (compatibility fallback). Security is enabled but the '%s' license excludes " + .formatted(licenseType) + + "DLS/FLS. Epic D would require a Platinum or Enterprise license — a separate infra " + + "decision that should precede any commitment to it."; + } + return "Inconclusive. The license tier could not be mapped to a DLS/FLS entitlement; inspect " + + "the license and settings in this report before recording a decision."; + } + + // --------------------------------------------------------------------------- + // Probes + // --------------------------------------------------------------------------- + + /** Reads the caller's own cluster privileges. A POST, but an evaluation rather than a write. */ + private Map clusterPrivileges() { + String body = + """ + {"cluster":[%s],"index":[{"names":["%s"],"privileges":["read","view_index_metadata"]}]}""" + .formatted( + PROBED_CLUSTER_PRIVILEGES.stream() + .map("\"%s\""::formatted) + .collect(Collectors.joining(",")), + probeIndex()); + + ProbeResult result = probe(HttpMethod.POST, "/_security/user/_has_privileges", body, Map.of()); + if (result.status() != 200 || !result.body().has("cluster")) { + return Map.of(); + } + Map privileges = new LinkedHashMap<>(); + JsonObject cluster = result.body().getAsJsonObject("cluster"); + for (String privilege : PROBED_CLUSTER_PRIVILEGES) { + if (cluster.has(privilege)) { + privileges.put(privilege, cluster.get(privilege).getAsBoolean()); + } + } + return privileges; + } + + /** + * Returns the cluster's security-relevant settings. + * + *

A cluster reports around fifty {@code xpack.security.*} defaults, nearly all of which — + * cache TTLs, thread-pool sizes, SSL handshake timeouts, hashing algorithms — say nothing about + * whether a capability is available, and including them buries the handful that do. So defaults + * are filtered to {@link #REPORTED_SECURITY_DEFAULTS}, while any {@code xpack.security.*} value + * an operator has explicitly set as a persistent or transient override is always reported: the + * fact that someone configured it is itself worth seeing. + */ + private Map securitySettings() { + ProbeResult result = + probe(HttpMethod.GET, "/_cluster/settings?include_defaults=true&flat_settings=true"); + if (result.status() != 200) { + return Map.of(); + } + Map settings = new TreeMap<>(); + // Later sections win: a persistent or transient override beats the default. + for (String section : List.of("defaults", "persistent", "transient")) { + if (!result.body().has(section) || !result.body().get(section).isJsonObject()) { + continue; + } + boolean isDefault = "defaults".equals(section); + for (Map.Entry entry : + result.body().getAsJsonObject(section).entrySet()) { + if (entry.getValue().isJsonPrimitive() + && isReportableSecuritySetting(entry.getKey(), isDefault)) { + settings.put(entry.getKey(), entry.getValue().getAsString()); + } + } + } + return settings; + } + + private boolean isReportableSecuritySetting(String key, boolean isDefault) { + if (!key.startsWith("xpack.security") && !key.contains("dls_fls")) { + return false; + } + if (!isDefault) { + // Explicitly configured, so report it whatever it is — except the audit-logfile settings, + // which are voluminous and describe log formatting rather than capability. + return !key.contains("audit.logfile"); + } + return REPORTED_SECURITY_DEFAULTS.contains(key); + } + + private Boolean securityEnabled(ProbeResult xpack, Map settings) { + if (xpack.status() == 200) { + JsonObject security = object(xpack.body(), "features", "security"); + if (security != null && security.has("enabled")) { + return security.get("enabled").getAsBoolean(); + } + } + String setting = settings.get("xpack.security.enabled"); + return setting == null ? null : Boolean.parseBoolean(setting); + } + + /** A 400/404/405 from the security API means security is off rather than merely restricted. */ + private boolean securityApiPresent(int authenticateStatus) { + return authenticateStatus == 200 || authenticateStatus == 401 || authenticateStatus == 403; + } + + private ElasticSearchCapabilityReport unreachableReport(List notes) { + logWarn("Elasticsearch capability probe could not reach the cluster"); + notes.add("The cluster could not be reached, so no capability could be determined."); + return new ElasticSearchCapabilityReport( + null, + null, + null, + "unknown", + null, + null, + null, + null, + null, + List.of(), + Map.of(), + Map.of(), + false, + List.of( + new ElasticSearchCapability( + "Cluster reachability", + CapabilityVerdict.UNKNOWN, + "The configured Elasticsearch cluster did not respond to GET /.", + "GET /")), + "Not assessed — the cluster was unreachable.", + "Not assessed — the cluster was unreachable.", + notes); + } + + private ProbeResult probe(String method, String path) { + return probe(method, path, null, Map.of()); + } + + private ProbeResult probe(String method, String path, String body, Map headers) { + Request request = new Request(method, path); + if (body != null) { + request.setJsonEntity(body); + } + if (!headers.isEmpty()) { + RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); + headers.forEach(options::addHeader); + request.setOptions(options.build()); + } + try { + return toProbeResult(esClient.performRequest(request)); + } catch (ResponseException e) { + // A non-2xx is data here, not a failure: the status is frequently the finding itself. + return toProbeResult(e.getResponse()); + } catch (IOException | RuntimeException e) { + logWarn( + "Elasticsearch capability probe failed for %s %s: %s" + .formatted(method, path, e.getMessage())); + return new ProbeResult(0, new JsonObject()); + } + } + + private ProbeResult toProbeResult(Response response) { + int status = response.getStatusLine().getStatusCode(); + try { + if (response.getEntity() == null) { + return new ProbeResult(status, new JsonObject()); + } + String content = + new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8); + JsonElement parsed = JsonParser.parseString(content); + return new ProbeResult( + status, parsed.isJsonObject() ? parsed.getAsJsonObject() : new JsonObject()); + } catch (Exception e) { + return new ProbeResult(status, new JsonObject()); + } + } + + // --------------------------------------------------------------------------- + // JSON helpers + // --------------------------------------------------------------------------- + + private static JsonObject object(JsonObject source, String... path) { + JsonObject current = source; + for (String key : path) { + if (current == null || !current.has(key) || !current.get(key).isJsonObject()) { + return null; + } + current = current.getAsJsonObject(key); + } + return current; + } + + private static String string(JsonObject source, String... path) { + if (path.length == 0) { + return null; + } + JsonObject parent = + path.length == 1 ? source : object(source, Arrays.copyOf(path, path.length - 1)); + String key = path[path.length - 1]; + if (parent == null || !parent.has(key) || !parent.get(key).isJsonPrimitive()) { + return null; + } + return parent.get(key).getAsString(); + } + + private static String stringOrDefault(JsonObject source, String fallback, String... path) { + String value = string(source, path); + return value == null ? fallback : value; + } + + private static List stringList(JsonObject source, String key) { + if (source == null || !source.has(key) || !source.get(key).isJsonArray()) { + return List.of(); + } + List values = new ArrayList<>(); + source.getAsJsonArray(key).forEach(element -> values.add(element.getAsString())); + return values; + } + + private static String reason(JsonObject body) { + JsonObject error = object(body, "error"); + if (error != null && error.has("reason")) { + return error.get("reason").getAsString(); + } + return "no reason reported by the cluster"; + } + + /** A probe's HTTP status paired with its parsed body; a status of 0 means transport failure. */ + private record ProbeResult(int status, JsonObject body) {} +} diff --git a/src/main/resources/assets/api-docs.yaml b/src/main/resources/assets/api-docs.yaml index 03c1ae3fd..ecba13103 100644 --- a/src/main/resources/assets/api-docs.yaml +++ b/src/main/resources/assets/api-docs.yaml @@ -674,6 +674,8 @@ paths: $ref: './paths/publicFeatureById.yaml' /api/feature/{id}: $ref: './paths/adminFeatureById.yaml' + /api/elasticSearch/capabilities: + $ref: './paths/elasticSearchCapabilities.yaml' /api/institutions: $ref: './paths/institutions.yaml' /api/institutions/{id}: diff --git a/src/main/resources/assets/paths/elasticSearchCapabilities.yaml b/src/main/resources/assets/paths/elasticSearchCapabilities.yaml new file mode 100644 index 000000000..66768c96c --- /dev/null +++ b/src/main/resources/assets/paths/elasticSearchCapabilities.yaml @@ -0,0 +1,75 @@ +get: + summary: Elasticsearch security capability report + operationId: apiElasticSearchCapabilitiesGet + description: | + Report the security features of the Elasticsearch cluster this deployment is configured + against: version, edition, X-Pack Security, document-level security, field-level security, + API keys, and run_as impersonation. Requires Admin role. + + Because each environment runs its own deployment against its own cluster, calling this in + dev, staging, and production produces the per-environment inventory without needing direct + network access to the clusters or a copy of their credentials. + + By default all probes are non-destructive: nothing is created, modified, or deleted on the + cluster. The trade-off is certainty — DLS, FLS, and API-key support can only be proven by + creating a role or a key, so in that mode those verdicts are inferred from the license tier + and cluster settings and are returned with an INFERRED_ verdict. Only run_as and X-Pack + Security itself are reported as observed fact. + + Pass writeProbes=true to observe them instead. That mode mints a short-lived API key, + creates a role carrying a DLS query and an FLS grant, and checks that a key whose + role_descriptors carry those filters is actually enforced against the dataset index — + the only way to distinguish a cluster that accepts DLS descriptors from one that enforces + them, since a Basic-licensed cluster accepts them at creation and fails only at search time. + Everything created is namespaced duos-capability-probe / duos_dlsfls_probe, expires within + 10 minutes, and is torn down before the response is returned. + tags: + - ElasticSearch + parameters: + - name: runAsUser + in: query + description: | + Username to attempt the run_as probe against. Defaults to the deployment credential's + own principal, which still establishes whether the feature is licensed and enabled. + required: false + schema: + type: string + - name: writeProbes + in: query + description: | + Create and tear down a short-lived API key and role so that DLS, FLS, and API-key + support are observed rather than inferred. Requires the deployment credential to hold + manage_api_key (or grant_api_key) and manage_security; the report says which of those + it has under cluster_privileges. Ignored on OpenSearch or when security is disabled, + since there is nothing to probe. + required: false + schema: + type: boolean + default: false + responses: + 200: + description: The capability report + content: + application/json: + schema: + $ref: '../schemas/ElasticSearchCapabilityReport.yaml' + 401: + description: Unauthorized. + 403: + description: Forbidden - Admin role required + content: + application/json: + schema: + $ref: '../schemas/ErrorResponse.yaml' + 429: + description: Too Many Requests - rate limit exceeded + content: + application/json: + schema: + $ref: '../schemas/ErrorResponse.yaml' + 500: + description: Internal Server Error + content: + application/json: + schema: + $ref: '../schemas/ErrorResponse.yaml' diff --git a/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml b/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml new file mode 100644 index 000000000..28ce616d0 --- /dev/null +++ b/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml @@ -0,0 +1,222 @@ +type: object +title: ElasticSearchCapabilityReport +description: | + Security feature inventory for the Elasticsearch cluster the responding deployment is + configured against. Unless write probes were requested, every probe behind this report is + non-destructive, which is why the DLS, FLS, and API-key verdicts may be inferred rather than + observed; write_probes_run says which of the two this report is. + + The example is a real writeProbes=true response from a local trial-licensed cluster, where the + configured credential is the `elastic` superuser. A deployed environment's shared credential is + unlikely to hold manage_security or manage_api_key, so expect NOT_PERMITTED verdicts and a + narrower cluster_privileges block there rather than this all-true one. +properties: + cluster_name: + type: string + description: The cluster's own name, confirming which cluster was reached. + version: + type: string + description: Elasticsearch version. + examples: + - "9.4.4" + - "9.3.3" + distribution: + type: string + description: | + Distribution reported by the cluster. OpenSearch exposes different security APIs than + Elasticsearch and has no API-key endpoint. + examples: + - elasticsearch + - opensearch + edition: + type: string + description: OSS, a license tier, Elastic Cloud, or OpenSearch. + license_type: + type: string + description: License tier reported by the cluster. + examples: + - basic + - trial + - platinum + - enterprise + license_status: + type: string + examples: + - active + elastic_cloud: + type: boolean + description: Whether this deployment is configured with a cloud ID. + security_enabled: + type: boolean + description: Whether X-Pack Security is enabled. + authenticated_user: + type: string + description: The principal the deployment's shared credential resolves to. + authenticated_user_roles: + type: array + items: + type: string + cluster_privileges: + type: object + description: | + Cluster privileges the shared credential holds, from a read-only privilege check. These + are the privileges any per-request credential work would have to build on. The six probed + privileges are always all present as keys, so a false is a checked refusal rather than an + omission. + additionalProperties: + type: boolean + examples: + - manage_security: true + manage_api_key: true + grant_api_key: true + manage_own_api_key: true + read_security: true + monitor: true + security_settings: + type: object + description: | + Security-relevant cluster settings. Defaults are filtered to the values that gate a + capability or describe the authentication posture, since a cluster reports around fifty + xpack.security.* defaults that are pure tuning. Any setting explicitly configured as a + persistent or transient override is always included. + additionalProperties: + type: string + examples: + - xpack.security.audit.enabled: "false" + xpack.security.authc.anonymous.username: _anonymous + xpack.security.authc.api_key.enabled: "true" + xpack.security.authc.reserved_realm.enabled: "true" + xpack.security.authc.run_as.enabled: "true" + xpack.security.authc.token.enabled: "false" + xpack.security.dls_fls.enabled: "true" + xpack.security.enabled: "true" + xpack.security.fips_mode.enabled: "false" + xpack.security.http.ssl.enabled: "false" + xpack.security.operator_privileges.enabled: "false" + xpack.security.transport.ssl.enabled: "false" + write_probes_run: + type: boolean + description: | + Whether write probes ran. When false, the DLS, FLS, and API-key verdicts are inferred from + the license tier rather than observed — read this field before reading those verdicts. + capabilities: + type: array + description: | + One entry per probed capability, always in the same order: X-Pack Security, API keys, + DLS, FLS, run_as impersonation. + items: + type: object + title: ElasticSearchCapability + required: + - name + - verdict + properties: + name: + type: string + enum: + - X-Pack Security + - API keys + - Document-level security (DLS) + - Field-level security (FLS) + - run_as impersonation + verdict: + type: string + description: | + SUPPORTED and UNAVAILABLE are observed. LICENSE_BLOCKED and NOT_PERMITTED are + observed refusals. The INFERRED_ values are derived from license tier and cluster + settings because proving them would require writing to the cluster. + enum: + - SUPPORTED + - UNAVAILABLE + - LICENSE_BLOCKED + - NOT_PERMITTED + - INFERRED_SUPPORTED + - INFERRED_UNAVAILABLE + - UNKNOWN + detail: + type: string + evidence: + type: string + description: The probe the verdict is drawn from, so it can be re-run by hand. + rest_client_compatibility: + type: string + description: Whether the bundled elasticsearch-rest-client can drive the security APIs. + recommendation: + type: string + description: Which implementation path the findings point to. + notes: + type: array + description: Caveats required to interpret the report correctly. + items: + type: string +examples: + - cluster_name: docker-cluster + version: "9.4.4" + distribution: elasticsearch + edition: trial + license_type: trial + license_status: active + elastic_cloud: false + security_enabled: true + authenticated_user: elastic + authenticated_user_roles: + - superuser + cluster_privileges: + manage_security: true + manage_api_key: true + grant_api_key: true + manage_own_api_key: true + read_security: true + monitor: true + security_settings: + xpack.security.audit.enabled: "false" + xpack.security.authc.anonymous.username: _anonymous + xpack.security.authc.api_key.enabled: "true" + xpack.security.authc.reserved_realm.enabled: "true" + xpack.security.authc.run_as.enabled: "true" + xpack.security.authc.token.enabled: "false" + xpack.security.dls_fls.enabled: "true" + xpack.security.enabled: "true" + xpack.security.fips_mode.enabled: "false" + xpack.security.http.ssl.enabled: "false" + xpack.security.operator_privileges.enabled: "false" + xpack.security.transport.ssl.enabled: "false" + write_probes_run: true + capabilities: + - name: X-Pack Security + verdict: SUPPORTED + detail: X-Pack Security is enabled and the /_security API responds. + evidence: GET /_xpack (status 200), GET /_security/_authenticate + - name: API keys + verdict: SUPPORTED + detail: "Observed: a key was created, authenticated as 'elastic', and invalidated." + evidence: POST /_security/api_key -> 200; GET /_security/_authenticate as the key -> 200 + - name: Document-level security (DLS) + verdict: SUPPORTED + detail: "Proven end to end: a match_none DLS key returned 0 of 1158 documents." + evidence: >- + GET /dataset/_search?size=1 through a match_none DLS key -> 200; 0 of 1158 documents + visible + - name: Field-level security (FLS) + verdict: SUPPORTED + detail: >- + Proven end to end: documents came back carrying only the granted 'datasetIdentifier' + field. + evidence: >- + GET /dataset/_search?size=1 through a key granting only 'datasetIdentifier' -> 200 + - name: run_as impersonation + verdict: SUPPORTED + detail: The cluster honoured the run_as header and resolved the request to 'elastic'. + evidence: "GET /_security/_authenticate with es-security-runas-user: elastic -> 200" + rest_client_compatibility: >- + Compatible: elasticsearch-rest-client 9.4.4 matches cluster major 9. Security calls go + through RestClient.performRequest(Request) with no typed-client dependency. + recommendation: >- + Epic D (native DLS/FLS) is viable on this cluster, and this was observed rather than + inferred: a probe role and API key carrying DLS/FLS descriptors were accepted and enforced. + Keep Epic E in scope only if another environment cannot support DLS/FLS. + notes: + - >- + Write probes ran: 3 short-lived API key(s) and one probe role were created under the + duos-capability-probe / duos_dlsfls_probe names and removed again. The DLS, FLS, and + API-key verdicts below are observed rather than inferred. diff --git a/src/test/java/org/broadinstitute/consent/http/ConsentModuleTest.java b/src/test/java/org/broadinstitute/consent/http/ConsentModuleTest.java index 1db7ad1fb..d1eef788c 100644 --- a/src/test/java/org/broadinstitute/consent/http/ConsentModuleTest.java +++ b/src/test/java/org/broadinstitute/consent/http/ConsentModuleTest.java @@ -1,5 +1,6 @@ package org.broadinstitute.consent.http; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -59,6 +60,7 @@ import org.broadinstitute.consent.http.service.DataAccessRequestService; import org.broadinstitute.consent.http.service.DatasetRegistrationService; import org.broadinstitute.consent.http.service.DatasetService; +import org.broadinstitute.consent.http.service.ElasticSearchCapabilityService; import org.broadinstitute.consent.http.service.ElasticSearchService; import org.broadinstitute.consent.http.service.ElectionService; import org.broadinstitute.consent.http.service.EmailService; @@ -91,6 +93,7 @@ import org.broadinstitute.consent.http.util.CountryValidator; import org.broadinstitute.consent.http.util.HttpClientUtil; import org.broadinstitute.consent.http.util.InstitutionUtil; +import org.elasticsearch.client.RestClient; import org.jdbi.v3.core.Jdbi; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -338,15 +341,67 @@ void testExecutorServiceShutdownNowWhenStopIsInterrupted() throws Exception { /** * The module registers several lifecycle-managed objects (the JDBI data source, Jersey client - * executors, and the shared executor shutdown hook). Find the module's own anonymous Managed. + * executors, the shared executor shutdown hook, and the Elasticsearch client shutdown). Find the + * module's own anonymous Managed — the executor hook is the only one of those still anonymous. */ private Managed findExecutorManaged() { + return managedByModule() + .filter(m -> !(m instanceof ConsentModule.ElasticSearchClientShutdown)) + .findFirst() + .orElseThrow(); + } + + private java.util.stream.Stream managedByModule() { return environment.lifecycle().getManagedObjects().stream() .filter(JettyManaged.class::isInstance) .map(JettyManaged.class::cast) .map(JettyManaged::getManaged) - .filter(m -> m.getClass().getName().startsWith(ConsentModule.class.getName())) - .findFirst() - .orElseThrow(); + .filter(m -> m.getClass().getName().startsWith(ConsentModule.class.getName())); + } + + /** + * One client for the whole application. Each RestClient owns a connection pool and background + * threads, so a client per consumer would multiply pools against the same cluster. + */ + @Test + void testProvidesASingleSharedElasticSearchClient() { + RestClient restClient = injector.getInstance(RestClient.class); + + assertNotNull(restClient); + assertSame(restClient, injector.getInstance(RestClient.class)); + } + + @Test + void testElasticSearchServicesShareThatOneClient() { + RestClient restClient = injector.getInstance(RestClient.class); + + // Both services are constructed with the injected client rather than building their own, so + // resolving them must not add any further clients to close. + injector.getInstance(ElasticSearchService.class); + injector.getInstance(ElasticSearchCapabilityService.class); + + assertEquals( + 1, + managedByModule() + .filter(ConsentModule.ElasticSearchClientShutdown.class::isInstance) + .count(), + "exactly one Elasticsearch client should be registered for shutdown"); + assertSame(restClient, injector.getInstance(RestClient.class)); + } + + /** Without this the client's connections and threads would outlive the application. */ + @Test + void testElasticSearchClientIsClosedOnShutdown() throws Exception { + RestClient restClient = injector.getInstance(RestClient.class); + assertTrue(restClient.isRunning()); + + Managed managed = + managedByModule() + .filter(ConsentModule.ElasticSearchClientShutdown.class::isInstance) + .findFirst() + .orElseThrow(); + managed.stop(); + + assertFalse(restClient.isRunning(), "the shared Elasticsearch client should be closed"); } } diff --git a/src/test/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResourceTest.java b/src/test/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResourceTest.java new file mode 100644 index 000000000..b57c7dd8e --- /dev/null +++ b/src/test/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResourceTest.java @@ -0,0 +1,108 @@ +package org.broadinstitute.consent.http.resources; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import jakarta.ws.rs.core.Response; +import java.util.Date; +import java.util.List; +import java.util.Map; +import org.broadinstitute.consent.http.AbstractTestHelper; +import org.broadinstitute.consent.http.models.AuthUser; +import org.broadinstitute.consent.http.models.DuosUser; +import org.broadinstitute.consent.http.models.User; +import org.broadinstitute.consent.http.models.elastic_search.CapabilityVerdict; +import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchCapability; +import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchCapabilityReport; +import org.broadinstitute.consent.http.service.ElasticSearchCapabilityService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticSearchCapabilityResourceTest extends AbstractTestHelper { + + @Mock private ElasticSearchCapabilityService capabilityService; + + private ElasticSearchCapabilityResource resource; + private final AuthUser authUser = new AuthUser("admin@test.com"); + private final User user = new User(1, "admin@test.com", "Admin", new Date()); + private final DuosUser duosUser = new DuosUser(authUser, user); + + @BeforeEach + void setUp() { + resource = new ElasticSearchCapabilityResource(capabilityService); + } + + private ElasticSearchCapabilityReport report() { + return new ElasticSearchCapabilityReport( + "duos-cluster", + "9.3.3", + "elasticsearch", + "trial", + "trial", + "active", + false, + true, + "consent", + List.of("superuser"), + Map.of("manage_api_key", true), + Map.of("xpack.security.enabled", "true"), + false, + List.of( + new ElasticSearchCapability( + "X-Pack Security", CapabilityVerdict.SUPPORTED, "enabled", "GET /_xpack")), + "Compatible", + "Epic D", + List.of("All probes are non-destructive.")); + } + + @Test + void testGetCapabilities() { + when(capabilityService.getCapabilityReport(null, false)).thenReturn(report()); + + Response response = resource.getCapabilities(duosUser, null, false); + + assertEquals(200, response.getStatus()); + assertNotNull(response.getEntity()); + assertEquals(report(), response.getEntity()); + verify(capabilityService).getCapabilityReport(null, false); + } + + @Test + void testGetCapabilitiesPassesRunAsUserThrough() { + when(capabilityService.getCapabilityReport("someone-else", false)).thenReturn(report()); + + Response response = resource.getCapabilities(duosUser, "someone-else", false); + + assertEquals(200, response.getStatus()); + verify(capabilityService).getCapabilityReport("someone-else", false); + } + + /** Write probes must never be a side effect of calling the endpoint — only of asking for them. */ + @Test + void testWriteProbesAreOnlyRunWhenExplicitlyRequested() { + when(capabilityService.getCapabilityReport(null, true)).thenReturn(report()); + + Response response = resource.getCapabilities(duosUser, null, true); + + assertEquals(200, response.getStatus()); + verify(capabilityService).getCapabilityReport(null, true); + verify(capabilityService, never()).getCapabilityReport(null, false); + } + + @Test + void testGetCapabilitiesHandlesServiceFailure() { + when(capabilityService.getCapabilityReport(null, false)) + .thenThrow(new RuntimeException("cluster exploded")); + + Response response = resource.getCapabilities(duosUser, null, false); + + assertEquals(500, response.getStatus()); + } +} diff --git a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java new file mode 100644 index 000000000..7f9ffeff5 --- /dev/null +++ b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java @@ -0,0 +1,1036 @@ +package org.broadinstitute.consent.http.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.http.HttpVersion; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicRequestLine; +import org.apache.http.message.BasicStatusLine; +import org.broadinstitute.consent.http.configurations.ElasticSearchConfiguration; +import org.broadinstitute.consent.http.models.elastic_search.CapabilityVerdict; +import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchCapability; +import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchCapabilityReport; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ElasticSearchCapabilityServiceTest { + + private static final String ROOT = "/"; + private static final String XPACK = "/_xpack"; + private static final String LICENSE = "/_license"; + private static final String SETTINGS = + "/_cluster/settings?include_defaults=true&flat_settings=true"; + private static final String AUTHENTICATE = "/_security/_authenticate"; + private static final String RUN_AS = "/_security/_authenticate#runas"; + // A POST, so it is keyed with its method like the other write-shaped calls. + private static final String HAS_PRIVILEGES = "POST /_security/user/_has_privileges"; + + // Write-probe endpoints. The three key creations share a method and endpoint, so they are keyed + // by the role descriptor in the body — which is also what distinguishes them on a real cluster. + private static final String CREATE_KEY = "POST /_security/api_key"; + private static final String CREATE_DLS_KEY = "POST /_security/api_key#dls"; + private static final String CREATE_FLS_KEY = "POST /_security/api_key#fls"; + private static final String INVALIDATE_KEY = "DELETE /_security/api_key"; + private static final String CREATE_ROLE = "PUT /_security/role"; + private static final String DELETE_ROLE = "DELETE /_security/role"; + private static final String COUNT = "/dataset/_count"; + private static final String SEARCH = "/dataset/_search?size=1"; + + private static final String ROOT_BODY = + """ + {"cluster_name":"duos-cluster","version":{"number":"9.3.3","build_flavor":"default"}}"""; + + @Mock private RestClient esClient; + + private ElasticSearchConfiguration config; + + /** Canned responses keyed by endpoint; the run_as probe is keyed separately by header. */ + private Map stubs; + + /** Every request the service issued, so teardown can be asserted on rather than assumed. */ + private List requests; + + private RestClient keyClient; + private String activeApiKey; + + @BeforeEach + void setUp() { + config = new ElasticSearchConfiguration(); + config.setDatasetIndexName("dataset"); + config.setIndexName("ontology"); + stubs = new HashMap<>(); + requests = new java.util.ArrayList<>(); + } + + private ElasticSearchCapabilityService service() throws IOException { + stubClient(esClient, this::keyFor); + return new ElasticSearchCapabilityService(esClient, config, this::apiKeyClient); + } + + /** + * Answers from {@link #stubs}, turning a stubbed non-2xx into the ResponseException ES throws. + */ + private void stubClient(RestClient client, java.util.function.Function keying) + throws IOException { + when(client.performRequest(any(Request.class))) + .thenAnswer( + invocation -> { + Request request = invocation.getArgument(0); + requests.add(request); + String key = keying.apply(request); + StubResponse stub = stubs.get(key); + if (stub == null) { + throw new IOException("no stub for " + key); + } + Response response = response(stub.status(), stub.body()); + if (stub.status() >= 300) { + throw new ResponseException(response); + } + return response; + }); + } + + /** + * A client that authenticates as an API key. One mock serves all of them, routing on which key + * was most recently handed to the factory — safe because the probes run sequentially, and it + * keeps the DLS key's search distinguishable from the FLS key's. + */ + private RestClient apiKeyClient(String encodedApiKey) { + activeApiKey = encodedApiKey; + if (keyClient != null) { + return keyClient; + } + keyClient = mock(RestClient.class); + try { + stubClient(keyClient, request -> activeApiKey + "|" + request.getEndpoint()); + } catch (IOException e) { + throw new AssertionError(e); + } + return keyClient; + } + + private String keyFor(Request request) { + boolean isRunAs = + request.getOptions().getHeaders().stream() + .anyMatch(h -> h.getName().equals("es-security-runas-user")); + String endpoint = request.getEndpoint(); + if (isRunAs && endpoint.equals(AUTHENTICATE)) { + return RUN_AS; + } + String method = request.getMethod(); + if (endpoint.equals("/_security/api_key") && method.equals("POST")) { + String body = bodyOf(request); + if (body.contains("dls_probe")) { + return CREATE_DLS_KEY; + } + return body.contains("fls_probe") ? CREATE_FLS_KEY : CREATE_KEY; + } + // Probe role names carry a timestamp, so match the family rather than the exact name. + if (endpoint.startsWith("/_security/role/")) { + return method + " /_security/role"; + } + return method.equals("GET") ? endpoint : method + " " + endpoint; + } + + private String bodyOf(Request request) { + try { + return request.getEntity() == null + ? "" + : new String( + request.getEntity().getContent().readAllBytes(), + java.nio.charset.StandardCharsets.UTF_8); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private List requestsTo(String method, String endpointPrefix) { + return requests.stream() + .filter(r -> r.getMethod().equals(method) && r.getEndpoint().startsWith(endpointPrefix)) + .toList(); + } + + /** + * Stubs are lenient because which accessors get touched depends on the status: only the error + * path builds a ResponseException, which is what reads the request line. + */ + private Response response(int status, String body) { + Response response = mock(Response.class); + lenient() + .when(response.getStatusLine()) + .thenReturn(new BasicStatusLine(HttpVersion.HTTP_1_1, status, "reason")); + lenient() + .when(response.getEntity()) + .thenReturn(new StringEntity(body, ContentType.APPLICATION_JSON)); + lenient() + .when(response.getRequestLine()) + .thenReturn(new BasicRequestLine("GET", "/", HttpVersion.HTTP_1_1)); + return response; + } + + private void stub(String key, int status, String body) { + stubs.put(key, new StubResponse(status, body)); + } + + /** + * Baseline: a cluster with security switched off, as the local docker-compose one is. X-Pack is + * still installed and still reports a license — it has shipped in every default distribution + * since 6.3 — so this fixture matches what the local cluster was actually measured to return + * rather than the OSS shape. A missing /_xpack endpoint is a different cluster, covered by the + * OpenSearch case. + */ + private void stubSecurityDisabledCluster() { + stub(ROOT, 200, ROOT_BODY); + stub( + XPACK, + 200, + """ + {"features":{"security":{"available":true,"enabled":false}}}"""); + stub( + LICENSE, + 200, + """ + {"license":{"status":"active","type":"basic"}}"""); + stub( + SETTINGS, + 200, + """ + {"defaults":{"xpack.security.enabled":"false"},"persistent":{},"transient":{}}"""); + stub( + AUTHENTICATE, + 400, + """ + {"error":{"reason":"Security must be explicitly enabled"}}"""); + } + + /** A security-enabled cluster; the license tier is what varies between DLS/FLS outcomes. */ + private void stubSecurityEnabledCluster(String licenseType) { + stub(ROOT, 200, ROOT_BODY); + stub( + XPACK, + 200, + """ + {"features":{"security":{"available":true,"enabled":true}}}"""); + stub( + LICENSE, + 200, + """ + {"license":{"status":"active","type":"%s"}}""" + .formatted(licenseType)); + stub( + SETTINGS, + 200, + """ + {"defaults":{"xpack.security.enabled":"true","xpack.security.dls_fls.enabled":"true", + "xpack.security.authc.api_key.enabled":"true", + "xpack.security.audit.logfile.events.emit_request_body":"false", + "xpack.security.crypto.thread_pool.size":"4", + "xpack.security.authc.api_key.cache.ttl":"24h"}, + "persistent":{"xpack.security.authc.password_hashing.algorithm":"PBKDF2"}, + "transient":{}}"""); + stub( + AUTHENTICATE, + 200, + """ + {"username":"consent","roles":["superuser"]}"""); + stub( + HAS_PRIVILEGES, + 200, + """ + {"cluster":{"manage_security":true,"manage_api_key":true,"grant_api_key":true, + "manage_own_api_key":true,"read_security":true,"monitor":true}}"""); + stub( + RUN_AS, + 200, + """ + {"username":"consent","roles":["superuser"]}"""); + } + + private ElasticSearchCapability capability(ElasticSearchCapabilityReport report, String name) { + return report.capabilities().stream() + .filter(c -> c.name().equals(name)) + .findFirst() + .orElseThrow(() -> new AssertionError("no capability named " + name)); + } + + @Test + void testSecurityDisabledClusterReportsEverythingUnavailable() throws IOException { + stubSecurityDisabledCluster(); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("9.3.3", report.version()); + assertEquals("elasticsearch", report.distribution()); + assertEquals("duos-cluster", report.clusterName()); + assertEquals(Boolean.FALSE, report.securityEnabled()); + // X-Pack is installed and licensed here; only security is switched off. Calling that "OSS" + // would hide the fact that this cluster could have security enabled by configuration alone. + assertEquals("basic", report.edition()); + for (ElasticSearchCapability capability : report.capabilities()) { + assertEquals( + CapabilityVerdict.UNAVAILABLE, + capability.verdict(), + capability.name() + " should be unavailable when security is off"); + } + assertTrue(report.recommendation().contains("Epic E")); + // No privilege probe should be attempted when the security API is absent. + assertTrue(report.clusterPrivileges().isEmpty()); + } + + @Test + void testBasicLicenseBlocksDlsFlsButAllowsApiKeysAndRunAs() throws IOException { + stubSecurityEnabledCluster("basic"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals(Boolean.TRUE, report.securityEnabled()); + assertEquals("basic", report.licenseType()); + assertEquals("active", report.licenseStatus()); + assertEquals( + CapabilityVerdict.LICENSE_BLOCKED, + capability(report, "Document-level security (DLS)").verdict()); + assertEquals( + CapabilityVerdict.LICENSE_BLOCKED, + capability(report, "Field-level security (FLS)").verdict()); + assertEquals(CapabilityVerdict.INFERRED_SUPPORTED, capability(report, "API keys").verdict()); + assertEquals(CapabilityVerdict.SUPPORTED, capability(report, "run_as impersonation").verdict()); + assertTrue(report.recommendation().contains("Epic E")); + } + + @Test + void testTrialLicenseMakesNativePathViable() throws IOException { + stubSecurityEnabledCluster("trial"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals( + CapabilityVerdict.INFERRED_SUPPORTED, + capability(report, "Document-level security (DLS)").verdict()); + assertEquals( + CapabilityVerdict.INFERRED_SUPPORTED, + capability(report, "Field-level security (FLS)").verdict()); + assertEquals(CapabilityVerdict.SUPPORTED, capability(report, "X-Pack Security").verdict()); + assertTrue(report.recommendation().contains("Epic D")); + } + + @Test + void testSecuritySettingsAreFilteredToTheInformativeOnes() throws IOException { + stubSecurityEnabledCluster("trial"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertTrue(report.securitySettings().containsKey("xpack.security.enabled")); + assertTrue(report.securitySettings().containsKey("xpack.security.dls_fls.enabled")); + // Audit-logfile, cache, and tuning defaults are noise for a capability question. A real + // cluster reports around fifty of these, which would bury the few that gate a capability. + assertFalse( + report + .securitySettings() + .containsKey("xpack.security.audit.logfile.events.emit_request_body")); + assertFalse(report.securitySettings().containsKey("xpack.security.authc.api_key.cache.ttl")); + assertFalse(report.securitySettings().containsKey("xpack.security.crypto.thread_pool.size")); + // But an operator having explicitly overridden a setting is itself informative, so a + // persistent value is reported even though the same key as a default would be filtered out. + assertEquals( + "PBKDF2", + report.securitySettings().get("xpack.security.authc.password_hashing.algorithm"), + "explicitly configured security settings should always be reported"); + } + + @Test + void testCredentialWithoutKeyMintingPrivilegesIsReportedAsNotPermitted() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + HAS_PRIVILEGES, + 200, + """ + {"cluster":{"manage_security":false,"manage_api_key":false,"grant_api_key":false, + "manage_own_api_key":true,"read_security":false,"monitor":true}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + ElasticSearchCapability apiKeys = capability(report, "API keys"); + assertEquals(CapabilityVerdict.NOT_PERMITTED, apiKeys.verdict()); + assertTrue(apiKeys.detail().contains("manage_api_key")); + assertEquals(Boolean.FALSE, report.clusterPrivileges().get("grant_api_key")); + } + + @Test + void testRunAsDeniedByPrivilegeIsDistinguishedFromLicenseBlock() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + RUN_AS, + 403, + """ + {"error":{"reason":"action [cluster:admin/xpack/security/user/authenticate] is unauthorized + for user [consent] because user is unauthorized to run as [other]"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport("other", false); + + assertEquals( + CapabilityVerdict.NOT_PERMITTED, capability(report, "run_as impersonation").verdict()); + } + + /** + * The run_as target is caller-supplied and ends up in a request header on the deployment's own + * credential. The transport does not validate header values, so a CRLF would let a caller append + * requests of their own to the one this service sends. + */ + @Test + void testRunAsUserCarryingHeaderTerminatorsIsRejectedBeforeAnyRequestIsSent() throws IOException { + stubSecurityEnabledCluster("trial"); + + ElasticSearchCapabilityReport report = + service().getCapabilityReport("victim\r\nDELETE /dataset HTTP/1.1", false); + + ElasticSearchCapability runAs = capability(report, "run_as impersonation"); + assertEquals(CapabilityVerdict.UNKNOWN, runAs.verdict()); + assertTrue(runAs.detail().contains("not a valid Elasticsearch username"), runAs.detail()); + assertTrue( + requests.stream() + .noneMatch( + r -> + r.getOptions().getHeaders().stream() + .anyMatch(h -> h.getName().equals("es-security-runas-user"))), + "no run_as request should have been issued for an invalid username"); + } + + @Test + void testRunAsBlockedByLicenseIsReportedAsSuch() throws IOException { + stubSecurityEnabledCluster("basic"); + stub( + RUN_AS, + 403, + """ + {"error":{"reason":"current license is non-compliant for [run_as]"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport("other", false); + + assertEquals( + CapabilityVerdict.LICENSE_BLOCKED, capability(report, "run_as impersonation").verdict()); + } + + @Test + void testOpenSearchIsCalledOutRatherThanMisreported() throws IOException { + stub( + ROOT, + 200, + """ + {"cluster_name":"duos-cluster","version":{"number":"2.19.1","distribution":"opensearch"}}"""); + stub( + XPACK, + 400, + """ + {"error":{"reason":"no handler found"}}"""); + stub( + LICENSE, + 400, + """ + {"error":{"reason":"no handler found"}}"""); + stub( + SETTINGS, + 200, + """ + {"defaults":{},"persistent":{},"transient":{}}"""); + stub( + AUTHENTICATE, + 401, + """ + {"error":{"reason":"unauthorized"}}"""); + stub( + HAS_PRIVILEGES, + 404, + """ + {"error":{"reason":"no handler found"}}"""); + stub( + RUN_AS, + 401, + """ + {"error":{"reason":"unauthorized"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("opensearch", report.distribution()); + assertEquals("OpenSearch (Apache 2.0 / security plugin)", report.edition()); + assertEquals(CapabilityVerdict.UNAVAILABLE, capability(report, "API keys").verdict()); + assertEquals( + CapabilityVerdict.UNKNOWN, capability(report, "Document-level security (DLS)").verdict()); + assertTrue(report.restClientCompatibility().contains("OpenSearch")); + assertTrue(report.notes().stream().anyMatch(n -> n.contains("OpenSearch"))); + } + + @Test + void testUnreachableClusterProducesAReportRatherThanAnException() throws IOException { + when(esClient.performRequest(any(Request.class))) + .thenThrow(new IOException("connection refused")); + + ElasticSearchCapabilityReport report = + new ElasticSearchCapabilityService(esClient, config, this::apiKeyClient) + .getCapabilityReport(null, false); + + assertNotNull(report); + assertEquals("unknown", report.edition()); + assertEquals(1, report.capabilities().size()); + assertEquals(CapabilityVerdict.UNKNOWN, report.capabilities().get(0).verdict()); + assertTrue(report.recommendation().contains("unreachable")); + } + + @Test + void testEveryCapabilityCarriesEvidence() throws IOException { + stubSecurityEnabledCluster("trial"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + List capabilities = report.capabilities(); + assertEquals(5, capabilities.size()); + for (ElasticSearchCapability capability : capabilities) { + assertNotNull(capability.evidence(), capability.name() + " must cite its probe"); + assertFalse(capability.evidence().isBlank()); + assertNotNull(capability.detail()); + } + } + + // --------------------------------------------------------------------------- + // Write probes + // --------------------------------------------------------------------------- + + /** Stubs a cluster on which every write probe succeeds and DLS/FLS are genuinely enforced. */ + private void stubWorkingWriteProbes() { + stub( + CREATE_KEY, + 200, + """ + {"id":"plain-key-id","api_key":"plain-secret","encoded":"cGxhaW4="}"""); + stub( + "cGxhaW4=|" + AUTHENTICATE, + 200, + """ + {"username":"consent"}"""); + stub( + CREATE_ROLE, + 200, + """ + {"role":{"created":true}}"""); + stub( + COUNT, + 200, + """ + {"count":2}"""); + stub( + CREATE_DLS_KEY, + 200, + """ + {"id":"dls-key-id","api_key":"dls-secret","encoded":"ZGxz"}"""); + // A match_none DLS key must see none of the two documents. + stub( + "ZGxz|" + SEARCH, + 200, + """ + {"hits":{"total":{"value":0},"hits":[]}}"""); + stub( + CREATE_FLS_KEY, + 200, + """ + {"id":"fls-key-id","api_key":"fls-secret","encoded":"Zmxz"}"""); + // A key granting only datasetIdentifier must return only that field. + stub( + "Zmxz|" + SEARCH, + 200, + """ + {"hits":{"total":{"value":2},"hits":[{"_source":{"datasetIdentifier":"DUOS-000001"}}]}}"""); + stub( + INVALIDATE_KEY, + 200, + """ + {"invalidated_api_keys":["x"]}"""); + stub( + DELETE_ROLE, + 200, + """ + {"found":true}"""); + } + + @Test + void testWriteProbesObserveApiKeysDlsAndFlsRatherThanInferringThem() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertEquals(Boolean.TRUE, report.writeProbesRun()); + assertEquals(CapabilityVerdict.SUPPORTED, capability(report, "API keys").verdict()); + assertEquals( + CapabilityVerdict.SUPPORTED, capability(report, "Document-level security (DLS)").verdict()); + assertEquals( + CapabilityVerdict.SUPPORTED, capability(report, "Field-level security (FLS)").verdict()); + // The point of the exercise: proven, not inferred. + assertTrue( + capability(report, "Document-level security (DLS)").detail().contains("Proven end to end"), + "DLS should report end-to-end proof, not license inference"); + assertTrue(capability(report, "Document-level security (DLS)").detail().contains("0 of 2")); + // The evidence line has to stay a request an operator can paste into curl. + assertTrue( + capability(report, "Document-level security (DLS)").evidence().contains("_search?size=1"), + "evidence must cite the real query string: " + + capability(report, "Document-level security (DLS)").evidence()); + assertTrue( + capability(report, "Field-level security (FLS)").detail().contains("Proven end to end")); + assertTrue(report.recommendation().contains("Epic D")); + assertTrue(report.recommendation().contains("observed rather than")); + assertFalse( + report.notes().stream().anyMatch(n -> n.contains("non-destructive")), + "a write-probe run must not claim to have been non-destructive"); + } + + @Test + void testWriteProbesTearDownEveryResourceTheyCreate() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + List invalidations = requestsTo("DELETE", "/_security/api_key"); + assertEquals(3, invalidations.size(), "each of the three probe keys must be invalidated"); + String invalidated = invalidations.stream().map(this::bodyOf).reduce("", String::concat); + assertTrue(invalidated.contains("plain-key-id")); + assertTrue(invalidated.contains("dls-key-id")); + assertTrue(invalidated.contains("fls-key-id")); + assertEquals( + 1, requestsTo("DELETE", "/_security/role/").size(), "the probe role must be deleted"); + // Nothing was left behind, so nothing should be reported as left behind. + assertFalse(report.notes().stream().anyMatch(n -> n.contains("could not be"))); + } + + @Test + void testProbeResourcesAreNamespacedAndShortLived() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + + service().getCapabilityReport(null, true); + + for (Request creation : requestsTo("POST", "/_security/api_key")) { + String body = bodyOf(creation); + assertTrue(body.contains("duos-capability-probe-"), "probe keys must be namespaced: " + body); + assertTrue(body.contains("\"expiration\":\"10m\""), "probe keys must expire: " + body); + } + Request role = requestsTo("PUT", "/_security/role/").get(0); + assertTrue(role.getEndpoint().contains("duos_dlsfls_probe_"), role.getEndpoint()); + } + + /** + * A key created without role_descriptors inherits a snapshot of the creating credential's own + * permissions, which on a cluster where the deployment authenticates broadly would make the + * round-trip probe mint a live copy of that credential for ten minutes. Every probe key must + * carry a descriptor bounding what it can do. + */ + @Test + void testEveryProbeKeyIsScopedByARoleDescriptor() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + + service().getCapabilityReport(null, true); + + List creations = requestsTo("POST", "/_security/api_key"); + assertEquals(3, creations.size()); + for (Request creation : creations) { + assertTrue( + bodyOf(creation).contains("\"role_descriptors\""), + "a probe key must not inherit the deployment credential's privileges: " + + bodyOf(creation)); + } + // The plain round-trip key needs no privileges at all to prove it can authenticate. + String plainKey = + creations.stream() + .map(this::bodyOf) + .filter(b -> !b.contains("dls_probe") && !b.contains("fls_probe")) + .findFirst() + .orElseThrow(); + assertTrue( + plainKey.contains("\"cluster\":[]") && plainKey.contains("\"indices\":[]"), + "the round-trip probe key should grant nothing: " + plainKey); + } + + @Test + void testFailedTeardownIsReportedInTheNotesRatherThanSwallowed() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + DELETE_ROLE, + 500, + """ + {"error":{"reason":"boom"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertTrue( + report.notes().stream().anyMatch(n -> n.contains("could not be deleted")), + "an operator must be told a probe role was left on the cluster"); + assertTrue(report.notes().stream().anyMatch(n -> n.contains("duos_dlsfls_probe_"))); + } + + @Test + void testDlsFilterAcceptedButNotEnforcedIsReportedAsUnavailable() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + // The dangerous case: the cluster took the filter and ignored it. + stub( + "ZGxz|" + SEARCH, + 200, + """ + {"hits":{"total":{"value":2},"hits":[{"_source":{"x":1}}]}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.UNAVAILABLE, dls.verdict()); + assertTrue(dls.detail().contains("Not enforced")); + assertTrue(dls.detail().contains("2 of 2")); + assertTrue(report.recommendation().contains("Epic E"), report.recommendation()); + } + + @Test + void testFlsGrantAcceptedButNotProjectedIsReportedAsUnavailable() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + "Zmxz|" + SEARCH, + 200, + """ + {"hits":{"total":{"value":2},"hits":[{"_source":{"datasetIdentifier":"D","secret":"leaked"}}]}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability fls = capability(report, "Field-level security (FLS)"); + assertEquals(CapabilityVerdict.UNAVAILABLE, fls.verdict()); + assertTrue(fls.detail().contains("Not enforced")); + assertTrue( + fls.detail().contains("secret"), "the leaked field should be named: " + fls.detail()); + } + + @Test + void testBasicLicenseRefusesTheDlsFlsRoleAndIsReportedAsLicenseBlocked() throws IOException { + stubSecurityEnabledCluster("basic"); + stubWorkingWriteProbes(); + stub( + CREATE_ROLE, + 403, + """ + {"error":{"reason":"current license is non-compliant for [field and document level security]"}}"""); + // A Basic cluster accepts the key and only fails at search time — the failure mode the record + // doc calls out, so the probe has to reach the same verdict from a 403 on search. + stub( + "ZGxz|" + SEARCH, + 403, + """ + {"error":{"reason":"current license is non-compliant for [field and document level security]"}}"""); + stub( + "Zmxz|" + SEARCH, + 403, + """ + {"error":{"reason":"current license is non-compliant for [field and document level security]"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertEquals( + CapabilityVerdict.LICENSE_BLOCKED, + capability(report, "Document-level security (DLS)").verdict()); + assertEquals( + CapabilityVerdict.LICENSE_BLOCKED, + capability(report, "Field-level security (FLS)").verdict()); + // API keys are a Basic feature, so they should still come back proven. + assertEquals(CapabilityVerdict.SUPPORTED, capability(report, "API keys").verdict()); + assertTrue(report.recommendation().contains("Epic E")); + // The role was refused, so there is nothing to delete and no note claiming otherwise. + assertTrue(requestsTo("DELETE", "/_security/role/").isEmpty()); + } + + @Test + void testCredentialThatCannotMintKeysIsReportedAsNotPermitted() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + CREATE_KEY, + 403, + """ + {"error":{"reason":"action [cluster:admin/xpack/security/api_key/create] is unauthorized"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability apiKeys = capability(report, "API keys"); + assertEquals(CapabilityVerdict.NOT_PERMITTED, apiKeys.verdict()); + assertTrue(apiKeys.detail().contains("refused")); + // Without a key, the enforcement probes cannot run, so no key-scoped search is attempted. + assertTrue(requestsTo("GET", "/dataset/_search").isEmpty()); + // The role probe still answers the license question on its own. + assertEquals( + CapabilityVerdict.SUPPORTED, capability(report, "Document-level security (DLS)").verdict()); + } + + /** + * The likeliest real-world case: the deployment's shared credential holds neither manage_security + * nor manage_api_key. The probes then describe the credential, not the cluster, and must not be + * read as a verdict against the native path. + */ + @Test + void testCredentialWithoutWritePrivilegesYieldsInconclusiveRatherThanEpicE() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + CREATE_KEY, + 403, + """ + {"error":{"reason":"action [cluster:admin/xpack/security/api_key/create] is unauthorized"}}"""); + stub( + CREATE_ROLE, + 403, + """ + {"error":{"reason":"action [cluster:admin/xpack/security/role/put] is unauthorized"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertEquals( + CapabilityVerdict.NOT_PERMITTED, + capability(report, "Document-level security (DLS)").verdict()); + assertTrue( + report.recommendation().contains("Inconclusive from the write probes"), + "a privilege refusal says nothing about the cluster: " + report.recommendation()); + // It should still surface what the license implies rather than leaving the reader with nothing. + assertTrue(report.recommendation().contains("Epic D"), report.recommendation()); + assertTrue(report.recommendation().contains("cluster_privileges")); + } + + @Test + void testNoteDoesNotClaimResourcesWereCreatedWhenNothingWas() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + CREATE_KEY, + 403, + """ + {"error":{"reason":"unauthorized"}}"""); + stub( + CREATE_ROLE, + 403, + """ + {"error":{"reason":"unauthorized"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertTrue( + report.notes().stream().anyMatch(n -> n.contains("accepted nothing")), + "notes must say nothing was created: " + report.notes()); + assertFalse( + report.notes().stream().anyMatch(n -> n.contains("were created under")), + "notes must not claim resources were created and removed when none were"); + assertTrue(requestsTo("DELETE", "/_security/api_key").isEmpty()); + assertTrue(requestsTo("DELETE", "/_security/role/").isEmpty()); + } + + /** + * A dropped connection is not a licensing finding. Reporting one as LICENSE_BLOCKED would push + * the recommendation to Epic E off the back of a network blip. + */ + @Test + void testTransportFailureDuringEnforcementSearchIsUnknownNotLicenseBlocked() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + // No stub for the DLS key's search: the client throws, which is how a transport failure + // arrives. + stubs.remove("ZGxz|" + SEARCH); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.UNKNOWN, dls.verdict()); + assertTrue(dls.detail().contains("transport failure"), dls.detail()); + } + + /** + * An API key is capped by its owner's privileges, so a search 403 need not be about licensing. + */ + @Test + void testPrivilegeRefusalDuringEnforcementSearchIsNotReportedAsLicenseBlocked() + throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + "ZGxz|" + SEARCH, + 403, + """ + {"error":{"reason":"action [indices:data/read/search] is unauthorized for API key"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.NOT_PERMITTED, dls.verdict()); + assertTrue(dls.detail().contains("privilege rather than licensing"), dls.detail()); + // And a privilege refusal must not be read as a verdict against the native path. + assertTrue(report.recommendation().contains("Inconclusive"), report.recommendation()); + } + + /** + * "Not enforced" is the gravest verdict this report can return — a filter accepted and ignored. + * It must never come from a response shape the parser simply did not recognise. + */ + @Test + void testUnparseableSearchResponseIsUnknownRatherThanNotEnforced() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + "ZGxz|" + SEARCH, + 200, + """ + {"took":1,"timed_out":false}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.UNKNOWN, dls.verdict()); + assertFalse(dls.detail().contains("Not enforced"), dls.detail()); + assertTrue(dls.detail().contains("no hit total"), dls.detail()); + } + + @Test + void testFlsWithNoInspectableFieldsFallsBackAndSaysWhy() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + // A hit with no _source at all: nothing to check the projection against. + stub( + "Zmxz|" + SEARCH, + 200, + """ + {"hits":{"total":{"value":2},"hits":[{"_id":"1"}]}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + // Role acceptance stands, but the report must not let that read as proven enforcement. + assertEquals( + CapabilityVerdict.SUPPORTED, capability(report, "Field-level security (FLS)").verdict()); + assertFalse(capability(report, "Field-level security (FLS)").detail().contains("Proven")); + assertTrue( + report.notes().stream().anyMatch(n -> n.contains("no document fields to inspect")), + "the reader must be told the projection check was inconclusive: " + report.notes()); + } + + @Test + void testMissingProbeKeyIsSaidToLimitTheDlsAndFlsVerdicts() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + CREATE_KEY, + 403, + """ + {"error":{"reason":"unauthorized"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertTrue( + report.notes().stream().anyMatch(n -> n.contains("No usable probe key")), + "role acceptance must not be presented as enforcement: " + report.notes()); + } + + @Test + void testProbeNamesAreUniquePerRunNotPerMillisecond() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + + service().getCapabilityReport(null, true); + String first = requestsTo("PUT", "/_security/role/").get(0).getEndpoint(); + requests.clear(); + service().getCapabilityReport(null, true); + String second = requestsTo("PUT", "/_security/role/").get(0).getEndpoint(); + + assertNotEquals(first, second, "two runs must not be able to collide on a probe role name"); + } + + @Test + void testEmptyIndexFallsBackToRoleAcceptanceAndSaysSo() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + COUNT, + 200, + """ + {"count":0}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + // Role acceptance still stands, but it must not be dressed up as end-to-end proof. + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.SUPPORTED, dls.verdict()); + assertFalse(dls.detail().contains("Proven end to end")); + assertTrue( + report.notes().stream().anyMatch(n -> n.contains("empty")), + "an empty index must be called out rather than read as a pass"); + assertTrue(requestsTo("GET", "/dataset/_search").isEmpty()); + } + + @Test + void testUnreadableIndexIsCalledOutRatherThanTreatedAsEnforcement() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + COUNT, + 403, + """ + {"error":{"reason":"unauthorized"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertTrue(report.notes().stream().anyMatch(n -> n.contains("not readable"))); + assertEquals( + CapabilityVerdict.SUPPORTED, capability(report, "Document-level security (DLS)").verdict()); + } + + @Test + void testWriteProbesAreNotRunWhenSecurityIsDisabled() throws IOException { + stubSecurityDisabledCluster(); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertEquals(Boolean.FALSE, report.writeProbesRun()); + assertTrue( + report.notes().stream().anyMatch(n -> n.contains("requested but not run")), + "asking for write probes on a security-disabled cluster must be answered explicitly"); + assertTrue(requestsTo("POST", "/_security/api_key").isEmpty()); + for (ElasticSearchCapability capability : report.capabilities()) { + assertEquals(CapabilityVerdict.UNAVAILABLE, capability.verdict()); + } + } + + @Test + void testReadOnlyRunCreatesNothing() throws IOException { + stubSecurityEnabledCluster("trial"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals(Boolean.FALSE, report.writeProbesRun()); + assertTrue(requestsTo("POST", "/_security/api_key").isEmpty()); + assertTrue(requestsTo("PUT", "/_security/role/").isEmpty()); + assertTrue(requestsTo("DELETE", "/_security/api_key").isEmpty()); + assertTrue(report.notes().stream().anyMatch(n -> n.contains("non-destructive"))); + } + + private record StubResponse(int status, String body) {} +} From 71309ff00c8185d4e63ae30f245ef3be59b1e999 Mon Sep 17 00:00:00 2001 From: Elliot Otchet Date: Wed, 29 Jul 2026 19:13:53 +0000 Subject: [PATCH 02/10] fix: semgrep config for non-key keys. --- .../consent/http/service/ElasticSearchCapabilityService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java index 9fe9a6239..cd9a6d7c3 100644 --- a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java +++ b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java @@ -103,6 +103,7 @@ public class ElasticSearchCapabilityService implements ConsentLogger { "xpack.security.transport.ssl.enabled"); private static final String AUTHENTICATE_PATH = "/_security/_authenticate"; + // nosemgrep - an endpoint path, not a key private static final String API_KEY_PATH = "/_security/api_key"; private static final String RUN_AS_HEADER = "es-security-runas-user"; @@ -114,12 +115,14 @@ public class ElasticSearchCapabilityService implements ConsentLogger { */ private static final Pattern RUN_AS_USERNAME = Pattern.compile("[A-Za-z0-9._@+\\-]{1,255}"); + // nosemgrep - a capability label in the report, not a key private static final String API_KEYS = "API keys"; private static final String DLS = "Document-level security (DLS)"; private static final String FLS = "Field-level security (FLS)"; private static final String RUN_AS = "run_as impersonation"; /** How long a probe API key lives even if teardown never runs. */ + // nosemgrep - a duration, not a key private static final String PROBE_KEY_EXPIRATION = "10m"; /** The field a probe FLS grant is scoped to; a real field of the dataset index. */ @@ -132,6 +135,7 @@ public class ElasticSearchCapabilityService implements ConsentLogger { * An empty descriptor grants nothing, and {@code GET /_security/_authenticate} needs nothing, so * the round trip is proven just as well by a key that can do nothing else. */ + // nosemgrep - an empty privilege grant, not a key private static final String PRIVILEGE_FREE_DESCRIPTOR = """ {"probe":{"cluster":[],"indices":[]}}"""; @@ -341,6 +345,7 @@ private ElasticSearchCapability apiKeyCapability( "Security is disabled, so API keys cannot be issued.", "GET " + AUTHENTICATE_PATH); } + // nosemgrep - a cluster setting name, not a key if ("false".equals(settings.get("xpack.security.authc.api_key.enabled"))) { return new ElasticSearchCapability( API_KEYS, From ff91e0f88cea6ab590d85760e4730e20409a8743 Mon Sep 17 00:00:00 2001 From: Elliot Otchet Date: Wed, 29 Jul 2026 19:38:46 +0000 Subject: [PATCH 03/10] feedback: copilot feedback. --- docs/plans/es-security-capability-record.md | 10 +++ .../consent/http/ConsentModule.java | 8 ++- .../http/health/ElasticSearchHealthCheck.java | 23 +++---- .../ElasticSearchCapabilityResource.java | 5 ++ .../ElasticSearchCapabilityService.java | 31 +++++++-- .../paths/elasticSearchCapabilities.yaml | 11 +++- .../ElasticSearchCapabilityReport.yaml | 20 +++++- .../consent/http/ConsentModuleTest.java | 9 ++- .../health/ElasticSearchHealthCheckTest.java | 3 +- .../ElasticSearchCapabilityServiceTest.java | 63 +++++++++++++++---- 10 files changed, 140 insertions(+), 43 deletions(-) diff --git a/docs/plans/es-security-capability-record.md b/docs/plans/es-security-capability-record.md index da1bb3b74..8f4c22971 100644 --- a/docs/plans/es-security-capability-record.md +++ b/docs/plans/es-security-capability-record.md @@ -82,6 +82,16 @@ Three fields carry most of the interpretive weight: - **`security_settings`** — filtered to the dozen or so values that gate a capability, out of the ~50 defaults a cluster reports. +**Scope: these are X-Pack probes, so they measure Elasticsearch and not OpenSearch.** OpenSearch's +security works differently — DLS and FLS come from its security plugin under `/_plugins/_security`, +and there is no `POST /_security/api_key` — so the endpoint identifies the distribution from +`GET /` and reports the X-Pack capabilities as out of scope rather than drawing X-Pack conclusions +from probes that do not fit: DLS/FLS come back `UNKNOWN` (not inspected), API keys `UNAVAILABLE` +(no such endpoint), and write probes never run whatever `writeProbes` says. Those verdicts are not +license inferences and `writeProbes=true` will not change them. Measuring an OpenSearch cluster +properly would mean a second set of `/_plugins/_security` probes, which is out of scope here — every +environment in the inventory below runs Elasticsearch. + ## Environment inventory ### Local (`config/docker-compose.yaml`) — measured 2026-07-29 with `writeProbes=true` diff --git a/src/main/java/org/broadinstitute/consent/http/ConsentModule.java b/src/main/java/org/broadinstitute/consent/http/ConsentModule.java index ecec9d5d9..4fa63523a 100644 --- a/src/main/java/org/broadinstitute/consent/http/ConsentModule.java +++ b/src/main/java/org/broadinstitute/consent/http/ConsentModule.java @@ -40,6 +40,7 @@ import org.broadinstitute.consent.http.db.VoteDAO; import org.broadinstitute.consent.http.filters.ClaimsCache; import org.broadinstitute.consent.http.filters.RateLimitFilter; +import org.broadinstitute.consent.http.health.ElasticSearchHealthCheck; import org.broadinstitute.consent.http.mail.SendGridAPI; import org.broadinstitute.consent.http.mail.freemarker.FreeMarkerTemplateHelper; import org.broadinstitute.consent.http.matching.DataUseMatcherV4; @@ -367,8 +368,11 @@ private DataUseMatcherV4 providesDataUseMatcherV4(DataUseUtil dataUseUtil) { /** * The application's Elasticsearch client. A singleton because each {@link RestClient} owns its * own connection pool and background threads: building one per consumer multiplies pools against - * the same cluster for no benefit. Closed on shutdown, since nothing else releases those - * connections. + * the same cluster for no benefit. Every consumer in the application — {@link + * ElasticSearchService}, {@link ElasticSearchCapabilityService}, and {@link + * ElasticSearchHealthCheck} — takes this instance by injection rather than calling {@code + * ElasticSearchSupport.createRestClient} itself. Closed on shutdown, since nothing else releases + * those connections. */ @Provides @Singleton diff --git a/src/main/java/org/broadinstitute/consent/http/health/ElasticSearchHealthCheck.java b/src/main/java/org/broadinstitute/consent/http/health/ElasticSearchHealthCheck.java index 5e02d56aa..35ac367ab 100644 --- a/src/main/java/org/broadinstitute/consent/http/health/ElasticSearchHealthCheck.java +++ b/src/main/java/org/broadinstitute/consent/http/health/ElasticSearchHealthCheck.java @@ -6,34 +6,27 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.inject.Inject; -import io.dropwizard.lifecycle.Managed; import java.io.IOException; import java.nio.charset.Charset; import org.apache.commons.io.IOUtils; -import org.broadinstitute.consent.http.configurations.ElasticSearchConfiguration; import org.broadinstitute.consent.http.service.ontology.ElasticSearchSupport; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; -public class ElasticSearchHealthCheck extends HealthCheck implements Managed { +public class ElasticSearchHealthCheck extends HealthCheck { private final RestClient client; - @Override - public void start() throws Exception {} - - @Override - public void stop() throws Exception { - if (client != null) { - client.close(); - } - } - + /** + * Takes the application's shared {@link RestClient} rather than building one: a health check that + * opened its own connection pool would double the pools held against the same cluster. The client + * is closed by the module that provides it, so nothing is closed here. + */ @Inject - public ElasticSearchHealthCheck(ElasticSearchConfiguration config) { - this.client = ElasticSearchSupport.createRestClient(config); + public ElasticSearchHealthCheck(RestClient client) { + this.client = client; } @Override diff --git a/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java b/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java index 07e414e04..2d6e7ae2c 100644 --- a/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java +++ b/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java @@ -23,6 +23,11 @@ * network access to the clusters or a copy of their credentials — the application already holds * them. * + *

The probes behind this endpoint are Elasticsearch X-Pack probes. Against an OpenSearch + * cluster, whose security works through a different plugin, the report identifies the distribution + * and reports the X-Pack capabilities as not applicable rather than drawing conclusions from probes + * that do not fit — see {@link ElasticSearchCapabilityService}. + * *

The probes behind this endpoint are read-only by default. Pass {@code writeProbes=true} to * additionally create and tear down a short-lived API key and role, which is the only way to * observe DLS, FLS, and API-key support rather than infer it from the license tier — see {@link diff --git a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java index cd9a6d7c3..620d69c68 100644 --- a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java +++ b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java @@ -36,6 +36,17 @@ * Inventories the security features of the Elasticsearch cluster this deployment is configured * against: version, edition, X-Pack Security, DLS, FLS, API keys, and {@code run_as}. * + *

Every probe here targets Elasticsearch's X-Pack security APIs, not OpenSearch. + * OpenSearch implements security differently — DLS and FLS live in its security plugin under {@code + * /_plugins/_security}, and it has no {@code POST /_security/api_key} at all — so none of the + * X-Pack probes are meaningful there. This service detects that case from {@code + * version.distribution} on {@code GET /} and reports it rather than probing: the DLS/FLS verdicts + * come back {@code UNKNOWN} because this probe does not inspect the OpenSearch security plugin, the + * API-key verdict comes back {@code UNAVAILABLE} because that endpoint does not exist, and write + * probes never run. Those verdicts are scope statements, not license inferences, and {@code + * writeProbes=true} does not change them. Covering OpenSearch properly would mean a separate set of + * {@code /_plugins/_security} probes. + * *

The default pass is non-destructive. Nothing is created, updated, or deleted, so it is * safe to run anywhere — but DLS, FLS, and API keys cannot be *proven* without creating a role or a * key, so a read-only pass reports those with an {@code INFERRED_} verdict reasoned from the @@ -230,16 +241,21 @@ public ElasticSearchCapabilityReport getCapabilityReport(String runAsUser, boole } if (isOpenSearch) { notes.add( - "This cluster is OpenSearch, not Elasticsearch. The X-Pack security APIs do not exist " - + "here: DLS/FLS come from the OpenSearch security plugin under /_plugins/_security, " - + "and there is no POST /_security/api_key."); + "This cluster is OpenSearch, not Elasticsearch. Every probe in this report targets the " + + "X-Pack security APIs, which do not exist here: DLS/FLS come from the OpenSearch " + + "security plugin under /_plugins/_security, and there is no POST " + + "/_security/api_key. The DLS, FLS, and API-key verdicts below are therefore " + + "statements about what this probe covers on OpenSearch, not license inferences, " + + "and no value of writeProbes changes them."); } boolean writeProbesRan = writeProbes && securityApiPresent && !isOpenSearch; if (!securityApiPresent) { notes.add( "The /_security API is not available on this cluster, so no security feature can be " + "exercised. Every security verdict below follows from that one fact."); - } else if (!writeProbesRan) { + } else if (!writeProbesRan && !isOpenSearch) { + // Only an Elasticsearch cluster has verdicts that a write probe would convert from inferred + // to observed; on OpenSearch nothing here is inferred and nothing would be written. notes.add( "DLS, FLS, and API-key verdicts are inferred from the license tier and cluster " + "settings. Re-run with writeProbes=true to create and tear down a short-lived key " @@ -247,8 +263,11 @@ public ElasticSearchCapabilityReport getCapabilityReport(String runAsUser, boole } if (writeProbes && !writeProbesRan) { notes.add( - "Write probes were requested but not run: they need the /_security API on an " - + "Elasticsearch distribution."); + isOpenSearch + ? "Write probes were requested but not run: they create an X-Pack role and API key, " + + "which OpenSearch does not provide." + : "Write probes were requested but not run: they need the /_security API, which this " + + "cluster did not answer as a security-enabled cluster would."); } WriteProbeOutcome writeProbeOutcome = writeProbesRan ? runWriteProbes(notes) : null; diff --git a/src/main/resources/assets/paths/elasticSearchCapabilities.yaml b/src/main/resources/assets/paths/elasticSearchCapabilities.yaml index 66768c96c..de26fd6b6 100644 --- a/src/main/resources/assets/paths/elasticSearchCapabilities.yaml +++ b/src/main/resources/assets/paths/elasticSearchCapabilities.yaml @@ -10,6 +10,12 @@ get: dev, staging, and production produces the per-environment inventory without needing direct network access to the clusters or a copy of their credentials. + The probes are Elasticsearch X-Pack probes. OpenSearch secures itself differently — DLS and + FLS come from its security plugin under /_plugins/_security, and there is no + POST /_security/api_key — so against an OpenSearch cluster the report identifies the + distribution, returns UNKNOWN for the DLS/FLS verdicts it does not inspect, and skips write + probes entirely rather than reporting X-Pack conclusions that would not apply. + By default all probes are non-destructive: nothing is created, modified, or deleted on the cluster. The trade-off is certainty — DLS, FLS, and API-key support can only be proven by creating a role or a key, so in that mode those verdicts are inferred from the license tier @@ -40,8 +46,9 @@ get: Create and tear down a short-lived API key and role so that DLS, FLS, and API-key support are observed rather than inferred. Requires the deployment credential to hold manage_api_key (or grant_api_key) and manage_security; the report says which of those - it has under cluster_privileges. Ignored on OpenSearch or when security is disabled, - since there is nothing to probe. + it has under cluster_privileges. Ignored on OpenSearch, which has no X-Pack role or + API-key endpoints to write to, and when security is disabled, since there is nothing to + probe. In both cases a note in the response says the probes were requested but not run. required: false schema: type: boolean diff --git a/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml b/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml index 28ce616d0..5b5ee1c3b 100644 --- a/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml +++ b/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml @@ -6,6 +6,11 @@ description: | non-destructive, which is why the DLS, FLS, and API-key verdicts may be inferred rather than observed; write_probes_run says which of the two this report is. + The probes target Elasticsearch's X-Pack security APIs only. Against an OpenSearch cluster — + which secures itself through a plugin under /_plugins/_security and has no + POST /_security/api_key — the report says so in distribution and notes, returns UNKNOWN for + DLS/FLS because it does not inspect that plugin, and never runs write probes. + The example is a real writeProbes=true response from a local trial-licensed cluster, where the configured credential is the `elastic` superuser. A deployed environment's shared credential is unlikely to hold manage_security or manage_api_key, so expect NOT_PERMITTED verdicts and a @@ -97,13 +102,19 @@ properties: write_probes_run: type: boolean description: | - Whether write probes ran. When false, the DLS, FLS, and API-key verdicts are inferred from - the license tier rather than observed — read this field before reading those verdicts. + Whether write probes ran. When false on an Elasticsearch cluster, the DLS, FLS, and API-key + verdicts are inferred from the license tier rather than observed — read this field before + reading those verdicts. On OpenSearch it is always false and nothing is inferred: those + verdicts report that the X-Pack probes do not apply. capabilities: type: array description: | One entry per probed capability, always in the same order: X-Pack Security, API keys, DLS, FLS, run_as impersonation. + + An unreachable cluster is the one exception to that shape: when GET / does not answer, no + capability can be probed at all, so the array instead holds the single Cluster reachability + entry with an UNKNOWN verdict and the rest of the report is empty. items: type: object title: ElasticSearchCapability @@ -119,12 +130,15 @@ properties: - Document-level security (DLS) - Field-level security (FLS) - run_as impersonation + - Cluster reachability verdict: type: string description: | SUPPORTED and UNAVAILABLE are observed. LICENSE_BLOCKED and NOT_PERMITTED are observed refusals. The INFERRED_ values are derived from license tier and cluster - settings because proving them would require writing to the cluster. + settings because proving them would require writing to the cluster. UNKNOWN also + covers what the probe does not reach — DLS/FLS on OpenSearch, or any capability on + a cluster that did not respond. enum: - SUPPORTED - UNAVAILABLE diff --git a/src/test/java/org/broadinstitute/consent/http/ConsentModuleTest.java b/src/test/java/org/broadinstitute/consent/http/ConsentModuleTest.java index d1eef788c..5de5c74d6 100644 --- a/src/test/java/org/broadinstitute/consent/http/ConsentModuleTest.java +++ b/src/test/java/org/broadinstitute/consent/http/ConsentModuleTest.java @@ -45,6 +45,7 @@ import org.broadinstitute.consent.http.db.UserRoleDAO; import org.broadinstitute.consent.http.db.VoteDAO; import org.broadinstitute.consent.http.filters.ClaimsCache; +import org.broadinstitute.consent.http.health.ElasticSearchHealthCheck; import org.broadinstitute.consent.http.mail.SendGridAPI; import org.broadinstitute.consent.http.mail.freemarker.FreeMarkerTemplateHelper; import org.broadinstitute.consent.http.matching.DataUseMatcherV4; @@ -372,13 +373,15 @@ void testProvidesASingleSharedElasticSearchClient() { } @Test - void testElasticSearchServicesShareThatOneClient() { + void testElasticSearchConsumersShareThatOneClient() { RestClient restClient = injector.getInstance(RestClient.class); - // Both services are constructed with the injected client rather than building their own, so - // resolving them must not add any further clients to close. + // Every consumer is constructed with the injected client rather than building its own, so + // resolving them must not add any further clients to close. The health check is included + // because it used to open a second pool of its own. injector.getInstance(ElasticSearchService.class); injector.getInstance(ElasticSearchCapabilityService.class); + injector.getInstance(ElasticSearchHealthCheck.class); assertEquals( 1, diff --git a/src/test/java/org/broadinstitute/consent/http/health/ElasticSearchHealthCheckTest.java b/src/test/java/org/broadinstitute/consent/http/health/ElasticSearchHealthCheckTest.java index f55b48905..3730b0af7 100644 --- a/src/test/java/org/broadinstitute/consent/http/health/ElasticSearchHealthCheckTest.java +++ b/src/test/java/org/broadinstitute/consent/http/health/ElasticSearchHealthCheckTest.java @@ -12,6 +12,7 @@ import java.util.Collections; import org.broadinstitute.consent.http.WireMockTestHelper; import org.broadinstitute.consent.http.configurations.ElasticSearchConfiguration; +import org.broadinstitute.consent.http.service.ontology.ElasticSearchSupport; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -36,7 +37,7 @@ private void initHealthCheck(String status, Integer statusCode) { wireMockServer.stubFor( any(anyUrl()).willReturn(aResponse().withStatus(statusCode).withBody(stringResponse))); - healthCheck = new ElasticSearchHealthCheck(config); + healthCheck = new ElasticSearchHealthCheck(ElasticSearchSupport.createRestClient(config)); } catch (Exception e) { fail(e.getMessage()); } diff --git a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java index 7f9ffeff5..d69e3bfb0 100644 --- a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java @@ -173,7 +173,10 @@ private List requestsTo(String method, String endpointPrefix) { /** * Stubs are lenient because which accessors get touched depends on the status: only the error - * path builds a ResponseException, which is what reads the request line. + * path builds a ResponseException, which is what reads the request line. Under Mockito's default + * strictness the unread stub fails the test with an UnnecessaryStubbingException that says + * nothing about the code under test — dropping {@code lenient()} here fails every test that stubs + * a 2xx response. */ private Response response(int status, String body) { Response response = mock(Response.class); @@ -433,6 +436,50 @@ void testRunAsBlockedByLicenseIsReportedAsSuch() throws IOException { @Test void testOpenSearchIsCalledOutRatherThanMisreported() throws IOException { + stubOpenSearchClusterWithSecurityPresent(); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("opensearch", report.distribution()); + assertEquals("OpenSearch (Apache 2.0 / security plugin)", report.edition()); + assertEquals(CapabilityVerdict.UNAVAILABLE, capability(report, "API keys").verdict()); + assertEquals( + CapabilityVerdict.UNKNOWN, capability(report, "Document-level security (DLS)").verdict()); + assertTrue(report.restClientCompatibility().contains("OpenSearch")); + assertTrue(report.notes().stream().anyMatch(n -> n.contains("OpenSearch"))); + } + + /** + * A 401 from /_security/_authenticate reads as "security is present but this credential was + * refused", which is true on OpenSearch too. That must not pull in the Elasticsearch note about + * inferred verdicts: on OpenSearch the DLS, FLS, and API-key verdicts are not license inferences + * and write probes never run, so telling the reader to re-run with writeProbes=true would be + * advice that changes nothing. + */ + @Test + void testOpenSearchIsNotDescribedWithTheElasticsearchInferenceNote() throws IOException { + stubOpenSearchClusterWithSecurityPresent(); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertFalse(report.writeProbesRun()); + assertTrue( + report.notes().stream().noneMatch(n -> n.contains("writeProbes=true")), + "no note should suggest write probes against OpenSearch"); + assertTrue( + report.notes().stream().noneMatch(n -> n.startsWith("DLS, FLS, and API-key verdicts are ")), + "the inferred-verdict note is an Elasticsearch statement"); + assertTrue( + report.notes().stream() + .anyMatch(n -> n.contains("Write probes were requested but not run")), + "a requested write probe that did not run has to be said out loud"); + // Nothing was written, and in particular no attempt was made against endpoints OpenSearch + // does not have. + assertTrue(requestsTo("POST", "/_security/api_key").isEmpty()); + assertTrue(requestsTo("PUT", "/_security/role").isEmpty()); + } + + private void stubOpenSearchClusterWithSecurityPresent() { stub( ROOT, 200, @@ -453,6 +500,7 @@ void testOpenSearchIsCalledOutRatherThanMisreported() throws IOException { 200, """ {"defaults":{},"persistent":{},"transient":{}}"""); + // OpenSearch answers this path from its own security plugin, and refuses rather than 404s. stub( AUTHENTICATE, 401, @@ -468,16 +516,6 @@ void testOpenSearchIsCalledOutRatherThanMisreported() throws IOException { 401, """ {"error":{"reason":"unauthorized"}}"""); - - ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); - - assertEquals("opensearch", report.distribution()); - assertEquals("OpenSearch (Apache 2.0 / security plugin)", report.edition()); - assertEquals(CapabilityVerdict.UNAVAILABLE, capability(report, "API keys").verdict()); - assertEquals( - CapabilityVerdict.UNKNOWN, capability(report, "Document-level security (DLS)").verdict()); - assertTrue(report.restClientCompatibility().contains("OpenSearch")); - assertTrue(report.notes().stream().anyMatch(n -> n.contains("OpenSearch"))); } @Test @@ -493,6 +531,9 @@ void testUnreachableClusterProducesAReportRatherThanAnException() throws IOExcep assertEquals("unknown", report.edition()); assertEquals(1, report.capabilities().size()); assertEquals(CapabilityVerdict.UNKNOWN, report.capabilities().get(0).verdict()); + // The published schema enumerates the capability names, so this one entry — the only shape + // the report takes other than the five probed capabilities — has to stay in that enum. + assertEquals("Cluster reachability", report.capabilities().get(0).name()); assertTrue(report.recommendation().contains("unreachable")); } From 39dea4d05669577dd5279dbdfc317398ae7ad30c Mon Sep 17 00:00:00 2001 From: Elliot Otchet Date: Wed, 29 Jul 2026 20:43:12 +0000 Subject: [PATCH 04/10] feedback: remove lenient stubbing. --- .../ElasticSearchCapabilityServiceTest.java | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java index d69e3bfb0..fc9fe734d 100644 --- a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java @@ -6,7 +6,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -172,23 +171,19 @@ private List requestsTo(String method, String endpointPrefix) { } /** - * Stubs are lenient because which accessors get touched depends on the status: only the error - * path builds a ResponseException, which is what reads the request line. Under Mockito's default - * strictness the unread stub fails the test with an UnnecessaryStubbingException that says - * nothing about the code under test — dropping {@code lenient()} here fails every test that stubs - * a 2xx response. + * Only the error path builds a {@link ResponseException}, which is what reads the request line; + * the 2xx path reads just the status and entity. Stubbing the request line only for non-2xx + * responses keeps every stub read by the test it's created for, so no stub needs to be lenient. */ private Response response(int status, String body) { Response response = mock(Response.class); - lenient() - .when(response.getStatusLine()) + when(response.getStatusLine()) .thenReturn(new BasicStatusLine(HttpVersion.HTTP_1_1, status, "reason")); - lenient() - .when(response.getEntity()) - .thenReturn(new StringEntity(body, ContentType.APPLICATION_JSON)); - lenient() - .when(response.getRequestLine()) - .thenReturn(new BasicRequestLine("GET", "/", HttpVersion.HTTP_1_1)); + when(response.getEntity()).thenReturn(new StringEntity(body, ContentType.APPLICATION_JSON)); + if (status >= 300) { + when(response.getRequestLine()) + .thenReturn(new BasicRequestLine("GET", "/", HttpVersion.HTTP_1_1)); + } return response; } From dd36cbabe68466343419ebbef47fb4611ffd0140 Mon Sep 17 00:00:00 2001 From: Elliot Otchet Date: Wed, 29 Jul 2026 21:40:47 +0000 Subject: [PATCH 05/10] fix: address sonar, be clearer that we're not supporting opensearch by removing refs to it, improve coverage. --- docs/plans/es-security-capability-record.md | 18 +- .../ElasticSearchCapabilityReport.java | 4 +- .../ElasticSearchCapabilityResource.java | 6 +- .../ElasticSearchCapabilityService.java | 314 +++++----- .../paths/elasticSearchCapabilities.yaml | 12 +- .../ElasticSearchCapabilityReport.yaml | 21 +- .../ElasticSearchCapabilityServiceTest.java | 560 +++++++++++++++--- 7 files changed, 645 insertions(+), 290 deletions(-) diff --git a/docs/plans/es-security-capability-record.md b/docs/plans/es-security-capability-record.md index 8f4c22971..25f8e94df 100644 --- a/docs/plans/es-security-capability-record.md +++ b/docs/plans/es-security-capability-record.md @@ -82,15 +82,9 @@ Three fields carry most of the interpretive weight: - **`security_settings`** — filtered to the dozen or so values that gate a capability, out of the ~50 defaults a cluster reports. -**Scope: these are X-Pack probes, so they measure Elasticsearch and not OpenSearch.** OpenSearch's -security works differently — DLS and FLS come from its security plugin under `/_plugins/_security`, -and there is no `POST /_security/api_key` — so the endpoint identifies the distribution from -`GET /` and reports the X-Pack capabilities as out of scope rather than drawing X-Pack conclusions -from probes that do not fit: DLS/FLS come back `UNKNOWN` (not inspected), API keys `UNAVAILABLE` -(no such endpoint), and write probes never run whatever `writeProbes` says. Those verdicts are not -license inferences and `writeProbes=true` will not change them. Measuring an OpenSearch cluster -properly would mean a second set of `/_plugins/_security` probes, which is out of scope here — every -environment in the inventory below runs Elasticsearch. +**Scope: these are X-Pack probes, so they measure Elasticsearch only.** The endpoint identifies the +distribution from `GET /` and is otherwise scoped to Elasticsearch deployments; every environment +in the inventory below runs Elasticsearch. ## Environment inventory @@ -105,7 +99,7 @@ inference — this is the first environment where all five capabilities came bac | Capability | Verdict | Evidence | | --- | --- | --- | | Elasticsearch version | 9.4.4 | `GET /` → `version.number` | -| Distribution | elasticsearch (not OpenSearch) | `GET /` → `version.distribution` | +| Distribution | elasticsearch | `GET /` → `version.distribution` | | Edition / license | Trial (Platinum-equivalent), `status: active`, expires 2026-08-28 | `GET /_license` → `type: trial` | | X-Pack Security enabled | **`SUPPORTED`** | `GET /_xpack` 200; `GET /_security/_authenticate` 200 | | DLS | **`SUPPORTED` — enforced, not merely accepted** | a `match_none` DLS key returned **0 of 1158** documents from `GET /dataset/_search` | @@ -318,9 +312,7 @@ between client and cluster, which is what a deployed environment on an older min No dependency change is needed. The low-level `RestClient` is a version-agnostic HTTP transport with no typed request model, so security endpoints are reached with `RestClient.performRequest(Request)` and a JSON entity — neither the high-level REST client -(removed in 8.x) nor the new typed Java API client is required. The one caveat is that this -holds for Elasticsearch; against OpenSearch there is no `POST /_security/api_key` at all, and -the endpoint flags that case explicitly. +(removed in 8.x) nor the new typed Java API client is required. `ElasticSearchCapabilityService` is the standing demonstration of that conclusion, which is why no separate feasibility test is kept: it drives the same security APIs from inside the application diff --git a/src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapabilityReport.java b/src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapabilityReport.java index 5d60584a5..cfe527141 100644 --- a/src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapabilityReport.java +++ b/src/main/java/org/broadinstitute/consent/http/models/elastic_search/ElasticSearchCapabilityReport.java @@ -14,8 +14,8 @@ * * @param clusterName the cluster's own name, to confirm which cluster was reached * @param version Elasticsearch version, e.g. {@code 9.3.3} - * @param distribution {@code elasticsearch} or {@code opensearch}; the security APIs differ - * @param edition OSS, Basic, Enterprise, Elastic Cloud, or OpenSearch + * @param distribution the distribution reported by the cluster, e.g. {@code elasticsearch} + * @param edition OSS, a license tier, or Elastic Cloud * @param licenseType license tier reported by the cluster * @param licenseStatus whether that license is active * @param elasticCloud whether the deployment is configured with a cloud ID diff --git a/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java b/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java index 2d6e7ae2c..1d1ea7f5a 100644 --- a/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java +++ b/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java @@ -23,10 +23,8 @@ * network access to the clusters or a copy of their credentials — the application already holds * them. * - *

The probes behind this endpoint are Elasticsearch X-Pack probes. Against an OpenSearch - * cluster, whose security works through a different plugin, the report identifies the distribution - * and reports the X-Pack capabilities as not applicable rather than drawing conclusions from probes - * that do not fit — see {@link ElasticSearchCapabilityService}. + *

The probes behind this endpoint are Elasticsearch X-Pack probes — see {@link + * ElasticSearchCapabilityService}. * *

The probes behind this endpoint are read-only by default. Pass {@code writeProbes=true} to * additionally create and tear down a short-lived API key and role, which is the only way to diff --git a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java index 620d69c68..8e2dc9465 100644 --- a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java +++ b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java @@ -36,16 +36,8 @@ * Inventories the security features of the Elasticsearch cluster this deployment is configured * against: version, edition, X-Pack Security, DLS, FLS, API keys, and {@code run_as}. * - *

Every probe here targets Elasticsearch's X-Pack security APIs, not OpenSearch. - * OpenSearch implements security differently — DLS and FLS live in its security plugin under {@code - * /_plugins/_security}, and it has no {@code POST /_security/api_key} at all — so none of the - * X-Pack probes are meaningful there. This service detects that case from {@code - * version.distribution} on {@code GET /} and reports it rather than probing: the DLS/FLS verdicts - * come back {@code UNKNOWN} because this probe does not inspect the OpenSearch security plugin, the - * API-key verdict comes back {@code UNAVAILABLE} because that endpoint does not exist, and write - * probes never run. Those verdicts are scope statements, not license inferences, and {@code - * writeProbes=true} does not change them. Covering OpenSearch properly would mean a separate set of - * {@code /_plugins/_security} probes. + *

Every probe here targets Elasticsearch's X-Pack security APIs. This service is scoped + * to Elasticsearch deployments only; it does not detect or account for other search engines. * *

The default pass is non-destructive. Nothing is created, updated, or deleted, so it is * safe to run anywhere — but DLS, FLS, and API keys cannot be *proven* without creating a role or a @@ -92,6 +84,12 @@ public class ElasticSearchCapabilityService implements ConsentLogger { "read_security", "monitor"); + // nosemgrep - a cluster setting name, not a key + private static final String API_KEY_ENABLED_SETTING = "xpack.security.authc.api_key.enabled"; + private static final String VERSION_FIELD = "version"; + private static final String LICENSE_FIELD = "license"; + private static final String USERNAME_FIELD = "username"; + /** * The cluster-default security settings worth reporting: each one either gates a capability this * report covers, or describes the authentication posture a reader needs in order to interpret the @@ -101,7 +99,7 @@ public class ElasticSearchCapabilityService implements ConsentLogger { Set.of( "xpack.security.enabled", "xpack.security.dls_fls.enabled", - "xpack.security.authc.api_key.enabled", + API_KEY_ENABLED_SETTING, "xpack.security.authc.run_as.enabled", "xpack.security.authc.token.enabled", "xpack.security.authc.anonymous.username", @@ -148,8 +146,7 @@ public class ElasticSearchCapabilityService implements ConsentLogger { */ // nosemgrep - an empty privilege grant, not a key private static final String PRIVILEGE_FREE_DESCRIPTOR = - """ - {"probe":{"cluster":[],"indices":[]}}"""; + "{\"probe\":{\"cluster\":[],\"indices\":[]}}"; /** * Builds a client that authenticates as an API key instead of the deployment's shared credential. @@ -214,48 +211,88 @@ public ElasticSearchCapabilityReport getCapabilityReport(String runAsUser, boole return unreachableReport(notes); } - String version = string(root.body(), "version", "number"); - String distribution = stringOrDefault(root.body(), "elasticsearch", "version", "distribution"); - boolean isOpenSearch = "opensearch".equalsIgnoreCase(distribution); + String version = string(root.body(), VERSION_FIELD, "number"); + String distribution = + stringOrDefault(root.body(), "elasticsearch", VERSION_FIELD, "distribution"); ProbeResult xpack = probe(HttpMethod.GET, "/_xpack"); ProbeResult license = probe(HttpMethod.GET, "/_license"); - String licenseType = string(license.body(), "license", "type"); - String licenseStatus = string(license.body(), "license", "status"); + String licenseType = string(license.body(), LICENSE_FIELD, "type"); + String licenseStatus = string(license.body(), LICENSE_FIELD, "status"); Map securitySettings = securitySettings(); Boolean securityEnabled = securityEnabled(xpack, securitySettings); ProbeResult authenticate = probe(HttpMethod.GET, AUTHENTICATE_PATH); boolean securityApiPresent = securityApiPresent(authenticate.status()); - String authenticatedUser = string(authenticate.body(), "username"); + String authenticatedUser = string(authenticate.body(), USERNAME_FIELD); List roles = stringList(authenticate.body(), "roles"); Map clusterPrivileges = securityApiPresent ? clusterPrivileges() : Map.of(); boolean elasticCloud = esConfig.getCloudId() != null && !esConfig.getCloudId().trim().isEmpty(); + boolean writeProbesRan = writeProbes && securityApiPresent; + notes.addAll(deploymentNotes(elasticCloud, securityApiPresent, writeProbes, writeProbesRan)); + + WriteProbeOutcome writeProbeOutcome = writeProbesRan ? runWriteProbes(notes) : null; + + List capabilities = + buildCapabilities( + writeProbeOutcome, + securityEnabled, + securityApiPresent, + xpack, + securitySettings, + clusterPrivileges, + licenseType, + authenticatedUser, + runAsUser); + + return new ElasticSearchCapabilityReport( + string(root.body(), "cluster_name"), + version, + distribution, + edition( + elasticCloud, + string(root.body(), VERSION_FIELD, "build_flavor"), + xpack.status(), + licenseType), + licenseType, + licenseStatus, + elasticCloud, + securityEnabled, + authenticatedUser, + roles, + clusterPrivileges, + securitySettings, + writeProbesRan, + capabilities, + restClientCompatibility(version), + recommendation(securityApiPresent, licenseType, writeProbeOutcome), + notes); + } + + /** + * The advisory notes that depend only on the shape of the deployment and cluster, not on any + * probe result — split out of {@link #getCapabilityReport} so its branching does not compound + * with the rest of that method's own. + */ + private List deploymentNotes( + boolean elasticCloud, + boolean securityApiPresent, + boolean writeProbes, + boolean writeProbesRan) { + List notes = new ArrayList<>(); if (elasticCloud) { notes.add( "This deployment is configured with a cloud ID, so the cluster is Elastic Cloud and " + "X-Pack Security is always present."); } - if (isOpenSearch) { - notes.add( - "This cluster is OpenSearch, not Elasticsearch. Every probe in this report targets the " - + "X-Pack security APIs, which do not exist here: DLS/FLS come from the OpenSearch " - + "security plugin under /_plugins/_security, and there is no POST " - + "/_security/api_key. The DLS, FLS, and API-key verdicts below are therefore " - + "statements about what this probe covers on OpenSearch, not license inferences, " - + "and no value of writeProbes changes them."); - } - boolean writeProbesRan = writeProbes && securityApiPresent && !isOpenSearch; if (!securityApiPresent) { notes.add( "The /_security API is not available on this cluster, so no security feature can be " + "exercised. Every security verdict below follows from that one fact."); - } else if (!writeProbesRan && !isOpenSearch) { - // Only an Elasticsearch cluster has verdicts that a write probe would convert from inferred - // to observed; on OpenSearch nothing here is inferred and nothing would be written. + } else if (!writeProbesRan) { notes.add( "DLS, FLS, and API-key verdicts are inferred from the license tier and cluster " + "settings. Re-run with writeProbes=true to create and tear down a short-lived key " @@ -263,57 +300,43 @@ public ElasticSearchCapabilityReport getCapabilityReport(String runAsUser, boole } if (writeProbes && !writeProbesRan) { notes.add( - isOpenSearch - ? "Write probes were requested but not run: they create an X-Pack role and API key, " - + "which OpenSearch does not provide." - : "Write probes were requested but not run: they need the /_security API, which this " - + "cluster did not answer as a security-enabled cluster would."); + "Write probes were requested but not run: they need the /_security API, which this " + + "cluster did not answer as a security-enabled cluster would."); } + return notes; + } - WriteProbeOutcome writeProbeOutcome = writeProbesRan ? runWriteProbes(notes) : null; - + /** + * The five capability verdicts, each preferring an observed write-probe outcome over the inferred + * read-only one — split out of {@link #getCapabilityReport} so its own branching does not + * compound with the rest of that method's. + */ + private List buildCapabilities( + WriteProbeOutcome writeProbeOutcome, + Boolean securityEnabled, + boolean securityApiPresent, + ProbeResult xpack, + Map securitySettings, + Map clusterPrivileges, + String licenseType, + String authenticatedUser, + String runAsUser) { List capabilities = new ArrayList<>(); capabilities.add(securityCapability(securityEnabled, securityApiPresent, xpack)); capabilities.add( writeProbeOutcome != null ? writeProbeOutcome.apiKeys() - : apiKeyCapability( - securityApiPresent, isOpenSearch, securitySettings, clusterPrivileges)); + : apiKeyCapability(securityApiPresent, securitySettings, clusterPrivileges)); capabilities.add( writeProbeOutcome != null ? writeProbeOutcome.dls() - : dlsFlsCapability( - DLS, securityApiPresent, isOpenSearch, licenseType, securitySettings)); + : dlsFlsCapability(DLS, securityApiPresent, licenseType, securitySettings)); capabilities.add( writeProbeOutcome != null ? writeProbeOutcome.fls() - : dlsFlsCapability( - FLS, securityApiPresent, isOpenSearch, licenseType, securitySettings)); + : dlsFlsCapability(FLS, securityApiPresent, licenseType, securitySettings)); capabilities.add(runAsCapability(securityApiPresent, authenticatedUser, runAsUser)); - - return new ElasticSearchCapabilityReport( - string(root.body(), "cluster_name"), - version, - distribution, - edition( - elasticCloud, - isOpenSearch, - string(root.body(), "version", "build_flavor"), - xpack.status(), - licenseType), - licenseType, - licenseStatus, - elasticCloud, - securityEnabled, - authenticatedUser, - roles, - clusterPrivileges, - securitySettings, - writeProbesRan, - capabilities, - restClientCompatibility(version, isOpenSearch), - recommendation(securityApiPresent, isOpenSearch, licenseType, writeProbeOutcome), - notes); + return capabilities; } // --------------------------------------------------------------------------- @@ -346,17 +369,7 @@ private ElasticSearchCapability securityCapability( } private ElasticSearchCapability apiKeyCapability( - boolean securityApiPresent, - boolean isOpenSearch, - Map settings, - Map privileges) { - if (isOpenSearch) { - return new ElasticSearchCapability( - API_KEYS, - CapabilityVerdict.UNAVAILABLE, - "OpenSearch has no POST /_security/api_key endpoint.", - "GET / -> version.distribution"); - } + boolean securityApiPresent, Map settings, Map privileges) { if (!securityApiPresent) { return new ElasticSearchCapability( API_KEYS, @@ -365,7 +378,7 @@ private ElasticSearchCapability apiKeyCapability( "GET " + AUTHENTICATE_PATH); } // nosemgrep - a cluster setting name, not a key - if ("false".equals(settings.get("xpack.security.authc.api_key.enabled"))) { + if ("false".equals(settings.get(API_KEY_ENABLED_SETTING))) { return new ElasticSearchCapability( API_KEYS, CapabilityVerdict.UNAVAILABLE, @@ -392,23 +405,11 @@ private ElasticSearchCapability apiKeyCapability( "Security is enabled and API keys are a Basic-tier feature, so key creation is expected " + "to work. Not proven: creating a key is a write.", "xpack.security.authc.api_key.enabled=%s; POST /_security/user/_has_privileges" - .formatted(settings.getOrDefault("xpack.security.authc.api_key.enabled", "not-set"))); + .formatted(settings.getOrDefault(API_KEY_ENABLED_SETTING, "not-set"))); } private ElasticSearchCapability dlsFlsCapability( - String name, - boolean securityApiPresent, - boolean isOpenSearch, - String licenseType, - Map settings) { - if (isOpenSearch) { - return new ElasticSearchCapability( - name, - CapabilityVerdict.UNKNOWN, - "OpenSearch provides DLS/FLS through its security plugin, which this probe does not " - + "inspect. Check /_plugins/_security instead.", - "GET / -> version.distribution"); - } + String name, boolean securityApiPresent, String licenseType, Map settings) { if (!securityApiPresent) { return new ElasticSearchCapability( name, @@ -483,7 +484,7 @@ private ElasticSearchCapability runAsCapability( .formatted(AUTHENTICATE_PATH, RUN_AS_HEADER, target, result.status()); if (result.status() == 200) { - String resolved = string(result.body(), "username"); + String resolved = string(result.body(), USERNAME_FIELD); if (target.equals(resolved)) { return new ElasticSearchCapability( RUN_AS, @@ -666,7 +667,7 @@ private ElasticSearchCapability apiKeyRoundTripProbe(String stamp, List API_KEYS, CapabilityVerdict.SUPPORTED, "Observed: a key was created, authenticated as '%s', and invalidated." - .formatted(string(asKey.body(), "username")), + .formatted(string(asKey.body(), USERNAME_FIELD)), evidence + "; GET %s as the key -> 200".formatted(AUTHENTICATE_PATH)); } return new ElasticSearchCapability( @@ -735,13 +736,14 @@ private Optional dlsEnforcementProbe( .formatted(index); EnforcementAttempt attempt = attemptEnforcement( - DLS, - "DLS", - "duos-capability-probe-dls-" + stamp, - descriptor, - "a DLS role_descriptor", - "a match_none DLS key", - index, + new EnforcementRequest( + DLS, + "DLS", + "duos-capability-probe-dls-" + stamp, + descriptor, + "a DLS role_descriptor", + "a match_none DLS key", + index), createdKeyIds); if (attempt.settled() != null) { return Optional.of(attempt.settled()); @@ -796,13 +798,14 @@ private Optional flsProjectionProbe( .formatted(index, FLS_GRANT_FIELD); EnforcementAttempt attempt = attemptEnforcement( - FLS, - "FLS", - "duos-capability-probe-fls-" + stamp, - descriptor, - "an FLS role_descriptor", - "a key granting only '%s'".formatted(FLS_GRANT_FIELD), - index, + new EnforcementRequest( + FLS, + "FLS", + "duos-capability-probe-fls-" + stamp, + descriptor, + "an FLS role_descriptor", + "a key granting only '%s'".formatted(FLS_GRANT_FIELD), + index), createdKeyIds); if (attempt.settled() != null) { return Optional.of(attempt.settled()); @@ -857,43 +860,53 @@ static EnforcementAttempt searched(JsonObject body, String evidence) { } /** - * The half the DLS and FLS enforcement checks share: mint a key carrying the filter under test, - * then search the real index through it. Only the reading of a successful response differs - * between the two, so only that is left to the callers. + * Everything {@link #attemptEnforcement} needs to mint a probe key and search through it, bundled + * so the method itself takes a request and the list of created key ids to reconcile, rather than + * one parameter per fact about the attempt. * * @param descriptorLabel how the {@code role_descriptors} block is described in evidence * @param keyLabel how the key is described in evidence, e.g. {@code a match_none DLS key} */ - private EnforcementAttempt attemptEnforcement( + private record EnforcementRequest( String name, String shortName, String keyName, String descriptor, String descriptorLabel, String keyLabel, - String index, - List createdKeyIds) { - KeyCreation key = createProbeKey(keyName, descriptor, createdKeyIds); + String index) {} + + /** + * The half the DLS and FLS enforcement checks share: mint a key carrying the filter under test, + * then search the real index through it. Only the reading of a successful response differs + * between the two, so only that is left to the callers. + */ + private EnforcementAttempt attemptEnforcement( + EnforcementRequest request, List createdKeyIds) { + KeyCreation key = createProbeKey(request.keyName(), request.descriptor(), createdKeyIds); if (!created(key.response())) { // On a license-blocked cluster the key may be refused here rather than at search time. return EnforcementAttempt.settled( new ElasticSearchCapability( - name, + request.name(), refusalVerdict(key.response()), - "A key carrying %s was refused: ".formatted(descriptorLabel) + "A key carrying %s was refused: ".formatted(request.descriptorLabel()) + reason(key.response().body()), - "POST %s with %s -> %d".formatted(API_KEY_PATH, descriptorLabel, key.status()))); + "POST %s with %s -> %d" + .formatted(API_KEY_PATH, request.descriptorLabel(), key.status()))); } if (key.encoded() == null) { return EnforcementAttempt.inconclusive(); } - String searchPath = "/%s/_search?size=1".formatted(index); + String searchPath = "/%s/_search?size=1".formatted(request.index()); ProbeResult search = probeAsApiKey(key.encoded(), HttpMethod.GET, searchPath); - String evidence = "GET %s through %s -> %d".formatted(searchPath, keyLabel, search.status()); + String evidence = + "GET %s through %s -> %d".formatted(searchPath, request.keyLabel(), search.status()); return search.status() == 200 ? EnforcementAttempt.searched(search.body(), evidence) - : EnforcementAttempt.settled(searchFailure(name, shortName, search, evidence)); + : EnforcementAttempt.settled( + searchFailure(request.name(), request.shortName(), search, evidence)); } /** @@ -915,8 +928,7 @@ int status() { private KeyCreation createProbeKey( String keyName, String roleDescriptors, List createdKeyIds) { String body = - """ - {"name":"%s","expiration":"%s","role_descriptors":%s}""" + "{\"name\":\"%s\",\"expiration\":\"%s\",\"role_descriptors\":%s}" .formatted(keyName, PROBE_KEY_EXPIRATION, roleDescriptors); ProbeResult created = probe(HttpMethod.POST, API_KEY_PATH, body, Map.of()); if (!created(created)) { @@ -939,13 +951,7 @@ private void tearDownProbeResources( List createdKeyIds, String roleName, List notes) { for (String keyId : createdKeyIds) { ProbeResult result = - probe( - HttpMethod.DELETE, - API_KEY_PATH, - """ - {"ids":["%s"]}""" - .formatted(keyId), - Map.of()); + probe(HttpMethod.DELETE, API_KEY_PATH, "{\"ids\":[\"%s\"]}".formatted(keyId), Map.of()); if (result.status() != 200) { logWarn( "Failed to invalidate probe API key %s (status %d)".formatted(keyId, result.status())); @@ -1043,7 +1049,7 @@ private CapabilityVerdict refusalVerdict(ProbeResult result) { */ private static boolean licenseRefusal(String reason) { String lowered = reason.toLowerCase(); - return lowered.contains("non-compliant") || lowered.contains("license"); + return lowered.contains("non-compliant") || lowered.contains(LICENSE_FIELD); } /** Whether the cluster created what was asked of it; PUT role returns 200 or 201. */ @@ -1125,17 +1131,10 @@ private ProbeResult probeAsApiKey(String encodedApiKey, String method, String pa * when the cluster reports it. */ private String edition( - boolean elasticCloud, - boolean isOpenSearch, - String buildFlavor, - int xpackStatus, - String licenseType) { + boolean elasticCloud, String buildFlavor, int xpackStatus, String licenseType) { if (elasticCloud) { return "Elastic Cloud (X-Pack always present)"; } - if (isOpenSearch) { - return "OpenSearch (Apache 2.0 / security plugin)"; - } if ("oss".equalsIgnoreCase(buildFlavor) || xpackStatus == 400 || xpackStatus == 404) { return "OSS (no X-Pack endpoint)"; } @@ -1147,12 +1146,8 @@ private String edition( * low-level client is a version-agnostic HTTP transport with no typed request model, so the only * real compatibility axis is major-version skew. */ - private String restClientCompatibility(String clusterVersion, boolean isOpenSearch) { + private String restClientCompatibility(String clusterVersion) { String clientVersion = RestClient.class.getPackage().getImplementationVersion(); - if (isOpenSearch) { - return "Incompatible path: OpenSearch has no POST /_security/api_key. The low-level client " - + "can still reach /_plugins/_security, but the X-Pack API-key design does not apply."; - } if (clientVersion == null || clusterVersion == null) { return "Could not determine the client or cluster version at runtime. This report was itself " + "produced through RestClient.performRequest, so the transport reaches /_security."; @@ -1170,15 +1165,12 @@ private String restClientCompatibility(String clusterVersion, boolean isOpenSear } private String recommendation( - boolean securityApiPresent, - boolean isOpenSearch, - String licenseType, - WriteProbeOutcome writeProbeOutcome) { + boolean securityApiPresent, String licenseType, WriteProbeOutcome writeProbeOutcome) { // An observation outranks the license inference in either direction: a cluster that actually // enforced DLS settles the question, and one that refused on licensing grounds settles it just // as firmly. A refusal on *privilege* grounds settles nothing about the cluster, so that case // falls through to the license reading rather than being read as a verdict against Epic D. - if (writeProbeOutcome != null && !isOpenSearch && securityApiPresent) { + if (writeProbeOutcome != null && securityApiPresent) { if (writeProbeOutcome.dlsUsable()) { return "Epic D (native DLS/FLS) is viable on this cluster, and this was observed rather " + "than inferred: a probe role and API key carrying DLS/FLS descriptors were accepted " @@ -1196,10 +1188,6 @@ private String recommendation( + "was not usable: %s Epic D would need that resolved first." .formatted(writeProbeOutcome.dls().detail()); } - if (isOpenSearch) { - return "Epic E (compatibility fallback). This cluster is OpenSearch, where the X-Pack " - + "API-key and role-descriptor design behind Epic D does not exist."; - } if (!securityApiPresent) { return "Epic E (compatibility fallback) is the only path available on this cluster. " + "Security is not enabled, so DLS, FLS, API keys, and run_as cannot be used at all. " @@ -1235,13 +1223,13 @@ private String licenseBasedRecommendation(String licenseType) { /** Reads the caller's own cluster privileges. A POST, but an evaluation rather than a write. */ private Map clusterPrivileges() { String body = - """ - {"cluster":[%s],"index":[{"names":["%s"],"privileges":["read","view_index_metadata"]}]}""" - .formatted( - PROBED_CLUSTER_PRIVILEGES.stream() - .map("\"%s\""::formatted) - .collect(Collectors.joining(",")), - probeIndex()); + "{\"cluster\":[%s],\"index\":[{\"names\":[\"%s\"]," + + "\"privileges\":[\"read\",\"view_index_metadata\"]}]}" + .formatted( + PROBED_CLUSTER_PRIVILEGES.stream() + .map("\"%s\""::formatted) + .collect(Collectors.joining(",")), + probeIndex()); ProbeResult result = probe(HttpMethod.POST, "/_security/user/_has_privileges", body, Map.of()); if (result.status() != 200 || !result.body().has("cluster")) { diff --git a/src/main/resources/assets/paths/elasticSearchCapabilities.yaml b/src/main/resources/assets/paths/elasticSearchCapabilities.yaml index de26fd6b6..0a4c4420b 100644 --- a/src/main/resources/assets/paths/elasticSearchCapabilities.yaml +++ b/src/main/resources/assets/paths/elasticSearchCapabilities.yaml @@ -10,11 +10,8 @@ get: dev, staging, and production produces the per-environment inventory without needing direct network access to the clusters or a copy of their credentials. - The probes are Elasticsearch X-Pack probes. OpenSearch secures itself differently — DLS and - FLS come from its security plugin under /_plugins/_security, and there is no - POST /_security/api_key — so against an OpenSearch cluster the report identifies the - distribution, returns UNKNOWN for the DLS/FLS verdicts it does not inspect, and skips write - probes entirely rather than reporting X-Pack conclusions that would not apply. + The probes are Elasticsearch X-Pack probes; this endpoint is scoped to Elasticsearch + deployments only. By default all probes are non-destructive: nothing is created, modified, or deleted on the cluster. The trade-off is certainty — DLS, FLS, and API-key support can only be proven by @@ -46,9 +43,8 @@ get: Create and tear down a short-lived API key and role so that DLS, FLS, and API-key support are observed rather than inferred. Requires the deployment credential to hold manage_api_key (or grant_api_key) and manage_security; the report says which of those - it has under cluster_privileges. Ignored on OpenSearch, which has no X-Pack role or - API-key endpoints to write to, and when security is disabled, since there is nothing to - probe. In both cases a note in the response says the probes were requested but not run. + it has under cluster_privileges. Ignored when security is disabled, since there is + nothing to probe — a note in the response says the probes were requested but not run. required: false schema: type: boolean diff --git a/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml b/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml index 5b5ee1c3b..9cac01dd0 100644 --- a/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml +++ b/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml @@ -6,10 +6,8 @@ description: | non-destructive, which is why the DLS, FLS, and API-key verdicts may be inferred rather than observed; write_probes_run says which of the two this report is. - The probes target Elasticsearch's X-Pack security APIs only. Against an OpenSearch cluster — - which secures itself through a plugin under /_plugins/_security and has no - POST /_security/api_key — the report says so in distribution and notes, returns UNKNOWN for - DLS/FLS because it does not inspect that plugin, and never runs write probes. + The probes target Elasticsearch's X-Pack security APIs only; this report covers Elasticsearch + deployments exclusively. The example is a real writeProbes=true response from a local trial-licensed cluster, where the configured credential is the `elastic` superuser. A deployed environment's shared credential is @@ -28,14 +26,12 @@ properties: distribution: type: string description: | - Distribution reported by the cluster. OpenSearch exposes different security APIs than - Elasticsearch and has no API-key endpoint. + Distribution reported by the cluster. examples: - elasticsearch - - opensearch edition: type: string - description: OSS, a license tier, Elastic Cloud, or OpenSearch. + description: OSS, a license tier, or Elastic Cloud. license_type: type: string description: License tier reported by the cluster. @@ -102,10 +98,8 @@ properties: write_probes_run: type: boolean description: | - Whether write probes ran. When false on an Elasticsearch cluster, the DLS, FLS, and API-key - verdicts are inferred from the license tier rather than observed — read this field before - reading those verdicts. On OpenSearch it is always false and nothing is inferred: those - verdicts report that the X-Pack probes do not apply. + Whether write probes ran. When false, the DLS, FLS, and API-key verdicts are inferred from + the license tier rather than observed — read this field before reading those verdicts. capabilities: type: array description: | @@ -137,8 +131,7 @@ properties: SUPPORTED and UNAVAILABLE are observed. LICENSE_BLOCKED and NOT_PERMITTED are observed refusals. The INFERRED_ values are derived from license tier and cluster settings because proving them would require writing to the cluster. UNKNOWN also - covers what the probe does not reach — DLS/FLS on OpenSearch, or any capability on - a cluster that did not respond. + covers any capability on a cluster that did not respond. enum: - SUPPORTED - UNAVAILABLE diff --git a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java index fc9fe734d..d685c4f9e 100644 --- a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java @@ -195,8 +195,7 @@ private void stub(String key, int status, String body) { * Baseline: a cluster with security switched off, as the local docker-compose one is. X-Pack is * still installed and still reports a license — it has shipped in every default distribution * since 6.3 — so this fixture matches what the local cluster was actually measured to return - * rather than the OSS shape. A missing /_xpack endpoint is a different cluster, covered by the - * OpenSearch case. + * rather than the OSS shape. */ private void stubSecurityDisabledCluster() { stub(ROOT, 200, ROOT_BODY); @@ -429,90 +428,6 @@ void testRunAsBlockedByLicenseIsReportedAsSuch() throws IOException { CapabilityVerdict.LICENSE_BLOCKED, capability(report, "run_as impersonation").verdict()); } - @Test - void testOpenSearchIsCalledOutRatherThanMisreported() throws IOException { - stubOpenSearchClusterWithSecurityPresent(); - - ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); - - assertEquals("opensearch", report.distribution()); - assertEquals("OpenSearch (Apache 2.0 / security plugin)", report.edition()); - assertEquals(CapabilityVerdict.UNAVAILABLE, capability(report, "API keys").verdict()); - assertEquals( - CapabilityVerdict.UNKNOWN, capability(report, "Document-level security (DLS)").verdict()); - assertTrue(report.restClientCompatibility().contains("OpenSearch")); - assertTrue(report.notes().stream().anyMatch(n -> n.contains("OpenSearch"))); - } - - /** - * A 401 from /_security/_authenticate reads as "security is present but this credential was - * refused", which is true on OpenSearch too. That must not pull in the Elasticsearch note about - * inferred verdicts: on OpenSearch the DLS, FLS, and API-key verdicts are not license inferences - * and write probes never run, so telling the reader to re-run with writeProbes=true would be - * advice that changes nothing. - */ - @Test - void testOpenSearchIsNotDescribedWithTheElasticsearchInferenceNote() throws IOException { - stubOpenSearchClusterWithSecurityPresent(); - - ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); - - assertFalse(report.writeProbesRun()); - assertTrue( - report.notes().stream().noneMatch(n -> n.contains("writeProbes=true")), - "no note should suggest write probes against OpenSearch"); - assertTrue( - report.notes().stream().noneMatch(n -> n.startsWith("DLS, FLS, and API-key verdicts are ")), - "the inferred-verdict note is an Elasticsearch statement"); - assertTrue( - report.notes().stream() - .anyMatch(n -> n.contains("Write probes were requested but not run")), - "a requested write probe that did not run has to be said out loud"); - // Nothing was written, and in particular no attempt was made against endpoints OpenSearch - // does not have. - assertTrue(requestsTo("POST", "/_security/api_key").isEmpty()); - assertTrue(requestsTo("PUT", "/_security/role").isEmpty()); - } - - private void stubOpenSearchClusterWithSecurityPresent() { - stub( - ROOT, - 200, - """ - {"cluster_name":"duos-cluster","version":{"number":"2.19.1","distribution":"opensearch"}}"""); - stub( - XPACK, - 400, - """ - {"error":{"reason":"no handler found"}}"""); - stub( - LICENSE, - 400, - """ - {"error":{"reason":"no handler found"}}"""); - stub( - SETTINGS, - 200, - """ - {"defaults":{},"persistent":{},"transient":{}}"""); - // OpenSearch answers this path from its own security plugin, and refuses rather than 404s. - stub( - AUTHENTICATE, - 401, - """ - {"error":{"reason":"unauthorized"}}"""); - stub( - HAS_PRIVILEGES, - 404, - """ - {"error":{"reason":"no handler found"}}"""); - stub( - RUN_AS, - 401, - """ - {"error":{"reason":"unauthorized"}}"""); - } - @Test void testUnreachableClusterProducesAReportRatherThanAnException() throws IOException { when(esClient.performRequest(any(Request.class))) @@ -1068,5 +983,478 @@ void testReadOnlyRunCreatesNothing() throws IOException { assertTrue(report.notes().stream().anyMatch(n -> n.contains("non-destructive"))); } + @Test + void testElasticCloudDeploymentIsReportedAsSuch() throws IOException { + config.setCloudId("duos-cloud:ZXhhbXBsZQ=="); + stubSecurityEnabledCluster("trial"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("Elastic Cloud (X-Pack always present)", report.edition()); + assertTrue(report.elasticCloud()); + assertTrue( + report.notes().stream().anyMatch(n -> n.contains("cloud ID")), + "an Elastic Cloud deployment must be called out in the notes"); + } + + @Test + void testOssDistributionIsReportedWhenXPackEndpointIsMissing() throws IOException { + stubSecurityDisabledCluster(); + stub( + XPACK, + 404, + """ + {"error":{"reason":"no handler found"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("OSS (no X-Pack endpoint)", report.edition()); + } + + @Test + void testSecurityReportedEnabledButApiUnreachableIsDistinguishedFromDisabled() + throws IOException { + stubSecurityDisabledCluster(); + stub( + XPACK, + 200, + """ + {"features":{"security":{"available":true,"enabled":true}}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + ElasticSearchCapability security = capability(report, "X-Pack Security"); + assertEquals(CapabilityVerdict.UNAVAILABLE, security.verdict()); + assertTrue( + security.detail().contains("reports itself enabled"), + "the two signals disagreeing must be described, not silently resolved: " + + security.detail()); + } + + @Test + void testApiKeysExplicitlyDisabledByClusterSettingIsReportedAsUnavailable() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + SETTINGS, + 200, + """ + {"defaults":{"xpack.security.enabled":"true","xpack.security.dls_fls.enabled":"true", + "xpack.security.authc.api_key.enabled":"false"},"persistent":{},"transient":{}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + ElasticSearchCapability apiKeys = capability(report, "API keys"); + assertEquals(CapabilityVerdict.UNAVAILABLE, apiKeys.verdict()); + assertTrue(apiKeys.detail().contains("explicitly disabled by cluster setting")); + } + + @Test + void testUnmappedLicenseTierYieldsUnknownDlsFlsVerdict() throws IOException { + stubSecurityEnabledCluster("some-future-tier"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.UNKNOWN, dls.verdict()); + assertTrue(dls.detail().contains("could not be mapped")); + assertTrue(report.recommendation().contains("Inconclusive")); + } + + @Test + void testRunAsAcceptedButResolvedToADifferentUserIsUnknown() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + RUN_AS, + 200, + """ + {"username":"someone-else","roles":["other"]}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + ElasticSearchCapability runAs = capability(report, "run_as impersonation"); + assertEquals(CapabilityVerdict.UNKNOWN, runAs.verdict()); + assertTrue(runAs.detail().contains("still resolved to")); + } + + @Test + void testRunAsUnexpectedStatusIsUnknown() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + RUN_AS, + 500, + """ + {"error":{"reason":"internal server error"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + ElasticSearchCapability runAs = capability(report, "run_as impersonation"); + assertEquals(CapabilityVerdict.UNKNOWN, runAs.verdict()); + assertTrue(runAs.detail().contains("unexpected status")); + } + + @Test + void testApiKeyCreatedButUnableToAuthenticateIsReportedAsUnknown() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + "cGxhaW4=|" + AUTHENTICATE, + 403, + """ + {"error":{"reason":"unauthorized"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability apiKeys = capability(report, "API keys"); + assertEquals(CapabilityVerdict.UNKNOWN, apiKeys.verdict()); + assertTrue(apiKeys.detail().contains("could not authenticate")); + } + + @Test + void testMalformedProbeKeyResponseLeavesEnforcementInconclusive() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + // Created, but the response carries neither an encoded key nor an id/secret pair to build one. + stub( + CREATE_DLS_KEY, + 200, + """ + {"id":"dls-key-id"}"""); + stub( + CREATE_FLS_KEY, + 200, + """ + {"id":"fls-key-id"}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertTrue( + report.notes().stream() + .anyMatch(n -> n.contains("DLS enforcement check could not be run to a conclusion"))); + assertTrue( + report.notes().stream() + .anyMatch(n -> n.contains("FLS projection check returned no document fields"))); + } + + @Test + void testFailedKeyInvalidationIsReportedInTheNotesRatherThanSwallowed() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + INVALIDATE_KEY, + 500, + """ + {"error":{"reason":"boom"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertTrue( + report.notes().stream().anyMatch(n -> n.contains("could not be invalidated")), + "an operator must be told a probe key was left on the cluster: " + report.notes()); + } + + @Test + void testRoleRefusalWithNoLicenseOrPrivilegeReasonIsUnknownRatherThanMisclassified() + throws IOException { + stubSecurityEnabledCluster("trial"); + // No usable probe key, so the enforcement probes are skipped and the DLS/FLS verdicts stand at + // role acceptance alone rather than being overwritten by an end-to-end enforcement result. + stub( + CREATE_KEY, + 500, + """ + {"error":{"reason":"internal server error"}}"""); + stub( + CREATE_ROLE, + 500, + """ + {"error":{"reason":"internal server error"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.UNKNOWN, dls.verdict()); + assertTrue(dls.detail().contains("The role was rejected")); + } + + @Test + void testEncodedApiKeyIsBuiltFromIdAndSecretWhenEncodedFieldIsAbsent() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + CREATE_KEY, + 200, + """ + {"id":"plain-key-id","api_key":"plain-secret"}"""); + String builtKey = + java.util.Base64.getEncoder() + .encodeToString( + "plain-key-id:plain-secret".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + stub( + builtKey + "|" + AUTHENTICATE, + 200, + """ + {"username":"consent"}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertEquals(CapabilityVerdict.SUPPORTED, capability(report, "API keys").verdict()); + } + + @Test + void testUnrecognisedSearchResponseShapeFallsBackRatherThanFailingTheProbe() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + // No "total" under "hits" at all: hitCount() must return -1 rather than throw. + stub( + "ZGxz|" + SEARCH, + 200, + """ + {"hits":{}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.UNKNOWN, dls.verdict()); + assertTrue(dls.detail().contains("no hit total")); + } + + @Test + void testRunAsWithNoAuthenticatedOrRequestedUserIsUnknown() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + AUTHENTICATE, + 200, + """ + {"roles":["consent"]}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + ElasticSearchCapability runAs = capability(report, "run_as impersonation"); + assertEquals(CapabilityVerdict.UNKNOWN, runAs.verdict()); + assertTrue(runAs.detail().contains("No target user was available")); + } + + @Test + void testApiKeyCreatedWithNoUsableCredentialAtAllIsReportedAsUnknown() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + CREATE_KEY, + 200, + """ + {"name":"probe-key"}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability apiKeys = capability(report, "API keys"); + assertEquals(CapabilityVerdict.UNKNOWN, apiKeys.verdict()); + assertTrue(apiKeys.detail().contains("neither an encoded form nor an id and")); + } + + @Test + void testDlsKeyCreationRefusedOnLicenseGroundsIsReportedAtCreationRatherThanSearch() + throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + CREATE_DLS_KEY, + 403, + """ + {"error":{"reason":"current license is non-compliant for [field and document level security]"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.LICENSE_BLOCKED, dls.verdict()); + assertTrue(dls.detail().contains("A key carrying")); + } + + @Test + void testDlsSearchUnexpectedStatusIsUnknownRatherThanMisclassified() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + "ZGxz|" + SEARCH, + 500, + """ + {"error":{"reason":"internal server error"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.UNKNOWN, dls.verdict()); + assertTrue(dls.detail().contains("unexpected status")); + } + + @Test + void testHitCountReadsAPlainNumericTotalAsWellAsAnObjectShapedOne() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + "ZGxz|" + SEARCH, + 200, + """ + {"hits":{"total":0,"hits":[]}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.SUPPORTED, dls.verdict()); + } + + @Test + void testFlsSearchWithNoHitsIsReportedAsInconclusiveRatherThanUnprojected() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + // No "hits" array under "hits" at all: firstHitSourceFields() must fall back rather than throw. + stub( + "Zmxz|" + SEARCH, + 200, + """ + {"hits":{"total":{"value":0}}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertTrue( + report.notes().stream() + .anyMatch(n -> n.contains("FLS projection check returned no document fields"))); + } + + @Test + void testFlsSearchWithAnEmptyHitsArrayIsReportedAsInconclusive() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + // "hits" is present as an array, but empty: no document to read fields from. + stub( + "Zmxz|" + SEARCH, + 200, + """ + {"hits":{"total":{"value":0},"hits":[]}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertTrue( + report.notes().stream() + .anyMatch(n -> n.contains("FLS projection check returned no document fields"))); + } + + @Test + void testProbeResponseBodyThatIsNotValidJsonFallsBackToAnEmptyBodyRatherThanThrowing() + throws IOException { + stubSecurityEnabledCluster("trial"); + stub(HAS_PRIVILEGES, 200, "not valid json"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertTrue(report.clusterPrivileges().isEmpty()); + } + + @Test + void testReasonFallsBackToAGenericMessageWhenTheErrorBodyHasNoReasonField() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + RUN_AS, + 500, + """ + {"error":{}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + ElasticSearchCapability runAs = capability(report, "run_as impersonation"); + assertTrue(runAs.detail().contains("no reason reported by the cluster")); + } + + @Test + void testClusterPrivilegesProbeFailureYieldsAnEmptyMapRatherThanAnException() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + HAS_PRIVILEGES, + 500, + """ + {"error":{"reason":"internal server error"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertTrue(report.clusterPrivileges().isEmpty()); + } + + @Test + void testSecuritySettingsProbeFailureYieldsAnEmptyMapRatherThanAnException() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + SETTINGS, + 500, + """ + {"error":{"reason":"internal server error"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertTrue(report.securitySettings().isEmpty()); + } + + @Test + void testSecuritySettingsSkipsSectionsThatAreAbsentOrNotAnObject() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + SETTINGS, + 200, + """ + {"defaults":{"xpack.security.enabled":"true","cluster.name":"duos-cluster"}, + "persistent":"not-an-object"}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("true", report.securitySettings().get("xpack.security.enabled")); + assertFalse( + report.securitySettings().containsKey("cluster.name"), + "a non-security default must be filtered out rather than reported"); + } + + @Test + void testSecurityEnabledFallsBackToClusterSettingWhenXPackIsUnreachable() throws IOException { + stubSecurityDisabledCluster(); + stub( + XPACK, + 404, + """ + {"error":{"reason":"no handler found"}}"""); + stub( + SETTINGS, + 200, + """ + {"defaults":{"xpack.security.enabled":"true"},"persistent":{},"transient":{}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals(Boolean.TRUE, report.securityEnabled()); + } + + @Test + void testMissingClusterVersionCannotBeCompatibilityChecked() throws IOException { + stubSecurityDisabledCluster(); + stub( + ROOT, + 200, + """ + {"cluster_name":"duos-cluster"}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertTrue(report.restClientCompatibility().contains("Could not determine")); + } + + @Test + void testMajorVersionSkewIsCalledOutRatherThanAssumedCompatible() throws IOException { + stubSecurityDisabledCluster(); + stub( + ROOT, + 200, + """ + {"cluster_name":"duos-cluster","version":{"number":"8.1.2","build_flavor":"default"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertTrue(report.restClientCompatibility().contains("Major-version skew")); + } + private record StubResponse(int status, String body) {} } From a32b48b05dee9dd6bb8e2d7673060ee2619cb867 Mon Sep 17 00:00:00 2001 From: Elliot Otchet Date: Thu, 30 Jul 2026 13:34:55 +0000 Subject: [PATCH 06/10] feedback: sonar - final two items to resolve. --- .../ElasticSearchCapabilityService.java | 75 ++++++++++++------- .../ElasticSearchCapabilityServiceTest.java | 24 ++++++ 2 files changed, 73 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java index 8e2dc9465..0b2a44a87 100644 --- a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java +++ b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java @@ -239,14 +239,15 @@ public ElasticSearchCapabilityReport getCapabilityReport(String runAsUser, boole List capabilities = buildCapabilities( writeProbeOutcome, - securityEnabled, - securityApiPresent, - xpack, - securitySettings, - clusterPrivileges, - licenseType, - authenticatedUser, - runAsUser); + new ClusterObservations( + securityEnabled, + securityApiPresent, + xpack, + securitySettings, + clusterPrivileges, + licenseType, + authenticatedUser, + runAsUser)); return new ElasticSearchCapabilityReport( string(root.body(), "cluster_name"), @@ -307,12 +308,11 @@ private List deploymentNotes( } /** - * The five capability verdicts, each preferring an observed write-probe outcome over the inferred - * read-only one — split out of {@link #getCapabilityReport} so its own branching does not - * compound with the rest of that method's. + * What the read-only pass observed about the cluster, plus the impersonation target it was asked + * to probe: the whole of what a capability verdict is reasoned from when no write probe settled + * the question. Grouped so the verdict builder takes the cluster's state as one value. */ - private List buildCapabilities( - WriteProbeOutcome writeProbeOutcome, + private record ClusterObservations( Boolean securityEnabled, boolean securityApiPresent, ProbeResult xpack, @@ -320,22 +320,45 @@ private List buildCapabilities( Map clusterPrivileges, String licenseType, String authenticatedUser, - String runAsUser) { + String runAsUser) {} + + /** + * The five capability verdicts, each preferring an observed write-probe outcome over the inferred + * read-only one — split out of {@link #getCapabilityReport} so its own branching does not + * compound with the rest of that method's. + */ + private List buildCapabilities( + WriteProbeOutcome writeProbeOutcome, ClusterObservations observed) { List capabilities = new ArrayList<>(); - capabilities.add(securityCapability(securityEnabled, securityApiPresent, xpack)); + capabilities.add( + securityCapability( + observed.securityEnabled(), observed.securityApiPresent(), observed.xpack())); capabilities.add( writeProbeOutcome != null ? writeProbeOutcome.apiKeys() - : apiKeyCapability(securityApiPresent, securitySettings, clusterPrivileges)); + : apiKeyCapability( + observed.securityApiPresent(), + observed.securitySettings(), + observed.clusterPrivileges())); capabilities.add( writeProbeOutcome != null ? writeProbeOutcome.dls() - : dlsFlsCapability(DLS, securityApiPresent, licenseType, securitySettings)); + : dlsFlsCapability( + DLS, + observed.securityApiPresent(), + observed.licenseType(), + observed.securitySettings())); capabilities.add( writeProbeOutcome != null ? writeProbeOutcome.fls() - : dlsFlsCapability(FLS, securityApiPresent, licenseType, securitySettings)); - capabilities.add(runAsCapability(securityApiPresent, authenticatedUser, runAsUser)); + : dlsFlsCapability( + FLS, + observed.securityApiPresent(), + observed.licenseType(), + observed.securitySettings())); + capabilities.add( + runAsCapability( + observed.securityApiPresent(), observed.authenticatedUser(), observed.runAsUser())); return capabilities; } @@ -1223,13 +1246,13 @@ private String licenseBasedRecommendation(String licenseType) { /** Reads the caller's own cluster privileges. A POST, but an evaluation rather than a write. */ private Map clusterPrivileges() { String body = - "{\"cluster\":[%s],\"index\":[{\"names\":[\"%s\"]," - + "\"privileges\":[\"read\",\"view_index_metadata\"]}]}" - .formatted( - PROBED_CLUSTER_PRIVILEGES.stream() - .map("\"%s\""::formatted) - .collect(Collectors.joining(",")), - probeIndex()); + ("{\"cluster\":[%s],\"index\":[{\"names\":[\"%s\"]," + + "\"privileges\":[\"read\",\"view_index_metadata\"]}]}") + .formatted( + PROBED_CLUSTER_PRIVILEGES.stream() + .map("\"%s\""::formatted) + .collect(Collectors.joining(",")), + probeIndex()); ProbeResult result = probe(HttpMethod.POST, "/_security/user/_has_privileges", body, Map.of()); if (result.status() != 200 || !result.body().has("cluster")) { diff --git a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java index d685c4f9e..c03a3aaa8 100644 --- a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java @@ -373,6 +373,30 @@ void testCredentialWithoutKeyMintingPrivilegesIsReportedAsNotPermitted() throws assertEquals(Boolean.FALSE, report.clusterPrivileges().get("grant_api_key")); } + /** + * A privileges body that failed to interpolate would be rejected by the cluster, and a rejected + * probe degrades quietly to "no privileges reported" rather than to a visible error — so the body + * itself is asserted on, not just the verdict it produces. + */ + @Test + void testPrivilegeProbeBodyIsFullyInterpolated() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + HAS_PRIVILEGES, + 200, + """ + {"cluster":{"manage_api_key":true}}"""); + + service().getCapabilityReport(null, false); + + List probes = requestsTo("POST", "/_security/user/_has_privileges"); + assertEquals(1, probes.size()); + String body = bodyOf(probes.getFirst()); + assertFalse(body.contains("%s"), "every placeholder should have been substituted: " + body); + assertTrue(body.contains("\"manage_api_key\""), body); + assertTrue(body.contains("\"names\":[\"dataset\"]"), body); + } + @Test void testRunAsDeniedByPrivilegeIsDistinguishedFromLicenseBlock() throws IOException { stubSecurityEnabledCluster("trial"); From dd283a3b02dde1656bb3c504311e6458a33e77f2 Mon Sep 17 00:00:00 2001 From: Elliot Otchet Date: Thu, 30 Jul 2026 18:19:47 +0000 Subject: [PATCH 07/10] feedback: address feedback from reviewers. --- docs/plans/es-security-capability-record.md | 77 ++++++--- .../ElasticSearchCapabilityResource.java | 57 +++++-- .../ElasticSearchCapabilityService.java | 161 +++++++++++++++--- .../paths/elasticSearchCapabilities.yaml | 89 +++++++--- .../ElasticSearchCapabilityReport.yaml | 17 +- .../ElasticSearchCapabilityResourceTest.java | 46 ++++- .../ElasticSearchCapabilityServiceTest.java | 153 ++++++++++++++++- .../ElasticSearchBasicLicenseTest.java | 7 + .../ElasticSearchContainerTests.java | 2 + .../integration/ElasticSearchTestCluster.java | 50 ++++++ .../consent/integration/README.md | 7 + 11 files changed, 566 insertions(+), 100 deletions(-) diff --git a/docs/plans/es-security-capability-record.md b/docs/plans/es-security-capability-record.md index 25f8e94df..ed7216088 100644 --- a/docs/plans/es-security-capability-record.md +++ b/docs/plans/es-security-capability-record.md @@ -8,9 +8,14 @@ Companion to ## How this record is produced -One tool: [`GET /api/elasticSearch/capabilities`](../../src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java), -which reports the full inventory for whichever cluster a deployment is pointed at — inferred, or with -`writeProbes=true` proven. All it needs is an Admin token for that environment. +One tool: [`/api/elasticSearch/capabilities`](../../src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java), +which reports the full inventory for whichever cluster a deployment is pointed at — inferred on `GET`, +proven on `POST`. All it needs is an Admin token for that environment. + +The two methods are the two modes. `GET` creates nothing; `POST` runs the write probes. The split is +deliberate rather than a query flag: minting credentials on a cluster is a side effect, and a URL +that does it on `GET` is one a prefetcher, a monitoring crawler, or a shared bookmark can fire +without anyone deciding to. Each environment already runs its own Consent deployment holding its own cluster credential, so that token yields the per-environment record without anyone obtaining cluster network access or a copy of a @@ -30,8 +35,8 @@ here" are worse than one that is tested. To capture a report file, redirect the endpoint's JSON: ```shell -curl -s -H "Authorization: Bearer $(gcloud auth print-access-token)" \ - 'https:///api/elasticSearch/capabilities?writeProbes=true' \ +curl -s -X POST -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + 'https:///api/elasticSearch/capabilities' \ | tee "es-capability--$(date +%F).json" | jq ``` @@ -42,25 +47,27 @@ endpoint against that. ### Running the capability endpoint ```shell -# Read-only. Safe anywhere, but DLS/FLS/API-key verdicts are inferred from the license tier. +# GET: read-only. Safe anywhere, but DLS/FLS/API-key verdicts are inferred from the license tier. curl -s -H "Authorization: Bearer $(gcloud auth print-access-token)" \ https:///api/elasticSearch/capabilities | jq -# Proven instead of inferred: creates and tears down a short-lived key and role. -curl -s -H "Authorization: Bearer $(gcloud auth print-access-token)" \ - 'https:///api/elasticSearch/capabilities?writeProbes=true' | jq +# POST: proven instead of inferred — creates and tears down a short-lived key and role. +curl -s -X POST -H "Authorization: Bearer $(gcloud auth print-access-token)" \ + 'https:///api/elasticSearch/capabilities' | jq # Optionally probe run_as against a specific username rather than the credential's own principal +# (accepted by both methods) curl -s -H "Authorization: Bearer $(gcloud auth print-access-token)" \ 'https:///api/elasticSearch/capabilities?runAsUser=some-user' | jq ``` -**Read-only mode** creates, modifies, and deletes nothing. That safety is what costs certainty: DLS, -FLS, and API-key support cannot be proven without writing, so they come back as `INFERRED_SUPPORTED` -/ `LICENSE_BLOCKED` reasoned from the license tier. Only `run_as` (a header on a read-only request) -and X-Pack Security itself are observed. +**Read-only mode (`GET`)** creates, modifies, and deletes nothing. That safety is what costs certainty: +DLS, FLS, and API-key support cannot be proven without writing, so they come back as +`INFERRED_SUPPORTED` / `LICENSE_BLOCKED` reasoned from the license tier and the cluster's +`xpack.security.dls_fls.enabled` setting. Only `run_as` (a header on a read-only request) and X-Pack +Security itself are observed. -**`writeProbes=true`** mints a short-lived API key and authenticates as it, creates a role carrying +**Write-probe mode (`POST`)** mints a short-lived API key and authenticates as it, creates a role carrying both a DLS query and an FLS grant, then uses keys whose `role_descriptors` carry those filters against the real dataset index to check the cluster *enforces* them: a `match_none` DLS key must return zero of the documents the shared credential can see, and a key granting one field must return @@ -76,11 +83,20 @@ Three fields carry most of the interpretive weight: - **`write_probes_run`** — read this first. It tells you whether the DLS/FLS/API-key verdicts below are observations or inferences. - **`cluster_privileges`** — what the deployment's *own* shared credential may do, which is the - constraint Epic D has to work within. If it holds neither `manage_security` nor `manage_api_key`, - the write probes cannot run and the report says so explicitly rather than reading their refusal as - a verdict against the native path (see the decision table below). + constraint Epic D has to work within. If it holds neither `manage_security` nor a key-minting + privilege (`manage_own_api_key` or `manage_api_key`), the write probes cannot run and the report + says so explicitly rather than reading their refusal as a verdict against the native path (see the + decision table below). - **`security_settings`** — filtered to the dozen or so values that gate a capability, out of the - ~50 defaults a cluster reports. + ~50 defaults a cluster reports. `xpack.security.dls_fls.enabled` is the one to read alongside the + license: set to `false` it switches DLS and FLS off cluster-wide whatever the tier entitles the + cluster to, and the report treats it as decisive. + +One verdict distinction to keep straight in a write-probe run: `SUPPORTED` for DLS or FLS means the +filters were *enforced* end to end, while `INFERRED_SUPPORTED` means the cluster accepted them and +the enforcement check did not complete — an empty or unreadable index, or no usable probe key. A +cluster can store a DLS query and ignore it at search time, so acceptance is not enforcement, and +`notes` says which of the two you are reading. **Scope: these are X-Pack probes, so they measure Elasticsearch only.** The endpoint identifies the distribution from `GET /` and is otherwise scoped to Elasticsearch deployments; every environment @@ -88,7 +104,7 @@ in the inventory below runs Elasticsearch. ## Environment inventory -### Local (`config/docker-compose.yaml`) — measured 2026-07-29 with `writeProbes=true` +### Local (`config/docker-compose.yaml`) — measured 2026-07-29 with the write probes Ticket A-0 is closed: the compose file now sets `xpack.security.enabled` to **true** by default (overridable per-run with `ES_SECURITY_ENABLED=false`) and self-generates a **trial** license, so the @@ -202,7 +218,7 @@ The endpoint was run against the control clusters in both modes and under both l 9.3.3 and 9.4.4. Its read-only inferences agree with the tier-by-tier measurements above, and its own write probes independently reproduce them — so the verdicts have been checked rather than trusted: -| Capability | Basic, read-only | Basic, `writeProbes` | Trial, read-only | Trial, `writeProbes` | +| Capability | Basic, `GET` | Basic, `POST` | Trial, `GET` | Trial, `POST` | | --- | --- | --- | --- | --- | | X-Pack Security | `SUPPORTED` | `SUPPORTED` | `SUPPORTED` | `SUPPORTED` | | API keys | `INFERRED_SUPPORTED` | `SUPPORTED` — created, authenticated, invalidated | `INFERRED_SUPPORTED` | `SUPPORTED` | @@ -239,7 +255,7 @@ falls back to the license reading and says which of the two you are looking at. ### `dev` — not yet measured -> Call `GET /api/elasticSearch/capabilities?writeProbes=true` against dev with an Admin token and +> Call `POST /api/elasticSearch/capabilities` against dev with an Admin token and > paste the findings here. Dev is the right place to run the write probes first in a *deployed* > environment — teardown has been confirmed on the control clusters and locally, so what dev adds is > confirmation under a real shared credential rather than a superuser one. @@ -271,7 +287,7 @@ falls back to the license reading and says which of the two you are looking at. ### `production` — not yet measured Call the endpoint read-only first — in that mode it creates nothing, so it cannot leave anything -behind on the production cluster. Only add `writeProbes=true` after the same call has been run in dev +behind on the production cluster. Only `POST` it after the same call has been run in dev and staging and the teardown has been confirmed there; the probes are designed to be safe in production (namespaced, 10-minute expiry, torn down in a `finally`, failures reported in `notes`), but production is not the place to find out. @@ -331,8 +347,10 @@ fixed in advance so the measurement determines the outcome: | Security enabled, license lacks DLS/FLS | **Epic E**, and raise the Platinum/Enterprise upgrade as a separate infra decision before committing to Epic D. | | Security disabled anywhere | **Epic E** now; Epic D stays blocked on infra enabling X-Pack Security in that environment. | | Environments disagree | **Both** — Epic E as the portable path, Epic D where licensed. The access contract from Ticket A-2 must be identical either way, so the enforcement layer stays swappable. | -| Write probes refused for lack of privileges | **Not a decision.** The probes measured the credential, not the cluster; fall back to the license reading and treat the missing `manage_security` / `manage_api_key` grant as its own prerequisite for Epic D. | +| Write probes refused for lack of privileges | **Not a decision.** The probes measured the credential, not the cluster; fall back to the license reading and treat the missing `manage_security` / `manage_own_api_key` grant as its own prerequisite for Epic D. | | DLS/FLS accepted but **not enforced** | **Epic E**, and treat it as a defect report to infra: a filter that is accepted and silently ignored is worse than one that is refused, and Epic D cannot be built on it. | +| DLS/FLS accepted, enforcement **not checked** (`INFERRED_SUPPORTED` from a write-probe run) | **Not a decision.** Acceptance is not enforcement. Fix what the notes say stopped the check — usually an empty or unreadable dataset index — and re-run the write probes before recording anything. | +| DLS/FLS licensed but `xpack.security.dls_fls.enabled=false` | **Epic E** until the setting is enabled. The license entitles the cluster to the feature; the setting switches it off cluster-wide, so it is an infra change, not a license one. | Known so far: the local environment now falls in the **first** row — security enabled, DLS and FLS licensed *and* observed enforced — which closes Ticket A-0 but decides nothing on its own. The rule @@ -344,13 +362,18 @@ any of them use. - The shared `authUser` almost certainly does **not** hold `manage_security` or `manage_api_key`. The report's `cluster_privileges` block says exactly which it has, via `POST /_security/user/_has_privileges`. If it lacks them, that is itself a finding: Epic D's - per-request key minting needs at minimum `grant_api_key` (preferred, since it mints keys on - behalf of a user without full `manage_api_key`). + per-request key minting goes through `POST /_security/api_key`, which needs at minimum + `manage_own_api_key` — the narrowest grant that authorises it, and so the one to ask infra for. +- `grant_api_key` is **not** a substitute. It authorises `POST /_security/api_key/grant`, which mints + a key on behalf of another user from that user's own credentials — a different endpoint and a + different design, and not the one these probes or Epic D use. The report's privilege check is + scoped to the endpoint actually called, so a credential holding only `grant_api_key` is reported as + unable to mint rather than predicted to work and then refused. - Because the endpoint authenticates as the deployment's own configured credential, that block *is* the shared credential's privileges — there is no way to accidentally record an admin's instead, which is what Epic D actually has to work with at runtime. When the credential holds neither - `manage_api_key` nor `grant_api_key`, API keys come back `NOT_PERMITTED` rather than supported: - the distinction between "the cluster can" and "we can." + `manage_own_api_key` nor `manage_api_key`, API keys come back `NOT_PERMITTED` rather than + supported: the distinction between "the cluster can" and "we can." - The end-to-end DLS check needs a non-empty index. The endpoint uses the configured `datasetIndexName` automatically, and says so explicitly when that index is empty or unreadable rather than reporting a false pass — an empty index makes a `match_none` key return zero documents diff --git a/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java b/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java index 1d1ea7f5a..efb739d9b 100644 --- a/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java +++ b/src/main/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResource.java @@ -3,8 +3,8 @@ import com.google.inject.Inject; import io.dropwizard.auth.Auth; import jakarta.annotation.security.RolesAllowed; -import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; import jakarta.ws.rs.QueryParam; @@ -26,10 +26,15 @@ *

The probes behind this endpoint are Elasticsearch X-Pack probes — see {@link * ElasticSearchCapabilityService}. * - *

The probes behind this endpoint are read-only by default. Pass {@code writeProbes=true} to - * additionally create and tear down a short-lived API key and role, which is the only way to - * observe DLS, FLS, and API-key support rather than infer it from the license tier — see {@link - * ElasticSearchCapabilityService} for what each mode establishes. + *

Two modes, split by HTTP method rather than by a query parameter. {@code GET} is read-only: + * nothing is created, modified, or deleted on the cluster. {@code POST} additionally creates and + * tears down a short-lived API key and role, which is the only way to observe DLS, FLS, and API-key + * support rather than infer it from the license tier — see {@link ElasticSearchCapabilityService} + * for what each mode establishes. + * + *

The split is the point: minting credentials is a side effect, and a URL that mints them on + * {@code GET} is one a link prefetcher, a monitoring crawler, or a bookmark can fire without anyone + * deciding to. Behind {@code POST} it takes a deliberate request. */ @Path("api/elasticSearch") public class ElasticSearchCapabilityResource extends Resource { @@ -43,14 +48,12 @@ public ElasticSearchCapabilityResource(ElasticSearchCapabilityService capability /** * Report the cluster's security capabilities: version, edition, X-Pack Security, DLS, FLS, API - * keys, and run_as. + * keys, and run_as. Read-only — nothing is created, modified, or deleted on the cluster, so the + * DLS, FLS, and API-key verdicts are inferred from the license tier rather than observed. * * @param duosUser the authenticated admin * @param runAsUser optional username to attempt the run_as probe against; defaults to the * credential's own principal, which still establishes whether the feature is licensed - * @param writeProbes when true, create and tear down a short-lived API key and role so DLS, FLS, - * and API-key support are observed rather than inferred. Off by default: the caller has to - * ask for writes against the cluster their environment depends on. * @return the capability report */ @GET @@ -58,9 +61,39 @@ public ElasticSearchCapabilityResource(ElasticSearchCapabilityService capability @Produces(MediaType.APPLICATION_JSON) @RolesAllowed({ADMIN}) public Response getCapabilities( - @Auth DuosUser duosUser, - @QueryParam("runAsUser") String runAsUser, - @QueryParam("writeProbes") @DefaultValue("false") boolean writeProbes) { + @Auth DuosUser duosUser, @QueryParam("runAsUser") String runAsUser) { + return report(duosUser, runAsUser, false); + } + + /** + * The same report, with the write probes run: a short-lived API key and a role carrying DLS and + * FLS filters are created, used, and torn down, so those verdicts are observed rather than + * inferred. + * + *

A {@code POST} because it has side effects on the cluster — credentials are created and + * removed — even though the response body is a report. Reaching it therefore takes a deliberate + * call rather than anything that merely follows a link. + * + * @param duosUser the authenticated admin + * @param runAsUser optional username to attempt the run_as probe against; defaults to the + * credential's own principal, which still establishes whether the feature is licensed + * @return the capability report + */ + @POST + @Path("/capabilities") + @Produces(MediaType.APPLICATION_JSON) + @RolesAllowed({ADMIN}) + public Response runCapabilityProbes( + @Auth DuosUser duosUser, @QueryParam("runAsUser") String runAsUser) { + return report(duosUser, runAsUser, true); + } + + /** + * Audits the call and runs the report. Inside the {@code try} with the report itself so that + * anything thrown on the way — including reading the caller's id — still comes back through the + * resource's own error mapping rather than as an unmapped server error. + */ + private Response report(DuosUser duosUser, String runAsUser, boolean writeProbes) { try { // Worth an audit trail either way: this reports on the cluster's security posture, and with // write probes it also creates and removes credentials on that cluster. diff --git a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java index 0b2a44a87..a3e9729d9 100644 --- a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java +++ b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java @@ -20,12 +20,14 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.http.Header; +import org.apache.http.HttpHost; import org.apache.http.message.BasicHeader; import org.broadinstitute.consent.http.configurations.ElasticSearchConfiguration; import org.broadinstitute.consent.http.models.elastic_search.CapabilityVerdict; import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchCapability; import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchCapabilityReport; import org.broadinstitute.consent.http.util.ConsentLogger; +import org.elasticsearch.client.Node; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; @@ -57,6 +59,12 @@ * DLS descriptor and only fails later at search time. Everything created is namespaced, expires * within {@value #PROBE_KEY_EXPIRATION}, and is torn down in a {@code finally} block. * + *

Acceptance is not enforcement, and the DLS/FLS verdicts keep the two apart: {@code + * SUPPORTED} means an end-to-end check completed and the filter was applied, while a write-probe + * run that could only get as far as the cluster accepting the filters reports {@code + * INFERRED_SUPPORTED} with a note saying what stopped the check. Collapsing those two would hide + * the exact failure — a filter stored and ignored — that the enforcement probes exist to find. + * *

Because each environment's deployment already holds its own cluster credential, running this * with write probes in each environment produces the measured per-environment record without anyone * needing cluster network access or a copy of a secret. @@ -86,6 +94,14 @@ public class ElasticSearchCapabilityService implements ConsentLogger { // nosemgrep - a cluster setting name, not a key private static final String API_KEY_ENABLED_SETTING = "xpack.security.authc.api_key.enabled"; + + /** + * The setting that switches DLS and FLS off cluster-wide. Independent of the license: a Platinum + * cluster with this set to {@code false} enforces neither, so the license tier alone cannot + * settle the verdict. + */ + private static final String DLS_FLS_ENABLED_SETTING = "xpack.security.dls_fls.enabled"; + private static final String VERSION_FIELD = "version"; private static final String LICENSE_FIELD = "license"; private static final String USERNAME_FIELD = "username"; @@ -98,7 +114,7 @@ public class ElasticSearchCapabilityService implements ConsentLogger { private static final Set REPORTED_SECURITY_DEFAULTS = Set.of( "xpack.security.enabled", - "xpack.security.dls_fls.enabled", + DLS_FLS_ENABLED_SETTING, API_KEY_ENABLED_SETTING, "xpack.security.authc.run_as.enabled", "xpack.security.authc.token.enabled", @@ -177,18 +193,33 @@ public ElasticSearchCapabilityService(RestClient esClient, ElasticSearchConfigur } /** - * Points a second client at the same host the injected client uses, so an API key is exercised + * Points a second client at the same nodes the injected client uses, so an API key is exercised * against the same cluster — including when the deployment is configured by cloud ID, which the - * injected client has already resolved to a host. + * injected client has already resolved to hosts. + * + *

Every node is carried over rather than only the first, so a probe fails over between nodes + * the way an ordinary request does; taking node 0 alone would make a write-probe pass fail for no + * reason other than that one node of a multi-node cluster happened to be down. The list is read + * per key rather than once at construction, so a client whose nodes have since been re-resolved + * is followed rather than pinned to the set it started with. */ private static ApiKeyClientFactory defaultApiKeyClientFactory(RestClient esClient) { return encodedApiKey -> - RestClient.builder(esClient.getNodes().get(0).getHost()) + RestClient.builder(probeHosts(esClient)) .setDefaultHeaders( new Header[] {new BasicHeader("Authorization", "ApiKey " + encodedApiKey)}) .build(); } + /** + * Every node the injected client currently knows about. Package-private so the failover contract + * can be asserted without standing up a cluster — and the contract is precisely that this is the + * whole list rather than its first entry. + */ + static HttpHost[] probeHosts(RestClient esClient) { + return esClient.getNodes().stream().map(Node::getHost).toArray(HttpHost[]::new); + } + /** * Builds the capability report. * @@ -269,7 +300,8 @@ public ElasticSearchCapabilityReport getCapabilityReport(String runAsUser, boole writeProbesRan, capabilities, restClientCompatibility(version), - recommendation(securityApiPresent, licenseType, writeProbeOutcome), + recommendation( + securityApiPresent, licenseType, dlsFlsDisabled(securitySettings), writeProbeOutcome), notes); } @@ -411,15 +443,29 @@ private ElasticSearchCapability apiKeyCapability( // API keys are a Basic-tier feature, so a security-enabled cluster is expected to support // them regardless of license. What actually gates Consent is whether the shared credential // may mint them. + // + // Tested against the endpoint that actually mints a key here and in Epic D — POST + // /_security/api_key — which manage_own_api_key authorises on its own; manage_api_key is the + // broader grant that also covers other users' keys. grant_api_key is deliberately not part of + // this test: it authorises POST /_security/api_key/grant, a different endpoint, so reading it + // as "can mint" would predict success for a credential whose create-key call is refused. boolean canMint = Boolean.TRUE.equals(privileges.get("manage_api_key")) - || Boolean.TRUE.equals(privileges.get("grant_api_key")); + || Boolean.TRUE.equals(privileges.get("manage_own_api_key")); if (!privileges.isEmpty() && !canMint) { + String grantCaveat = + Boolean.TRUE.equals(privileges.get("grant_api_key")) + ? " It does hold grant_api_key, but that authorises only POST %s/grant — minting on " + .formatted(API_KEY_PATH) + + "behalf of another user, which is a different endpoint and a different design." + : ""; return new ElasticSearchCapability( API_KEYS, CapabilityVerdict.NOT_PERMITTED, "The cluster supports API keys, but the credential this deployment authenticates with " - + "holds neither manage_api_key nor grant_api_key, so it cannot mint per-request keys.", + + "holds neither manage_own_api_key nor manage_api_key, so it cannot mint per-request " + + "keys through POST %s.".formatted(API_KEY_PATH) + + grantCaveat, "POST /_security/user/_has_privileges"); } return new ElasticSearchCapability( @@ -441,7 +487,20 @@ private ElasticSearchCapability dlsFlsCapability( "GET " + AUTHENTICATE_PATH); } String license = licenseType == null ? "" : licenseType.toLowerCase(); - String dlsFlsSetting = settings.getOrDefault("xpack.security.dls_fls.enabled", "not-set"); + String dlsFlsSetting = settings.getOrDefault(DLS_FLS_ENABLED_SETTING, "not-set"); + // The setting overrides the license in one direction only: it can switch the feature off on a + // cluster whose license includes it, so a qualifying tier is not on its own enough to report + // the feature as expected to work. + if (dlsFlsDisabled(settings)) { + return new ElasticSearchCapability( + name, + CapabilityVerdict.UNAVAILABLE, + "%s=false switches DLS and FLS off cluster-wide, so no role or API key can carry a DLS " + .formatted(DLS_FLS_ENABLED_SETTING) + + "query or an FLS grant here — whatever the '%s' license includes." + .formatted(licenseType), + "GET /_cluster/settings -> %s=false".formatted(DLS_FLS_ENABLED_SETTING)); + } if (DLS_FLS_LICENSES.contains(license)) { return new ElasticSearchCapability( name, @@ -545,11 +604,27 @@ private ElasticSearchCapability runAsCapability( private record WriteProbeOutcome( ElasticSearchCapability apiKeys, ElasticSearchCapability dls, ElasticSearchCapability fls) { - /** Whether DLS came back usable, which is the pivot the Epic D / Epic E decision turns on. */ - boolean dlsUsable() { + /** + * Whether DLS was observed to be enforced, which is the pivot the Epic D / Epic E + * decision turns on. Only a completed end-to-end probe sets {@link CapabilityVerdict#SUPPORTED} + * here; a cluster that merely accepted the filters lands on {@link + * CapabilityVerdict#INFERRED_SUPPORTED} and is reported by {@link #dlsAcceptedNotEnforced()}. + */ + boolean dlsEnforced() { return dls.verdict() == CapabilityVerdict.SUPPORTED; } + /** + * Whether the cluster accepted the DLS filter but its enforcement was never observed — the + * verdict a pass lands on when the enforcement check could not be run to a conclusion. Distinct + * from {@link #dlsEnforced()} because acceptance does not imply enforcement: a cluster can + * store a DLS descriptor and ignore it at search time, which is the failure this probe exists + * to catch, so the recommendation must not read one as the other. + */ + boolean dlsAcceptedNotEnforced() { + return dls.verdict() == CapabilityVerdict.INFERRED_SUPPORTED; + } + /** * Whether the probes were stopped by the credential's privileges rather than by the cluster's * capability. In that case the DLS and FLS verdicts say nothing about what the cluster can do, @@ -654,8 +729,10 @@ private String writeProbeNote(int keysCreated, boolean roleCreated) { } return "Write probes ran: %d short-lived API key(s)%s were created under the " .formatted(keysCreated, roleCreated ? " and one probe role" : "") - + "duos-capability-probe / duos_dlsfls_probe names and removed again. The DLS, FLS, and " - + "API-key verdicts below are observed rather than inferred."; + + "duos-capability-probe / duos_dlsfls_probe names and removed again. The verdicts below " + + "come from what the cluster did rather than from its license tier — but a DLS or FLS " + + "verdict of INFERRED_SUPPORTED means only that the filters were accepted, with a note " + + "above saying what stopped the enforcement check."; } /** @@ -729,10 +806,16 @@ private RoleAcceptance dlsFlsRoleProbe(String roleName, String index) { "PUT %s carrying query and field_security -> %d".formatted(path, result.status()); if (created(result)) { + // Acceptance, not enforcement — so INFERRED_SUPPORTED, and an enforcement probe that runs to + // a conclusion is what upgrades it to SUPPORTED. A cluster can accept a role carrying a DLS + // query and then apply nothing at search time, which is the failure mode this whole + // write-probe pass exists to catch; reporting acceptance as SUPPORTED would hide exactly it. return new RoleAcceptance( true, - CapabilityVerdict.SUPPORTED, - "Observed: the cluster accepted a role carrying both a DLS query and an FLS grant.", + CapabilityVerdict.INFERRED_SUPPORTED, + "Observed: the cluster accepted a role carrying both a DLS query and an FLS grant, so its " + + "license permits the filters. Not observed: whether it enforces them — that needs " + + "an end-to-end enforcement probe.", evidence); } CapabilityVerdict verdict = refusalVerdict(result); @@ -1188,13 +1271,16 @@ private String restClientCompatibility(String clusterVersion) { } private String recommendation( - boolean securityApiPresent, String licenseType, WriteProbeOutcome writeProbeOutcome) { + boolean securityApiPresent, + String licenseType, + boolean dlsFlsDisabled, + WriteProbeOutcome writeProbeOutcome) { // An observation outranks the license inference in either direction: a cluster that actually // enforced DLS settles the question, and one that refused on licensing grounds settles it just // as firmly. A refusal on *privilege* grounds settles nothing about the cluster, so that case // falls through to the license reading rather than being read as a verdict against Epic D. if (writeProbeOutcome != null && securityApiPresent) { - if (writeProbeOutcome.dlsUsable()) { + if (writeProbeOutcome.dlsEnforced()) { return "Epic D (native DLS/FLS) is viable on this cluster, and this was observed rather " + "than inferred: a probe role and API key carrying DLS/FLS descriptors were accepted " + "and enforced. Keep Epic E in scope only if another environment cannot support " @@ -1204,8 +1290,21 @@ private String recommendation( return "Inconclusive from the write probes: this deployment's credential is not permitted " + "to create a role or an API key, so the DLS and FLS verdicts describe the credential " + "rather than the cluster (see cluster_privileges). Re-run with a credential holding " - + "manage_security and manage_api_key to settle it. On the license alone: " - + licenseBasedRecommendation(licenseType); + + "manage_security and manage_own_api_key to settle it. On the license alone: " + + licenseBasedRecommendation(licenseType, dlsFlsDisabled); + } + // Accepted but never exercised. Claiming Epic D off the back of that would be claiming + // enforcement from acceptance, which is the one inference this whole probe exists to refuse. + if (writeProbeOutcome.dlsAcceptedNotEnforced()) { + return "Not settled by the write probes. The cluster accepted a role carrying DLS and FLS " + + "filters, which establishes that its license permits them, but no end-to-end check " + + "was completed, so whether it *enforces* them is still unknown — see notes for what " + + "stopped the check (commonly an empty or unreadable '%s' index, or no usable probe " + .formatted(probeIndex()) + + "key). A cluster can accept a filter and silently ignore it, so re-run the write " + + "probes once that is resolved rather than recording Epic D on acceptance alone. On " + + "the license alone: " + + licenseBasedRecommendation(licenseType, dlsFlsDisabled); } return "Epic E (compatibility fallback). Write probes were run against this cluster and DLS " + "was not usable: %s Epic D would need that resolved first." @@ -1216,12 +1315,24 @@ private String recommendation( + "Security is not enabled, so DLS, FLS, API keys, and run_as cannot be used at all. " + "Epic D stays blocked until X-Pack Security is enabled here."; } - return licenseBasedRecommendation(licenseType); + return licenseBasedRecommendation(licenseType, dlsFlsDisabled); } - /** What the license tier alone implies, used when no write probe settled the question. */ - private String licenseBasedRecommendation(String licenseType) { + /** + * What the license tier and the cluster's DLS/FLS setting imply, used when no write probe settled + * the question. The setting is checked first: it switches the feature off whatever the license + * entitles the cluster to, so a Platinum tier with it disabled is not a viable Epic D cluster. + */ + private String licenseBasedRecommendation(String licenseType, boolean dlsFlsDisabled) { String license = licenseType == null ? "" : licenseType.toLowerCase(); + if (dlsFlsDisabled) { + return "Epic E (compatibility fallback). Security is enabled, but %s=false switches DLS and " + .formatted(DLS_FLS_ENABLED_SETTING) + + "FLS off cluster-wide, so neither can be enforced here whatever the '%s' license " + .formatted(licenseType) + + "includes. Epic D would need that setting enabled first — an infra change that should " + + "precede any commitment to it."; + } if (DLS_FLS_LICENSES.contains(license)) { return "Epic D (native DLS/FLS) is viable on this cluster: security is enabled and the '%s' " .formatted(licenseType) @@ -1302,6 +1413,14 @@ && isReportableSecuritySetting(entry.getKey(), isDefault)) { return settings; } + /** + * Whether DLS/FLS has been switched off cluster-wide. Only an explicit {@code false} counts: a + * setting the cluster did not report says nothing, and must not be read as the feature being off. + */ + private static boolean dlsFlsDisabled(Map settings) { + return "false".equals(settings.get(DLS_FLS_ENABLED_SETTING)); + } + private boolean isReportableSecuritySetting(String key, boolean isDefault) { if (!key.startsWith("xpack.security") && !key.contains("dls_fls")) { return false; diff --git a/src/main/resources/assets/paths/elasticSearchCapabilities.yaml b/src/main/resources/assets/paths/elasticSearchCapabilities.yaml index 0a4c4420b..5a2db28a5 100644 --- a/src/main/resources/assets/paths/elasticSearchCapabilities.yaml +++ b/src/main/resources/assets/paths/elasticSearchCapabilities.yaml @@ -1,5 +1,5 @@ get: - summary: Elasticsearch security capability report + summary: Elasticsearch security capability report (read-only) operationId: apiElasticSearchCapabilitiesGet description: | Report the security features of the Elasticsearch cluster this deployment is configured @@ -13,21 +13,15 @@ get: The probes are Elasticsearch X-Pack probes; this endpoint is scoped to Elasticsearch deployments only. - By default all probes are non-destructive: nothing is created, modified, or deleted on the - cluster. The trade-off is certainty — DLS, FLS, and API-key support can only be proven by - creating a role or a key, so in that mode those verdicts are inferred from the license tier - and cluster settings and are returned with an INFERRED_ verdict. Only run_as and X-Pack - Security itself are reported as observed fact. + This mode is non-destructive: nothing is created, modified, or deleted on the cluster. The + trade-off is certainty — DLS, FLS, and API-key support can only be proven by creating a role + or a key, so here those verdicts are inferred from the license tier and cluster settings and + are returned with an INFERRED_ verdict. Only run_as and X-Pack Security itself are reported as + observed fact. - Pass writeProbes=true to observe them instead. That mode mints a short-lived API key, - creates a role carrying a DLS query and an FLS grant, and checks that a key whose - role_descriptors carry those filters is actually enforced against the dataset index — - the only way to distinguish a cluster that accepts DLS descriptors from one that enforces - them, since a Basic-licensed cluster accepts them at creation and fails only at search time. - Everything created is namespaced duos-capability-probe / duos_dlsfls_probe, expires within - 10 minutes, and is torn down before the response is returned. + POST the same path to observe them instead. tags: - - ElasticSearch + - Admin parameters: - name: runAsUser in: query @@ -37,18 +31,69 @@ get: required: false schema: type: string - - name: writeProbes + responses: + 200: + description: The capability report + content: + application/json: + schema: + $ref: '../schemas/ElasticSearchCapabilityReport.yaml' + 401: + description: Unauthorized. + 403: + description: Forbidden - Admin role required + content: + application/json: + schema: + $ref: '../schemas/ErrorResponse.yaml' + 429: + description: Too Many Requests - rate limit exceeded + content: + application/json: + schema: + $ref: '../schemas/ErrorResponse.yaml' + 500: + description: Internal Server Error + content: + application/json: + schema: + $ref: '../schemas/ErrorResponse.yaml' +post: + summary: Elasticsearch security capability report, with write probes + operationId: apiElasticSearchCapabilitiesProbePost + description: | + The same report as GET, with the write probes run, so DLS, FLS, and API-key support are + observed rather than inferred. Requires Admin role. + + This mode mints a short-lived API key, creates a role carrying a DLS query and an FLS grant, + and checks that a key whose role_descriptors carry those filters is actually enforced against + the dataset index — the only way to distinguish a cluster that accepts DLS descriptors from + one that enforces them, since a Basic-licensed cluster accepts them at creation and fails only + at search time. Everything created is namespaced duos-capability-probe / duos_dlsfls_probe, + expires within 10 minutes, and is torn down before the response is returned. + + A POST rather than a GET flag because those creations are side effects on the cluster: they + should take a deliberate call, not something that follows a link. The response body is the + same report schema either way. + + Requires the deployment credential to hold manage_own_api_key (or the broader manage_api_key) + for POST /_security/api_key, and manage_security for the role probe; the report says which of + those it has under cluster_privileges. Note that grant_api_key is not sufficient — it + authorises POST /_security/api_key/grant, a different endpoint that these probes do not use. + + Where security is disabled there is nothing to probe, so the probes are skipped and a note in + the response says they were requested but not run. + tags: + - Admin + parameters: + - name: runAsUser in: query description: | - Create and tear down a short-lived API key and role so that DLS, FLS, and API-key - support are observed rather than inferred. Requires the deployment credential to hold - manage_api_key (or grant_api_key) and manage_security; the report says which of those - it has under cluster_privileges. Ignored when security is disabled, since there is - nothing to probe — a note in the response says the probes were requested but not run. + Username to attempt the run_as probe against. Defaults to the deployment credential's + own principal, which still establishes whether the feature is licensed and enabled. required: false schema: - type: boolean - default: false + type: string responses: 200: description: The capability report diff --git a/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml b/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml index 9cac01dd0..7eb92243a 100644 --- a/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml +++ b/src/main/resources/assets/schemas/ElasticSearchCapabilityReport.yaml @@ -9,10 +9,10 @@ description: | The probes target Elasticsearch's X-Pack security APIs only; this report covers Elasticsearch deployments exclusively. - The example is a real writeProbes=true response from a local trial-licensed cluster, where the + The example is a real write-probe (POST) response from a local trial-licensed cluster, where the configured credential is the `elastic` superuser. A deployed environment's shared credential is - unlikely to hold manage_security or manage_api_key, so expect NOT_PERMITTED verdicts and a - narrower cluster_privileges block there rather than this all-true one. + unlikely to hold manage_security or a key-minting privilege, so expect NOT_PERMITTED verdicts and + a narrower cluster_privileges block there rather than this all-true one. properties: cluster_name: type: string @@ -132,6 +132,11 @@ properties: observed refusals. The INFERRED_ values are derived from license tier and cluster settings because proving them would require writing to the cluster. UNKNOWN also covers any capability on a cluster that did not respond. + + For DLS and FLS, INFERRED_SUPPORTED also covers a write-probe run in which the cluster + accepted the filters but their enforcement could not be checked end to end — a cluster + can store a DLS query and ignore it at search time, so only SUPPORTED means enforced. + The notes say what stopped the check. enum: - SUPPORTED - UNAVAILABLE @@ -225,5 +230,7 @@ examples: notes: - >- Write probes ran: 3 short-lived API key(s) and one probe role were created under the - duos-capability-probe / duos_dlsfls_probe names and removed again. The DLS, FLS, and - API-key verdicts below are observed rather than inferred. + duos-capability-probe / duos_dlsfls_probe names and removed again. The verdicts below come + from what the cluster did rather than from its license tier — but a DLS or FLS verdict of + INFERRED_SUPPORTED means only that the filters were accepted, with a note above saying what + stopped the enforcement check. diff --git a/src/test/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResourceTest.java b/src/test/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResourceTest.java index b57c7dd8e..8b5d79232 100644 --- a/src/test/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResourceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/resources/ElasticSearchCapabilityResourceTest.java @@ -66,7 +66,7 @@ private ElasticSearchCapabilityReport report() { void testGetCapabilities() { when(capabilityService.getCapabilityReport(null, false)).thenReturn(report()); - Response response = resource.getCapabilities(duosUser, null, false); + Response response = resource.getCapabilities(duosUser, null); assertEquals(200, response.getStatus()); assertNotNull(response.getEntity()); @@ -78,30 +78,64 @@ void testGetCapabilities() { void testGetCapabilitiesPassesRunAsUserThrough() { when(capabilityService.getCapabilityReport("someone-else", false)).thenReturn(report()); - Response response = resource.getCapabilities(duosUser, "someone-else", false); + Response response = resource.getCapabilities(duosUser, "someone-else"); assertEquals(200, response.getStatus()); verify(capabilityService).getCapabilityReport("someone-else", false); } - /** Write probes must never be a side effect of calling the endpoint — only of asking for them. */ + /** + * The GET is the whole read-only contract: creating credentials on the cluster must not be + * reachable by anything that merely follows a link, so no argument to it can turn writes on. + */ @Test - void testWriteProbesAreOnlyRunWhenExplicitlyRequested() { + void testGetNeverRunsWriteProbes() { + when(capabilityService.getCapabilityReport(null, false)).thenReturn(report()); + + resource.getCapabilities(duosUser, null); + + verify(capabilityService, never()).getCapabilityReport(null, true); + } + + /** Write probes are what the POST is for, and the only thing that reaches them. */ + @Test + void testPostRunsWriteProbes() { when(capabilityService.getCapabilityReport(null, true)).thenReturn(report()); - Response response = resource.getCapabilities(duosUser, null, true); + Response response = resource.runCapabilityProbes(duosUser, null); assertEquals(200, response.getStatus()); + assertEquals(report(), response.getEntity()); verify(capabilityService).getCapabilityReport(null, true); verify(capabilityService, never()).getCapabilityReport(null, false); } + @Test + void testPostPassesRunAsUserThrough() { + when(capabilityService.getCapabilityReport("someone-else", true)).thenReturn(report()); + + Response response = resource.runCapabilityProbes(duosUser, "someone-else"); + + assertEquals(200, response.getStatus()); + verify(capabilityService).getCapabilityReport("someone-else", true); + } + @Test void testGetCapabilitiesHandlesServiceFailure() { when(capabilityService.getCapabilityReport(null, false)) .thenThrow(new RuntimeException("cluster exploded")); - Response response = resource.getCapabilities(duosUser, null, false); + Response response = resource.getCapabilities(duosUser, null); + + assertEquals(500, response.getStatus()); + } + + @Test + void testPostHandlesServiceFailure() { + when(capabilityService.getCapabilityReport(null, true)) + .thenThrow(new RuntimeException("cluster exploded")); + + Response response = resource.runCapabilityProbes(duosUser, null); assertEquals(500, response.getStatus()); } diff --git a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java index c03a3aaa8..f53dbf48e 100644 --- a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java @@ -13,6 +13,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.http.HttpHost; import org.apache.http.HttpVersion; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; @@ -22,6 +23,7 @@ import org.broadinstitute.consent.http.models.elastic_search.CapabilityVerdict; import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchCapability; import org.broadinstitute.consent.http.models.elastic_search.ElasticSearchCapabilityReport; +import org.elasticsearch.client.Node; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; @@ -363,7 +365,7 @@ void testCredentialWithoutKeyMintingPrivilegesIsReportedAsNotPermitted() throws 200, """ {"cluster":{"manage_security":false,"manage_api_key":false,"grant_api_key":false, - "manage_own_api_key":true,"read_security":false,"monitor":true}}"""); + "manage_own_api_key":false,"read_security":false,"monitor":true}}"""); ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); @@ -373,6 +375,101 @@ void testCredentialWithoutKeyMintingPrivilegesIsReportedAsNotPermitted() throws assertEquals(Boolean.FALSE, report.clusterPrivileges().get("grant_api_key")); } + /** + * The probe mints keys with POST /_security/api_key, which manage_own_api_key authorises on its + * own. Predicting a refusal for a credential that can in fact create its own keys would + * understate what Epic D has to work with. + */ + @Test + void testManageOwnApiKeyAloneIsEnoughToMintPerRequestKeys() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + HAS_PRIVILEGES, + 200, + """ + {"cluster":{"manage_security":false,"manage_api_key":false,"grant_api_key":false, + "manage_own_api_key":true,"read_security":false,"monitor":true}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals(CapabilityVerdict.INFERRED_SUPPORTED, capability(report, "API keys").verdict()); + } + + /** + * grant_api_key authorises POST /_security/api_key/grant, not the create-key endpoint this probe + * and Epic D use, so holding it alone must not be reported as being able to mint keys — the + * create-key call would be refused. + */ + @Test + void testGrantApiKeyAloneIsNotReadAsAbleToMintKeys() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + HAS_PRIVILEGES, + 200, + """ + {"cluster":{"manage_security":false,"manage_api_key":false,"grant_api_key":true, + "manage_own_api_key":false,"read_security":false,"monitor":true}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + ElasticSearchCapability apiKeys = capability(report, "API keys"); + assertEquals(CapabilityVerdict.NOT_PERMITTED, apiKeys.verdict()); + // And the reader is told why the privilege it does hold does not answer the question. + assertTrue(apiKeys.detail().contains("grant_api_key"), apiKeys.detail()); + assertTrue(apiKeys.detail().contains("/grant"), apiKeys.detail()); + } + + /** + * The setting overrides the license: a Platinum-equivalent tier with DLS/FLS switched off + * cluster-wide enforces neither, so reading the tier alone would report Epic D as viable on a + * cluster where it cannot work. + */ + @Test + void testDlsFlsDisabledByClusterSettingOverridesAQualifyingLicense() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + SETTINGS, + 200, + """ + {"defaults":{"xpack.security.enabled":"true","xpack.security.dls_fls.enabled":"false", + "xpack.security.authc.api_key.enabled":"true"},"persistent":{},"transient":{}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + for (String name : List.of("Document-level security (DLS)", "Field-level security (FLS)")) { + ElasticSearchCapability capability = capability(report, name); + assertEquals(CapabilityVerdict.UNAVAILABLE, capability.verdict(), name); + assertTrue(capability.detail().contains("dls_fls.enabled"), capability.detail()); + assertTrue(capability.evidence().contains("dls_fls.enabled=false"), capability.evidence()); + } + // And the recommendation has to follow the setting rather than the license tier. + assertTrue(report.recommendation().contains("Epic E"), report.recommendation()); + assertTrue( + report.recommendation().contains("dls_fls.enabled=false"), + "the recommendation must say what blocks the native path: " + report.recommendation()); + } + + /** + * Only an explicit false counts. A cluster that did not report the setting says nothing about it, + * and treating that silence as "disabled" would block Epic D on a missing value. + */ + @Test + void testUnreportedDlsFlsSettingIsNotReadAsDisabled() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + SETTINGS, + 200, + """ + {"defaults":{"xpack.security.enabled":"true"},"persistent":{},"transient":{}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals( + CapabilityVerdict.INFERRED_SUPPORTED, + capability(report, "Document-level security (DLS)").verdict()); + assertTrue(report.recommendation().contains("Epic D"), report.recommendation()); + } + /** * A privileges body that failed to interpolate would be rejected by the cluster, and a rejected * probe degrades quietly to "no privileges reported" rather than to a visible error — so the body @@ -471,6 +568,29 @@ void testUnreachableClusterProducesAReportRatherThanAnException() throws IOExcep assertTrue(report.recommendation().contains("unreachable")); } + /** + * The API-key client has to reach the cluster the same way the injected client does. Building it + * from the first node alone would make a write-probe pass fail whenever that one node of a + * multi-node cluster happened to be down — a failure about the topology, reported as a finding + * about the cluster's capabilities. + */ + @Test + void testTheApiKeyProbeClientCarriesEveryNodeNotJustTheFirst() { + when(esClient.getNodes()) + .thenReturn( + List.of( + new Node(new HttpHost("es-1", 9200, "https")), + new Node(new HttpHost("es-2", 9200, "https")), + new Node(new HttpHost("es-3", 9200, "https")))); + + HttpHost[] hosts = ElasticSearchCapabilityService.probeHosts(esClient); + + assertEquals(3, hosts.length, "a probe must be able to fail over like any other request"); + assertEquals("es-1", hosts[0].getHostName()); + assertEquals("es-3", hosts[2].getHostName()); + assertEquals("https", hosts[0].getSchemeName(), "the scheme must survive the copy"); + } + @Test void testEveryCapabilityCarriesEvidence() throws IOException { stubSecurityEnabledCluster("trial"); @@ -757,9 +877,11 @@ void testCredentialThatCannotMintKeysIsReportedAsNotPermitted() throws IOExcepti assertTrue(apiKeys.detail().contains("refused")); // Without a key, the enforcement probes cannot run, so no key-scoped search is attempted. assertTrue(requestsTo("GET", "/dataset/_search").isEmpty()); - // The role probe still answers the license question on its own. - assertEquals( - CapabilityVerdict.SUPPORTED, capability(report, "Document-level security (DLS)").verdict()); + // The role probe still answers the license question on its own — but only that question, so the + // verdict stays inferred rather than claiming an enforcement that was never exercised. + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.INFERRED_SUPPORTED, dls.verdict()); + assertTrue(dls.detail().contains("Not observed: whether it enforces them"), dls.detail()); } /** @@ -901,7 +1023,8 @@ void testFlsWithNoInspectableFieldsFallsBackAndSaysWhy() throws IOException { // Role acceptance stands, but the report must not let that read as proven enforcement. assertEquals( - CapabilityVerdict.SUPPORTED, capability(report, "Field-level security (FLS)").verdict()); + CapabilityVerdict.INFERRED_SUPPORTED, + capability(report, "Field-level security (FLS)").verdict()); assertFalse(capability(report, "Field-level security (FLS)").detail().contains("Proven")); assertTrue( report.notes().stream().anyMatch(n -> n.contains("no document fields to inspect")), @@ -923,6 +1046,13 @@ void testMissingProbeKeyIsSaidToLimitTheDlsAndFlsVerdicts() throws IOException { assertTrue( report.notes().stream().anyMatch(n -> n.contains("No usable probe key")), "role acceptance must not be presented as enforcement: " + report.notes()); + // And the verdict itself must say the same thing the note does. + assertEquals( + CapabilityVerdict.INFERRED_SUPPORTED, + capability(report, "Document-level security (DLS)").verdict()); + assertEquals( + CapabilityVerdict.INFERRED_SUPPORTED, + capability(report, "Field-level security (FLS)").verdict()); } @Test @@ -953,12 +1083,19 @@ void testEmptyIndexFallsBackToRoleAcceptanceAndSaysSo() throws IOException { // Role acceptance still stands, but it must not be dressed up as end-to-end proof. ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); - assertEquals(CapabilityVerdict.SUPPORTED, dls.verdict()); + assertEquals(CapabilityVerdict.INFERRED_SUPPORTED, dls.verdict()); assertFalse(dls.detail().contains("Proven end to end")); assertTrue( report.notes().stream().anyMatch(n -> n.contains("empty")), "an empty index must be called out rather than read as a pass"); assertTrue(requestsTo("GET", "/dataset/_search").isEmpty()); + // The recommendation is the part a reader acts on, so it must not claim enforcement either. + assertFalse( + report.recommendation().contains("enforced"), + "no enforcement was observed, so none may be claimed: " + report.recommendation()); + assertTrue( + report.recommendation().contains("Not settled by the write probes"), + report.recommendation()); } @Test @@ -975,7 +1112,9 @@ void testUnreadableIndexIsCalledOutRatherThanTreatedAsEnforcement() throws IOExc assertTrue(report.notes().stream().anyMatch(n -> n.contains("not readable"))); assertEquals( - CapabilityVerdict.SUPPORTED, capability(report, "Document-level security (DLS)").verdict()); + CapabilityVerdict.INFERRED_SUPPORTED, + capability(report, "Document-level security (DLS)").verdict()); + assertFalse(report.recommendation().contains("enforced"), report.recommendation()); } @Test diff --git a/src/test/java/org/broadinstitute/consent/integration/ElasticSearchBasicLicenseTest.java b/src/test/java/org/broadinstitute/consent/integration/ElasticSearchBasicLicenseTest.java index 46bec9768..3edd12961 100644 --- a/src/test/java/org/broadinstitute/consent/integration/ElasticSearchBasicLicenseTest.java +++ b/src/test/java/org/broadinstitute/consent/integration/ElasticSearchBasicLicenseTest.java @@ -51,6 +51,13 @@ class ElasticSearchBasicLicenseTest { ELASTIC.start(); CONFIGURATION = ElasticSearchTestCluster.configuration(ELASTIC, INDEX); CLIENT = ElasticSearchSupport.createRestClient(CONFIGURATION); + + try { + // The license is published after the container reports ready; see the helper's javadoc. + ElasticSearchTestCluster.awaitLicense(CLIENT); + } catch (Exception e) { + throw new ExceptionInInitializerError(e); + } } /** diff --git a/src/test/java/org/broadinstitute/consent/integration/ElasticSearchContainerTests.java b/src/test/java/org/broadinstitute/consent/integration/ElasticSearchContainerTests.java index 7aeb82d45..21eb69a3e 100644 --- a/src/test/java/org/broadinstitute/consent/integration/ElasticSearchContainerTests.java +++ b/src/test/java/org/broadinstitute/consent/integration/ElasticSearchContainerTests.java @@ -67,6 +67,8 @@ public abstract class ElasticSearchContainerTests { CLIENT = ElasticSearchSupport.createRestClient(CONFIGURATION); try { + // The basic license is published after the container reports ready; see the helper's javadoc. + ElasticSearchTestCluster.awaitLicense(CLIENT); // Required before any DLS/FLS grant will be honored; see the class javadoc. jsonResponse(new Request("POST", "/_license/start_trial?acknowledge=true")); } catch (Exception e) { diff --git a/src/test/java/org/broadinstitute/consent/integration/ElasticSearchTestCluster.java b/src/test/java/org/broadinstitute/consent/integration/ElasticSearchTestCluster.java index eb7f26100..ee9963b0c 100644 --- a/src/test/java/org/broadinstitute/consent/integration/ElasticSearchTestCluster.java +++ b/src/test/java/org/broadinstitute/consent/integration/ElasticSearchTestCluster.java @@ -50,6 +50,11 @@ final class ElasticSearchTestCluster { static final String USER = "elastic"; static final String PASSWORD = "devpassword"; + /** How long {@link #awaitLicense} waits for the self-generated license to become readable. */ + private static final long LICENSE_TIMEOUT_MILLIS = 60_000; + + private static final long LICENSE_POLL_INTERVAL_MILLIS = 250; + /** * The error Elasticsearch returns when a document- or field-level security grant is exercised * under a basic license. Asserted on directly: a future version that changed this to a silent @@ -81,6 +86,51 @@ static ElasticsearchContainer container(boolean securityEnabled) { .withPassword(PASSWORD); } + /** + * Blocks until the cluster will answer {@code GET /_license}, which happens strictly later than + * the container reporting itself started. + * + *

Testcontainers considers an {@code ElasticsearchContainer} ready as soon as the HTTP layer + * answers on port 9200, but the self-generated basic license is published to the cluster state + * after that point. In between, {@code GET /_license} returns {@code 404} with an empty body — + * documented as a transient of a master node still building cluster state, not a + * misconfiguration. The gap is on the order of a second on an idle machine and widens under the + * load of a full test run, where three of these containers boot alongside the Postgres one. + * + *

Any class that reads or changes the license immediately after {@code start()} must call this + * first, or its first request can fail on that 404. + */ + static void awaitLicense(RestClient client) throws IOException { + long deadline = System.nanoTime() + LICENSE_TIMEOUT_MILLIS * 1_000_000L; + ResponseException lastRejection; + while (true) { + try { + execute(client, new Request("GET", "/_license")); + return; + } catch (ResponseException e) { + if (e.getResponse().getStatusLine().getStatusCode() != 404) { + throw e; + } + lastRejection = e; + } + if (System.nanoTime() - deadline >= 0) { + throw new IOException( + "no license after " + + LICENSE_TIMEOUT_MILLIS + + "ms; last response: " + + bodyOf(lastRejection), + lastRejection); + } + try { + // Polling a cluster that has no readiness signal for this; see the javadoc above. + Thread.sleep(LICENSE_POLL_INTERVAL_MILLIS); // NOSONAR + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("interrupted while waiting for the license", e); + } + } + } + /** Configuration pointing at a running container, carrying the shared dev credentials. */ static ElasticSearchConfiguration configuration( ElasticsearchContainer container, String datasetIndexName) { diff --git a/src/test/java/org/broadinstitute/consent/integration/README.md b/src/test/java/org/broadinstitute/consent/integration/README.md index 92b533458..2a9afbcdb 100644 --- a/src/test/java/org/broadinstitute/consent/integration/README.md +++ b/src/test/java/org/broadinstitute/consent/integration/README.md @@ -49,6 +49,13 @@ Two things are easy to get wrong here: non-compliant for [field and document level security]`. The base class activates the 30-day trial in its static initializer. Note that `POST /_security/api_key` accepts a DLS/FLS role descriptor even on a basic license — the rejection only surfaces on the search request. +- **"Container started" is earlier than "license readable."** Testcontainers considers the container + ready once the HTTP layer answers on port 9200, but the self-generated basic license reaches the + cluster state a beat later, and until it does `GET /_license` returns `404` with an empty body. + A class that reads or changes the license in its static initializer must call + `ElasticSearchTestCluster.awaitLicense(client)` first. The window is about a second on an idle + machine and wider during a full test run, where three of these containers boot alongside the + Postgres one — which is why this surfaced as a full-suite-only failure. - **The container defaults to HTTPS.** For image versions 8.0.0 and above, `ElasticsearchContainer` automatically applies `withPassword("changeme")` and `withCertPath(...)`, serving the HTTP layer over TLS with a self-signed CA. `ElasticSearchSupport.createRestClient` has From 4e1054e390a8054ffd736ad29a06a34d69ebb79f Mon Sep 17 00:00:00 2001 From: Elliot Otchet Date: Fri, 31 Jul 2026 14:11:13 +0000 Subject: [PATCH 08/10] fix: pr feedback. --- .../consent/http/service/ElasticSearchCapabilityService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java index a3e9729d9..a69c34c2f 100644 --- a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java +++ b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java @@ -328,8 +328,8 @@ private List deploymentNotes( } else if (!writeProbesRan) { notes.add( "DLS, FLS, and API-key verdicts are inferred from the license tier and cluster " - + "settings. Re-run with writeProbes=true to create and tear down a short-lived key " - + "and role and observe them instead."); + + "settings. Re-run as POST /api/elasticSearch/capabilities to create and tear down " + + "a short-lived key and role and observe them instead."); } if (writeProbes && !writeProbesRan) { notes.add( From 7507a57633ba9e9b46dce03d51fd031d96f3d8fb Mon Sep 17 00:00:00 2001 From: Elliot Otchet Date: Wed, 5 Aug 2026 20:46:38 +0000 Subject: [PATCH 09/10] Create contract and adjust existing plans based on findings of current ES cluster information. --- DEVNOTES.md | 39 +- .../elasticsearch-service-duos-ui-usage.md | 445 +++++++++------ docs/plans/es-access-contract.md | 512 ++++++++++++++++++ docs/plans/es-security-capability-record.md | 170 ++++-- pom.xml | 7 +- .../consent/http/ConsentModule.java | 11 + .../ElasticSearchCapabilityService.java | 60 +- .../ElasticSearchCapabilityServiceTest.java | 52 +- src/test/resources/mvn.properties | 6 + 9 files changed, 1089 insertions(+), 213 deletions(-) create mode 100644 docs/plans/es-access-contract.md diff --git a/DEVNOTES.md b/DEVNOTES.md index 45da4baa9..8a0231f5a 100644 --- a/DEVNOTES.md +++ b/DEVNOTES.md @@ -72,15 +72,19 @@ and the defaults should be correct. ### Developing with a local Elastic Search instance: #### Running with X-Pack Security enabled -Security is **on by default**, which is what the security work (`GET /api/elasticSearch/capabilities` -and native DLS/FLS) needs. Two things make that work, and they have to agree: +`config/` is not version-controlled — it is rendered per developer (see "Render Configs" above) — so +none of this arrives with a `git pull`; it has to go into your own `config/docker-compose.yaml`. The +example stanza below turns security **on**, which is what the security work +(`GET /api/elasticSearch/capabilities` and native DLS/FLS) needs, and keeps a +`${ES_SECURITY_ENABLED:-true}` gate so you can drop back to an unauthenticated cluster for a single +run. Two things make it work, and they have to agree: * `ELASTIC_PASSWORD` in the compose `elastic` service bootstraps the `elastic` superuser. * `authUser` / `authPassword` in `config/consent.yaml` are the credentials Consent sends. They are harmless when security is off — the ES client only sends credentials in response to a 401 challenge, which a security-disabled cluster never issues. -DLS and FLS are Platinum features, so the compose file self-generates a 30-day **trial** license via +DLS and FLS are Platinum features, so the stanza self-generates a 30-day **trial** license via `xpack.license.self_generated.type`. That setting only applies the first time a cluster forms. If your `elastic` data volume predates it, the cluster keeps its `basic` license and DLS/FLS come back `LICENSE_BLOCKED`; activate the trial once, by hand: @@ -93,7 +97,7 @@ curl -s -u elastic:devpassword localhost:9200/_license # expect "type": "trial Note that transport SSL stays disabled. ES logs a bootstrap warning about it, which is expected and correct here — transport SSL is only required for multi-node clusters. -To get the old security-disabled cluster back for a run, without editing the committed file: +To get an unauthenticated cluster back for a run, without editing your compose file again: ```bash ES_SECURITY_ENABLED=false docker-compose -p consent -f config/docker-compose.yaml up @@ -114,18 +118,19 @@ An example docker-compose stanza for elastic: memory: 4gb environment: - "ES_JAVA_OPTS=-Xms2g -Xmx2g" - # X-Pack Security is OFF by default so the default `docker compose up` behaves exactly as - # before. Epics A-C and E (application-layer fallback) need no security. To work on Epic D - # (native DLS/FLS), start the stack with security on: + # X-Pack Security is ON here: the capability endpoint and Epic D (native DLS/FLS) need it. + # Epics A-C and E (application-layer fallback) do not, so to run one session unauthenticated: # - # ES_SECURITY_ENABLED=true docker-compose -p consent -f config/docker-compose.yaml up - # - # DLS/FLS is a Platinum feature, so also activate the 30-day trial license once per cluster: - # - # curl -u elastic:devpassword -XPOST 'localhost:9200/_license/start_trial?acknowledge=true' + # ES_SECURITY_ENABLED=false docker-compose -p consent -f config/docker-compose.yaml up # # See DEVNOTES.md ("Developing with a local Elastic Search instance") for the full workflow. - xpack.security.enabled=${ES_SECURITY_ENABLED:-true} + # DLS/FLS is a Platinum feature, so self-generate a trial rather than the default basic + # license. Only applies the first time a cluster forms — on an existing `elastic` volume, + # activate it by hand instead: + # + # curl -u elastic:devpassword -XPOST 'localhost:9200/_license/start_trial?acknowledge=true' + - xpack.license.self_generated.type=${ES_LICENSE_TYPE:-trial} # Bootstraps the `elastic` superuser password when security is on; ignored when it is off. # Must match authUser/authPassword in consent.yaml. - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-devpassword} @@ -137,13 +142,11 @@ An example docker-compose stanza for elastic: I also suggest changing the default bucket location so uploaded ontology files do not interfere with other dev environments. -#### Running local Elastic Search with security enabled - -By default the local cluster runs with `xpack.security.enabled=true`: requests are -now authenticated, which is what most work needs. Only work on native Elasticsearch document- and -field-level security (DLS/FLS) requires security to be on. Application-layer authorization work -does not — it never touches Elasticsearch security. +#### Which work actually needs security enabled +Only the capability endpoint and native Elasticsearch document- and field-level security (DLS/FLS) +require it. Application-layer authorization work does not — it never touches Elasticsearch security — +so if you are not working on those, running with `ES_SECURITY_ENABLED=false` is fine. #### Enabling DLS/FLS locally (trial license required) diff --git a/docs/plans/elasticsearch-service-duos-ui-usage.md b/docs/plans/elasticsearch-service-duos-ui-usage.md index ab6ab6550..cd927842f 100644 --- a/docs/plans/elasticsearch-service-duos-ui-usage.md +++ b/docs/plans/elasticsearch-service-duos-ui-usage.md @@ -397,12 +397,21 @@ The dataset index stores `DatasetTerm` documents with nested objects used by sea - `dataTypes` - `assets` (`Map`; includes study asset collections) - `data` (`Map`) +- `externalIdentifier` +- `externalIdentifierType` ### UserTerm and DacTerm nested fields - `submitter` / `updateUser` (`UserTerm`): `userId`, `displayName`, `institution` + (`InstitutionTerm`: `id`, `name`) - `dac` (`DacTerm`): `dacId`, `dacName`, `dacEmail` +> **Access classification lives elsewhere.** Every path above is classified SEARCH-VISIBLE or +> INTERNAL in [`es-access-contract.md`](es-access-contract.md) §B, which is enumerated from the model +> classes and is the authoritative list. Adding a field to `DatasetTerm`, `StudyTerm`, `UserTerm`, +> `DacTerm`, or `InstitutionTerm` requires classifying it there in the same change — an unclassified +> field is dropped by the E-3 allowlist and omitted from the D-3 field grant. + ## Test, Config, and Docs Touchpoints In duos-ui Test files with stubs/intercepts tied to these paths: @@ -447,9 +456,9 @@ Size key: **S** ≈ 1 day, **M** ≈ 2–3 days, **L** ≈ 4–5 days, **XL** | Epic | Name | Owner | Blocked by | Blocks | | --- | --- | --- | --- | --- | | A | Discovery & Contract | Infra + Backend | — | B, C, D, E | -| B | Index Schema & Indexing Pipeline | Backend | A | D, E, F | -| C | Auth Context Service | Backend | A | D, E | -| D | Native DLS/FLS Path | Backend + Infra | A, B, C | F | +| B | Index Schema & Indexing Pipeline | Backend | A-2, A-3 (not A-1) | D, E, F | +| C | Auth Context Service | Backend | A-2 (not A-1) | D, E | +| D | Native DLS/FLS Path | Backend + Infra | A-1, B, C | F | | E | Compatibility Fallback | Backend | B, C | F | | F | API Hardening | Backend | D or E | G | | G | Frontend Alignment | Frontend | F | — | @@ -461,7 +470,19 @@ Size key: **S** ≈ 1 day, **M** ≈ 2–3 days, **L** ≈ 4–5 days, **XL** **Goal**: Establish facts about the Elasticsearch cluster's security capabilities, evaluate the local developer configuration changes needed, and define the formal access contract that all later -epics are built on. All other epics are blocked on A-1. +epics are built on. + +**Status**: A-0 closed. A-1 has local, control-cluster, and production measurements recorded in +[`es-security-capability-record.md`](es-security-capability-record.md); dev and staging remain, and +they decide Epic D vs. E. A-2 is delivered and complete as +[`es-access-contract.md`](es-access-contract.md) — every dimension and field is decided, and its +remaining OPEN items are proposed *changes* to current behavior, each with a preserve-today default, +so none of them blocks Epics B or C. + +**Note on the blocking relationships**: only the *enforcement mechanism* (Epic D vs. E) is blocked on +A-1. The access contract is not, and was deliberately written to be mechanism-neutral — the rules +must be identical under native DLS/FLS and under the mediated fallback, or the fallback becomes a +hole. Epics B and C are blocked on A-2, not A-1. --- @@ -522,8 +543,12 @@ layer and requires no Elasticsearch configuration changes. ##### A-0 Outcome -**Decision: Option B** — security is env-var gated in `config/docker-compose.yaml`, default off. -Findings below were verified empirically against `elasticsearch:9.4.4` using throwaway containers +**Decision: Option B** — security is env-var gated in `config/docker-compose.yaml` rather than +unconditionally on. The recommended default *inside* that gate later moved from off to on, once the +capability endpoint made a secured local cluster useful beyond Epic D (DEVNOTES.md); since `/config/` +is not version-controlled (see the note at the end of this section), that default is per-developer +either way, and the gate is the part of the decision that carries. Findings below were verified +empirically against `elasticsearch:9.4.4` using throwaway containers running the exact env block from `config/docker-compose.yaml`. (Originally established on 9.3.3 and re-verified on 9.4.4 when the pin moved; every finding reproduced unchanged, including the exact error strings. The only observed delta was the bundled Lucene version, 10.3.2 → 10.4.0.) @@ -575,9 +600,10 @@ Consequences: **Verified compose behavior (both modes)** -The `elastic` service now uses `xpack.security.enabled=${ES_SECURITY_ENABLED:-false}`, +The env block verified here was `xpack.security.enabled=${ES_SECURITY_ENABLED:-false}`, `ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-devpassword}`, and explicit -`xpack.security.http.ssl.enabled=false`. Confirmed with the exact env block: +`xpack.security.http.ssl.enabled=false`. Both modes were exercised, so the table holds whichever way +the gate defaults — and the recommended default has since moved to `:-true` (DEVNOTES.md): | Mode | Unauthenticated | Authenticated | HTTPS on 9200 | | --- | --- | --- | --- | @@ -702,25 +728,53 @@ per-request credential work can begin, we need the cluster edition and security in the Elasticsearch index. This ticket defines the contract that shapes both the `accessPolicy` nested object (Ticket B-1) and the auth context resolver (Ticket C-1). -**Acceptance criteria**: -- Completed matrix: for each dimension (`publicVisibility`, ADMIN bypass, creator, custodian, DAC - member/chair, institution allowlist, policy tags) record: data source, enforcement level (DLS - filter vs. FLS field bundle), and whether persistent backing currently exists. -- Field-level security groupings decided: e.g. `"public"` profile grants `datasetName`, - `datasetId`, `study.studyName`; `"privileged"` additionally grants `study.dataCustodianEmail`, - `study.dataSubmitterEmail`. -- `publicVisibility` DLS semantics decided: invisible to non-privileged callers, or visible with - field redaction? +**Deliverable**: [`es-access-contract.md`](es-access-contract.md) — **written**. The sections below +record what it settled and what it deliberately did not. + +**Acceptance criteria** (all met except where noted): +- ✅ Completed matrix: for each dimension (`publicVisibility`, ADMIN bypass, dataset creator, study + creator, custodian, DAC member/chair, institution allowlist, policy tags) record data source, + enforcement level, persistent backing, **and whether the contract preserves current behavior or + expands it** — contract §A. Fourteen dimensions; eight PRESERVE, six DEFER. +- ✅ Field-level security groupings decided — contract §B, classifying **every** indexed path from + the model classes into SEARCH-VISIBLE or INTERNAL, with dynamic maps (`data`, `assets`) INTERNAL + and no wildcard grants permitted — including for admins, who bypass document filtering but not + field filtering. +- ✅ `publicVisibility` DLS semantics decided — contract Decision 1: restricted documents are + **invisible**, not redacted. Redaction is also not expressible in native FLS; see below. +- ✅ `publicVisibility = NULL` resolved — contract §A.1. It first looked like an undecidable policy + question because the two code paths read a null differently, but `study.public_visibility` is + `NOT NULL` and the nulls that actually occur come from the summary query's `LEFT JOIN` — i.e. they + are the "dataset has no study" case, which both paths already allow. Nothing was left for an owner + to decide. + +**Findings that change other tickets**: +- **A per-document `fieldAccessProfile` cannot drive native FLS.** Field grants live in the + credential's index privileges and apply uniformly to every document a search request matches; + nothing re-selects a grant per hit. B-2 is invalidated as written and B-1 must drop the field — + contract Decision 2. +- **Four dimensions in B-1's `AccessPolicyTerm` grant no access today** (DAC membership/chair, + institution, policy tags, principal allowlists). Populating them into a DLS filter would be a + silent authorization expansion, so they are DEFERred pending OPEN-3/OPEN-5. +- **Dataset creator and study creator are distinct** (different columns, both privileged paths); + B-1/B-3 must index both document-side IDs, and D-3/E-2 must compare the caller's one user ID + against both. +- **Custodian matching is case-sensitive and trims only the stored side**, so `Alice@x.org` fails + against a stored `alice@x.org` — contract §A.2, OPEN-6. **Implementation notes**: -- Start from the `StudyTerm` fields listed in the Indexed Elements section of this document. Flag - every field containing PII or internal-only data. -- `dataCustodianEmail` is currently parsed from the study property bag in - `DatasetService.isCreatorOrCustodian` (L224–238), not a dedicated DB column — note this as a - storage gap candidate. - -**Dependencies**: A-1. -**Size**: S +- `dataCustodianEmail` is parsed from the study property bag in `DatasetService.isCreatorOrCustodian` + (L220–236), not a dedicated DB column. Contract §C records what that does and does not cost, and + corrects A-3's "no new storage needed" framing. +- The Indexed Elements section of this document is incomplete (it omits `study.externalIdentifier`, + `study.externalIdentifierType`, and `UserTerm.institution` sub-fields). Contract §B is enumerated + from the model classes and supersedes it. + +**Dependencies**: A-1 — **satisfied for this ticket's purposes.** A-1's outstanding dev/staging rows +choose the enforcement *mechanism* (Epic D vs. E); the contract is stated in mechanism-neutral terms +because it must be identical either way, so it was not held for them. +**Size**: S — **actual: L.** The exhaustive field classification and the behavior-preservation audit +were the bulk of it. --- @@ -740,10 +794,23 @@ source for every field. Dimensions such as `allowedInstitutionIds`, `allowedPrin separate dataset-to-institution mapping table. **Implementation notes**: -- `dacId` already exists on `Dataset`; `dataCustodianEmail` already exists in the study property - bag — no new storage needed for those dimensions. +- `dacId` already exists as a column on `Dataset` — genuinely no new storage. +- `dataCustodianEmail` is **not** the same case, and the earlier "no new storage needed" framing was + wrong. Persistent backing exists, but as an unstructured JSON array inside the `study_property` + bag: no referential integrity (access is granted to a string, not a principal), no normalization, + no index, and no defined behavior for malformed values. [`es-access-contract.md`](es-access-contract.md) + §C records this in full. It does **not** block Epics B–E, because B-3 denormalizes custodian emails + into `accessPolicy` at index time — but it does make reindex-on-custodian-change a correctness + requirement (B-4), it leaves the non-search endpoints parsing the bag, and OPEN-6 (case + normalization) has to be answered either way. Whether custodianship should become a first-class + relation is this ticket's call to make. - `allowedPrincipalIds` (explicit user allowlists) and `policyTags` (consent-code-based access - tags) are the most likely to require new storage. + tags) are the most likely to require new storage — but answer contract **OPEN-5** first: none of + them corresponds to a current requirement, and if the answer is "not now," the storage question + does not arise and B-1 should drop the fields. +- Institution allowlists cannot be derived from `User.institutionId` alone: there is **no + dataset-to-institution mapping** of any kind today, so this dimension needs a storage decision + before it can mean anything at all. **Dependencies**: A-2. **Size**: M @@ -770,17 +837,30 @@ Currently it carries no structured access metadata; all visibility logic lives i This ticket adds the schema without populating it yet (population is B-3). **Acceptance criteria**: -- New `AccessPolicyTerm` class with fields: - - `publicVisibility: boolean` - - `creatorUserId: Integer` - - `creatorEmail: String` - - `custodianEmails: List` - - `dacId: Integer` - - `dacApproval: Boolean` - - `allowedInstitutionIds: List` - - `allowedPrincipalIds: List` - - `policyTags: List` - - `fieldAccessProfile: String` — `"public"` or `"privileged"` +- New `AccessPolicyTerm` class. **Field set revised by the A-2 contract** — see + [`es-access-contract.md`](es-access-contract.md) §D: + - `publicVisibility: Boolean` + - `hasStudy: Boolean` — carries contract §A row 5 (a dataset with no study is readable by + everyone today). Required because the filter treats a null `publicVisibility` on a study-bearing + document as *not* public, so "no study" cannot be expressed as an absent visibility. + - `datasetCreatorUserId: Integer` — the dataset's creator (`dataset.create_user_id`) + - `studyCreatorUserId: Integer` — the study's creator (`study.create_user_id`), a *different* + privileged path; contract §A rows 6 and 7 + - `custodianEmails: List` — trim surrounding whitespace on each stored value, matching + today's `custodian.trim()` behavior, but preserve case (contract §A.2). Lowercasing or otherwise + normalizing here would authorize case-mismatched custodians through search while the dataset + endpoints still rejected them; OPEN-6 proposes fixing both paths together. + - `dacId: Integer` — indexed for display/filtering parity, **not** consulted for authorization + while contract rows 9–10 are DEFERred + - ~~`creatorEmail`~~ — dropped; creator matching is by user ID (contract §A.2) + - ~~`dacApproval`~~ — dropped; it is a display attribute, not authorization (contract row 11) + - ~~`allowedInstitutionIds`, `allowedPrincipalIds`, `policyTags`~~ — **do not add** until OPEN-5 + establishes that they are requirements. None has storage or current behavior; shipping them + unpopulated invites a later reader to treat them as enforcement. + - ~~`fieldAccessProfile`~~ — **removed.** Native FLS cannot select a field grant per document + (contract Decision 2). +- Class-level comment recording that **every** `accessPolicy` path is INTERNAL and must never appear + in a field grant (contract §B.4). - `DatasetTerm` gains an `accessPolicy: AccessPolicyTerm` field. - Elasticsearch index mapping updated with `accessPolicy` as a `nested` (or `object`) type — confirm with A-1 outcome which is required for the DLS query approach. @@ -799,26 +879,26 @@ This ticket adds the schema without populating it yet (population is B-3). --- -#### Ticket B-2 — Add `fieldAccessProfile` marker to `DatasetTerm` +#### Ticket B-2 — ~~Add `fieldAccessProfile` marker to `DatasetTerm`~~ — **CANCELLED by A-2** -**Summary**: Add a `fieldAccessProfile` string field (inside `AccessPolicyTerm`) to signal which -FLS field bundle applies to the document. +**Do not implement.** The mechanism this ticket specifies does not exist in Elasticsearch. -**Context**: FLS in Elasticsearch requires knowing which fields each document grants to which -caller profile. A profile marker on the document allows the auth context (C-1) to request a -field-grant list matched to the document's declared profile, without enumerating fields per -document in the query path. +It assumed the auth context could read a profile marker off each document and request a matching +field grant. Field grants are declared in the credential's index privileges and are evaluated when +the request is authorized, then applied uniformly to every document that privilege matches; there is +no stage at which the cluster inspects a hit and re-selects a grant for it. +([Elastic: controlling access at document and field level](https://www.elastic.co/docs/deploy-manage/users-roles/cluster-or-deployment-auth/controlling-access-at-document-field-level)) -**Acceptance criteria**: -- `AccessPolicyTerm.fieldAccessProfile` present (this may overlap with B-1; keep as a separate - deliverable to track separately). -- Valid values: `"public"` (`publicVisibility=true`), `"privileged"` (`publicVisibility=false` or - restricted fields present). -- Unit test: `fieldAccessProfile` is `"public"` for a dataset whose study has - `publicVisibility=true`, and `"privileged"` for `publicVisibility=false`. - -**Dependencies**: B-1. -**Size**: S +The obvious replacement — a request-wide privileged bundle selected from the caller — is also +unsafe. Creator and custodian privilege is document-scoped, while the same DLS request also returns +unrelated public datasets. A privileged request-wide grant would therefore expose privileged fields +from those unrelated documents. See [`es-access-contract.md`](es-access-contract.md) Decision 2. + +**Replacement work**: D-3 and E-3 apply the single SEARCH-VISIBLE allowlist to every caller, +including ADMIN. C-1 derives document-visibility context only; it derives no field bundle. No +separate ticket is needed. + +**Size**: — (removed from the plan) --- @@ -832,16 +912,27 @@ and `institutionDAO` (L58–66). All data needed for `accessPolicy` is reachable the mapping. **Acceptance criteria**: -- `accessPolicy.publicVisibility` ← `dataset.getStudy().getPublicVisibility()` (null-safe). -- `accessPolicy.creatorUserId` ← `dataset.getCreateUserId()`. -- `accessPolicy.creatorEmail` ← `userDAO.getUserById(createUserId).getEmail()`. -- `accessPolicy.custodianEmails` ← parsed from study property bag using the same logic as - `DatasetService.isCreatorOrCustodian` (L224–238). +- `accessPolicy.publicVisibility` ← `dataset.getStudy().getPublicVisibility()` (null-safe). The + column is `NOT NULL`, so a study-bearing dataset always has a real value; the filter treats an + unexpected null as *not* public (contract §A.1). +- `accessPolicy.hasStudy` ← `dataset.getStudyId() != null`. This is what carries contract §A row 5 + — a dataset with no study is readable by everyone today, and that must not be expressed as a null + `publicVisibility`. +- `accessPolicy.datasetCreatorUserId` ← `dataset.getCreateUserId()`. +- `accessPolicy.studyCreatorUserId` ← `dataset.getStudy().getCreateUserId()` — a separate privileged + path from the dataset creator (contract §A rows 6/7), not a duplicate of it. +- `accessPolicy.custodianEmails` ← parsed from the study property bag as in + `DatasetService.isCreatorOrCustodian` (L220–236), preserving that method's exact matching + semantics — apply the same `trim()` on the stored side and **no** case normalization (contract + §A.2). Malformed or non-array property values must not throw out of indexing. - `accessPolicy.dacId` ← `dataset.getDacId()`. -- `accessPolicy.dacApproval` ← `dataset.getDacApproval()`. -- `accessPolicy.fieldAccessProfile` ← `"public"` if `publicVisibility=true`, else `"privileged"`. -- `allowedInstitutionIds` ← per outcome of A-3; empty list if not yet implemented. -- Unit test: null study → `accessPolicy.publicVisibility` defaults to `false`, no NPE. +- ~~`creatorEmail`, `dacApproval`, `fieldAccessProfile`, `allowedInstitutionIds`~~ — not populated; + see the revised B-1 field set. +- **A dataset with no study must remain readable by everyone** (contract §A row 5 — that is current + behavior). Do not default a null study to `publicVisibility=false`; that would hide datasets that + are visible today. Represent "no study" explicitly so the DLS filter can match it. +- Unit test: null study → no NPE, and the resulting document is readable by a caller with no + relationship to it. **Implementation notes**: - `isCreatorOrCustodian` in `DatasetService` (L224–238) parses custodian email from @@ -851,7 +942,8 @@ the mapping. - Guard all `study` accesses — datasets created outside the registration flow may have a null study reference. -**Dependencies**: B-1, B-2, A-3. +**Dependencies**: B-1. A-3 is not required for the current field set; its speculative dimensions are +deferred pending OPEN-5. **Size**: M --- @@ -926,16 +1018,21 @@ cannot add nested types to a live index. The active index is identified by and `toStudyTerm` before any auth enforcement code is written against them. **Acceptance criteria**: -- `publicVisibility=true` → `fieldAccessProfile == "public"`. -- `publicVisibility=false` → `fieldAccessProfile == "privileged"`. +- `publicVisibility` flows through for `true` / `false`. +- A dataset with no study indexes `hasStudy=false` and remains readable by an unrelated caller + (contract §A row 5) — the case a null `publicVisibility` would otherwise have to carry. - `custodianEmails` populated when study has `dataCustodianEmail` property. - `custodianEmails` is empty (not null) when study has no custodian property. -- Null study → `publicVisibility` defaults to `false`, no NPE. -- `dacId`, `dacApproval`, `creatorUserId`, `creatorEmail` flow through from dataset and user DAO. +- `custodianEmails` preserves case: `" Alice@X.org "` in the property bag indexes as `Alice@X.org` + (trimmed, not lowercased), so search authorizes exactly the callers `DatasetService` does today. +- Malformed `dataCustodianEmail` (not a JSON array, unparseable) does not throw out of indexing. +- Null study → no NPE, and the document remains readable by an unrelated caller (contract row 5). +- `dacId`, `datasetCreatorUserId`, `studyCreatorUserId` flow through — with a case where the dataset + creator and study creator are **different users**, since conflating them is the likely bug. +- No `accessPolicy` field is present in the SEARCH-VISIBLE projection (contract §B.4). **Implementation notes**: - Mirror the existing mock-heavy pattern in `ElasticSearchServiceTest` — mock all DAO calls. -- Use `@ParameterizedTest` for the `fieldAccessProfile` cases. **Dependencies**: B-3. **Size**: M @@ -964,24 +1061,32 @@ not once per document. **Acceptance criteria**: - `DatasetSearchAuthContext` (record or immutable class) with: - - `Integer userId` - - `String userEmail` - - `Integer institutionId` + - `Integer userId` — matches **both** the dataset creator and the study creator dimensions, which + are distinct columns and distinct privileged paths ([`es-access-contract.md`](es-access-contract.md) + §A rows 6 and 7); the filter must test both, not one + - `String userEmail` — passed through **unnormalized**, for the exact custodian matching the + dataset endpoints do today (contract §A.2) - `boolean isAdmin` - - `Set dacMemberships` — all DAC IDs the user belongs to - - `Set dacChairScopes` — DAC IDs where the user is chair - - `List policyTagGrants` — initially empty, placeholder for future policy-tag grants + - **No field-bundle field.** Search serves one bundle to every caller (contract Decision 2), so + there is nothing per-caller to derive. A per-caller `privileged` bundle was considered and + rejected: creator/custodian privilege is document-scoped, and a caller privileged on one dataset + also receives every public dataset through the same DLS filter, so a request-wide privileged + grant would project privileged fields out of unrelated documents. + - **No institution, DAC-membership/chair, principal-allowlist, or policy-tag fields.** None is + consumed by current read authorization; resolving speculative context adds queries and invites a + later implementation to feed it into DLS accidentally. Add a field only with the signed-off + requirement that consumes it (contract rows 9–14, OPEN-3/OPEN-5). - `DatasetSearchAuthContextResolver` service: accepts a `DuosUser`, returns a `DatasetSearchAuthContext`. - `isAdmin` is `true` when user has `UserRoles.ADMIN` (L13 in `UserRoles.java`). -- `dacMemberships` loaded from `DacDAO` — do not pull in `DacService` as a dependency to keep the - graph flat. +- No DAC lookup is performed; DAC membership and chair status grant no search read access today. **Implementation notes**: - `DuosUser.getRoles()` returns the role set; check for `UserRoles.ADMIN`. -- Keep the resolver stateless; all DB calls happen eagerly in the constructor/factory, not lazily. -- `DacService` (L51) already resolves DAC memberships — use `dacDAO` directly to avoid - introducing a circular dependency through `DacService`. +- Keep the resolver stateless and derive the context entirely from the supplied `DuosUser`; the + current contract requires no DAO lookup. +- Do not inject `DacDAO` or `DacService` until OPEN-3 is approved. The current resolver needs no + DAC dependency. **Dependencies**: A-2. **Size**: M @@ -1025,17 +1130,19 @@ the two code paths from diverging. `DatasetSearchAuthContext` and `DatasetAccessPolicy`. **Acceptance criteria**: -- Test matrix: `ADMIN`, public reader (`RESEARCHER`/`MEMBER`/`SIGNINGOFFICIAL`), dataset creator, - study custodian, DAC chair for the dataset's DAC, DAC member (not chair), - institution-restricted user, user with no matching institution. -- Each combination tested for `canRead`, `isCreator`, `isCustodian`. +- Positive matrix: `ADMIN`, public reader (`RESEARCHER`/`MEMBER`/`SIGNINGOFFICIAL`), dataset + creator, study creator, and study custodian. +- Negative matrix: `CHAIRPERSON` or `MEMBER` with a DAC relationship but no creator/custodian + relationship, and users sharing an institution with the submitter. These remain ordinary public + readers; DAC and institution do not feed the auth context or policy. +- Each applicable combination tested for `canRead`, `isCreator`, and `isCustodian`. - Edge cases: null study, dataset with no DAC, custodian email list empty, user with multiple roles. **Implementation notes**: - Use `@ParameterizedTest` with a method source building `DatasetSearchAuthContext` + `AccessPolicyTerm` pairs with expected `canRead` outcomes. -- Mock `DacDAO` in `DatasetSearchAuthContextResolver` tests to control DAC membership data. +- Assert that `DatasetSearchAuthContextResolver` has no DAC or institution DAO dependency. **Dependencies**: C-1, C-2. **Size**: M @@ -1056,8 +1163,8 @@ context available). #### Ticket D-1 — Extend `ElasticSearchConfiguration` with security-mode settings -**Summary**: Add `securityMode`, privileged service-account credential fields, and -`fieldAccessProfiles` to `ElasticSearchConfiguration`. +**Summary**: Add `securityMode`, privileged service-account credential fields, and the single search +field allowlist to `ElasticSearchConfiguration`. **Context**: `ElasticSearchConfiguration` currently holds a single shared `authUser`/`authPassword` (L22–24). The native path requires either a privileged account for API-key generation or a @@ -1068,16 +1175,18 @@ context available). - `String securityMode` — `"none"`, `"fallback"`, `"shadow"`, or `"native-dls"`. - `String serviceAccountUser` / `String serviceAccountPassword` — may reuse `authUser`/ `authPassword` if the same account has sufficient privilege. - - `Map> fieldAccessProfiles` — maps profile name to list of allowed field - glob patterns (e.g. `"public"` → `["datasetId", "datasetName", "study.studyName", ...]`). + - `List searchVisibleFields` — the one bundle's **literal** allowed paths, transcribed from + [`es-access-contract.md`](es-access-contract.md) §B. Not glob patterns: contract §B.5 forbids + wildcards, because `data`, `study.data`, and `study.assets` are dynamic maps and a + `study.*`-style grant would publish whatever a future registration schema puts in them. - Application starts cleanly with `securityMode: none` (legacy behavior unchanged). - Startup validation: if `securityMode` is `"native-dls"` and `serviceAccountUser` is blank, throw with a descriptive error. **Implementation notes**: - Use Dropwizard `@JsonProperty` / `@NotNull` pattern consistent with existing fields. -- `fieldAccessProfiles` default should include at minimum `"public"` and `"privileged"` entries - reflecting the field lists decided in A-2. +- `searchVisibleFields` defaults to the complete SEARCH-VISIBLE list from contract §B. There is no + public/privileged split and no caller-specific override. - Document new keys in the config YAML schema or `docs/`. **Dependencies**: A-1, A-2. @@ -1103,7 +1212,8 @@ using the existing low-level `RestClient` — no new client instance is needed p DLS query from D-3 and FLS field list; embed the resulting key as `Authorization: ApiKey `. - `run_as` approach (simpler fallback within D): set header `es-security-runas-user: ` on a service-account-authenticated request. -- Unit test: given `isAdmin=true`, generated credential grants unrestricted access. +- Unit test: given `isAdmin=true`, generated credential grants unrestricted **document** access but + uses the same SEARCH-VISIBLE FLS grant as every other caller. - Unit test: given non-admin context, credential includes DLS query and FLS field list. **Implementation notes**: @@ -1122,31 +1232,47 @@ using the existing low-level `RestClient` — no new client instance is needed p **Summary**: Build `DlsQueryBuilder` and `FlsGrantBuilder` that translate `DatasetSearchAuthContext` into an Elasticsearch DLS query string and an FLS field-grant list. -**Context**: The DLS query must express: return documents where `accessPolicy.publicVisibility=true` -OR `accessPolicy.creatorUserId = ` OR `accessPolicy.custodianEmails` contains `` -OR `accessPolicy.dacId` is in ``. ADMIN bypasses all filters. +**Context**: The DLS query expresses the contract's document-visibility rules +([`es-access-contract.md`](es-access-contract.md) §A rows 1–3, 5–8). The FLS grant is **constant** — +one bundle for every caller including admins (contract Decision 2), so `FlsGrantBuilder` takes no +caller input at all beyond validating that it was asked for the one bundle that exists. **Acceptance criteria**: - `DlsQueryBuilder.buildForContext(DatasetSearchAuthContext ctx)` → JSON string. - ADMIN → `{"match_all": {}}`. - - Non-admin → `bool` with `should` clauses for `publicVisibility`, creator, custodian, DAC - membership; `minimum_should_match: 1`. -- `FlsGrantBuilder.buildForContext(DatasetSearchAuthContext ctx, - Map> profiles)` → `List` field patterns. - - ADMIN → `["*"]`. - - Non-admin → field list from the caller's applicable profile. -- Unit tests for each access dimension and combinations. + - Non-admin → `bool` with `minimum_should_match: 1` over exactly these clauses: + `publicVisibility` true; **dataset has no study**; dataset creator; study creator; custodian. + - **No DAC clause, no institution clause, no policy-tag clause.** Contract rows 9–14 are DEFERred: + none of them grants dataset read access today, and adding them here is an authorization + expansion pending OPEN-3/OPEN-5. +- `FlsGrantBuilder.build()` → `List`: the SEARCH-VISIBLE literal paths from contract §B. + - **Same list for admins** — no `["*"]`. An admin wildcard would serve `accessPolicy.*` and the + dynamic property maps, contradicting contract §B.4/§B.5/§B.7. ADMIN is a document-visibility + bypass, not a projection bypass. + - No wildcard or `except` form anywhere in the grant (contract §B.5). +- Unit tests per dimension, plus negative tests: a DAC member who is not creator/custodian does + **not** match a non-public dataset; an admin grant contains no `accessPolicy` path. **Implementation notes**: -- Example DLS query for non-admin: +- DLS query for a non-admin caller: ```json {"bool": {"should": [ {"term": {"accessPolicy.publicVisibility": true}}, - {"term": {"accessPolicy.creatorUserId": 42}}, - {"terms": {"accessPolicy.custodianEmails": ["user@example.com"]}}, - {"terms": {"accessPolicy.dacId": [1, 3]}} + {"term": {"accessPolicy.hasStudy": false}}, + {"term": {"accessPolicy.datasetCreatorUserId": 42}}, + {"term": {"accessPolicy.studyCreatorUserId": 42}}, + {"terms": {"accessPolicy.custodianEmails": ["user@example.com"]}} ], "minimum_should_match": 1}} ``` +- `accessPolicy.hasStudy: false` is what keeps the currently-public "dataset with no study" case + readable (contract §A row 5). Without it that case is silently denied — it cannot be carried by a + null `publicVisibility`, because the filter treats a null on a study-bearing document as *not* + public (contract §A.1). +- Both creator clauses are required and they are different columns; matching only one denies + legitimate access to the other kind of creator. +- Custodian matching is **exact** — `keyword` term match, no lowercase normalizer on the field, and + the caller email passed through unnormalized (contract §A.2). This preserves today's + case-sensitive behavior; changing it is OPEN-6 and must move both paths at once. - If `accessPolicy` is mapped as `nested`, terms must be wrapped in a `nested` query — confirm with B-1 mapping decision. @@ -1193,8 +1319,10 @@ document filtering and field omission. - Test: non-admin search → `publicVisibility=false` document absent from results. - Test: dataset creator search → sees own `publicVisibility=false` document. - Test: ADMIN → sees all documents. -- Test: FLS — non-privileged caller's response does not contain `study.dataCustodianEmail`. -- Test: ADMIN response contains all fields. +- Test: FLS — every caller receives the same SEARCH-VISIBLE fields, including + `study.dataCustodianEmail`, which is published by search today. +- Test: ADMIN sees all documents but receives the same SEARCH-VISIBLE fields; `accessPolicy`, + `data`, `study.data`, and `study.assets` are absent. **Implementation notes**: - Use `testcontainers` with `docker.elastic.co/elasticsearch/elasticsearch:9.x` and @@ -1262,7 +1390,9 @@ alongside a server-built access policy filter derived from `DatasetSearchAuthCon - Non-admin → access policy filter equivalent to DLS query from D-3 (same logic, different execution path — reuse `DlsQueryBuilder.buildForContext` if D-3 has shipped, otherwise implement a standalone `AccessFilterBuilder` and unify later). -- Unit tests for each access dimension: public reader, creator, custodian, DAC member. +- Unit tests for each enforced access dimension: public reader, dataset creator, study creator, and + custodian. Add a negative DAC-member/chair case proving that DAC relationship alone does not add a + clause or grant a restricted document. - Negative test: crafted client query attempting to retrieve `publicVisibility=false` documents is blocked by the injected filter. @@ -1279,28 +1409,38 @@ alongside a server-built access policy filter derived from `DatasetSearchAuthCon #### Ticket E-3 — Server-managed field allowlist applied to search responses -**Summary**: Strip fields from Elasticsearch response documents that the caller is not permitted -to see, based on their `fieldAccessProfile`. +**Summary**: Strip every field outside the single SEARCH-VISIBLE allowlist from Elasticsearch +response documents, for every caller. **Context**: The fallback path cannot rely on native FLS. The server must remove restricted fields from hit `_source` objects before returning the response. **Acceptance criteria**: -- `ResponseFieldFilter.applyProfile(String responseJson, String callerProfile, - Map> profileDefs)` → `String`: - - For each hit in `hits.hits[*]._source`, removes fields not in the caller's profile grant list. - - ADMIN profile → no fields removed. - - `"public"` profile → only fields in the `"public"` grant list retained. -- Unit test: response with `study.dataCustodianEmail` has that field stripped for `"public"` - profile caller. -- Unit test: ADMIN caller receives the full document. +- `ResponseFieldFilter.apply(String responseJson, List searchVisibleFields)` → `String`: + - For each hit in `hits.hits[*]._source`, retains only the SEARCH-VISIBLE paths from contract §B + and drops everything else, including unrecognized paths. + - **ADMIN is filtered identically** — no bypass. Admins see every *document* (DLS `match_all`), + not every *field*; contract §B.7. +- Unit test: `accessPolicy` is absent from the response for **every** caller, admin included + (contract §B.4). +- Unit test: `data`, `study.data`, and `study.assets` are absent for every caller (contract §B.5). +- Unit test: a path not present in §B — simulating a newly added model field — is dropped rather + than passed through. **Implementation notes**: -- Walk `hits.hits[*]._source` as `Map` and apply a recursive filter against the - profile's allowed field glob patterns (e.g. `"study.*"` allows all `study` sub-fields). -- The caller's profile derives from `DatasetSearchAuthContext.isAdmin` → `"admin"`, otherwise - use the document's `accessPolicy.fieldAccessProfile` or a per-caller override from config. -- The `profileDefs` map comes from `ElasticSearchConfiguration.fieldAccessProfiles` (D-1). +- Walk `hits.hits[*]._source` as `Map` and apply a recursive **allowlist** filter: + retain only paths present in the bundle, drop everything else — including paths the filter does not + recognize. A denylist, or a glob like `"study.*"`, fails open on every field added later; contract + §B.5 forbids both, because `study.data` / `study.assets` / `data` are dynamic maps whose keys are + populated wholesale from property bags. +- There is one bundle for all callers (contract Decision 2) — it comes neither from the document + (cancelled B-2) nor from the caller. Admins included; see contract §B.7. +- `accessPolicy.*` must be stripped from every response regardless of bundle (contract §B.4). The + fallback path retrieves whole `_source` objects, so this is the ticket where an enforcement-input + field would otherwise be handed back to the caller. +- Bundle definitions must be generated from, or checked against, contract §B — the same source D-3 + builds its FLS grant from. Two hand-maintained lists will diverge, and the divergence will only be + visible in whichever environment runs the fallback. **Dependencies**: E-2, D-1. **Size**: M @@ -1315,7 +1455,7 @@ from hit `_source` objects before returning the response. **Acceptance criteria**: - When `securityMode == "fallback"`: - Client DSL processed by `SearchQueryMediator.mediate(clientDsl, ctx)` before ES submission. - - Response processed by `ResponseFieldFilter.applyProfile(...)` before returning to caller. + - Response processed by `ResponseFieldFilter.apply(...)` before returning to caller. - When `securityMode == "none"`: existing behavior unchanged. - Both `searchDatasets` (L212) and `searchDatasetsStream` (L230) updated. - `DatasetResource` callers at L425 and L439 updated to pass `duosUser` → resolved @@ -1340,14 +1480,15 @@ mediator and response filter. **Acceptance criteria**: - `SearchQueryMediator` tests: DSL passthrough for ADMIN, correct `bool.must` injection for each non-admin role/dimension. -- `ResponseFieldFilter` tests: field stripping per profile, ADMIN bypass, nested field handling - (`study.dataCustodianEmail`), null/missing source fields are not errors. +- `ResponseFieldFilter` tests: the same allowlist is applied to every caller including ADMIN; + nested SEARCH-VISIBLE fields such as `study.dataCustodianEmail` are retained; INTERNAL fields and + unknown fields are removed; null/missing source fields are not errors. - Negative test: client DSL with injected `_source` override does not expose restricted fields after full mediation pipeline. **Implementation notes**: - Use `JSONAssert` or Jackson-based assertions for comparing query structure. -- Test the full pipeline end-to-end: `mediate(...)` → mock response JSON → `applyProfile(...)` → +- Test the full pipeline end-to-end: `mediate(...)` → mock response JSON → `apply(...)` → assert final field set. **Dependencies**: E-4. @@ -1485,8 +1626,8 @@ access-policy grounds should be cleaned up. **Implementation notes**: - `BucketUtils.ts:L337` — inspect the query for any visibility-related terms. -- `DACDatasets.jsx:L52` — verify it does not filter by DAC membership client-side (the server - handles this via auth context for CHAIRPERSON callers after F-1). +- `DACDatasets.jsx:L52` — document any DAC filtering it performs. The server does **not** grant read + access from DAC membership or chair status; contract rows 9–10 defer that expansion. **Dependencies**: G-1, F-1. **Size**: M @@ -1644,7 +1785,7 @@ it affects users. ``` A-1 → A-2 → A-3 ↓ ↓ - B-1→B-2→B-3→B-4→B-5→B-6 C-1→C-2→C-3 + B-1→B-3→B-4→B-5→B-6 C-1→C-2→C-3 ↓ ↓ D-1→D-2→D-3→D-4→D-5 (or E-1→E-2→E-3→E-4→E-5) ↓ @@ -1664,17 +1805,17 @@ supports DLS/FLS. | Task | Size | Owner | | --- | --- | --- | | 0.1 Confirm target Elasticsearch edition, DLS/FLS availability, API-key/run-as support, and operational model for per-request credentials | S | Infra + Backend | -| 0.2 Define formal access contract: enumerate all dimensions (publicVisibility, ADMIN, creator, custodian, DAC, institution, allowlist, policy tags) and decide which are document-level vs. field-level | S | Backend (policy lead) | +| 0.2 Define formal access contract — **done and unblocked**: [`es-access-contract.md`](es-access-contract.md). Remaining OPEN items are proposed behavior changes, each defaulting to preserve-today, so Phase 1 can start | S → L | Backend (policy lead) | | 0.3 Inventory storage gaps: determine which new dimensions (institution allowlists, explicit user/group lists, policy tags) lack persistent backing and decide whether they go in existing Study/Dataset properties, new DB tables, or external config | M | Backend + DB | ### Phase 1 — Index Schema and Indexing Pipeline -*~Parallel with Phase 0 once contract is defined. Steps 1.1→1.2→1.3 are sequential; 1.4–1.6 parallel after 1.3.* +*~Parallel with Phase 0 once contract is defined. Steps 1.1→1.3 are sequential; 1.4–1.6 run in parallel after 1.3. Cancelled step 1.2 is retained below only as a decision record.* | Task | Size | Owner | | --- | --- | --- | -| 1.1 Add `accessPolicy` nested object to `DatasetTerm` carrying all DLS-needed fields: `publicVisibility`, `creatorUserId`, `creatorEmail`, `custodianEmails`, `datasetCreatorUserId`, `dacId`, `dacApproval`, `allowedInstitutionIds`, `allowedPrincipalIds`, `policyTags` | M | Backend | -| 1.2 Add field-access profile marker to `DatasetTerm` to control FLS (e.g. `fieldAccessProfile: "public" \| "privileged"`) | S | Backend (depends on 1.1) | +| 1.1 Add `accessPolicy` nested object to `DatasetTerm` carrying the DLS-needed fields the contract authorizes: `publicVisibility`, `hasStudy`, `datasetCreatorUserId`, `studyCreatorUserId`, `custodianEmails`, `dacId`. The speculative allowlist/tag fields are held back pending OPEN-5 | M | Backend | +| ~~1.2 Add field-access profile marker to `DatasetTerm` to control FLS~~ — **cancelled**: ES cannot select a field grant per document, and a request-wide caller-specific privileged bundle leaks fields from unrelated public documents. Search uses one allowlist for every caller (contract Decision 2) | — | — | | 1.3 Update `ElasticSearchService.toDatasetTerm` and `toStudyTerm` to populate all new `accessPolicy` fields from Dataset/Study/User data | M | Backend (depends on 1.1 and 0.3) | | 1.4 Update all reindex trigger paths (dataset registration, dataset update, study update, DAC externalization, explicit reindex endpoint) to ensure `accessPolicy` is always current | M | Backend (depends on 1.3) | | 1.5 Design versioned index migration: new index name + Elasticsearch alias cutover + full background reindex strategy; write the reindex script/job | M | Backend + Infra (depends on 1.3) | @@ -1686,9 +1827,9 @@ supports DLS/FLS. | Task | Size | Owner | | --- | --- | --- | -| 2.1 Create `DatasetSearchAuthContext` (or similar) that resolves an authenticated `DuosUser` into: userId, email, institutionId, global roles, DAC memberships by dacId, DAC chair scopes, and any policy-tag grants | M | Backend | +| 2.1 Create `DatasetSearchAuthContext` (or similar) with only the currently enforced inputs: userId, unnormalized email, and global roles. Do not resolve DAC, institution, allowlist, or policy-tag context until a signed-off requirement consumes it | M | Backend | | 2.2 Normalize existing Consent read rules from `DatasetService.verifyPublicVisibilityAccess` / `canReadStudy` / `isCreatorOrCustodian` into a shared policy evaluator usable by both search mediation and native DLS role generation; avoid duplicating the logic | S | Backend (depends on 2.1) | -| 2.3 Unit-test auth context for each role/dimension combination: ADMIN, public reader, creator, custodian, DAC chair, institution-restricted, explicit allowlist | M | Backend (depends on 2.1, 2.2) | +| 2.3 Unit-test auth context and policy evaluation for ADMIN, public reader, dataset creator, study creator, and custodian; add negative cases proving DAC membership/chair status and institution alone grant no access | M | Backend (depends on 2.1, 2.2) | ### Phase 3A — Native Elasticsearch DLS/FLS Path @@ -1696,11 +1837,11 @@ supports DLS/FLS. | Task | Size | Owner | | --- | --- | --- | -| 3A.1 Extend `ElasticSearchConfiguration` with security-mode flag, impersonation/API-key settings, and field-security profile definitions | S | Backend + Infra | +| 3A.1 Extend `ElasticSearchConfiguration` with security-mode flag, impersonation/API-key settings, and the single SEARCH-VISIBLE field allowlist | S | Backend + Infra | | 3A.2 Update `ElasticSearchSupport` to support per-request credential construction: either generate API keys with inline role descriptors or set run-as headers from a privileged service account | L | Backend (depends on 2.1, 3A.1) | | 3A.3 Build role/query descriptor generator that translates `DatasetSearchAuthContext` into Elasticsearch DLS query (wrapping index's `accessPolicy` fields) and FLS field-grant list | L | Backend (depends on 2.2, 3A.2) | | 3A.4 Wire per-request credentials into `ElasticSearchService.searchDatasets` and `searchDatasetsStream` so they use the secured client rather than the shared service credential | M | Backend (depends on 3A.3) | -| 3A.5 Add integration tests against a security-enabled Elasticsearch instance to validate DLS and FLS enforcement: document filtering, field omission, and admin bypass | L | Backend + QA (depends on 3A.4) | +| 3A.5 Add integration tests against a security-enabled Elasticsearch instance to validate DLS and FLS enforcement: document filtering, field omission, and admin document-bypass — asserting that the admin bypass is document-scoped only and that no caller receives `accessPolicy` or the dynamic maps | L | Backend + QA (depends on 3A.4) | ### Phase 3B — Compatibility Fallback @@ -1709,10 +1850,10 @@ supports DLS/FLS. | Task | Size | Owner | | --- | --- | --- | | 3B.1 Build `SearchQueryMediator` that accepts client DSL, strips unsafe response-shaping surfaces (`_source`, `docvalue_fields`, `script_fields`, `explain`, `profile`), and wraps the client query inside a server-built `bool` filter | M | Backend | -| 3B.2 Add mandatory authorization filter injection to `SearchQueryMediator` using `DatasetSearchAuthContext`: emit a `must` bool clause enforcing publicVisibility/creator/custodian/DAC/institution constraints as Elasticsearch terms/bool queries against indexed `accessPolicy` fields | L | Backend (depends on 2.1, 3B.1) | -| 3B.3 Add server-managed field allowlist per caller profile applied to search responses; strip sensitive fields server-side, not in the client | M | Backend (depends on 3B.2) | +| 3B.2 Add mandatory authorization filter injection to `SearchQueryMediator` using `DatasetSearchAuthContext`: emit a `must` bool clause enforcing publicVisibility / no-study / dataset-creator / study-creator / custodian as terms queries against indexed `accessPolicy` fields. **No DAC or institution clause** — contract rows 9–12 are DEFERred and adding them expands authorization | L | Backend (depends on 2.1, 3B.1) | +| 3B.3 Add the server-managed field allowlist (contract §B, one bundle for all callers) to search responses; strip internal fields server-side, not in the client, and apply it to admins too | M | Backend (depends on 3B.2) | | 3B.4 Wire `SearchQueryMediator` into both `searchDatasets` and `searchDatasetsStream` in `ElasticSearchService` | S | Backend (depends on 3B.2, 3B.3) | -| 3B.5 Unit-test `SearchQueryMediator` for each access dimension and confirm sensitive fields are absent from responses, not blanked on the client side | M | Backend (depends on 3B.4) | +| 3B.5 Unit-test `SearchQueryMediator` for each enforced dimension, add negative DAC/institution cases, and confirm INTERNAL fields are absent from responses rather than blanked on the client side | M | Backend (depends on 3B.4) | ### Phase 4 — API Hardening and Long-term Contract @@ -1759,9 +1900,10 @@ support. Phases 3B and 4.2/4.3 can be deferred if the cluster unambiguously supp `accessPolicy` metadata and backend-generated per-request Elasticsearch auth context. - **Required fallback**: server-owned query mediation and field allowlisting if native cluster capabilities or rollout timing block immediate DLS/FLS adoption. -- **Included scope**: `publicVisibility` enforcement on the server side, explicit allow lists, - institution restrictions, DAC-scoped access, policy tags/system-defined criteria, streaming - endpoint behavior, reindex strategy, testing, and duos-ui alignment. +- **Included scope**: `publicVisibility`, creator, and custodian enforcement on the server side; + explicit inventory and deferral of institution, DAC, principal-allowlist, and policy-tag access; + streaming endpoint behavior, reindex strategy, testing, and duos-ui alignment. Deferred dimensions + become enforcement scope only after their OPEN decision is approved. - **Excluded scope**: redesign of non-search Consent endpoints, unrelated UI behavior changes, and implementation of arbitrary policy-authoring UX unless policy storage gaps force a minimal admin/data-model addition. @@ -1774,10 +1916,11 @@ support. Phases 3B and 4.2/4.3 can be deferred if the cluster unambiguously supp 1. **Search API direction**: Option A — harden existing raw-DSL endpoints first for compatibility. Option B — add a server-owned search API in parallel and migrate clients over time. Recommendation: do both, but treat the server-owned API as the long-term destination. -2. **Field-level policy granularity**: Option A — role/profile-based field bundles (e.g. - `public-reader` vs `privileged-reader`). Option B — fully policy-tag-driven per-document field - exposure. Recommendation: start with profile-based bundles to control complexity, then evolve to - tag-driven rules if required. +2. **Field-level policy granularity**: the current contract deliberately has one SEARCH-VISIBLE + bundle for every caller. Role/profile-based or policy-tag-driven field exposure would be + document-scoped for creators and custodians and therefore cannot be implemented safely by native + FLS in a single search. If that requirement appears, reopen Decision 2 and use application-owned + per-document projection rather than adding another FLS profile. 3. **Institution restrictions source of truth**: Option A — user institution alone. Option B — institution plus library-card or other status-derived qualifiers. Recommendation: separate identity context from eligibility state so document policy remains stable even if login checks diff --git a/docs/plans/es-access-contract.md b/docs/plans/es-access-contract.md new file mode 100644 index 000000000..d7683daee --- /dev/null +++ b/docs/plans/es-access-contract.md @@ -0,0 +1,512 @@ +# Elasticsearch Access Contract — Ticket A-2 + +The formal access contract for dataset search: every access dimension, what it is allowed to do, +and which fields each class of caller may see. Companion to +[`elasticsearch-service-duos-ui-usage.md`](elasticsearch-service-duos-ui-usage.md) (the ticket plan) +and [`es-security-capability-record.md`](es-security-capability-record.md) (Ticket A-1, what the +clusters can enforce). + +This is the document B-1 (`accessPolicy` schema), C-1 (auth context resolver), D-3 (DLS/FLS +generator), and E-2/E-3 (fallback filter and allowlist) are implemented against. Where it says +**DECISION**, the question is settled and downstream tickets may rely on it. Where it says **OPEN**, +it needs a named owner's sign-off and is listed in §E — those are policy choices, not engineering +ones, and this document deliberately does not invent them. + +## Why this is not blocked on the rest of A-1 + +A-1 has production measured and dev/staging outstanding. Those rows choose the *enforcement +mechanism* — native DLS/FLS (Epic D) where licensed, mediated queries and response filtering +(Epic E) where not. They do not change the contract. The whole point of writing this separately is +that the answer to "who may see what" must be identical under either mechanism, or the two +enforcement paths will drift apart and the fallback will become a hole. So this contract is stated +in mechanism-neutral terms and each rule is annotated with how it lands in both. + +## DECISION 1 — Restricted documents are invisible, not redacted + +A caller who is not authorized for a dataset does not see a redacted version of it. The document is +absent from their results, and absent from the total hit count. + +The alternative — return every document and blank the restricted fields — was rejected for three +reasons: + +1. **It leaks by counting.** A result total, a facet count, or an aggregation bucket that includes + documents the caller may not read tells them those datasets exist and how many there are. +2. **It cannot be expressed in native FLS.** See Decision 2: field grants do not vary per document, + so "visible but redacted for restricted documents only" is not a thing a single search can do. +3. **It changes current behavior.** Today `DatasetService.verifyPublicVisibilityAccess` drops + unauthorized datasets from the list entirely (`DatasetService.java:147-171`). Invisibility + preserves that; redaction would be a new, more permissive posture adopted by accident. + +Landing in each mechanism: **Epic D** — a DLS `query` in the role descriptor, so the cluster filters +before scoring and counting. **Epic E** — a mandatory `filter` clause injected into the query's +boolean context, which must be non-removable by the caller-supplied DSL (E-2's sanitization is what +makes that true). + +## DECISION 2 — Search serves one field bundle to every caller + +Two designs are ruled out first, because both were in the plan and both are unsound. + +### A per-document FLS marker cannot work + +`fieldAccessProfile` as specified in B-1/B-2 — a per-document marker naming the FLS bundle that +applies to that document — **cannot work, and must be removed from the design.** + +Elasticsearch field-level security is declared in a role descriptor's index privileges: + +```json +{"indices": [{"names": ["dataset"], "privileges": ["read"], + "field_security": {"grant": ["datasetName", "study.studyName"]}}]} +``` + +The grant is bound to the *index privilege*, evaluated when the request is authorized, and applied +uniformly to every document that privilege matches. There is no point in the search lifecycle at +which the cluster reads a field off a hit and re-selects a field grant for it. A per-document marker +therefore has nothing to bind to. +([Elastic: controlling access at document and field level](https://www.elastic.co/docs/deploy-manage/users-roles/cluster-or-deployment-auth/controlling-access-at-document-field-level)) + +### Why a per-*caller* privileged bundle does not rescue it either + +The obvious repair — keep one bundle per request, but pick `privileged` when the caller is a creator +or custodian of *something* — is also wrong, and it fails in a way worth spelling out because it is +easy to talk yourself into. + +Creator and custodian privilege is **document-scoped**. The DLS filter is a disjunction: a custodian +of restricted dataset X receives X *and* every public dataset Y, because `publicVisibility = true` is +one of the `should` clauses. A request-wide `privileged` grant would then project privileged fields +out of all those unrelated public Y documents — datasets the caller has no relationship to +whatsoever. Being privileged on one document would buy privileged fields on every document in the +result set. + +So the constraint is real and has no clever workaround: **native FLS cannot express document-scoped +field access.** Anything that needs it must be built somewhere else: + +| Approach | Verdict | +| --- | --- | +| Separate indices per profile | Does not even apply. The profile here is caller-*relative* ("am I the custodian of this one?"), and an index split can only encode document-absolute properties. | +| Two searches — privileged-scoped and public-scoped — merged server-side | Technically possible; breaks scoring, paging, and aggregations across the merged set. | +| Application-owned projection per document after retrieval | Works, and is genuinely document-scoped. It is what Epic E does, and it is the upgrade path if the requirement ever appears. Doing it in the native path too would forfeit Epic D's reason to exist. | + +**What we do instead: DECISION 2 — search serves one field bundle to every caller.** + +The field grant does not vary at all — not by document, not by caller, not for admins. DLS varies +(who sees which documents); FLS does not. That is exactly the shape native FLS can enforce +correctly, so there is nothing left to get wrong. + +This is only a sound decision if no field genuinely needs document-scoped exposure through search, +so that was checked against the consumers rather than assumed (§B.0). It holds: every field the +catalog publishes through search is published to *all* authenticated callers today — duos-ui's +dataset table renders PI name, custodian emails, and data location unconditionally, and its +client-side filter matches on submitter and DAC emails. There is no existing privileged-in-search +tier to preserve. The fields that are genuinely internal are consumed by nobody through search and +simply leave the projection. + +Consequently the classification in §B has **two** tiers, not three: served by search, or not served +by search. Privileged, document-scoped data continues to reach privileged callers through the +per-dataset endpoints, which already do document-scoped checks in `DatasetService` and are not +affected by this contract. + +**If a future requirement does need a privileged field in search results**, this decision has to be +reopened, not worked around: the answer is application projection (Epic E's mechanism, applied to +the native path as well), and the cost is that Epic D stops being sufficient on its own. + +## DECISION 3 — The contract preserves today's authorization; every expansion is explicit + +The default for every dimension is **PRESERVE**: reproduce what the application does today, exactly, +including the parts that look accidental. Dimensions that would *grant new access* (DAC membership, +institution, policy tags) are **DEFER**red — schema may be reserved for them, but no enforcement +path may consult them until they are signed off in §E. + +This is the rule that keeps a security refactor from becoming a silent authorization change. The +matrix below states it per dimension so no implementer has to infer it. + +--- + +## §A — Access dimension matrix + +"Current effect" is what the application does **today**, from +`DatasetService.verifyPublicVisibilityAccess` / `canReadStudy` / `isCreatorOrCustodian` +(`DatasetService.java:147-237`). "Level" is document (who sees the dataset at all) or field (which +bundle they get). Under Decision 2 the field bundle is constant, so **every row's Level is +Document** — no dimension varies field access. The column is kept to make that explicit rather than +implied. + +| # | Dimension | Source of truth | Current effect on read | Level | Persistent backing | Contract | +| --- | --- | --- | --- | --- | --- | --- | +| 1 | ADMIN role | `user_role` → `UserRoles.ADMIN` | Full bypass: every dataset, no filtering | Document **only** | Yes — `user_role` table | **PRESERVE.** DLS filter is `match_all`. Field grant is the same as everyone else's — admin is not a projection bypass (§B.7). | +| 2 | Study `publicVisibility = TRUE` | `study.public_visibility` | Readable by any authenticated caller | Document | Yes — column | **PRESERVE.** | +| 3 | Study `publicVisibility = FALSE` | `study.public_visibility` | Readable only via #1, #6, #7, or #8 | Document | Yes — column | **PRESERVE.** | +| 4 | Study `publicVisibility = NULL` | `study.public_visibility` | **Unreachable** — the column is `NOT NULL`; the NULL seen in summary rows is the LEFT JOIN for #5 (§A.1) | Document | n/a | **RESOLVED.** Filter treats a null on a study-bearing document as not-public (fails closed). | +| 5 | Dataset has no study (`studyId IS NULL`) | `dataset.study_id` | Readable by everyone — "can't verify visibility, so return the dataset" | Document | Yes — column | **PRESERVE**, and see OPEN-2: this is fail-open. | +| 6 | Dataset creator | `dataset.create_user_id` = `user.user_id` | Readable | Document | Yes — column | **PRESERVE.** Distinct from #7. | +| 7 | Study creator | `study.create_user_id` = `user.user_id` | Readable | Document | Yes — column | **PRESERVE.** Distinct from #6. | +| 8 | Study data custodian | `study_property['dataCustodianEmail']` (JSON array) contains `user.email` | Readable | Document | **Partial — see §C.** Unstructured JSON in a property bag. | **PRESERVE**, with the matching rules in §A.2 made explicit. | +| 9 | DAC membership | `dac_user` via `DacDAO` | **No dataset read access today** | — | Yes — table | **DEFER.** Reserve `accessPolicy.dacId`; no enforcement path reads it. OPEN-3. | +| 10 | DAC chair | `dac_user` role | **No dataset read access today** | — | Yes — table | **DEFER.** OPEN-3. | +| 11 | `dacApproval` | `dataset.dac_approval` | **No read effect today** — a display/filter attribute the UI applies client-side (G-1) | — | Yes — column | **DEFER.** Indexing it as an `accessPolicy` field invites it to be read as authorization; it is not. OPEN-4. | +| 12 | Institution | `user.institution_id`; no dataset-side counterpart | **No dataset read access today** | — | User side only; **no dataset-to-institution mapping exists** | **DEFER.** Needs A-3 storage decision before it can mean anything. OPEN-5. | +| 13 | Policy tags | — | **Does not exist** | — | **None** | **DEFER.** OPEN-5. | +| 14 | Explicit principal allowlist | — | **Does not exist** | — | **None** | **DEFER.** OPEN-5. | + +Rows 9–14 are the ones the review flagged: B-1's `AccessPolicyTerm` lists them as fields alongside +the dimensions that do carry authorization today, with nothing recording that four of them grant +nothing. Reserving schema for them is fine. Wiring them into a DLS filter without sign-off would +grant DAC members, institution peers, and tag holders read access to non-public datasets that they +do not have today. + +### §A.1 — `publicVisibility = NULL` — RESOLVED by the schema, not by policy + +This looked like an unresolvable policy question and was briefly recorded as one. It is not: the +database settles it, and the apparent disagreement between the two code paths is unreachable. + +**`study.public_visibility` is `NOT NULL`** (`changelog-consent-2023-04-20-create-study.xml:22-24`, +never relaxed by a later changeset). A persisted study cannot have a null visibility, so +`canReadStudy` — which only ever runs against a loaded `Study` — cannot observe one. + +The NULL that *does* occur is in the summary list, and it comes from a join, not from a study: + +```sql +FROM dataset d LEFT JOIN study s ON s.study_id = d.study_id -- DatasetDAO.java:119-129 +``` + +`summary.public_visibility()` is NULL exactly when the dataset has **no study** — which is §A row 5, +a case both paths already agree on. Trace it: `Boolean.TRUE.equals(null)` is false, the creator +checks fail for a stranger, `summary.study_id() != null` is false, and control reaches the final +`else`, which adds the dataset. Readable. The single-dataset path returns the dataset for the same +reason (`studyId == null` → return). **The two paths agree.** + +**DECISION**: the contract has one filter, and it is the two-clause form already implied by rows 2–5: +`publicVisibility` is true, *or* the dataset has no study, *or* the caller is privileged on it. A +null or missing `publicVisibility` on a document that *does* have a study is unreachable, and the +filter must therefore treat it as **not public** — the unreachable branch fails closed rather than +open. B-3's "no study" marker (§D) is what keeps row 5 working under that rule, rather than leaning +on a null to mean "visible." + +This unblocks B-3, D-3, E-2, and G-1, which were waiting on a decision that did not need to be made. + +

+The divergence that prompted the question, kept for the record + +The two paths do read a null differently, and if the `NOT NULL` constraint were ever dropped they +would diverge in production. C-2 should normalize them while it is consolidating the predicates.
+ +**Single-dataset path** — `canReadStudy` (`DatasetService.java:193-205`): + +```java +if (!Boolean.FALSE.equals(study.getPublicVisibility())) { + return true; // NULL is readable by anyone +} +``` + +**Summary-list path** — `verifyPublicVisibilityAccess(List, User)` +(`DatasetService.java:147-171`): + +```java +if (Boolean.TRUE.equals(summary.public_visibility())) { // NULL is NOT public here + ... +} else if (...creator checks...) { +} else if (summary.study_id() != null) { + // falls through to a creator/custodian check, which a stranger fails +} +``` + +Were a null ever to reach these, a caller who is neither creator nor custodian **could** read the +study through `findMinimalDatasetByIdentifier` and **could not** see it in a dataset summary list. +The `NOT NULL` constraint is the only thing preventing that, which is a thin guarantee to leave +implicit — hence the normalization ask above. + +### §A.2 — Identity matching rules + +These are the details that decide whether a re-implementation actually reproduces current behavior. +All four are **PRESERVE**, meaning the DLS filter and the fallback must match today's semantics +exactly — but three of them are also latent defects, flagged as OPEN-6. + +| Rule | Current behavior | Note | +| --- | --- | --- | +| Creator matching (#6, #7) | Integer user ID equality | Sound. | +| Custodian matching (#8) | String equality on email: `user.getEmail().equals(custodian.trim())` | **Case-sensitive**, and only the *stored* value is trimmed — the user's email is not. `Alice@x.org` does not match a stored `alice@x.org`. | +| Custodian property parse | `Gson.fromJson(prop.getValue().toString(), List)` | Throws on malformed JSON; no defined behavior for a non-array value. | +| Missing custodian property | `Optional.empty()` → not a custodian | Correct and safe. | + +**DECISION: the index preserves today's exact matching.** B-3 stores custodian emails with only the +`trim()` the current code applies to the stored side, and C-1 compares the caller's email with +`equals`, case-sensitively. A case-mismatched email that does not authorize today must not authorize +through search either. + +This is deliberate, and it means shipping a known defect on purpose. Normalizing here would newly +authorize case-mismatched custodians — a real authorization expansion, arrived at as a side effect of +an indexing decision, and applying to the search path only while the non-search endpoints kept +rejecting the same user. That asymmetry is worse than the defect. **OPEN-6** proposes fixing it +properly: normalize in `DatasetService` and in the index together, under C-2's shared predicates, as +one reviewed change with parity across both paths. + +Note also that ES `keyword` fields match exactly by default, so exact matching is what the index does +without any special handling; normalization would be the thing requiring extra work, not the reverse. + +--- + +## §B — Field classification + +**Two tiers**, per Decision 2. Every indexed path gets exactly one; there is no unclassified residue, +because a field that nobody classified is a field that leaks by default. + +- **SEARCH-VISIBLE** — in the one bundle search serves. Every authenticated caller receives it, + including admins, and no caller receives more. (All search callers are authenticated: + `DatasetResource.java:442-462` is `@PermitAll` with `@Auth DuosUser`, so this is "any logged-in + user," not "anonymous.") +- **INTERNAL** — never served through search, to anyone, admins included. Present in the index only + because enforcement needs it, or because it is indexed and nothing consumes it. + +There is no third, privileged tier: Decision 2 establishes that native FLS cannot scope fields per +document, and §B.0 establishes that nothing needs it. + +### §B.0 — How these were classified, and what it means to move one + +The tier is **not** a judgment about how sensitive a field looks. It is a record of what the product +publishes today, checked against the consumers: + +- `duos-ui/src/components/data_search/DatasetSearchTableConstants.tsx` — the catalog table renders + `study.piName`, `study.dataCustodianEmail`, `dataset.dataLocation`, and `dataset.url` in + unconditional columns, for every caller. +- `duos-ui/src/libs/utils.ts:498-522` — the client-side search filter additionally matches on + `study.dataSubmitterEmail`, `dac.dacEmail`, and `createUserDisplayName`. + +So PI names, custodian emails, submitter emails, and DAC emails are **already public to every +authenticated user** in the DUOS catalog. Classifying them as privileged would not have protected +them; it would have broken the product — `dataCustodianEmail.join(', ')` in the table is not +null-guarded and throws outright if the field is stripped. This is the check that turned a plausible +three-tier design into a wrong one, and it is why the two tiers below are grounded in consumers +rather than in intuition about PII. + +Two directions of movement, with different costs: + +- **INTERNAL → SEARCH-VISIBLE** is an exposure change. It needs review here first. +- **SEARCH-VISIBLE → INTERNAL** is a tightening, and therefore a *product* change plus a duos-ui + change: something on screen today would disappear, and an unguarded consumer may throw. Do not do + it as part of the enforcement work. **OPEN-7** is where that conversation belongs. + +Every path below is a **literal Elasticsearch field path**, because that is what an FLS `grant` and +the E-3 allowlist consume. Object fields are enumerated to their leaves — `dataUse.primary` is an +array of objects, so `dataUse.primary.code` and `dataUse.primary.description` are the grantable +paths and `dataUse.primary[]` is not a usable path at all. + +### §B.1 — `DatasetTerm` (root) + +| Path | Tier | Note | +| --- | --- | --- | +| `datasetId` | SEARCH-VISIBLE | | +| `datasetIdentifier` | SEARCH-VISIBLE | | +| `datasetName` | SEARCH-VISIBLE | | +| `participantCount` | SEARCH-VISIBLE | Rendered and summed in the table. | +| `dataUse.primary.code` | SEARCH-VISIBLE | Filtered on client-side (`utils.ts:500`). | +| `dataUse.primary.description` | SEARCH-VISIBLE | | +| `dataUse.secondary.code` | SEARCH-VISIBLE | | +| `dataUse.secondary.description` | SEARCH-VISIBLE | | +| `accessManagement` | SEARCH-VISIBLE | Drives selectability and the access-type column. | +| `dacId` | SEARCH-VISIBLE | | +| `dacApproval` | SEARCH-VISIBLE | Display attribute, not authorization (§A row 11). | +| `hasInstitutionCertification` | SEARCH-VISIBLE | | +| `dataLocation` | SEARCH-VISIBLE | Rendered in the data-location column. | +| `url` | SEARCH-VISIBLE | Rendered as the data-location link; client-side filter matches it. | +| `createUserDisplayName` (deprecated) | SEARCH-VISIBLE | Client-side filter matches it. Deprecated in favour of `submitter.displayName`; if it is removed from the index, remove the consumer in the same change. | +| `requestLocation` | **INTERNAL** | Indexed, no consumer found in duos-ui. | +| `deletable` | **INTERNAL** | Lifecycle state; discloses whether a dataset is in use. No consumer. | +| `createUserId` | **INTERNAL** | Internal user ID. No consumer. | +| `data.*` | **INTERNAL** | Dynamic map — §B.5. | + +### §B.2 — `study` (`StudyTerm`) + +| Path | Tier | Note | +| --- | --- | --- | +| `study.studyId` | SEARCH-VISIBLE | Used as a row key. | +| `study.studyName` | SEARCH-VISIBLE | | +| `study.description` | SEARCH-VISIBLE | Client-side filter matches it. | +| `study.phsId` | SEARCH-VISIBLE | dbGaP accession. | +| `study.phenotype` | SEARCH-VISIBLE | | +| `study.species` | SEARCH-VISIBLE | Client-side filter matches it. | +| `study.dataTypes` | SEARCH-VISIBLE | Array of keywords; the path is grantable as-is. | +| `study.piName` | SEARCH-VISIBLE | **Rendered in an unconditional "PI" column.** Published today. | +| `study.dataCustodianEmail` | SEARCH-VISIBLE | **Rendered in an unconditional "Data Custodian" column**, and the consumer is not null-guarded. Also an authorization input — that does not make it secret, but it does mean B-4 must reindex on change. | +| `study.dataSubmitterEmail` | SEARCH-VISIBLE | Client-side filter matches it (`utils.ts:505`). | +| `study.externalIdentifier` | SEARCH-VISIBLE | | +| `study.externalIdentifierType` | SEARCH-VISIBLE | | +| `study.dataSubmitterId` | **INTERNAL** | Internal user ID. No consumer. | +| `study.throughBioId` | **INTERNAL** | Internal cross-system identifier. No consumer. | +| `study.publicVisibility` | **INTERNAL** | Under Decision 1 a caller only receives documents they may read, so it carries no information they need. G-1 removes the client-side filtering that reads it — sequence accordingly. | +| `study.assets.*` | **INTERNAL** | Dynamic map — §B.5. | +| `study.data.*` | **INTERNAL** | Dynamic map — §B.5. | + +### §B.3 — `submitter` / `updateUser` (`UserTerm`) and `dac` (`DacTerm`) + +| Path | Tier | Note | +| --- | --- | --- | +| `dac.dacId` | SEARCH-VISIBLE | | +| `dac.dacName` | SEARCH-VISIBLE | Client-side filter matches it. | +| `dac.dacEmail` | SEARCH-VISIBLE | Client-side filter matches it. Usually a shared mailbox. | +| `submitter.userId` | **INTERNAL** | Internal user ID. | +| `submitter.displayName` | **INTERNAL** | No consumer — the table uses the deprecated `createUserDisplayName` instead. Promote if a consumer appears; do not grant it speculatively. | +| `submitter.institution.id` | **INTERNAL** | | +| `submitter.institution.name` | **INTERNAL** | | +| `updateUser.userId` | **INTERNAL** | | +| `updateUser.displayName` | **INTERNAL** | | +| `updateUser.institution.id` | **INTERNAL** | | +| `updateUser.institution.name` | **INTERNAL** | | + +`updateUser` has no consumer at all and discloses who last edited a dataset. B-1 should consider +dropping it from the index rather than indexing it and then filtering it out of every response. + +### §B.4 — `accessPolicy` (added by B-1) + +**Every path under `accessPolicy.*` is INTERNAL, without exception, for every caller including +admins.** It is enforcement input: creator IDs, custodian emails, and whatever allowlists §E +eventually authorizes. Granting any of it would hand back identity data, and an allowlist field would +additionally disclose *who else* has access. + +This is stated as a rule rather than a list because B-1's field set is expected to grow. In the +fallback path (E-3) it matters most: that path retrieves whole `_source` objects, so `accessPolicy` +is returned by Elasticsearch and must be stripped by the server. + +### §B.5 — Dynamic maps are INTERNAL, and grants may not contain wildcards + +`DatasetTerm.data`, `StudyTerm.data`, and `StudyTerm.assets` are `Map` populated +wholesale from property-bag values (`ElasticSearchService.java:294-297, 514-517`). Their key sets are +not fixed, not validated, and not reviewable at design time — a new registration schema field lands +in them with no code change here. + +**Rule: dynamic maps are INTERNAL, and no field grant may contain a wildcard.** A grant of `data.*` +publishes whatever a future schema puts there, which is the precise failure this classification +exists to prevent. Individual keys may be promoted only by being enumerated explicitly — +`data.someKnownKey` — after review of what populates them. + +Two consequences for implementation: + +- **The grant list is an allowlist of literal paths.** ES FLS supports `grant` with `except` + patterns; do not use `{"grant": ["*"], "except": [...]}`, which fails open on every field added + later. Enumerate. This applies to the admin grant too — see §B.7. +- **Epic E must apply the same rule** in `ResponseFieldFilter` (E-3): filter by allowlist, dropping + unrecognized paths, rather than by denylist. + +### §B.6 — Keep this list in step with the model classes + +§B is enumerated from `DatasetTerm.java`, `StudyTerm.java`, `UserTerm.java`, `DacTerm.java`, +`InstitutionTerm.java`, `DataUseSummary.java`, and `DataUseTerm.java`. The "Indexed Elements" section +of the plan document was incomplete (it omitted `study.externalIdentifier`, +`study.externalIdentifierType`, and the `UserTerm.institution` sub-fields); it has been corrected, but +§B supersedes it either way. + +**Whoever adds a field to any of those classes must classify it here in the same change.** An +unclassified field is dropped by the E-3 allowlist and omitted from the D-3 grant — it will appear to +"not work" rather than to leak, which is the safe failure but an confusing one if this rule is not +known. A test asserting that every serialized path appears in §B's list would enforce it mechanically +and is worth adding in B-6. + +### §B.7 — Admins get the same bundle, not a wildcard + +Admins bypass **document** filtering (§A row 1: DLS is `match_all`). They do not bypass field +filtering. The D-3 FLS grant and the E-3 response filter must use the same SEARCH-VISIBLE list for +admins as for everyone else. + +A wildcard admin grant would serve `accessPolicy.*` and both dynamic maps to admin callers, directly +contradicting §B.4 and §B.5 — and it would do so through the same UI, which has no use for them. +Admin tooling that genuinely needs internal fields should use the admin endpoints, which are outside +this contract. "Admin" is a document-visibility bypass, not a projection bypass. + +--- + +## §C — Storage gaps, and a correction to A-3 + +A-3 currently states that `dataCustodianEmail` needs "no new storage." That is true in the narrow +sense and misleading in the way that matters, and A-2 flagged it as a storage-gap candidate for a +reason. Both are recorded here as the contract's position: + +**Persistent backing exists, but not in a queryable, constrained form.** The value lives in +`study_property` under the key `dataCustodianEmail`, typed `PropertyType.Json`, holding a JSON array +of strings (`DatasetService.java:213-237`; written at `DatasetService.java:573-582`). That means: + +- **No referential integrity.** A custodian email need not correspond to any DUOS user. Access is + granted to a string, not to a principal. +- **No constraint or normalization.** Case, whitespace, and duplicates are whatever was written; + §A.2 records the asymmetric trim and case-sensitive comparison that result. +- **No index.** Answering "which studies is this user custodian of" means parsing every property + bag, which is why the current code can only ask it one study at a time. +- **No defined behavior for malformed content.** A non-array or unparseable value throws from Gson + inside an authorization check. + +For Epics B–E this is workable, because B-3 denormalizes custodian emails into `accessPolicy` at +index time and the search path reads the index rather than the property bag. **The gap does not +block this work.** It stays a gap for the non-search endpoints, which keep parsing the bag, and it +makes reindex-on-custodian-change a correctness requirement rather than a nicety (B-4). + +Decisions that remain open on it are OPEN-6 (normalization) and, separately from this contract, +whether custodianship should become a first-class relation. The latter is A-3's call, not A-2's; +what A-2 requires is that A-3 stop describing it as a solved case. + +### Other dimensions' storage, restated + +| Dimension | Storage today | Needed for contract | +| --- | --- | --- | +| `publicVisibility` | `study.public_visibility` column | Nothing new. | +| Dataset creator | `dataset.create_user_id` column | Nothing new. | +| Study creator | `study.create_user_id` column | Nothing new. | +| `dacId` | `dataset.dac_id` column | Nothing new. | +| DAC membership / chair | `dac_user` table | Nothing new — but DEFERred (row 9/10), so nothing consumes it yet. | +| Institution allowlist | **User side only.** No dataset-to-institution mapping exists. | Blocked: A-3 must decide mapping table vs. derivation before OPEN-5 can be answered. | +| Policy tags | **None.** | Blocked on A-3 and OPEN-5. | +| Explicit principal allowlist | **None.** | Blocked on A-3 and OPEN-5. | + +--- + +## §D — What downstream tickets must change + +| Ticket | Required change | +| --- | --- | +| **B-1** | Drop `fieldAccessProfile` (Decision 2). Omit `allowedInstitutionIds`, `allowedPrincipalIds`, `policyTags` until OPEN-5 says they are requirements. Comment on the class that every field is INTERNAL (§B.4). Custodian emails are trimmed on the stored side but preserve case (§A.2); do not lowercase or otherwise normalize them. | +| **B-2** | **Cancelled.** It specifies a per-document FLS marker, which Elasticsearch cannot honour, and the per-caller repair leaks across documents. Deriving a bundle is no longer needed at all: there is one bundle (Decision 2). | +| **B-3** | Populate rows 1–3 and 5–8. Write an explicit **no-study marker** (`accessPolicy.hasStudy`, §A.1) so row 5 does not depend on a null meaning "visible." Do not normalize custodian emails. | +| **B-4** | Custodian changes must trigger reindex — see §C; the index is now the authorization source for search. | +| **C-1** | Carry only caller-side inputs: the caller's user ID, email **unnormalized** for exact matching, and global roles. Dataset/study creator IDs remain distinct document-side `accessPolicy` fields (rows 6/7); the filter compares the one caller ID with both. Do **not** resolve DAC, institution, allowlist, or policy-tag context while rows 9–14 are DEFERred. **No field-bundle field**: the bundle is constant (Decision 2). | +| **C-2** | Normalize the two `verifyPublicVisibilityAccess` overloads so their null handling cannot diverge if the `NOT NULL` constraint is ever relaxed (§A.1). | +| **D-3** | DLS filter from rows 1–3, 5–8 — plus the no-study clause. **No DAC clause** (row 9 is DEFERred). FLS grant = the single SEARCH-VISIBLE list from §B, literal paths, no wildcards, **including for admins** (§B.7). | +| **E-2 / E-3** | Same filter, same allowlist, same source document. E-3 must strip `accessPolicy.*` and both dynamic maps from every response including admin responses, and must filter by allowlist rather than denylist. | +| **G-1** | Client-side `publicVisibility`/`dacApproval` filtering is removed once the server filter implements rows 2–5 identically. `study.publicVisibility` is INTERNAL (§B.2), so the client stops receiving it — sequence the two changes together. | +| **G-2 / G-4** | Verify no duos-ui consumer reads an INTERNAL field. §B.0 records the audit done here; G-2 is where it is confirmed against the whole app rather than the search components. | + +--- + +## §E — Open decisions requiring sign-off + +These are policy, not engineering. **None of them blocks implementation** — every one has a stated +default that preserves current behavior, so Epics B and C can proceed while they are answered. What +they block is *changing* behavior in the direction each describes. + +| ID | Decision | Blocks | Default if unanswered | +| --- | --- | --- | --- | +| **OPEN-2** | Should a dataset with no study remain readable by everyone (row 5)? It is fail-open today. | Nothing — PRESERVE is implementable now | PRESERVE (readable), as the contract states — but worth a conscious confirmation rather than inheritance. | +| **OPEN-3** | Should DAC members and chairs gain read access to non-public datasets in their DAC? They have none today. | Rows 9–10 | DEFER. | +| **OPEN-4** | Is `dacApproval` ever authorization, or purely display? | Row 11, G-1 | Display only. | +| **OPEN-5** | Do institution allowlists, policy tags, and explicit principal allowlists exist as requirements at all? All three are speculative, none has storage, and B-1 reserved fields for them. | Rows 12–14, A-3 | DEFER — and if the answer is "not now," drop them from `AccessPolicyTerm` rather than shipping unpopulated fields that read as enforcement. | +| **OPEN-6** | Fix the custodian email case-sensitivity defect (§A.2)? The contract preserves today's exact matching, so this is a proposal, not a blocker. If approved it must land in `DatasetService` and the index **together**, via C-2 — normalizing only the search path would authorize a user through search whom the dataset endpoints still reject. | Nothing | Keep exact matching; fix as a separate reviewed change. | +| **OPEN-7** | Should the catalog **stop** publishing `study.piName`, `study.dataCustodianEmail`, `study.dataSubmitterEmail`, and `dac.dacEmail` to all authenticated callers? They are on screen today (§B.0), so this contract keeps them SEARCH-VISIBLE. Restricting them is a product decision, and needs a duos-ui change in the same release — the table's `dataCustodianEmail.join(', ')` throws if the field is absent. | Nothing — this is a proposed tightening, not a gap | Keep publishing. Changing it is product scope, not enforcement scope. | + +--- + +## Status + +**Complete and unblocked.** + +Every dimension in §A and every path in §B is decided, and no OPEN item stands between this contract +and B-1/B-3/C-1. That is a change from the first draft, which recorded `publicVisibility = NULL` as +an unanswerable policy question: the schema answers it (§A.1), and checking rather than escalating +turned the last blocker into a fact. + +What the OPEN items in §E now represent is proposed *changes* — expanding access (OPEN-3, OPEN-5), +tightening it (OPEN-7), or fixing a defect (OPEN-6) — each with a default that preserves what the +application does today. They can be answered on their own schedule. + +Two things this contract does **not** cover, deliberately: + +- **Document-scoped field access.** Decision 2 establishes that native FLS cannot do it and that + nothing currently needs it. If that changes, this document is where the change starts, and the + answer is application projection rather than a cleverer FLS grant. +- **The non-search endpoints.** `DatasetService` keeps its own document-scoped checks and its own + richer projections. This contract governs the search index only; C-2 is what keeps the two from + drifting apart. diff --git a/docs/plans/es-security-capability-record.md b/docs/plans/es-security-capability-record.md index ed7216088..8d4e9e906 100644 --- a/docs/plans/es-security-capability-record.md +++ b/docs/plans/es-security-capability-record.md @@ -104,13 +104,22 @@ in the inventory below runs Elasticsearch. ## Environment inventory -### Local (`config/docker-compose.yaml`) — measured 2026-07-29 with the write probes - -Ticket A-0 is closed: the compose file now sets `xpack.security.enabled` to **true** by default -(overridable per-run with `ES_SECURITY_ENABLED=false`) and self-generates a **trial** license, so the -security features are exercisable locally as shipped. The endpoint has now been run against the -running local cluster in write-probe mode, so every row below is an observation rather than an -inference — this is the first environment where all five capabilities came back `SUPPORTED`: +### Local (rendered `config/docker-compose.yaml`) — measured 2026-07-29 with the write probes + +Ticket A-0 is closed. Be precise about what that does and does not mean for anyone else's machine: +`/config/` is git-ignored (`.gitignore` L149) and rendered per developer by the Broad-internal +`firecloud-develop`, so **nothing in this repository sets any Elasticsearch default** — there is no +committed compose file for a change to land in. The cluster measured below is a local rendered copy, +edited to set `xpack.security.enabled` to **true** (overridable per-run with +`ES_SECURITY_ENABLED=false`) and to self-generate a **trial** license. The durable form of that +change is the `firecloud-develop` compose template, which is outside this repo and still needs an +owner — see the A-0 outcome in +[`elasticsearch-service-duos-ui-usage.md`](elasticsearch-service-duos-ui-usage.md). Until it lands, +each developer applies these settings themselves; see the notice below. + +The endpoint has now been run against that local cluster in write-probe mode, so every row below is +an observation rather than an inference — this is the first environment where all five capabilities +came back `SUPPORTED`: | Capability | Verdict | Evidence | | --- | --- | --- | @@ -145,12 +154,12 @@ work, not as a preview of the deployed rows. #### Notice: developers must update their own local configuration -A local cluster does **not** pick these settings up on its own. `config/docker-compose.yaml` is -committed, but most people carry local edits to it (the bucket location, ports, memory limits) or run -a copy of their own, so a pull of this branch will not necessarily put these settings into the file -you actually start ES with. Each developer has to enable them in their own compose file before the -endpoint will report anything like the table above — and a local cluster that lags behind produces -`UNAVAILABLE` / `LICENSE_BLOCKED` verdicts that read like findings when they are only local drift. +A local cluster does **not** pick these settings up on its own, and pulling this branch will not put +them anywhere: `config/docker-compose.yaml` is git-ignored and rendered per developer, so the file you +actually start ES with is not in this repository at all. Each developer has to enable these settings +in their own rendered copy before the endpoint will report anything like the table above — and a local +cluster that lags behind produces `UNAVAILABLE` / `LICENSE_BLOCKED` verdicts that read like findings +when they are only local drift. What has to be true in your `config/docker-compose.yaml` (and any personal copy or override file you run instead of it): @@ -256,9 +265,11 @@ falls back to the license reading and says which of the two you are looking at. ### `dev` — not yet measured > Call `POST /api/elasticSearch/capabilities` against dev with an Admin token and -> paste the findings here. Dev is the right place to run the write probes first in a *deployed* -> environment — teardown has been confirmed on the control clusters and locally, so what dev adds is -> confirmation under a real shared credential rather than a superuser one. +> summarise the verdicts here — verdicts only, with the raw report attached to the ticket (see the +> note under `production` below). Production has since been measured and came back clean, so the +> write probes are no longer unproven in a *deployed* environment; what dev and staging now add is +> the other two thirds of the decision rule, and — if either runs a narrower credential than +> production's — the first reading of what a real least-privilege shared credential can do. | Capability | Verdict | Evidence | | --- | --- | --- | @@ -284,29 +295,93 @@ falls back to the license reading and says which of the two you are looking at. | API keys | | | | `run_as` | | | -### `production` — not yet measured +### `production` — measured 2026-08-05 with the write probes -Call the endpoint read-only first — in that mode it creates nothing, so it cannot leave anything -behind on the production cluster. Only `POST` it after the same call has been run in dev -and staging and the teardown has been confirmed there; the probes are designed to be safe in -production (namespaced, 10-minute expiry, torn down in a `finally`, failures reported in `notes`), -but production is not the place to find out. +Measured out of the order this document recommends: the write probes were run against production +before dev or staging. The run came back clean — five `SUPPORTED` verdicts, `write_probes_run: true`, +and no teardown failure in `notes` — so nothing was harmed by taking it first, but the sequencing +advice above stands for the environments still to be measured. -If production's shared credential turns out to lack `manage_security`, the write probes there will be -inconclusive by construction. That is a finding to record rather than a problem to work around: it -means Epic D's per-request key minting needs a privilege grant before it can work in production at -all. +Production is an **Elastic Cloud** deployment on an **enterprise** license, and every capability was +observed rather than inferred: | Capability | Verdict | Evidence | | --- | --- | --- | -| Elasticsearch version | | | -| Distribution | | | -| Edition / license | | | -| X-Pack Security enabled | | | -| DLS | | | -| FLS | | | -| API keys | | | -| `run_as` | | | +| Elasticsearch version | 9.x — one minor behind the pinned client, same major | `GET /` → `version.number` | +| Distribution | elasticsearch | `GET /` → `version.distribution` | +| Edition / license | Elastic Cloud, `enterprise`, `status: active` | `elastic_cloud: true`; `GET /_license` | +| X-Pack Security enabled | **`SUPPORTED`** | `GET /_xpack` 200; `GET /_security/_authenticate` 200 | +| DLS | **`SUPPORTED` — enforced, not merely accepted** | a `match_none` DLS key returned **none** of the documents the shared credential can see from `GET /dataset/_search` | +| FLS | **`SUPPORTED` — enforced** | a key granting only `datasetIdentifier` returned documents carrying only that field | +| API keys | **`SUPPORTED`** | key created, authenticated, invalidated | +| `run_as` | **`SUPPORTED`** (self-impersonation only — see below) | the `es-security-runas-user` header was honoured and the request resolved to the named principal | +| Credential privileges | the deployment's credential holds the key-minting privilege Epic D needs; the full block is on the ticket, not here | `POST /_security/user/_has_privileges` | +| `dataset` index | non-empty | so the `match_none` DLS result means enforcement rather than an empty index | +| `elasticsearch-rest-client` (POM) | 9.4.4 against a cluster one minor older — same major | `rest_client_compatibility` could not read the client version in the deployed jar at the time of this run; since fixed (see below) | +| Recommendation | Epic D viable here, **observed** | probe role and keys carrying DLS/FLS descriptors accepted *and* enforced | + +Teardown behaved as documented: three short-lived keys and one probe role were created under the +`duos-capability-probe` / `duos_dlsfls_probe` names and removed again, with no teardown failure +reported in `notes`. + +> **Environment specifics are deliberately not in this file, because this repository is public.** +> The full report — the principal the deployment authenticates as, its roles, the complete +> `cluster_privileges` block, and the `security_settings` dump including the cluster's audit setting +> — is attached to **DT-3826**, where access is already scoped. What is kept here is what the Epic D +> / Epic E decision rule actually consumes: whether security is on, whether DLS and FLS are licensed +> *and enforced*, and whether the deployment can mint keys. When filling in the rows for dev and +> staging below, summarise the verdicts the same way and attach the raw report to the ticket rather +> than pasting it here. + +Four things in this run are worth carrying forward as their own findings. + +**Epic D's per-request key minting is not privilege-blocked in production.** The note further down +predicted the opposite — that the shared `authUser` almost certainly does not hold a key-minting +grant — and production contradicts it in the permissive direction, so there is no privilege to request +from infra before Epic D can proceed there. Two caveats travel with that. The run says nothing about +what a *least-privilege* credential could do, exactly as the local superuser run said nothing about +the deployed ones; and the breadth of what that credential does hold is worth reviewing on its own +terms, independent of this work — raised on DT-3826. Epic D itself needs only `manage_own_api_key`, +which is the grant to ask for if that credential is ever narrowed. + +**The `run_as` evidence is self-impersonation.** The probe resolved the credential's own principal to +itself, which shows the cluster accepts and honours the header but not that it will resolve a +*different* principal. If Epic D's design leans on `run_as` rather than on per-request keys, re-run +with `?runAsUser=` to settle it; the DLS/FLS verdicts do not depend on this. + +**Do not assume cluster-side audit logging.** Under Epic D the per-request API key *is* the +access-control decision, so on a cluster with auditing off nothing on the cluster side records which +key read what. Whatever audit trail the access contract needs must therefore come from the Consent +side unless auditing is known to be enabled on that deployment — read +`xpack.security.audit.enabled` out of the report per environment rather than assuming either way, and +treat enabling it as an infra ask. + +**Elastic Cloud reserves some cluster settings to Elastic's operators.** That gates none of DLS, FLS, +API keys, or `run_as` — all four were observed working — but it does mean any future infra ask +(enabling audit, for instance) is a support request rather than a settings change. + +#### Why `rest_client_compatibility` came back indeterminate — since fixed + +The field reported that it "could not determine the client or cluster version at runtime," which reads +like a gap but was not a finding about the cluster. The cluster version was known (it is in the +report's own `version` field); the *client* version was not, because +`ElasticSearchCapabilityService` read it from +`RestClient.class.getPackage().getImplementationVersion()`, and the shade plugin strips dependency +manifests (`META-INF/MANIFEST*`, `**/pom.properties`) when assembling the deployable uber jar. Locally, +where the client keeps its own jar, the same code reported the 9.4.4 match — so the field would have +stayed indeterminate in *every* deployed environment while working fine everywhere it did not matter. + +Fixed: the version now falls back to `elasticsearch.rest.client.version` in `mvn.properties`, the +build-time property file `properties-maven-plugin` already generates from the pom (the same mechanism +`SwaggerResource` uses), with the dependency version promoted to a pom property so it lands there. +The package lookup is still tried first, since it reports the jar actually loaded rather than the one +the build pinned. Verified against the built `consent.jar`: the package version is null there, +reproducing the production symptom exactly, and the fallback resolves 9.4.4. **A re-run against +production should now name the client version rather than declining to.** + +The substantive question it would have answered is settled anyway: the pinned 9.4.4 client ran against +a cluster one minor older, a gap within the same major and the same shape as the 9.3.3 control-cluster +run recorded below, and every security call in this run succeeded. ## REST client compatibility — resolved @@ -338,7 +413,7 @@ itself evidence that the transport reaches `/_security` there. The report says a ## Decision -**Pending** — blocked on the `dev`, `staging`, and `production` rows above. The decision rule, +**Pending** — blocked on the `dev` and `staging` rows above; `production` is measured. The decision rule, fixed in advance so the measurement determines the outcome: | Measured state of the deployed clusters | Decision | @@ -352,14 +427,29 @@ fixed in advance so the measurement determines the outcome: | DLS/FLS accepted, enforcement **not checked** (`INFERRED_SUPPORTED` from a write-probe run) | **Not a decision.** Acceptance is not enforcement. Fix what the notes say stopped the check — usually an empty or unreadable dataset index — and re-run the write probes before recording anything. | | DLS/FLS licensed but `xpack.security.dls_fls.enabled=false` | **Epic E** until the setting is enabled. The license entitles the cluster to the feature; the setting switches it off cluster-wide, so it is an infra change, not a license one. | -Known so far: the local environment now falls in the **first** row — security enabled, DLS and FLS -licensed *and* observed enforced — which closes Ticket A-0 but decides nothing on its own. The rule -above is about the deployed clusters, and local's superuser credential is not the shared credential -any of them use. +Known so far: **local and production both fall in the first row** — security enabled, DLS and FLS +licensed *and* observed enforced. Production is the one that counts, since it is a deployed cluster +measured through its own deployment's credential; local closed Ticket A-0 but decides nothing on its +own. + +That leaves the decision genuinely pending rather than merely unrecorded. The rule turns on all three +deployed clusters, and two of them are unmeasured — if dev or staging is on a lower license tier, or +has `dls_fls.enabled` off, the outcome is the "environments disagree" row (**both** epics, Epic E as +the portable path) rather than a clean Epic D. Production being on Elastic Cloud enterprise makes +that a real possibility rather than a formality: it is the environment most likely to carry the +strongest license, so the others cannot be assumed to match it. + +One caveat that no further measurement will remove: production's credential is broadly privileged, so +its write probes have the same limitation local's did. They prove what the *cluster* licenses and +enforces — which is what the decision rule asks — but not what a least-privilege service credential +could do there. If that credential is ever narrowed, Epic D's minimum grant is `manage_own_api_key`. ## Notes for whoever runs this against the deployed clusters -- The shared `authUser` almost certainly does **not** hold `manage_security` or `manage_api_key`. +- The shared `authUser` may or may not hold `manage_security` or `manage_api_key` — this was expected + to be the binding constraint, and in production it turned out not to be. Do not carry that forward + as an assumption about dev or staging; measure each, and keep the per-environment specifics on the + ticket rather than in this file. The report's `cluster_privileges` block says exactly which it has, via `POST /_security/user/_has_privileges`. If it lacks them, that is itself a finding: Epic D's per-request key minting goes through `POST /_security/api_key`, which needs at minimum diff --git a/pom.xml b/pom.xml index b4d2578ae..312078093 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,11 @@ 4.2.16.Final 3.8.0 12.1.11 + + 9.4.4 UTF-8 UTF-8 @@ -903,7 +908,7 @@ org.elasticsearch.client elasticsearch-rest-client - 9.4.4 + ${elasticsearch.rest.client.version} diff --git a/src/main/java/org/broadinstitute/consent/http/ConsentModule.java b/src/main/java/org/broadinstitute/consent/http/ConsentModule.java index 4fa63523a..18d6ca6d9 100644 --- a/src/main/java/org/broadinstitute/consent/http/ConsentModule.java +++ b/src/main/java/org/broadinstitute/consent/http/ConsentModule.java @@ -373,6 +373,17 @@ private DataUseMatcherV4 providesDataUseMatcherV4(DataUseUtil dataUseUtil) { * ElasticSearchHealthCheck} — takes this instance by injection rather than calling {@code * ElasticSearchSupport.createRestClient} itself. Closed on shutdown, since nothing else releases * those connections. + * + *

Ordering dependency: the shutdown hook is registered here, inside the provider, so it + * is registered only when this client is first provisioned. That is safe today because every + * consumer is reached from {@code ConsentApplication.run}, which resolves the resources and + * health checks before the server starts — so provisioning always happens while the lifecycle is + * still accepting registrations. It stops being safe if this client ever becomes reachable only + * from a path that first runs after startup: Dropwizard will not call {@code start()} on a {@link + * Managed} added after the lifecycle has started, and depending on when it was added it may not + * be stopped either, which would leak the connection pool and its threads. If that day comes, + * register the hook eagerly from {@code configure()} rather than moving the client's construction + * around. */ @Provides @Singleton diff --git a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java index a69c34c2f..7fe8076a3 100644 --- a/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java +++ b/src/main/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityService.java @@ -1,11 +1,13 @@ package org.broadinstitute.consent.http.service; +import com.google.common.annotations.VisibleForTesting; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.inject.Inject; import jakarta.ws.rs.HttpMethod; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -14,6 +16,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Properties; import java.util.Set; import java.util.TreeMap; import java.util.UUID; @@ -102,6 +105,12 @@ public class ElasticSearchCapabilityService implements ConsentLogger { */ private static final String DLS_FLS_ENABLED_SETTING = "xpack.security.dls_fls.enabled"; + /** Build-time properties written from the pom by {@code properties-maven-plugin}. */ + private static final String BUILD_PROPERTIES_RESOURCE = "/mvn.properties"; + + /** The pom property carrying the pinned {@code elasticsearch-rest-client} version. */ + private static final String REST_CLIENT_VERSION_PROPERTY = "elasticsearch.rest.client.version"; + private static final String VERSION_FIELD = "version"; private static final String LICENSE_FIELD = "license"; private static final String USERNAME_FIELD = "username"; @@ -1005,7 +1014,11 @@ private EnforcementAttempt attemptEnforcement( return EnforcementAttempt.inconclusive(); } - String searchPath = "/%s/_search?size=1".formatted(request.index()); + // track_total_hits because the default caps hits.total.value at 10000: without it, a DLS filter + // that was accepted and then ignored on a larger index reports "10000 of N" rather than the + // number actually visible. The verdict is a 0-vs-non-0 test either way, but the evidence string + // is what gets read and quoted, so it has to be the real count. + String searchPath = "/%s/_search?size=1&track_total_hits=true".formatted(request.index()); ProbeResult search = probeAsApiKey(key.encoded(), HttpMethod.GET, searchPath); String evidence = "GET %s through %s -> %d".formatted(searchPath, request.keyLabel(), search.status()); @@ -1253,7 +1266,7 @@ private String edition( * real compatibility axis is major-version skew. */ private String restClientCompatibility(String clusterVersion) { - String clientVersion = RestClient.class.getPackage().getImplementationVersion(); + String clientVersion = clientVersion(); if (clientVersion == null || clusterVersion == null) { return "Could not determine the client or cluster version at runtime. This report was itself " + "produced through RestClient.performRequest, so the transport reaches /_security."; @@ -1270,6 +1283,49 @@ private String restClientCompatibility(String clusterVersion) { + "over HTTP, but confirm the skew is within the supported range."; } + /** + * The version of the bundled {@code elasticsearch-rest-client}, from whichever source survives + * the build in hand. + * + *

The client's own {@code Package} is the more truthful of the two, since it reports the jar + * actually loaded rather than the version the build pinned — but it is only populated when that + * jar keeps its manifest. The shade plugin strips dependency manifests when assembling the + * deployable uber jar, so in every deployed environment this returns null and the build-time + * property is all there is. Falling back keeps the report from going indeterminate exactly where + * it is being used to make a decision. + */ + private String clientVersion() { + String packageVersion = RestClient.class.getPackage().getImplementationVersion(); + if (packageVersion != null) { + return packageVersion; + } + return buildProperty(REST_CLIENT_VERSION_PROPERTY); + } + + /** + * Reads a value from {@code mvn.properties}, which {@code properties-maven-plugin} writes from + * the pom at build time into both {@code target/classes} and {@code target/test-classes}. Returns + * null rather than throwing: an unreadable build property costs the report one field, and is not + * a reason to fail an inventory whose other verdicts were measured against the live cluster. + */ + @VisibleForTesting + String buildProperty(String name) { + try (InputStream is = getClass().getResourceAsStream(BUILD_PROPERTIES_RESOURCE)) { + if (is == null) { + return null; + } + Properties properties = new Properties(); + properties.load(is); + String value = properties.getProperty(name); + return (value == null || value.isBlank()) ? null : value; + } catch (IOException e) { + logWarn( + "Could not read %s from %s: %s" + .formatted(name, BUILD_PROPERTIES_RESOURCE, e.getMessage())); + return null; + } + } + private String recommendation( boolean securityApiPresent, String licenseType, diff --git a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java index f53dbf48e..1aa427944 100644 --- a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -55,8 +56,9 @@ class ElasticSearchCapabilityServiceTest { private static final String INVALIDATE_KEY = "DELETE /_security/api_key"; private static final String CREATE_ROLE = "PUT /_security/role"; private static final String DELETE_ROLE = "DELETE /_security/role"; + private static final String REST_CLIENT_VERSION_PROPERTY = "elasticsearch.rest.client.version"; private static final String COUNT = "/dataset/_count"; - private static final String SEARCH = "/dataset/_search?size=1"; + private static final String SEARCH = "/dataset/_search?size=1&track_total_hits=true"; private static final String ROOT_BODY = """ @@ -89,6 +91,11 @@ private ElasticSearchCapabilityService service() throws IOException { return new ElasticSearchCapabilityService(esClient, config, this::apiKeyClient); } + /** For the checks that never reach the cluster, so the client stub would go unused. */ + private ElasticSearchCapabilityService serviceWithoutClientStubs() { + return new ElasticSearchCapabilityService(esClient, config, this::apiKeyClient); + } + /** * Answers from {@link #stubs}, turning a stubbed non-2xx into the ResponseException ES throws. */ @@ -1619,5 +1626,48 @@ void testMajorVersionSkewIsCalledOutRatherThanAssumedCompatible() throws IOExcep assertTrue(report.restClientCompatibility().contains("Major-version skew")); } + /** + * Guards the pom wiring rather than the Java. The client version is read from the shaded jar's + * missing manifest in every deployed environment, so {@code mvn.properties} is the only source + * that survives packaging — and it only carries the version because the pom declares it as a + * property instead of inline on the dependency. Inlining it again would silently return the + * report to "Could not determine" in exactly the environments it is used to make a decision in, + * and nothing else in the suite would notice. + */ + @Test + void testRestClientVersionSurvivesPackagingAsABuildProperty() { + String version = serviceWithoutClientStubs().buildProperty(REST_CLIENT_VERSION_PROPERTY); + + assertNotNull( + version, + "elasticsearch.rest.client.version is missing from mvn.properties; it must be declared as a " + + "pom property (not inline on the dependency) and listed in src/test/resources/" + + "mvn.properties, which shadows the generated file on the test classpath"); + assertTrue(version.matches("\\d+\\.\\d+.*"), "expected a version number, got: " + version); + } + + @Test + void testUnknownBuildPropertyIsReportedAsAbsentRatherThanThrowing() { + assertNull(serviceWithoutClientStubs().buildProperty("no.such.property")); + } + + @Test + void testMatchingMajorVersionNamesTheResolvedClientVersion() throws IOException { + stubSecurityDisabledCluster(); + String clientMajor = + serviceWithoutClientStubs().buildProperty(REST_CLIENT_VERSION_PROPERTY).split("\\.")[0]; + stub( + ROOT, + 200, + """ + {"cluster_name":"duos-cluster","version":{"number":"%s.0.0","build_flavor":"default"}}""" + .formatted(clientMajor)); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertTrue(report.restClientCompatibility().startsWith("Compatible:")); + assertFalse(report.restClientCompatibility().contains("Could not determine")); + } + private record StubResponse(int status, String body) {} } diff --git a/src/test/resources/mvn.properties b/src/test/resources/mvn.properties index 642f12b91..06f8b5c43 100644 --- a/src/test/resources/mvn.properties +++ b/src/test/resources/mvn.properties @@ -1,2 +1,8 @@ #Properties +# +# This file shadows the mvn.properties that properties-maven-plugin generates into +# target/test-classes, because test resources are copied after generate-resources. The deployed jar +# gets the plugin's full property set; tests get only what is listed here, so a property read at +# runtime has to be added in both places. swagger.ui.path=${swagger.ui.path} +elasticsearch.rest.client.version=${elasticsearch.rest.client.version} From 739fe465dba6a30c7bf9066be34de75bf2b0d265 Mon Sep 17 00:00:00 2001 From: Elliot Otchet Date: Wed, 5 Aug 2026 22:38:43 +0000 Subject: [PATCH 10/10] increase coverage. --- .../ElasticSearchCapabilityServiceTest.java | 737 ++++++++++++++++-- 1 file changed, 686 insertions(+), 51 deletions(-) diff --git a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java index e6d926094..dacb69322 100644 --- a/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java +++ b/src/test/java/org/broadinstitute/consent/http/service/ElasticSearchCapabilityServiceTest.java @@ -35,6 +35,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; @@ -205,12 +207,16 @@ private List requestsTo(String method, String endpointPrefix) { * Only the error path builds a {@link ResponseException}, which is what reads the request line; * the 2xx path reads just the status and entity. Stubbing the request line only for non-2xx * responses keeps every stub read by the test it's created for, so no stub needs to be lenient. + * + *

A null body is an entity-less response, which is a shape a real cluster returns and a + * distinct one from an empty JSON body: it is the null the parser has to survive. */ private Response response(int status, String body) { Response response = mock(Response.class); when(response.getStatusLine()) .thenReturn(new BasicStatusLine(HttpVersion.HTTP_1_1, status, "reason")); - when(response.getEntity()).thenReturn(new StringEntity(body, ContentType.APPLICATION_JSON)); + when(response.getEntity()) + .thenReturn(body == null ? null : new StringEntity(body, ContentType.APPLICATION_JSON)); if (status >= 300) { when(response.getRequestLine()) .thenReturn(new BasicRequestLine("GET", "/", HttpVersion.HTTP_1_1)); @@ -222,6 +228,11 @@ private void stub(String key, int status, String body) { stubs.put(key, new StubResponse(status, body)); } + /** A response that carries no entity at all, rather than an empty JSON one. */ + private void stubWithNoBody(String key, int status) { + stubs.put(key, new StubResponse(status, null)); + } + private void onceServed(String key, Runnable sideEffect) { sideEffects.put(key, sideEffect); } @@ -1064,29 +1075,6 @@ void testUnparseableSearchResponseIsUnknownRatherThanNotEnforced() throws IOExce assertTrue(dls.detail().contains("no hit total"), dls.detail()); } - @Test - void testFlsWithNoInspectableFieldsFallsBackAndSaysWhy() throws IOException { - stubSecurityEnabledCluster("trial"); - stubWorkingWriteProbes(); - // A hit with no _source at all: nothing to check the projection against. - stub( - "Zmxz|" + SEARCH, - 200, - """ - {"hits":{"total":{"value":2},"hits":[{"_id":"1"}]}}"""); - - ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); - - // Role acceptance stands, but the report must not let that read as proven enforcement. - assertEquals( - CapabilityVerdict.INFERRED_SUPPORTED, - capability(report, "Field-level security (FLS)").verdict()); - assertFalse(capability(report, "Field-level security (FLS)").detail().contains("Proven")); - assertTrue( - report.notes().stream().anyMatch(n -> n.contains("no document fields to inspect")), - "the reader must be told the projection check was inconclusive: " + report.notes()); - } - @Test void testMissingProbeKeyIsSaidToLimitTheDlsAndFlsVerdicts() throws IOException { stubSecurityEnabledCluster("trial"); @@ -1520,40 +1508,42 @@ void testHitCountReadsAPlainNumericTotalAsWellAsAnObjectShapedOne() throws IOExc assertEquals(CapabilityVerdict.SUPPORTED, dls.verdict()); } - @Test - void testFlsSearchWithNoHitsIsReportedAsInconclusiveRatherThanUnprojected() throws IOException { - stubSecurityEnabledCluster("trial"); - stubWorkingWriteProbes(); - // No "hits" array under "hits" at all: firstHitSourceFields() must fall back rather than throw. - stub( - "Zmxz|" + SEARCH, - 200, - """ - {"hits":{"total":{"value":0}}}"""); - - ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); - - assertTrue( - report.notes().stream() - .anyMatch(n -> n.contains("FLS projection check returned no document fields"))); - } - - @Test - void testFlsSearchWithAnEmptyHitsArrayIsReportedAsInconclusive() throws IOException { + /** + * Every search-response shape that leaves no document field to inspect. None of them is evidence + * about the projection either way, so each has to fall back to role acceptance and say so — and + * none may reach {@code firstHitSourceFields} as an exception or as a claim of enforcement. + */ + @ParameterizedTest(name = "{0}") + @ValueSource( + strings = { + // No "hits" object at all. + "{\"took\":1,\"timed_out\":false}", + // A "hits" object with no inner "hits" array. + "{\"hits\":{\"total\":{\"value\":0}}}", + // An inner "hits" that is present but not an array. + "{\"hits\":{\"total\":{\"value\":2},\"hits\":{\"unexpected\":\"shape\"}}}", + // An array with no documents in it. + "{\"hits\":{\"total\":{\"value\":0},\"hits\":[]}}", + // A first hit that is not an object. + "{\"hits\":{\"total\":{\"value\":2},\"hits\":[\"unexpected\"]}}", + // A hit with no _source, so nothing to check the projection against. + "{\"hits\":{\"total\":{\"value\":2},\"hits\":[{\"_id\":\"1\"}]}}" + }) + void testFlsSearchShapesWithNoInspectableFieldsFallBackAndSayWhy(String searchBody) + throws IOException { stubSecurityEnabledCluster("trial"); stubWorkingWriteProbes(); - // "hits" is present as an array, but empty: no document to read fields from. - stub( - "Zmxz|" + SEARCH, - 200, - """ - {"hits":{"total":{"value":0},"hits":[]}}"""); + stub("Zmxz|" + SEARCH, 200, searchBody); ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + ElasticSearchCapability fls = capability(report, "Field-level security (FLS)"); + assertEquals(CapabilityVerdict.INFERRED_SUPPORTED, fls.verdict(), searchBody); + assertFalse(fls.detail().contains("Proven"), fls.detail()); assertTrue( report.notes().stream() - .anyMatch(n -> n.contains("FLS projection check returned no document fields"))); + .anyMatch(n -> n.contains("FLS projection check returned no document fields")), + "the reader must be told the projection check was inconclusive: " + report.notes()); } @Test @@ -2005,5 +1995,650 @@ void testCapabilityReportNeverActivatesTheTrial() throws IOException { assertTrue(activationRequests().isEmpty(), "the capability report started a trial license"); } + // --------------------------------------------------------------------------- + // The default API-key client + // --------------------------------------------------------------------------- + + /** + * The injecting constructor's own factory, which every deployment uses and which the other write + * probe tests replace with a stub. Exercised against a closed port so no cluster is needed: what + * is under test is that the factory builds a usable client from the injected client's nodes at + * all, and that a client which cannot connect degrades to an unproven verdict rather than an + * exception escaping the report. + */ + @Test + void testTheDefaultApiKeyClientFactoryBuildsAClientFromTheInjectedClientsNodes() + throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + // The discard port: a connection here is refused immediately rather than hanging. + when(esClient.getNodes()).thenReturn(List.of(new Node(new HttpHost("127.0.0.1", 9, "http")))); + stubClient(esClient, this::keyFor); + + ElasticSearchCapabilityReport report = + new ElasticSearchCapabilityService(esClient, config).getCapabilityReport(null, true); + + ElasticSearchCapability apiKeys = capability(report, "API keys"); + assertEquals(CapabilityVerdict.UNKNOWN, apiKeys.verdict()); + assertTrue(apiKeys.detail().contains("could not authenticate"), apiKeys.detail()); + // The key was still minted, so it must still be invalidated. + assertEquals(1, requestsTo("DELETE", "/_security/api_key").size()); + } + + // --------------------------------------------------------------------------- + // License readings the cluster answers only partly + // --------------------------------------------------------------------------- + + /** + * A tier with no status is not the same as a tier that is inactive. Reading it as LICENSE_BLOCKED + * would state a finding about the cluster from a field the cluster never sent. + */ + @Test + void testMissingLicenseStatusLeavesDlsFlsUnknownRatherThanBlocked() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + LICENSE, + 200, + """ + {"license":{"type":"trial"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + for (String name : List.of("Document-level security (DLS)", "Field-level security (FLS)")) { + ElasticSearchCapability capability = capability(report, name); + assertEquals(CapabilityVerdict.UNKNOWN, capability.verdict(), name); + assertTrue(capability.detail().contains("license status could not be read"), name); + } + assertTrue(report.recommendation().contains("Inconclusive"), report.recommendation()); + assertTrue( + report.recommendation().contains("license status could not be read"), + report.recommendation()); + } + + /** A status of whitespace carries no more information than an absent one. */ + @Test + void testBlankLicenseStatusIsTreatedAsUnreadableRatherThanInactive() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + LICENSE, + 200, + """ + {"license":{"type":"trial","status":" "}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals( + CapabilityVerdict.UNKNOWN, capability(report, "Document-level security (DLS)").verdict()); + assertTrue(report.recommendation().contains("Inconclusive"), report.recommendation()); + } + + /** A status without a tier maps to no entitlement, and must not be guessed at either. */ + @Test + void testActiveLicenseWithNoTierReportedIsUnknownRatherThanBlocked() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + LICENSE, + 200, + """ + {"license":{"status":"active"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertNull(report.licenseType()); + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.UNKNOWN, dls.verdict()); + assertTrue(dls.detail().contains("could not be mapped"), dls.detail()); + } + + /** + * The edition is what a reader scans first, so a cluster whose license could not be read has to + * say "unknown" there rather than inherit a tier from somewhere else. + */ + @Test + void testUnreadableLicenseLeavesTheEditionUnknown() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + LICENSE, + 500, + """ + {"error":{"reason":"internal server error"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("unknown", report.edition()); + assertNull(report.licenseType()); + assertEquals( + CapabilityVerdict.UNKNOWN, capability(report, "Document-level security (DLS)").verdict()); + assertTrue(report.recommendation().contains("Inconclusive"), report.recommendation()); + } + + @Test + void testTrialStatusWithoutTheEligibilityFieldIsReportedAsUnknown() throws IOException { + stubLicense("basic", "active", null); + stub(TRIAL_STATUS, 200, "{}"); + + ElasticSearchLicenseStatus license = service().getLicenseStatus(); + + assertNull(license.trialAvailable(), "a missing flag must not read as an ineligible cluster"); + assertTrue(license.notes().stream().anyMatch(note -> note.contains("trial_status"))); + } + + @Test + void testANonPrimitiveEligibilityFlagIsReportedAsUnknown() throws IOException { + stubLicense("basic", "active", null); + stub( + TRIAL_STATUS, + 200, + """ + {"eligible_to_start_trial":{"unexpected":"shape"}}"""); + + ElasticSearchLicenseStatus license = service().getLicenseStatus(); + + assertNull(license.trialAvailable()); + } + + /** + * A 200 is not an activation. Only the cluster's own {@code trial_was_started} flag says a trial + * was started, so every 200 that does not carry it as a true boolean has to be reported as a + * non-activation — recording ACTIVATED from any of these would put an irreversible change that + * never happened into the per-environment record. + */ + @ParameterizedTest(name = "{0}") + @ValueSource( + strings = { + // The flag absent entirely. + "{\"acknowledged\":true}", + // The flag present and explicitly false. + "{\"acknowledged\":true,\"trial_was_started\":false}", + // The flag present but not a boolean. + "{\"trial_was_started\":{\"unexpected\":\"shape\"}}" + }) + void testA200WithoutATrueStartedFlagIsNotReportedAsActivated(String activationBody) + throws IOException { + stubLicense("basic", "active", true); + stub(START_TRIAL, 200, activationBody); + + ElasticSearchLicenseActivation activation = service().activateTrialLicense(); + + assertEquals(LicenseActivationOutcome.REFUSED, activation.outcome(), activationBody); + assertEquals("basic", activation.licenseAfter().licenseType()); + } + + // --------------------------------------------------------------------------- + // Edition and deployment shape + // --------------------------------------------------------------------------- + + /** + * {@code build_flavor} says the distribution directly, so an OSS build is reported as one even on + * a cluster whose {@code /_xpack} endpoint answers — the flavor is the more specific signal. + */ + @Test + void testAnOssBuildFlavorIsReportedAsOssEvenWhenTheXPackEndpointAnswers() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + ROOT, + 200, + """ + {"cluster_name":"duos-cluster","version":{"number":"9.3.3","build_flavor":"oss"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("OSS (no X-Pack endpoint)", report.edition()); + } + + /** Older clusters answer 400 rather than 404 for an endpoint they do not have. */ + @Test + void testAnXPack400IsReadAsAnOssBuildJustLikeA404() throws IOException { + stubSecurityDisabledCluster(); + stub( + XPACK, + 400, + """ + {"error":{"reason":"no handler found for uri [/_xpack]"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("OSS (no X-Pack endpoint)", report.edition()); + } + + /** + * A cloud ID present but empty is an unset configuration value, and reading it as Elastic Cloud + * would put a note in the report asserting a deployment shape that was never configured. + */ + @Test + void testABlankCloudIdIsNotReadAsAnElasticCloudDeployment() throws IOException { + config.setCloudId(" "); + stubSecurityEnabledCluster("trial"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertFalse(report.elasticCloud()); + assertEquals("trial", report.edition()); + assertTrue(report.notes().stream().noneMatch(n -> n.contains("cloud ID"))); + } + + /** + * A missing version cannot be compared, but nor may it be read as a cluster whose {@code version} + * block is simply shaped differently than expected. + */ + @Test + void testAPrimitiveWhereTheClusterShouldHaveSentAnObjectIsReadAsAbsent() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + ROOT, + 200, + """ + {"cluster_name":"duos-cluster","version":"9.3.3"}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertNull(report.version()); + assertTrue(report.restClientCompatibility().contains("Could not determine")); + } + + @Test + void testAnObjectWhereTheClusterShouldHaveSentAStringIsReadAsAbsent() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + ROOT, + 200, + """ + {"cluster_name":{"unexpected":"shape"},"version":{"number":"9.3.3"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertNull(report.clusterName()); + assertEquals("9.3.3", report.version()); + } + + @Test + void testRolesThatAreNotAnArrayAreReadAsNoRoles() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + AUTHENTICATE, + 200, + """ + {"username":"consent","roles":"superuser"}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("consent", report.authenticatedUser()); + assertTrue(report.authenticatedUserRoles().isEmpty()); + } + + // --------------------------------------------------------------------------- + // Probe scope and the security API's own answers + // --------------------------------------------------------------------------- + + /** + * {@code datasetIndexName} is {@code @NotNull} in the configuration, so this is only reachable + * from a hand-built one — and the fallback has to be a name no real index uses, because a probe + * that widened its own scope would run a DLS role and a search against a live index it was never + * pointed at. + */ + @Test + void testAnUnconfiguredIndexFallsBackToANamespacedNameRatherThanWideningScope() + throws IOException { + config.setDatasetIndexName(null); + stubSecurityEnabledCluster("trial"); + + service().getCapabilityReport(null, false); + + String body = bodyOf(requestsTo("POST", "/_security/user/_has_privileges").getFirst()); + assertTrue(body.contains("\"names\":[\"duos-capability-probe-index\"]"), body); + } + + @Test + void testABlankConfiguredIndexFallsBackTheSameWay() throws IOException { + config.setDatasetIndexName(" "); + stubSecurityEnabledCluster("trial"); + + service().getCapabilityReport(null, false); + + String body = bodyOf(requestsTo("POST", "/_security/user/_has_privileges").getFirst()); + assertTrue(body.contains("\"names\":[\"duos-capability-probe-index\"]"), body); + } + + /** + * A 401 is security answering, not security absent: an unauthenticated call to a security-enabled + * cluster is exactly what produces one. Reading it as an absent API would report every security + * verdict as UNAVAILABLE on a cluster that has security switched on. + */ + @Test + void testA401FromTheSecurityApiMeansSecurityIsPresentNotAbsent() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + AUTHENTICATE, + 401, + """ + {"error":{"reason":"missing authentication credentials"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals(CapabilityVerdict.SUPPORTED, capability(report, "X-Pack Security").verdict()); + assertEquals( + CapabilityVerdict.INFERRED_SUPPORTED, + capability(report, "Document-level security (DLS)").verdict()); + assertTrue( + report.notes().stream().noneMatch(n -> n.contains("/_security API is not available")), + report.notes().toString()); + } + + @Test + void testA403FromTheSecurityApiMeansSecurityIsPresentNotAbsent() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + AUTHENTICATE, + 403, + """ + {"error":{"reason":"action is unauthorized for user [consent]"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals(CapabilityVerdict.SUPPORTED, capability(report, "X-Pack Security").verdict()); + assertFalse(report.clusterPrivileges().isEmpty(), "the privilege probe should still be tried"); + } + + @Test + void testXPackAnswerWithoutASecurityFeatureBlockFallsBackToTheClusterSetting() + throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + XPACK, + 200, + """ + {"features":{}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals(Boolean.TRUE, report.securityEnabled()); + } + + /** + * Neither source answering leaves the question genuinely open. A null is not a false: reported as + * one it would state that security is off on a cluster that said nothing either way. + */ + @Test + void testSecurityEnabledIsUnknownWhenNeitherXPackNorTheSettingsSayAnything() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + XPACK, + 200, + """ + {"features":{"security":{"available":true}}}"""); + stub( + SETTINGS, + 200, + """ + {"defaults":{"xpack.security.authc.api_key.enabled":"true"},"persistent":{}, + "transient":{}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertNull(report.securityEnabled()); + // The /_security API still answered, so the feature verdicts stand on that rather than on a + // security-enabled flag nobody reported. + assertEquals(CapabilityVerdict.UNAVAILABLE, capability(report, "X-Pack Security").verdict()); + assertEquals( + CapabilityVerdict.INFERRED_SUPPORTED, + capability(report, "Document-level security (DLS)").verdict()); + } + + // --------------------------------------------------------------------------- + // Settings filtering + // --------------------------------------------------------------------------- + + /** A DLS/FLS setting outside the {@code xpack.security} namespace still gates the capability. */ + @Test + void testADlsFlsSettingOutsideTheXPackNamespaceIsStillReported() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + SETTINGS, + 200, + """ + {"defaults":{"xpack.security.enabled":"true"}, + "persistent":{"indices.dls_fls.enabled":"false"},"transient":{}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("false", report.securitySettings().get("indices.dls_fls.enabled")); + } + + /** + * Audit-logfile settings are filtered even when explicitly configured: they describe log + * formatting rather than capability, and there are enough of them to bury the settings that do. + */ + @Test + void testExplicitlyConfiguredAuditLogfileSettingsAreStillFilteredOut() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + SETTINGS, + 200, + """ + {"defaults":{"xpack.security.enabled":"true"}, + "persistent":{"xpack.security.audit.logfile.events.include":"access_granted", + "xpack.security.audit.enabled":"true"},"transient":{}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertFalse( + report.securitySettings().containsKey("xpack.security.audit.logfile.events.include"), + report.securitySettings().toString()); + assertEquals("true", report.securitySettings().get("xpack.security.audit.enabled")); + } + + /** Structured settings values are skipped rather than stringified into the report. */ + @Test + void testNonPrimitiveSettingValuesAreSkippedRatherThanStringified() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + SETTINGS, + 200, + """ + {"defaults":{"xpack.security.enabled":"true", + "xpack.security.authc.realms":{"native":{"native1":{"order":"0"}}}}, + "persistent":{},"transient":{}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertEquals("true", report.securitySettings().get("xpack.security.enabled")); + assertFalse(report.securitySettings().containsKey("xpack.security.authc.realms")); + } + + // --------------------------------------------------------------------------- + // run_as target selection + // --------------------------------------------------------------------------- + + @Test + void testABlankRunAsUserFallsBackToTheAuthenticatedUser() throws IOException { + stubSecurityEnabledCluster("trial"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(" ", false); + + assertEquals(CapabilityVerdict.SUPPORTED, capability(report, "run_as impersonation").verdict()); + assertTrue( + requests.stream() + .flatMap(r -> r.getOptions().getHeaders().stream()) + .filter(h -> h.getName().equals("es-security-runas-user")) + .allMatch(h -> h.getValue().equals("consent")), + "a blank request should impersonate the authenticated user, not blank"); + } + + @Test + void testABlankAuthenticatedUserLeavesRunAsWithNoTarget() throws IOException { + stubSecurityEnabledCluster("trial"); + stub( + AUTHENTICATE, + 200, + """ + {"username":" ","roles":[]}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(" ", false); + + ElasticSearchCapability runAs = capability(report, "run_as impersonation"); + assertEquals(CapabilityVerdict.UNKNOWN, runAs.verdict()); + assertTrue(runAs.detail().contains("No target user was available"), runAs.detail()); + } + + // --------------------------------------------------------------------------- + // Write-probe refusals and response shapes + // --------------------------------------------------------------------------- + + /** PUT role answers 201 as readily as 200, and both mean the cluster took the filters. */ + @Test + void testARoleAcceptedWith201IsTreatedAsCreatedAndTornDown() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + CREATE_ROLE, + 201, + """ + {"role":{"created":true}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertEquals( + CapabilityVerdict.SUPPORTED, capability(report, "Document-level security (DLS)").verdict()); + assertEquals(1, requestsTo("DELETE", "/_security/role/").size()); + } + + @Test + void testA401OnKeyCreationIsReadAsAPrivilegeRefusal() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + CREATE_KEY, + 401, + """ + {"error":{"reason":"missing authentication credentials"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertEquals(CapabilityVerdict.NOT_PERMITTED, capability(report, "API keys").verdict()); + } + + @Test + void testA401OnTheEnforcementSearchIsReadAsAPrivilegeRefusal() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + "ZGxz|" + SEARCH, + 401, + """ + {"error":{"reason":"missing authentication credentials"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.NOT_PERMITTED, dls.verdict()); + assertTrue(dls.detail().contains("privilege rather than licensing"), dls.detail()); + } + + /** + * The licence/privilege split is what the whole report turns on, and a cluster does not always + * use the word "non-compliant" when it refuses on licensing grounds. Missing this refusal would + * report a licensing limit as a privilege one and send someone to fix the wrong thing. + */ + @Test + void testARefusalNamingTheLicenseWithoutTheWordNonCompliantIsStillALicenseBlock() + throws IOException { + stubSecurityEnabledCluster("basic"); + stub( + CREATE_KEY, + 500, + """ + {"error":{"reason":"internal server error"}}"""); + stub( + CREATE_ROLE, + 403, + """ + {"error":{"reason":"field and document level security requires a platinum license"}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.LICENSE_BLOCKED, dls.verdict()); + assertTrue(dls.detail().contains("The license does not permit it"), dls.detail()); + } + + @Test + void testACountResponseWithoutACountFieldIsTreatedAsUnreadable() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub(COUNT, 200, "{}"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertTrue( + report.notes().stream().anyMatch(n -> n.contains("not readable")), + report.notes().toString()); + assertEquals( + CapabilityVerdict.INFERRED_SUPPORTED, + capability(report, "Document-level security (DLS)").verdict()); + assertTrue(requestsTo("GET", "/dataset/_search").isEmpty()); + } + + /** A hit total whose shape is unrecognised must never become the "not enforced" verdict. */ + @Test + void testATotalObjectWithoutAValueIsUnknownRatherThanNotEnforced() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + "ZGxz|" + SEARCH, + 200, + """ + {"hits":{"total":{}}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + ElasticSearchCapability dls = capability(report, "Document-level security (DLS)"); + assertEquals(CapabilityVerdict.UNKNOWN, dls.verdict()); + assertTrue(dls.detail().contains("no hit total"), dls.detail()); + } + + @Test + void testATotalOfAnUnexpectedJsonTypeIsUnknownRatherThanNotEnforced() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWorkingWriteProbes(); + stub( + "ZGxz|" + SEARCH, + 200, + """ + {"hits":{"total":["unexpected"]}}"""); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, true); + + assertEquals( + CapabilityVerdict.UNKNOWN, capability(report, "Document-level security (DLS)").verdict()); + } + + // --------------------------------------------------------------------------- + // Transport-level response shapes + // --------------------------------------------------------------------------- + + /** A response with no entity at all, which is not the same as one with an empty JSON body. */ + @Test + void testAResponseCarryingNoEntityIsHandledAsAnEmptyBody() throws IOException { + stubSecurityEnabledCluster("trial"); + stubWithNoBody(HAS_PRIVILEGES, 200); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertTrue(report.clusterPrivileges().isEmpty()); + // A credential whose privileges could not be read is not a credential known to lack them. + assertEquals(CapabilityVerdict.INFERRED_SUPPORTED, capability(report, "API keys").verdict()); + } + + /** Valid JSON that is not an object is as unusable as invalid JSON, and must not throw either. */ + @Test + void testAJsonBodyThatIsNotAnObjectIsHandledAsAnEmptyBody() throws IOException { + stubSecurityEnabledCluster("trial"); + stub(HAS_PRIVILEGES, 200, "[\"unexpected\"]"); + + ElasticSearchCapabilityReport report = service().getCapabilityReport(null, false); + + assertTrue(report.clusterPrivileges().isEmpty()); + } + private record StubResponse(int status, String body) {} }