fix: Register the s3:// scheme when an S3 connection is configured

CHANGELOG.md

@@ -14,9 +14,15 @@ All notable changes to this project will be documented in this file.
   previously accepted by the API server but failed reconciliation ([#731]).
 - Bump stackable-operator to 0.114.0 ([#735]).
 
+### Fixed
+
+- HMS now also handles table and schema locations using the `s3://` scheme, not just `s3a://`,
+  when an S3 connection is configured ([#736]).
+
 [#726]: https://github.com/stackabletech/hive-operator/pull/726
 [#731]: https://github.com/stackabletech/hive-operator/pull/731
 [#735]: https://github.com/stackabletech/hive-operator/pull/735
+[#736]: https://github.com/stackabletech/hive-operator/pull/736
 
 ## [26.7.0] - 2026-07-21
 

docs/modules/hive/pages/usage-guide/data-storage.adoc

@@ -4,10 +4,10 @@
 You can operate the Hive metastore service (HMS) without S3 or HDFS.
 Its whole purpose is to store metadata such as "Table foo has columns a, b and c and is stored as parquet in local://tmp/hive/foo".
 
-However, as soon as you start storing metadata in the HMS that refers to a `s3a://` or `hdfs://` locations, HMS will actually do some operations on the filesystem. This can be e.g. checking if the table location exists, creating it in case it is missing.
+However, as soon as you start storing metadata in the HMS that refers to `s3://`, `s3a://` or `hdfs://` locations, HMS will actually do some operations on the filesystem. This can be e.g. checking if the table location exists, creating it in case it is missing.
 
 So if you are storing tables in S3 (or HDFS for that matter), you need to give the HMS access to that filesystem as well.
-The Stackable Operator currently supports S3 and HFS.
+The Stackable Operator currently supports S3 and HDFS.
 
 [s3]
 == S3 support
@@ -29,6 +29,10 @@ clusterConfig:
         secretClass: simple-hive-s3-secret-class
 ----
 
+Table and schema locations can use either the `s3a://` or the `s3://` scheme.
+Both are served by the same filesystem implementation and read the connection details configured above, so no additional configuration is needed for either.
+The `s3://` scheme matters for clients that address data with `s3://` URIs, such as the Apache Iceberg connector of xref:trino:index.adoc[Trino].
+
 [hdfs]
 == Apache HDFS support
 

rust/operator-binary/src/controller/build/properties/hive_site.rs

@@ -30,6 +30,10 @@ const S3_REGION_NAME: &str = "fs.s3a.endpoint.region";
 const S3_SECRET_KEY: &str = "fs.s3a.secret.key";
 const S3_SSL_ENABLED: &str = "fs.s3a.connection.ssl.enabled";
 
+// Maps the `s3://` scheme onto the s3a filesystem implementation.
+const S3_SCHEME_IMPL: &str = "fs.s3.impl";
+const S3A_FILE_SYSTEM: &str = "org.apache.hadoop.fs.s3a.S3AFileSystem";
+
 #[derive(Debug, Snafu)]
 pub enum Error {
     #[snafu(display("failed to configure S3 connection"))]
@@ -98,6 +102,19 @@ pub fn build(
             S3_PATH_STYLE_ACCESS.to_string(),
             (s3.access_style == s3::v1alpha1::S3AccessStyle::Path).to_string(),
         );
+
+        // The bundled Hadoop only registers the `s3a://` scheme, so HMS has no filesystem for
+        // `s3://` locations. Clients that address data with `s3://` URIs, such as Trino's native
+        // S3 Iceberg connector, therefore fail once HMS itself touches the filesystem, e.g. when
+        // it creates the schema/table directory. Map the scheme onto the same implementation the
+        // `fs.s3a.*` settings above configure; `S3AFileSystem` reads those regardless of the scheme
+        // it is mounted under, so `s3a://` locations keep working unchanged.
+        //
+        // Only the `FileSystem` API is mapped, not `fs.AbstractFileSystem.s3.impl`: HMS resolves
+        // filesystems through `FileSystem.get`, and the `FileContext` adapter
+        // `org.apache.hadoop.fs.s3a.S3A` hardcoded `s3a` as its only supported scheme before
+        // Hadoop 3.3.6, so mapping it would fail on older bundled Hadoop versions.
+        data.insert(S3_SCHEME_IMPL.to_string(), S3A_FILE_SYSTEM.to_string());
     }
 
     // Kerberos entries (empty when Kerberos is disabled).
@@ -122,7 +139,8 @@ pub fn build(
 mod tests {
     use super::*;
     use crate::{
-        controller::build::properties::test_support::derby_cluster_config, crd::MetaStoreConfig,
+        controller::build::properties::test_support::{derby_cluster_config, s3_cluster_config},
+        crd::MetaStoreConfig,
     };
 
     #[test]
@@ -176,6 +194,71 @@ mod tests {
         );
     }
 
+    #[test]
+    fn s3_connection_emits_s3a_settings() {
+        let cluster_config = s3_cluster_config();
+        let merged = ValidatedMetaStoreConfig::from_merged_for_test(MetaStoreConfig::default());
+
+        let data = build(
+            &cluster_config,
+            "4.0.0",
+            &merged,
+            BTreeMap::new(),
+            BTreeMap::new(),
+        )
+        .expect("build hive-site");
+
+        assert_eq!(
+            data.get("fs.s3a.endpoint"),
+            Some(&"http://minio:9000/".to_string())
+        );
+        assert_eq!(
+            data.get("fs.s3a.path.style.access"),
+            Some(&"true".to_string())
+        );
+        assert_eq!(
+            data.get("fs.s3a.connection.ssl.enabled"),
+            Some(&"false".to_string())
+        );
+    }
+
+    #[test]
+    fn s3_connection_registers_the_s3_scheme() {
+        let cluster_config = s3_cluster_config();
+        let merged = ValidatedMetaStoreConfig::from_merged_for_test(MetaStoreConfig::default());
+
+        let data = build(
+            &cluster_config,
+            "4.0.0",
+            &merged,
+            BTreeMap::new(),
+            BTreeMap::new(),
+        )
+        .expect("build hive-site");
+
+        assert_eq!(
+            data.get("fs.s3.impl"),
+            Some(&"org.apache.hadoop.fs.s3a.S3AFileSystem".to_string())
+        );
+    }
+
+    #[test]
+    fn no_s3_connection_leaves_the_s3_scheme_unregistered() {
+        let cluster_config = derby_cluster_config();
+        let merged = ValidatedMetaStoreConfig::from_merged_for_test(MetaStoreConfig::default());
+
+        let data = build(
+            &cluster_config,
+            "4.0.0",
+            &merged,
+            BTreeMap::new(),
+            BTreeMap::new(),
+        )
+        .expect("build hive-site");
+
+        assert!(!data.contains_key("fs.s3.impl"));
+    }
+
     #[test]
     fn user_override_wins_over_everything() {
         let cluster_config = derby_cluster_config();

rust/operator-binary/src/controller/build/properties/mod.rs

@@ -39,6 +39,8 @@ mod tests {
 
 #[cfg(test)]
 pub(crate) mod test_support {
+    use stackable_operator::crd::s3;
+
     use crate::{
         controller::{ValidatedClusterConfig, test_support::DERBY_YAML},
         crd::{
@@ -78,4 +80,20 @@ pub(crate) mod test_support {
             kerberos_secret_class: None,
         }
     }
+
+    /// Build a Derby-backed [`ValidatedClusterConfig`] with an S3 connection for builder tests.
+    pub fn s3_cluster_config() -> ValidatedClusterConfig {
+        let s3_connection_spec: s3::v1alpha1::ConnectionSpec =
+            stackable_operator::utils::yaml_from_str_singleton_map(indoc::indoc! {r#"
+                host: minio
+                port: 9000
+                accessStyle: Path
+            "#})
+            .expect("valid S3 ConnectionSpec YAML");
+
+        ValidatedClusterConfig {
+            s3_connection_spec: Some(s3_connection_spec),
+            ..derby_cluster_config()
+        }
+    }
 }

tests/templates/kuttl/smoke/test_metastore.py

@@ -39,6 +39,30 @@ def table(db_name, table_name, location):
     return test_table
 
 
+def check_table(hive_client, db_name, table_name, location, label):
+    """Create the table if it does not exist yet and assert the metastore returns the
+    expected schema for it.
+
+    Returns the table metadata so callers can make additional assertions.
+    """
+    try:
+        hive_client.create_table(table(db_name, table_name, location))
+    except AlreadyExistsException:
+        print(f"[INFO]: Table {table_name} already existed")
+
+    schema = hive_client.get_schema(db_name=db_name, table_name=table_name)
+    expected = [FieldSchema(name="id", type="string", comment="col comment")]
+    if schema != expected:
+        print(
+            f"[ERROR]: Received {label} schema {schema} - expected schema: {expected}"
+        )
+        exit(-1)
+
+    # Positional arguments on purpose: get_table takes `dbname`/`tbl_name`, unlike
+    # get_schema above, which takes `db_name`/`table_name`.
+    return hive_client.get_table(db_name, table_name)
+
+
 if __name__ == "__main__":
     all_args = argparse.ArgumentParser(description="Test hive metastore.")
     all_args.add_argument("-p", "--port", help="Metastore server port", default="9083")
@@ -55,6 +79,7 @@ def table(db_name, table_name, location):
     host = args["metastore"]
     local_test_table_name = "one_column_table"
     s3_test_table_name = "s3_one_column_table"
+    s3_scheme_test_table_name = "s3_scheme_one_column_table"
     s3_test_table_name_wrong_bucket = "s3_one_column_table_wrong_buckets"
     # Creating database object using builder
     database = DatabaseBuilder(database_name).build()
@@ -63,48 +88,42 @@ def table(db_name, table_name, location):
         hive_client.create_database_if_not_exists(database)
 
         # Local access
-        try:
-            hive_client.create_table(
-                table(
-                    database_name,
-                    local_test_table_name,
-                    f"/stackable/warehouse/location_{database_name}_{local_test_table_name}",
-                )
-            )
-        except AlreadyExistsException:
-            print(f"[INFO]: Table {local_test_table_name} already existed")
-        schema = hive_client.get_schema(
-            db_name=database_name, table_name=local_test_table_name
+        check_table(
+            hive_client,
+            database_name,
+            local_test_table_name,
+            f"/stackable/warehouse/location_{database_name}_{local_test_table_name}",
+            "local",
         )
-        expected = [FieldSchema(name="id", type="string", comment="col comment")]
-        if schema != expected:
-            print(
-                "[ERROR]: Received local schema "
-                + str(schema)
-                + " - expected schema: "
-                + expected
-            )
-            exit(-1)
 
         # S3 access
-        try:
-            hive_client.create_table(
-                table(
-                    database_name, s3_test_table_name, "s3a://hive/s3_one_column_table/"
-                )
-            )
-        except AlreadyExistsException:
-            print(f"[INFO]: Table {s3_test_table_name} already existed")
-        schema = hive_client.get_schema(
-            db_name=database_name, table_name=s3_test_table_name
+        check_table(
+            hive_client,
+            database_name,
+            s3_test_table_name,
+            "s3a://hive/s3_one_column_table/",
+            "s3",
         )
-        expected = [FieldSchema(name="id", type="string", comment="col comment")]
-        if schema != expected:
+
+        # S3 access via the s3:// scheme. Clients such as Trino's native S3 Iceberg connector
+        # address data with s3:// instead of s3a:// URIs. Creating the table makes the metastore
+        # create the table directory itself, which fails unless the s3:// scheme is registered.
+        s3_scheme_table = check_table(
+            hive_client,
+            database_name,
+            s3_scheme_test_table_name,
+            "s3://hive/s3_scheme_one_column_table/",
+            "s3 scheme",
+        )
+
+        # The scheme must survive the round-trip. The metastore rewrites the stored location
+        # using the scheme of the filesystem it resolved, and clients read that location back to
+        # address the data. A location normalised to s3a:// would defeat registering the scheme.
+        s3_scheme_location = s3_scheme_table.sd.location
+        if not s3_scheme_location.startswith("s3://"):
             print(
-                "[ERROR]: Received s3 schema "
-                + str(schema)
-                + " - expected schema: "
-                + expected
+                f"[ERROR]: Expected the stored location to keep the s3:// scheme, "
+                f"got {s3_scheme_location}"
             )
             exit(-1)