DRAFT/Experiment to see how encrypted reads might look

Cargo.toml

@@ -39,6 +39,7 @@ repository = "https://github.com/apache/iceberg-rust"
 rust-version = "1.88"
 
 [workspace.dependencies]
+aes-gcm = "0.10"
 anyhow = "1.0.72"
 apache-avro = { version = "0.21", features = ["zstandard"] }
 array-init = "2"
@@ -102,7 +103,7 @@ num-bigint = "0.4.6"
 once_cell = "1.20"
 opendal = "0.55.0"
 ordered-float = "4"
-parquet = "57.0"
+parquet = { version = "57.0", features = ["encryption"] }
 pilota = "0.11.10"
 port_scanner = "0.1.5"
 pretty_assertions = "1.4"
@@ -131,5 +132,6 @@ typed-builder = "0.20"
 url = "2.5.7"
 uuid = { version = "1.18", features = ["v7"] }
 volo = "0.10.6"
+zeroize = "1.7"
 volo-thrift = "0.10.8"
 zstd = "0.13.3"

crates/iceberg/Cargo.toml

@@ -32,6 +32,7 @@ repository = { workspace = true }
 default = ["storage-memory", "storage-fs", "storage-s3"]
 storage-all = ["storage-memory", "storage-fs", "storage-s3", "storage-gcs"]
 
+encryption = ["parquet/encryption"]
 storage-azdls = ["opendal/services-azdls"]
 storage-fs = ["opendal/services-fs"]
 storage-gcs = ["opendal/services-gcs"]
@@ -41,6 +42,7 @@ storage-s3 = ["opendal/services-s3", "reqsign"]
 
 
 [dependencies]
+aes-gcm = { workspace = true }
 anyhow = { workspace = true }
 apache-avro = { workspace = true }
 array-init = { workspace = true }
@@ -65,13 +67,14 @@ flate2 = { workspace = true }
 fnv = { workspace = true }
 futures = { workspace = true }
 itertools = { workspace = true }
+lru = "0.16.3"
 moka = { version = "0.12.10", features = ["future"] }
 murmur3 = { workspace = true }
 num-bigint = { workspace = true }
 once_cell = { workspace = true }
 opendal = { workspace = true }
 ordered-float = { workspace = true }
-parquet = { workspace = true, features = ["async"] }
+parquet = { workspace = true, features = ["async", "encryption"] }
 rand = { workspace = true }
 reqsign = { version = "0.16.3", optional = true, default-features = false }
 reqwest = { workspace = true }
@@ -88,6 +91,7 @@ tokio = { workspace = true, optional = false, features = ["sync"] }
 typed-builder = { workspace = true }
 url = { workspace = true }
 uuid = { workspace = true }
+zeroize = { workspace = true }
 zstd = { workspace = true }
 
 [dev-dependencies]

crates/iceberg/src/arrow/caching_delete_file_loader.rs

@@ -940,6 +940,7 @@ mod tests {
             partition_spec: None,
             name_mapping: None,
             case_sensitive: false,
+            key_metadata: None,
         };
 
         // Load the deletes - should handle both types without error

crates/iceberg/src/arrow/delete_file_loader.rs

@@ -64,6 +64,7 @@ impl BasicDeleteFileLoader {
             self.file_io.clone(),
             false,
             None,
+            None, // TODO: Add key_metadata support for encrypted delete files
         )
         .await?
         .build()?

crates/iceberg/src/arrow/delete_filter.rs

@@ -412,6 +412,7 @@ pub(crate) mod tests {
                 partition_spec: None,
                 name_mapping: None,
                 case_sensitive: false,
+                key_metadata: None,
             },
             FileScanTask {
                 start: 0,
@@ -427,6 +428,7 @@ pub(crate) mod tests {
                 partition_spec: None,
                 name_mapping: None,
                 case_sensitive: false,
+                key_metadata: None,
             },
         ];
 
@@ -482,6 +484,7 @@ pub(crate) mod tests {
             partition_spec: None,
             name_mapping: None,
             case_sensitive: true,
+            key_metadata: None,
         };
 
         let filter = DeleteFilter::default();