diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index ffa3f2f08..a7deaffbd 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -235,6 +235,7 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/version.h" add_subdirectory(catalog) add_subdirectory(data) add_subdirectory(deletes) +add_subdirectory(encryption) add_subdirectory(expression) add_subdirectory(manifest) add_subdirectory(puffin) diff --git a/src/iceberg/catalog/rest/json_serde.cc b/src/iceberg/catalog/rest/json_serde.cc index 2eb0d19de..ac80e9f08 100644 --- a/src/iceberg/catalog/rest/json_serde.cc +++ b/src/iceberg/catalog/rest/json_serde.cc @@ -902,17 +902,29 @@ Result CommitTableRequestFromJson(const nlohmann::json& json } ICEBERG_ASSIGN_OR_RAISE(auto requirements_json, - GetJsonValue(json, kRequirements)); - for (const auto& req_json : requirements_json) { - ICEBERG_ASSIGN_OR_RAISE(auto requirement, TableRequirementFromJson(req_json)); - request.requirements.push_back(std::move(requirement)); + GetJsonValueOptional(json, kRequirements)); + if (requirements_json.has_value()) { + if (!requirements_json->is_array()) { + return JsonParseError("Expected '{}' to be an array, got {}", kRequirements, + SafeDumpJson(*requirements_json)); + } + for (const auto& req_json : *requirements_json) { + ICEBERG_ASSIGN_OR_RAISE(auto requirement, TableRequirementFromJson(req_json)); + request.requirements.push_back(std::move(requirement)); + } } ICEBERG_ASSIGN_OR_RAISE(auto updates_json, - GetJsonValue(json, kUpdates)); - for (const auto& update_json : updates_json) { - ICEBERG_ASSIGN_OR_RAISE(auto update, TableUpdateFromJson(update_json)); - request.updates.push_back(std::move(update)); + GetJsonValueOptional(json, kUpdates)); + if (updates_json.has_value()) { + if (!updates_json->is_array()) { + return JsonParseError("Expected '{}' to be an array, got {}", kUpdates, + SafeDumpJson(*updates_json)); + } + for (const auto& update_json : *updates_json) { + ICEBERG_ASSIGN_OR_RAISE(auto update, TableUpdateFromJson(update_json)); + request.updates.push_back(std::move(update)); + } } ICEBERG_RETURN_UNEXPECTED(request.Validate()); diff --git a/src/iceberg/encryption/CMakeLists.txt b/src/iceberg/encryption/CMakeLists.txt new file mode 100644 index 000000000..c493ee250 --- /dev/null +++ b/src/iceberg/encryption/CMakeLists.txt @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +iceberg_install_all_headers(iceberg/encryption) diff --git a/src/iceberg/encryption/encrypted_key.h b/src/iceberg/encryption/encrypted_key.h new file mode 100644 index 000000000..b65255bf5 --- /dev/null +++ b/src/iceberg/encryption/encrypted_key.h @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/encryption/encrypted_key.h +/// Encrypted table key metadata. + +#include +#include +#include + +#include "iceberg/iceberg_export.h" + +namespace iceberg { + +/// \brief Represents an encrypted table key entry. +struct ICEBERG_EXPORT EncryptedKey { + std::string key_id; + std::string encrypted_key_metadata; + std::optional encrypted_by_id = std::nullopt; + std::unordered_map properties; + + friend bool operator==(const EncryptedKey&, const EncryptedKey&) = default; +}; + +} // namespace iceberg diff --git a/src/iceberg/encryption/meson.build b/src/iceberg/encryption/meson.build new file mode 100644 index 000000000..ad6e142c2 --- /dev/null +++ b/src/iceberg/encryption/meson.build @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +install_headers(['encrypted_key.h'], subdir: 'iceberg/encryption') diff --git a/src/iceberg/json_serde.cc b/src/iceberg/json_serde.cc index c72b7da57..1f2b8f45c 100644 --- a/src/iceberg/json_serde.cc +++ b/src/iceberg/json_serde.cc @@ -43,6 +43,7 @@ #include "iceberg/table_update.h" #include "iceberg/transform.h" #include "iceberg/type.h" +#include "iceberg/util/base64.h" #include "iceberg/util/checked_cast.h" #include "iceberg/util/formatter.h" // IWYU pragma: keep #include "iceberg/util/json_util_internal.h" @@ -90,6 +91,8 @@ constexpr std::string_view kValueId = "value-id"; constexpr std::string_view kRequired = "required"; constexpr std::string_view kElementRequired = "element-required"; constexpr std::string_view kValueRequired = "value-required"; +constexpr std::string_view kEncryptedKeyMetadata = "encrypted-key-metadata"; +constexpr std::string_view kEncryptedById = "encrypted-by-id"; // Snapshot constants constexpr std::string_view kSpecId = "spec-id"; @@ -166,6 +169,7 @@ constexpr std::string_view kRefs = "refs"; constexpr std::string_view kStatistics = "statistics"; constexpr std::string_view kPartitionStatistics = "partition-statistics"; constexpr std::string_view kNextRowId = "next-row-id"; +constexpr std::string_view kEncryptionKeys = "encryption-keys"; constexpr std::string_view kMetadataFile = "metadata-file"; constexpr std::string_view kStatisticsPath = "statistics-path"; constexpr std::string_view kFileSizeInBytes = "file-size-in-bytes"; @@ -196,6 +200,8 @@ constexpr std::string_view kActionRemoveStatistics = "remove-statistics"; constexpr std::string_view kActionSetPartitionStatistics = "set-partition-statistics"; constexpr std::string_view kActionRemovePartitionStatistics = "remove-partition-statistics"; +constexpr std::string_view kActionAddEncryptionKey = "add-encryption-key"; +constexpr std::string_view kActionRemoveEncryptionKey = "remove-encryption-key"; // TableUpdate field constants constexpr std::string_view kUUID = "uuid"; @@ -210,6 +216,9 @@ constexpr std::string_view kRefName = "ref-name"; constexpr std::string_view kRef = "ref"; constexpr std::string_view kUpdates = "updates"; constexpr std::string_view kRemovals = "removals"; +constexpr std::string_view kUpdated = "updated"; +constexpr std::string_view kRemoved = "removed"; +constexpr std::string_view kEncryptionKey = "encryption-key"; // TableRequirement type constants constexpr std::string_view kRequirementAssertDoesNotExist = "assert-create"; @@ -851,6 +860,41 @@ Result MetadataLogEntryFromJson(const nlohmann::json& json) { return metadata_log_entry; } +nlohmann::json ToJson(const EncryptedKey& encrypted_key) { + nlohmann::json json; + json[kKeyId] = encrypted_key.key_id; + json[kEncryptedKeyMetadata] = Base64::Encode(encrypted_key.encrypted_key_metadata); + SetOptionalField(json, kEncryptedById, encrypted_key.encrypted_by_id); + if (!encrypted_key.properties.empty()) { + json[kProperties] = encrypted_key.properties; + } + return json; +} + +Result EncryptedKeyFromJson(const nlohmann::json& json) { + using StringMap = std::unordered_map; + + if (!json.is_object()) { + return JsonParseError("Invalid encryption key, must be non-null object: {}", + SafeDumpJson(json)); + } + + ICEBERG_ASSIGN_OR_RAISE(auto key_id, GetJsonValue(json, kKeyId)); + ICEBERG_ASSIGN_OR_RAISE(auto encoded_metadata, + GetJsonValue(json, kEncryptedKeyMetadata)); + ICEBERG_ASSIGN_OR_RAISE(auto encrypted_key_metadata, Base64::Decode(encoded_metadata)); + ICEBERG_ASSIGN_OR_RAISE(auto encrypted_by_id, + GetJsonValueOptional(json, kEncryptedById)); + ICEBERG_ASSIGN_OR_RAISE(auto properties, + GetJsonValueOrDefault(json, kProperties)); + return EncryptedKey{ + .key_id = std::move(key_id), + .encrypted_key_metadata = std::move(encrypted_key_metadata), + .encrypted_by_id = std::move(encrypted_by_id), + .properties = std::move(properties), + }; +} + nlohmann::json ToJson(const TableMetadata& table_metadata) { nlohmann::json json; @@ -917,6 +961,9 @@ nlohmann::json ToJson(const TableMetadata& table_metadata) { json[kSnapshots] = ToJsonList(table_metadata.snapshots); json[kStatistics] = ToJsonList(table_metadata.statistics); json[kPartitionStatistics] = ToJsonList(table_metadata.partition_statistics); + if (!table_metadata.encryption_keys.empty()) { + json[kEncryptionKeys] = ToJsonList(table_metadata.encryption_keys); + } json[kSnapshotLog] = ToJsonList(table_metadata.snapshot_log); json[kMetadataLog] = ToJsonList(table_metadata.metadata_log); @@ -1182,6 +1229,9 @@ Result> TableMetadataFromJson(const nlohmann::jso table_metadata->partition_statistics, FromJsonList(json, kPartitionStatistics, PartitionStatisticsFileFromJson)); + ICEBERG_ASSIGN_OR_RAISE( + table_metadata->encryption_keys, + FromJsonList(json, kEncryptionKeys, EncryptedKeyFromJson)); ICEBERG_ASSIGN_OR_RAISE( table_metadata->snapshot_log, FromJsonList(json, kSnapshotLog, SnapshotLogEntryFromJson)); @@ -1486,6 +1536,18 @@ nlohmann::json ToJson(const TableUpdate& update) { json[kSnapshotId] = u.snapshot_id(); break; } + case TableUpdate::Kind::kAddEncryptionKey: { + const auto& u = internal::checked_cast(update); + json[kAction] = kActionAddEncryptionKey; + json[kEncryptionKey] = ToJson(u.key()); + break; + } + case TableUpdate::Kind::kRemoveEncryptionKey: { + const auto& u = internal::checked_cast(update); + json[kAction] = kActionRemoveEncryptionKey; + json[kKeyId] = u.key_id(); + break; + } } return json; } @@ -1506,7 +1568,6 @@ nlohmann::json ToJson(const TableRequirement& requirement) { const auto& r = internal::checked_cast(requirement); json[kType] = kRequirementAssertRefSnapshotID; - // REST spec names this field "ref", not "ref-name". json[kRef] = r.ref_name(); if (r.snapshot_id().has_value()) { json[kSnapshotId] = r.snapshot_id().value(); @@ -1570,8 +1631,10 @@ Result> TableUpdateFromJson(const nlohmann::json& j ICEBERG_ASSIGN_OR_RAISE(auto schema_json, GetJsonValue(json, kSchema)); ICEBERG_ASSIGN_OR_RAISE(auto parsed_schema, SchemaFromJson(schema_json)); - ICEBERG_ASSIGN_OR_RAISE(auto last_column_id, - GetJsonValue(json, kLastColumnId)); + ICEBERG_ASSIGN_OR_RAISE(auto highest_field_id, parsed_schema->HighestFieldId()); + ICEBERG_ASSIGN_OR_RAISE( + auto last_column_id, + GetJsonValueOrDefault(json, kLastColumnId, highest_field_id)); return std::make_unique(std::move(parsed_schema), last_column_id); } if (action == kActionSetCurrentSchema) { @@ -1650,12 +1713,17 @@ Result> TableUpdateFromJson(const nlohmann::json& j } if (action == kActionSetProperties) { using StringMap = std::unordered_map; - ICEBERG_ASSIGN_OR_RAISE(auto updates, GetJsonValue(json, kUpdates)); + ICEBERG_ASSIGN_OR_RAISE(auto updates, + json.contains(kUpdates) || !json.contains(kUpdated) + ? GetJsonValue(json, kUpdates) + : GetJsonValue(json, kUpdated)); return std::make_unique(std::move(updates)); } if (action == kActionRemoveProperties) { ICEBERG_ASSIGN_OR_RAISE(auto removals_vec, - GetJsonValue>(json, kRemovals)); + json.contains(kRemovals) || !json.contains(kRemoved) + ? GetJsonValue>(json, kRemovals) + : GetJsonValue>(json, kRemoved)); std::unordered_set removals( std::make_move_iterator(removals_vec.begin()), std::make_move_iterator(removals_vec.end())); @@ -1688,6 +1756,17 @@ Result> TableUpdateFromJson(const nlohmann::json& j ICEBERG_ASSIGN_OR_RAISE(auto snapshot_id, GetJsonValue(json, kSnapshotId)); return std::make_unique(snapshot_id); } + if (action == kActionAddEncryptionKey) { + if (!json.contains(kEncryptionKey)) { + return JsonParseError("Invalid encryption key, must be non-null object: null"); + } + ICEBERG_ASSIGN_OR_RAISE(auto key, EncryptedKeyFromJson(json.at(kEncryptionKey))); + return std::make_unique(std::move(key)); + } + if (action == kActionRemoveEncryptionKey) { + ICEBERG_ASSIGN_OR_RAISE(auto key_id, GetJsonValue(json, kKeyId)); + return std::make_unique(std::move(key_id)); + } return JsonParseError("Unknown table update action: {}", action); } diff --git a/src/iceberg/json_serde_internal.h b/src/iceberg/json_serde_internal.h index 8699e3dd1..351b41086 100644 --- a/src/iceberg/json_serde_internal.h +++ b/src/iceberg/json_serde_internal.h @@ -28,6 +28,7 @@ #include +#include "iceberg/encryption/encrypted_key.h" #include "iceberg/result.h" #include "iceberg/statistics_file.h" #include "iceberg/table_metadata.h" @@ -283,6 +284,18 @@ ICEBERG_EXPORT nlohmann::json ToJson(const MetadataLogEntry& metadata_log_entry) ICEBERG_EXPORT Result MetadataLogEntryFromJson( const nlohmann::json& json); +/// \brief Serializes an `EncryptedKey` object to JSON. +/// +/// \param encrypted_key The `EncryptedKey` object to be serialized. +/// \return A JSON object representing the `EncryptedKey`. +ICEBERG_EXPORT nlohmann::json ToJson(const EncryptedKey& encrypted_key); + +/// \brief Deserializes a JSON object into an `EncryptedKey` object. +/// +/// \param json The JSON object representing an `EncryptedKey`. +/// \return An `EncryptedKey` object or an error if the conversion fails. +ICEBERG_EXPORT Result EncryptedKeyFromJson(const nlohmann::json& json); + /// \brief Serializes a `TableMetadata` object to JSON. /// /// \param table_metadata The `TableMetadata` object to be serialized. diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build index 75fae754e..c7f75ce62 100644 --- a/src/iceberg/meson.build +++ b/src/iceberg/meson.build @@ -292,6 +292,7 @@ install_headers( subdir('catalog') subdir('data') subdir('deletes') +subdir('encryption') subdir('expression') subdir('manifest') subdir('metrics') diff --git a/src/iceberg/table_metadata.cc b/src/iceberg/table_metadata.cc index 30f99f259..88bde4d50 100644 --- a/src/iceberg/table_metadata.cc +++ b/src/iceberg/table_metadata.cc @@ -176,6 +176,10 @@ std::vector> ChangesForCreate( std::make_unique(metadata.properties.configs())); } + for (const auto& key : metadata.encryption_keys) { + changes.push_back(std::make_unique(key)); + } + return changes; } } // namespace @@ -353,7 +357,7 @@ bool operator==(const TableMetadata& lhs, const TableMetadata& rhs) { SnapshotRefEquals(lhs.refs, rhs.refs) && SharedPtrVectorEquals(lhs.statistics, rhs.statistics) && SharedPtrVectorEquals(lhs.partition_statistics, rhs.partition_statistics) && - lhs.next_row_id == rhs.next_row_id; + lhs.next_row_id == rhs.next_row_id && lhs.encryption_keys == rhs.encryption_keys; } // TableMetadataCache implementation @@ -579,6 +583,10 @@ class TableMetadataBuilder::Impl { snapshots_by_id_.emplace(snapshot->snapshot_id, snapshot); } + for (const auto& key : metadata_.encryption_keys) { + keys_by_id_.emplace(key.key_id); + } + metadata_.last_updated_ms = kInvalidLastUpdatedMs; } @@ -627,6 +635,8 @@ class TableMetadataBuilder::Impl { Status SetPartitionStatistics( std::shared_ptr partition_statistics_file); Status RemovePartitionStatistics(int64_t snapshot_id); + Status AddEncryptionKey(EncryptedKey key); + Status RemoveEncryptionKey(std::string_view key_id); Result> Build(); @@ -699,6 +709,7 @@ class TableMetadataBuilder::Impl { std::unordered_map> specs_by_id_; std::unordered_map> sort_orders_by_id_; std::unordered_map> snapshots_by_id_; + std::unordered_set keys_by_id_; }; Status TableMetadataBuilder::Impl::AssignUUID(std::string_view uuid) { @@ -1247,6 +1258,29 @@ Status TableMetadataBuilder::Impl::RemovePartitionStatistics(int64_t snapshot_id return {}; } +Status TableMetadataBuilder::Impl::AddEncryptionKey(EncryptedKey key) { + if (keys_by_id_.contains(key.key_id)) { + return {}; + } + + keys_by_id_.emplace(key.key_id); + changes_.push_back(std::make_unique(key)); + metadata_.encryption_keys.push_back(std::move(key)); + return {}; +} + +Status TableMetadataBuilder::Impl::RemoveEncryptionKey(std::string_view key_id) { + std::string key_id_str(key_id); + if (keys_by_id_.erase(key_id_str) == 0) { + return {}; + } + + std::erase_if(metadata_.encryption_keys, + [&key_id_str](const auto& key) { return key.key_id == key_id_str; }); + changes_.push_back(std::make_unique(std::move(key_id_str))); + return {}; +} + std::unordered_set TableMetadataBuilder::Impl::IntermediateSnapshotIdSet( int64_t current_snapshot_id) const { std::unordered_set added_snapshot_ids; @@ -1702,13 +1736,14 @@ TableMetadataBuilder& TableMetadataBuilder::SetLocation(std::string_view locatio return *this; } -TableMetadataBuilder& TableMetadataBuilder::AddEncryptionKey( - std::shared_ptr key) { - throw IcebergError(std::format("{} not implemented", __FUNCTION__)); +TableMetadataBuilder& TableMetadataBuilder::AddEncryptionKey(EncryptedKey key) { + ICEBERG_BUILDER_RETURN_IF_ERROR(impl_->AddEncryptionKey(std::move(key))); + return *this; } TableMetadataBuilder& TableMetadataBuilder::RemoveEncryptionKey(std::string_view key_id) { - throw IcebergError(std::format("{} not implemented", __FUNCTION__)); + ICEBERG_BUILDER_RETURN_IF_ERROR(impl_->RemoveEncryptionKey(key_id)); + return *this; } Result> TableMetadataBuilder::Build() { diff --git a/src/iceberg/table_metadata.h b/src/iceberg/table_metadata.h index 335fedadc..06d636ef5 100644 --- a/src/iceberg/table_metadata.h +++ b/src/iceberg/table_metadata.h @@ -28,6 +28,7 @@ #include #include +#include "iceberg/encryption/encrypted_key.h" #include "iceberg/iceberg_export.h" #include "iceberg/table_properties.h" #include "iceberg/type_fwd.h" @@ -129,6 +130,8 @@ struct ICEBERG_EXPORT TableMetadata { std::vector> partition_statistics; /// A `long` higher than all assigned row IDs int64_t next_row_id; + /// A list of encrypted table key entries + std::vector encryption_keys; static Result> Make( const iceberg::Schema& schema, const iceberg::PartitionSpec& spec, @@ -446,7 +449,7 @@ class ICEBERG_EXPORT TableMetadataBuilder : public ErrorCollector { /// /// \param key The encryption key to add /// \return Reference to this builder for method chaining - TableMetadataBuilder& AddEncryptionKey(std::shared_ptr key); + TableMetadataBuilder& AddEncryptionKey(EncryptedKey key); /// \brief Remove an encryption key from the table by key ID /// diff --git a/src/iceberg/table_update.cc b/src/iceberg/table_update.cc index 01612d765..505bb517c 100644 --- a/src/iceberg/table_update.cc +++ b/src/iceberg/table_update.cc @@ -565,4 +565,48 @@ std::unique_ptr RemovePartitionStatistics::Clone() const { return std::make_unique(snapshot_id_); } +// AddEncryptionKey + +void AddEncryptionKey::ApplyTo(TableMetadataBuilder& builder) const { + builder.AddEncryptionKey(key_); +} + +void AddEncryptionKey::GenerateRequirements(TableUpdateContext& context) const { + // AddEncryptionKey doesn't generate any requirements +} + +bool AddEncryptionKey::Equals(const TableUpdate& other) const { + if (other.kind() != Kind::kAddEncryptionKey) { + return false; + } + const auto& other_add = internal::checked_cast(other); + return key_ == other_add.key_; +} + +std::unique_ptr AddEncryptionKey::Clone() const { + return std::make_unique(key_); +} + +// RemoveEncryptionKey + +void RemoveEncryptionKey::ApplyTo(TableMetadataBuilder& builder) const { + builder.RemoveEncryptionKey(key_id_); +} + +void RemoveEncryptionKey::GenerateRequirements(TableUpdateContext& context) const { + // RemoveEncryptionKey doesn't generate any requirements +} + +bool RemoveEncryptionKey::Equals(const TableUpdate& other) const { + if (other.kind() != Kind::kRemoveEncryptionKey) { + return false; + } + const auto& other_remove = internal::checked_cast(other); + return key_id_ == other_remove.key_id_; +} + +std::unique_ptr RemoveEncryptionKey::Clone() const { + return std::make_unique(key_id_); +} + } // namespace iceberg::table diff --git a/src/iceberg/table_update.h b/src/iceberg/table_update.h index c75c3fa6a..837134e2b 100644 --- a/src/iceberg/table_update.h +++ b/src/iceberg/table_update.h @@ -29,6 +29,7 @@ #include #include +#include "iceberg/encryption/encrypted_key.h" #include "iceberg/iceberg_export.h" #include "iceberg/snapshot.h" #include "iceberg/type_fwd.h" @@ -63,6 +64,8 @@ class ICEBERG_EXPORT TableUpdate { kRemoveStatistics, kSetPartitionStatistics, kRemovePartitionStatistics, + kAddEncryptionKey, + kRemoveEncryptionKey, }; virtual ~TableUpdate(); @@ -608,6 +611,48 @@ class ICEBERG_EXPORT RemovePartitionStatistics : public TableUpdate { int64_t snapshot_id_; }; +/// \brief Represents adding an encryption key to the table. +class ICEBERG_EXPORT AddEncryptionKey : public TableUpdate { + public: + explicit AddEncryptionKey(EncryptedKey key) : key_(std::move(key)) {} + + const EncryptedKey& key() const { return key_; } + + void ApplyTo(TableMetadataBuilder& builder) const override; + + void GenerateRequirements(TableUpdateContext& context) const override; + + Kind kind() const override { return Kind::kAddEncryptionKey; } + + bool Equals(const TableUpdate& other) const override; + + std::unique_ptr Clone() const override; + + private: + EncryptedKey key_; +}; + +/// \brief Represents removing an encryption key from the table. +class ICEBERG_EXPORT RemoveEncryptionKey : public TableUpdate { + public: + explicit RemoveEncryptionKey(std::string key_id) : key_id_(std::move(key_id)) {} + + const std::string& key_id() const { return key_id_; } + + void ApplyTo(TableMetadataBuilder& builder) const override; + + void GenerateRequirements(TableUpdateContext& context) const override; + + Kind kind() const override { return Kind::kRemoveEncryptionKey; } + + bool Equals(const TableUpdate& other) const override; + + std::unique_ptr Clone() const override; + + private: + std::string key_id_; +}; + } // namespace table } // namespace iceberg diff --git a/src/iceberg/test/json_serde_test.cc b/src/iceberg/test/json_serde_test.cc index 0dac226e9..88b44e3ab 100644 --- a/src/iceberg/test/json_serde_test.cc +++ b/src/iceberg/test/json_serde_test.cc @@ -35,6 +35,7 @@ #include "iceberg/table_update.h" #include "iceberg/test/matchers.h" #include "iceberg/transform.h" +#include "iceberg/util/base64.h" #include "iceberg/util/formatter.h" // IWYU pragma: keep #include "iceberg/util/macros.h" // IWYU pragma: keep #include "iceberg/util/timepoint.h" @@ -361,6 +362,23 @@ TEST(JsonInternalTest, TableUpdateAddSchema) { EXPECT_EQ(*actual->schema(), *update.schema()); } +TEST(JsonInternalTest, TableUpdateAddSchemaWithoutDeprecatedLastColumnId) { + auto schema = std::make_shared( + std::vector{SchemaField(1, "id", int64(), false), + SchemaField(3, "name", string(), true)}, + /*schema_id=*/1); + nlohmann::json json = { + {"action", "add-schema"}, + {"schema", ToJson(*schema)}, + }; + + auto parsed = TableUpdateFromJson(json); + ASSERT_THAT(parsed, IsOk()); + auto* actual = internal::checked_cast(parsed.value().get()); + EXPECT_EQ(*actual->schema(), *schema); + EXPECT_EQ(actual->last_column_id(), 3); +} + TEST(JsonInternalTest, TableUpdateSetCurrentSchema) { table::SetCurrentSchema update(1); nlohmann::json expected = R"({"action":"set-current-schema","schema-id":1})"_json; @@ -540,6 +558,23 @@ TEST(JsonInternalTest, TableUpdateSetProperties) { EXPECT_EQ(*internal::checked_cast(parsed.value().get()), update); } +TEST(JsonInternalTest, TableUpdateSetPropertiesLegacyUpdatedField) { + nlohmann::json json = + R"({"action":"set-properties","updated":{"key1":"value1","key2":"value2"}})"_json; + + auto parsed = TableUpdateFromJson(json); + ASSERT_THAT(parsed, IsOk()); + table::SetProperties expected({{"key1", "value1"}, {"key2", "value2"}}); + EXPECT_EQ(*internal::checked_cast(parsed.value().get()), + expected); +} + +TEST(JsonInternalTest, TableUpdateSetPropertiesMissingCanonicalField) { + auto parsed = TableUpdateFromJson(R"({"action":"set-properties"})"_json); + EXPECT_THAT(parsed, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(parsed, HasErrorMessage("Missing 'updates'")); +} + TEST(JsonInternalTest, TableUpdateRemoveProperties) { table::RemoveProperties update({"key1", "key2"}); @@ -553,6 +588,23 @@ TEST(JsonInternalTest, TableUpdateRemoveProperties) { update); } +TEST(JsonInternalTest, TableUpdateRemovePropertiesLegacyRemovedField) { + nlohmann::json json = + R"({"action":"remove-properties","removed":["key1","key2"]})"_json; + + auto parsed = TableUpdateFromJson(json); + ASSERT_THAT(parsed, IsOk()); + table::RemoveProperties expected({"key1", "key2"}); + EXPECT_EQ(*internal::checked_cast(parsed.value().get()), + expected); +} + +TEST(JsonInternalTest, TableUpdateRemovePropertiesMissingCanonicalField) { + auto parsed = TableUpdateFromJson(R"({"action":"remove-properties"})"_json); + EXPECT_THAT(parsed, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(parsed, HasErrorMessage("Missing 'removals'")); +} + TEST(JsonInternalTest, TableUpdateSetLocation) { table::SetLocation update("s3://bucket/warehouse/table"); nlohmann::json expected = @@ -649,6 +701,50 @@ TEST(JsonInternalTest, TableUpdateRemovePartitionStatistics) { update); } +TEST(JsonInternalTest, TableUpdateAddEncryptionKey) { + EncryptedKey key{ + .key_id = "key-1", + .encrypted_key_metadata = "secret-key-metadata", + .encrypted_by_id = "kek-1", + .properties = {{"scope", "table"}}, + }; + table::AddEncryptionKey update(key); + + nlohmann::json expected = { + {"action", "add-encryption-key"}, + {"encryption-key", + {{"key-id", "key-1"}, + {"encrypted-key-metadata", Base64::Encode("secret-key-metadata")}, + {"encrypted-by-id", "kek-1"}, + {"properties", {{"scope", "table"}}}}}, + }; + + EXPECT_EQ(ToJson(update), expected); + auto parsed = TableUpdateFromJson(expected); + ASSERT_THAT(parsed, IsOk()); + EXPECT_EQ(*internal::checked_cast(parsed.value().get()), + update); +} + +TEST(JsonInternalTest, TableUpdateAddEncryptionKeyRejectsNonObjectKey) { + nlohmann::json json = R"({"action":"add-encryption-key","encryption-key":null})"_json; + + auto parsed = TableUpdateFromJson(json); + EXPECT_THAT(parsed, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(parsed, HasErrorMessage("Invalid encryption key")); +} + +TEST(JsonInternalTest, TableUpdateRemoveEncryptionKey) { + table::RemoveEncryptionKey update("key-1"); + nlohmann::json expected = R"({"action":"remove-encryption-key","key-id":"key-1"})"_json; + + EXPECT_EQ(ToJson(update), expected); + auto parsed = TableUpdateFromJson(expected); + ASSERT_THAT(parsed, IsOk()); + EXPECT_EQ(*internal::checked_cast(parsed.value().get()), + update); +} + TEST(JsonInternalTest, TableUpdateUnknownAction) { nlohmann::json json = R"({"action":"unknown-action"})"_json; auto result = TableUpdateFromJson(json); diff --git a/src/iceberg/test/metadata_serde_test.cc b/src/iceberg/test/metadata_serde_test.cc index 2c2b2bd1e..48f88f2bc 100644 --- a/src/iceberg/test/metadata_serde_test.cc +++ b/src/iceberg/test/metadata_serde_test.cc @@ -486,4 +486,57 @@ TEST(MetadataSerdeTest, DeserializeV2MissingSortOrder) { "sort-orders must exist"); } +TEST(MetadataSerdeTest, EncryptionKeysRoundTrip) { + nlohmann::json metadata_json = R"({ + "format-version": 2, + "table-uuid": "test-uuid-1234", + "location": "s3://bucket/test", + "last-sequence-number": 0, + "last-updated-ms": 0, + "last-column-id": 1, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "type": "int", "required": true} + ] + } + ], + "current-schema-id": 0, + "partition-specs": [{"spec-id": 0, "fields": []}], + "default-spec-id": 0, + "last-partition-id": 999, + "sort-orders": [{"order-id": 0, "fields": []}], + "default-sort-order-id": 0, + "properties": {}, + "current-snapshot-id": null, + "refs": {}, + "snapshots": [], + "statistics": [], + "partition-statistics": [], + "snapshot-log": [], + "metadata-log": [], + "encryption-keys": [ + { + "key-id": "key-1", + "encrypted-key-metadata": "c2VjcmV0LWtleS1tZXRhZGF0YQ==", + "encrypted-by-id": "kek-1", + "properties": {"scope": "table"} + } + ] + })"_json; + + auto metadata = TableMetadataFromJson(metadata_json); + ASSERT_THAT(metadata, IsOk()); + ASSERT_EQ(metadata.value()->encryption_keys.size(), 1); + EXPECT_EQ(metadata.value()->encryption_keys[0].key_id, "key-1"); + EXPECT_EQ(metadata.value()->encryption_keys[0].encrypted_key_metadata, + "secret-key-metadata"); + + auto serialized = ToJson(*metadata.value()); + ASSERT_TRUE(serialized.contains("encryption-keys")); + EXPECT_EQ(serialized["encryption-keys"], metadata_json["encryption-keys"]); +} + } // namespace iceberg diff --git a/src/iceberg/test/rest_json_serde_test.cc b/src/iceberg/test/rest_json_serde_test.cc index 86c852a1a..7304831c6 100644 --- a/src/iceberg/test/rest_json_serde_test.cc +++ b/src/iceberg/test/rest_json_serde_test.cc @@ -1236,7 +1236,31 @@ INSTANTIATE_TEST_SUITE_P( CommitTableRequestDeserializeParam{ .test_name = "MissingIdentifier", .json_str = R"({"requirements":[],"updates":[]})", - .expected_model = {}}), + .expected_model = {}}, + // Requirements field is missing (should deserialize to empty requirements) + CommitTableRequestDeserializeParam{ + .test_name = "MissingRequirements", + .json_str = + R"({"identifier":{"namespace":["ns1"],"name":"table1"},"updates":[]})", + .expected_model = {.identifier = TableIdentifier{Namespace{{"ns1"}}, + "table1"}}}, + // Updates field is missing (should deserialize to empty updates) + CommitTableRequestDeserializeParam{ + .test_name = "MissingUpdates", + .json_str = + R"({"identifier":{"namespace":["ns1"],"name":"table1"},"requirements":[]})", + .expected_model = {.identifier = TableIdentifier{Namespace{{"ns1"}}, + "table1"}}}, + // Null requirements and updates are treated as absent + CommitTableRequestDeserializeParam{ + .test_name = "NullRequirementsAndUpdates", + .json_str = + R"({"identifier":{"namespace":["ns1"],"name":"table1"},"requirements":null,"updates":null})", + .expected_model = {.identifier = TableIdentifier{Namespace{{"ns1"}}, + "table1"}}}, + // Empty JSON object + CommitTableRequestDeserializeParam{ + .test_name = "EmptyJson", .json_str = R"({})", .expected_model = {}}), [](const ::testing::TestParamInfo& info) { return info.param.test_name; }); @@ -1293,23 +1317,18 @@ INSTANTIATE_TEST_SUITE_P( .invalid_json_str = R"({"identifier":{"namespace":["ns1"],"name":"table1"},"requirements":[],"updates":[{"action":"assign-uuid"}]})", .expected_error_message = "Missing 'uuid'"}, - // Missing required requirements field + // Invalid requirements - must be an array CommitTableRequestInvalidParam{ - .test_name = "MissingRequirements", + .test_name = "InvalidRequirementsNotArray", .invalid_json_str = - R"({"identifier":{"namespace":["ns1"],"name":"table1"},"updates":[]})", - .expected_error_message = "Missing 'requirements'"}, - // Missing required updates field + R"({"identifier":{"namespace":["ns1"],"name":"table1"},"requirements":{"type":"assert-create"},"updates":[]})", + .expected_error_message = "Expected 'requirements' to be an array"}, + // Invalid updates - must be an array CommitTableRequestInvalidParam{ - .test_name = "MissingUpdates", + .test_name = "InvalidUpdatesNotArray", .invalid_json_str = - R"({"identifier":{"namespace":["ns1"],"name":"table1"},"requirements":[]})", - .expected_error_message = "Missing 'updates'"}, - // Empty JSON object - CommitTableRequestInvalidParam{ - .test_name = "EmptyJson", - .invalid_json_str = R"({})", - .expected_error_message = "Missing 'requirements'"}), + R"({"identifier":{"namespace":["ns1"],"name":"table1"},"requirements":[],"updates":{"action":"assign-uuid","uuid":"2cc52516-5e73-41f2-b139-545d41a4e151"}})", + .expected_error_message = "Expected 'updates' to be an array"}), [](const ::testing::TestParamInfo& info) { return info.param.test_name; }); diff --git a/src/iceberg/test/table_metadata_builder_test.cc b/src/iceberg/test/table_metadata_builder_test.cc index 68a980b9a..f3994f397 100644 --- a/src/iceberg/test/table_metadata_builder_test.cc +++ b/src/iceberg/test/table_metadata_builder_test.cc @@ -364,6 +364,40 @@ TEST(TableMetadataBuilderTest, RemoveProperties) { EXPECT_EQ(metadata->properties.configs().at("key3"), "value3"); } +TEST(TableMetadataBuilderTest, AddAndRemoveEncryptionKey) { + auto base = CreateBaseMetadata(); + EncryptedKey key{ + .key_id = "key-1", + .encrypted_key_metadata = "secret-key-metadata", + .encrypted_by_id = "kek-1", + .properties = {{"scope", "table"}}, + }; + + auto builder = TableMetadataBuilder::BuildFrom(base.get()); + builder->AddEncryptionKey(key); + builder->AddEncryptionKey(key); + + ICEBERG_UNWRAP_OR_FAIL(auto metadata, builder->Build()); + ASSERT_EQ(metadata->encryption_keys.size(), 1); + EXPECT_EQ(metadata->encryption_keys[0], key); + ASSERT_EQ(builder->changes().size(), 1); + EXPECT_EQ(builder->changes()[0]->kind(), TableUpdate::Kind::kAddEncryptionKey); + + builder = TableMetadataBuilder::BuildFrom(metadata.get()); + builder->RemoveEncryptionKey("missing-key"); + ICEBERG_UNWRAP_OR_FAIL(auto unchanged, builder->Build()); + ASSERT_EQ(unchanged->encryption_keys.size(), 1); + EXPECT_EQ(unchanged->encryption_keys[0], key); + EXPECT_TRUE(builder->changes().empty()); + + builder = TableMetadataBuilder::BuildFrom(metadata.get()); + builder->RemoveEncryptionKey("key-1"); + ICEBERG_UNWRAP_OR_FAIL(auto removed, builder->Build()); + EXPECT_TRUE(removed->encryption_keys.empty()); + ASSERT_EQ(builder->changes().size(), 1); + EXPECT_EQ(builder->changes()[0]->kind(), TableUpdate::Kind::kRemoveEncryptionKey); +} + TEST(TableMetadataBuilderTest, UpgradeFormatVersion) { auto builder = TableMetadataBuilder::BuildFromEmpty(1); auto schema = CreateTestSchema(); diff --git a/src/iceberg/type_fwd.h b/src/iceberg/type_fwd.h index 6b4160037..2ed2835a7 100644 --- a/src/iceberg/type_fwd.h +++ b/src/iceberg/type_fwd.h @@ -245,10 +245,7 @@ class HistoryTable; class MetadataTable; class SnapshotsTable; -/// ---------------------------------------------------------------------------- -/// TODO: Forward declarations below are not added yet. -/// ---------------------------------------------------------------------------- - -class EncryptedKey; +/// \brief Table encryption +struct EncryptedKey; } // namespace iceberg