diff --git a/docker/quickstart-flink/prepare_build.sh b/docker/quickstart-flink/prepare_build.sh index f2fc2ee7d4..f439270f5d 100755 --- a/docker/quickstart-flink/prepare_build.sh +++ b/docker/quickstart-flink/prepare_build.sh @@ -115,6 +115,7 @@ check_prerequisites() { "$PROJECT_ROOT/fluss-filesystems/fluss-fs-s3/target" "$PROJECT_ROOT/fluss-lake/fluss-lake-paimon/target" "$PROJECT_ROOT/fluss-lake/fluss-lake-iceberg/target" + "$PROJECT_ROOT/fluss-lake/fluss-lake-hudi/target" "$PROJECT_ROOT/fluss-flink/fluss-flink-tiering/target" ) diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/DataLakeFormat.java b/fluss-common/src/main/java/org/apache/fluss/metadata/DataLakeFormat.java index a0dfea4693..1dc2843358 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metadata/DataLakeFormat.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/DataLakeFormat.java @@ -21,7 +21,8 @@ public enum DataLakeFormat { PAIMON("paimon"), LANCE("lance"), - ICEBERG("iceberg"); + ICEBERG("iceberg"), + HUDI("hudi"); private final String value; diff --git a/fluss-dist/pom.xml b/fluss-dist/pom.xml index 5c8a00dea5..63e08b8640 100644 --- a/fluss-dist/pom.xml +++ b/fluss-dist/pom.xml @@ -96,6 +96,13 @@ ${project.version} provided + + + org.apache.fluss + fluss-lake-hudi + ${project.version} + provided + org.apache.flink diff --git a/fluss-dist/src/main/assemblies/plugins.xml b/fluss-dist/src/main/assemblies/plugins.xml index 8eb05e8a07..02459902cc 100644 --- a/fluss-dist/src/main/assemblies/plugins.xml +++ b/fluss-dist/src/main/assemblies/plugins.xml @@ -98,6 +98,12 @@ plugins/lance/ fluss-lake-lance-${project.version}.jar + + ../fluss-lake/fluss-lake-hudi/target/fluss-lake-hudi-${project.version}.jar + plugins/hudi/ + fluss-lake-hudi-${project.version}.jar + 0644 + \ No newline at end of file diff --git a/fluss-flink/fluss-flink-common/pom.xml b/fluss-flink/fluss-flink-common/pom.xml index 53ac05701a..dab880b58b 100644 --- a/fluss-flink/fluss-flink-common/pom.xml +++ b/fluss-flink/fluss-flink-common/pom.xml @@ -95,7 +95,6 @@ provided - org.apache.fluss diff --git a/fluss-lake/fluss-lake-hudi/pom.xml b/fluss-lake/fluss-lake-hudi/pom.xml new file mode 100644 index 0000000000..ddf6b1329d --- /dev/null +++ b/fluss-lake/fluss-lake-hudi/pom.xml @@ -0,0 +1,43 @@ + + + + + 4.0.0 + + org.apache.fluss + fluss-lake + 1.0-SNAPSHOT + + + fluss-lake-hudi + Fluss : Lake : Hudi + + jar + + + + + org.apache.fluss + fluss-common + ${project.version} + provided + + + + \ No newline at end of file diff --git a/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/HudiLakeStorage.java b/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/HudiLakeStorage.java new file mode 100644 index 0000000000..f0865de0ea --- /dev/null +++ b/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/HudiLakeStorage.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.hudi; + +import org.apache.fluss.config.Configuration; +import org.apache.fluss.lake.lakestorage.LakeCatalog; +import org.apache.fluss.lake.lakestorage.LakeStorage; +import org.apache.fluss.lake.source.LakeSource; +import org.apache.fluss.lake.writer.LakeTieringFactory; +import org.apache.fluss.metadata.TablePath; + +/** Hudi Implementation of {@link LakeStorage}. */ +public class HudiLakeStorage implements LakeStorage { + + private final Configuration hudiConfig; + + public HudiLakeStorage(Configuration configuration) { + this.hudiConfig = configuration; + } + + @Override + public LakeTieringFactory createLakeTieringFactory() { + throw new UnsupportedOperationException( + "HudiLakeStorage is currently a scaffold and does not support creating a " + + "LakeTieringFactory yet. Verify that Hudi lake storage was selected " + + "intentionally and that the required Hudi support/module is available."); + } + + @Override + public LakeCatalog createLakeCatalog() { + throw new UnsupportedOperationException( + "HudiLakeStorage is currently a scaffold and does not support creating a " + + "LakeCatalog yet. Verify that Hudi lake storage was selected " + + "intentionally and that the required Hudi support/module is available."); + } + + @Override + public LakeSource createLakeSource(TablePath tablePath) { + throw new UnsupportedOperationException( + "HudiLakeStorage is currently a scaffold and does not support creating a " + + "LakeSource for table '" + + tablePath + + "' yet. Verify that Hudi lake storage was selected intentionally " + + "and that the required Hudi support/module is available."); + } +} diff --git a/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/HudiLakeStoragePlugin.java b/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/HudiLakeStoragePlugin.java new file mode 100644 index 0000000000..124aa0a546 --- /dev/null +++ b/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/HudiLakeStoragePlugin.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.lake.hudi; + +import org.apache.fluss.config.Configuration; +import org.apache.fluss.lake.lakestorage.LakeStorage; +import org.apache.fluss.lake.lakestorage.LakeStoragePlugin; + +/** Hudi implementation of {@link LakeStoragePlugin}. */ +public class HudiLakeStoragePlugin implements LakeStoragePlugin { + + private static final String IDENTIFIER = "hudi"; + + @Override + public String identifier() { + return IDENTIFIER; + } + + @Override + public LakeStorage createLakeStorage(Configuration configuration) { + return new HudiLakeStorage(configuration); + } +} diff --git a/fluss-lake/fluss-lake-hudi/src/main/resources/META-INF/NOTICE b/fluss-lake/fluss-lake-hudi/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000..91800d5c87 --- /dev/null +++ b/fluss-lake/fluss-lake-hudi/src/main/resources/META-INF/NOTICE @@ -0,0 +1,7 @@ +fluss-lake-hudi +Copyright 2025-2026 The Apache Software Foundation + +This project includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) \ No newline at end of file diff --git a/fluss-lake/fluss-lake-hudi/src/main/resources/META-INF/services/org.apache.fluss.lake.lakestorage.LakeStoragePlugin b/fluss-lake/fluss-lake-hudi/src/main/resources/META-INF/services/org.apache.fluss.lake.lakestorage.LakeStoragePlugin new file mode 100644 index 0000000000..2932e74267 --- /dev/null +++ b/fluss-lake/fluss-lake-hudi/src/main/resources/META-INF/services/org.apache.fluss.lake.lakestorage.LakeStoragePlugin @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.fluss.lake.hudi.HudiLakeStoragePlugin \ No newline at end of file diff --git a/fluss-lake/fluss-lake-hudi/src/test/resources/log4j2-test.properties b/fluss-lake/fluss-lake-hudi/src/test/resources/log4j2-test.properties new file mode 100644 index 0000000000..2100a34e0b --- /dev/null +++ b/fluss-lake/fluss-lake-hudi/src/test/resources/log4j2-test.properties @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Set root logger level to OFF to not flood build logs +# set manually to INFO for debugging purposes +rootLogger.level=OFF +rootLogger.appenderRef.test.ref=TestLogger +appender.testlogger.name=TestLogger +appender.testlogger.type=CONSOLE +appender.testlogger.target=SYSTEM_ERR +appender.testlogger.layout.type=PatternLayout +appender.testlogger.layout.pattern=%-4r [%t] %-5p %c %x - %m%n \ No newline at end of file diff --git a/fluss-lake/fluss-lake-hudi/src/test/resources/org.junit.jupiter.api.extension.Extension b/fluss-lake/fluss-lake-hudi/src/test/resources/org.junit.jupiter.api.extension.Extension new file mode 100644 index 0000000000..ca0e907f6d --- /dev/null +++ b/fluss-lake/fluss-lake-hudi/src/test/resources/org.junit.jupiter.api.extension.Extension @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.fluss.testutils.common.TestLoggerExtension \ No newline at end of file diff --git a/fluss-lake/pom.xml b/fluss-lake/pom.xml index 1cd65133c6..1ee0dbb48f 100644 --- a/fluss-lake/pom.xml +++ b/fluss-lake/pom.xml @@ -79,6 +79,7 @@ fluss-lake-paimon fluss-lake-iceberg fluss-lake-lance + fluss-lake-hudi pom \ No newline at end of file diff --git a/pom.xml b/pom.xml index f0f0ef351d..dc5ec59d62 100644 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,7 @@ 15.0.0 1.3.1 1.10.1 + 1.1.0 1.3.0