From 5928388c89492b6f3135e356f4d3f58916281205 Mon Sep 17 00:00:00 2001 From: Samrat002 Date: Fri, 11 Sep 2026 17:20:51 +0530 Subject: [PATCH] [FLINK-40632][s3] Annotate native-s3-fs to PublicEvolving --- docs/content.zh/docs/deployment/filesystems/s3.md | 14 +++++--------- docs/content/docs/deployment/filesystems/s3.md | 14 +++++--------- .../fs/s3native/NativeS3AFileSystemFactory.java | 4 ++-- .../fs/s3native/NativeS3FileSystemFactory.java | 4 ++-- .../s3native/writer/NativeS3RecoverableWriter.java | 4 ++-- 5 files changed, 16 insertions(+), 24 deletions(-) diff --git a/docs/content.zh/docs/deployment/filesystems/s3.md b/docs/content.zh/docs/deployment/filesystems/s3.md index f8f7bad7a8e8e9..45f87ec9c51808 100644 --- a/docs/content.zh/docs/deployment/filesystems/s3.md +++ b/docs/content.zh/docs/deployment/filesystems/s3.md @@ -68,10 +68,10 @@ Note that these examples are *not* exhaustive and you can use S3 in other places Flink provides three independent S3 filesystem implementations: -| Implementation | Checkpointing | FileSink | Notes | -|---------------|:---:|:---:|-------| -| **Native S3** (`flink-s3-fs-native`) | ✓ | ✓ | **Experimental** in Flink 2.3. Built on AWS SDK v2; no Hadoop dependency. | -| **Presto S3** (`flink-s3-fs-presto`) | ✓ | x | Production-proven for checkpointing. | +| Implementation | Checkpointing | FileSink | Notes | +|---------------|:---:|:---:|--------------------------------------------------------------------------------------------| +| **Native S3** (`flink-s3-fs-native`) | ✓ | ✓ | Built on AWS SDK v2; no Hadoop dependency. Generally available since Flink 2.4. | +| **Presto S3** (`flink-s3-fs-presto`) | ✓ | x | Production-proven for checkpointing. | | **Hadoop S3** (`flink-s3-fs-hadoop`) | ✓ | ✓ | Mature; the only stable implementation that provides `RecoverableWriter` for the FileSink. | Previously, users had to choose between Presto (recommended for checkpointing throughput) and Hadoop (the only implementation with `RecoverableWriter`, required by the [FileSink]({{< ref "docs/connectors/datastream/filesystem" >}})). The Native S3 implementation unifies both capabilities in a single plugin and measurements show significant checkpoint throughput improvements over the Presto implementation. @@ -149,11 +149,7 @@ The legacy configuration key `s3.path.style.access` is still supported as a fall ## Implementation Details -### Native S3 FileSystem (Experimental) - -{{< hint warning >}} -**Experimental**: The Native S3 FileSystem is experimental in Flink 2.3. It is functionally complete and has demonstrated strong performance in benchmarks. -{{< /hint >}} +### Native S3 FileSystem The Native S3 FileSystem is a pure-Java implementation built on the AWS SDK v2 completely removing the dependency on Hadoop. It is registered under the schemes *s3://* and *s3a://*. It provides a drop-in replacement for the Presto and Hadoop implementations, supporting checkpointing, the [FileSink]({{< ref "docs/connectors/datastream/filesystem" >}}) (via `RecoverableWriter`), server-side encryption (SSE-S3, SSE-KMS), cross-account access via IAM role assumption, entropy injection, and bulk copy via S3TransferManager. diff --git a/docs/content/docs/deployment/filesystems/s3.md b/docs/content/docs/deployment/filesystems/s3.md index 36c2b55bad31e3..9f84f9e634968c 100644 --- a/docs/content/docs/deployment/filesystems/s3.md +++ b/docs/content/docs/deployment/filesystems/s3.md @@ -68,10 +68,10 @@ Note that these examples are *not* exhaustive and you can use S3 in other places Flink provides three independent S3 filesystem implementations: -| Implementation | Checkpointing | FileSink | Notes | -|---------------|:---:|:---:|-------| -| **Native S3** (`flink-s3-fs-native`) | ✓ | ✓ | **Experimental** in Flink 2.3. Built on AWS SDK v2; no Hadoop dependency. | -| **Presto S3** (`flink-s3-fs-presto`) | ✓ | x | Production-proven for checkpointing. | +| Implementation | Checkpointing | FileSink | Notes | +|---------------|:---:|:---:|--------------------------------------------------------------------------------------------| +| **Native S3** (`flink-s3-fs-native`) | ✓ | ✓ | Built on AWS SDK v2; no Hadoop dependency. Generally available since Flink 2.4. | +| **Presto S3** (`flink-s3-fs-presto`) | ✓ | x | Production-proven for checkpointing. | | **Hadoop S3** (`flink-s3-fs-hadoop`) | ✓ | ✓ | Mature; the only stable implementation that provides `RecoverableWriter` for the FileSink. | Previously, users had to choose between Presto (recommended for checkpointing throughput) and Hadoop (the only implementation with `RecoverableWriter`, required by the [FileSink]({{< ref "docs/connectors/datastream/filesystem" >}})). The Native S3 implementation unifies both capabilities in a single plugin and measurements show significant checkpoint throughput improvements over the Presto implementation. @@ -149,11 +149,7 @@ The legacy configuration key `s3.path.style.access` is still supported as a fall ## Implementation Details -### Native S3 FileSystem (Experimental) - -{{< hint warning >}} -**Experimental**: The Native S3 FileSystem is experimental in Flink 2.3. It is functionally complete and has demonstrated strong performance in benchmarks. -{{< /hint >}} +### Native S3 FileSystem The Native S3 FileSystem is a pure-Java implementation built on the AWS SDK v2 completely removing the dependency on Hadoop. It is registered under the schemes *s3://* and *s3a://*. It provides a drop-in replacement for the Presto and Hadoop implementations, supporting checkpointing, the [FileSink]({{< ref "docs/connectors/datastream/filesystem" >}}) (via `RecoverableWriter`), server-side encryption (SSE-S3, SSE-KMS), cross-account access via IAM role assumption, entropy injection, and bulk copy via S3TransferManager. diff --git a/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/NativeS3AFileSystemFactory.java b/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/NativeS3AFileSystemFactory.java index a91853ae440720..6f83af7cfe8009 100644 --- a/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/NativeS3AFileSystemFactory.java +++ b/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/NativeS3AFileSystemFactory.java @@ -18,7 +18,7 @@ package org.apache.flink.fs.s3native; -import org.apache.flink.annotation.Experimental; +import org.apache.flink.annotation.PublicEvolving; /** * Factory for the native S3 file system registered for the {@code s3a://} scheme. @@ -29,7 +29,7 @@ *

All configuration options are the same as for the {@code s3://} scheme. See {@link * NativeS3FileSystemFactory} for available options. */ -@Experimental +@PublicEvolving public class NativeS3AFileSystemFactory extends NativeS3FileSystemFactory { @Override diff --git a/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/NativeS3FileSystemFactory.java b/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/NativeS3FileSystemFactory.java index 63f6ac118ee98d..e057bf5ea271ff 100644 --- a/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/NativeS3FileSystemFactory.java +++ b/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/NativeS3FileSystemFactory.java @@ -18,7 +18,7 @@ package org.apache.flink.fs.s3native; -import org.apache.flink.annotation.Experimental; +import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.Configuration; @@ -61,7 +61,7 @@ * @see NativeS3FileSystem * @see org.apache.flink.core.fs.FileSystemFactory */ -@Experimental +@PublicEvolving public class NativeS3FileSystemFactory implements FileSystemFactory, MetricsAware { private static final Logger LOG = LoggerFactory.getLogger(NativeS3FileSystemFactory.class); diff --git a/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/writer/NativeS3RecoverableWriter.java b/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/writer/NativeS3RecoverableWriter.java index ab84189a6a3467..6ff0dd96cfbf72 100644 --- a/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/writer/NativeS3RecoverableWriter.java +++ b/flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/writer/NativeS3RecoverableWriter.java @@ -18,7 +18,7 @@ package org.apache.flink.fs.s3native.writer; -import org.apache.flink.annotation.Experimental; +import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.core.fs.Path; import org.apache.flink.core.fs.RecoverableFsDataOutputStream; import org.apache.flink.core.fs.RecoverableWriter; @@ -36,7 +36,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull; /** Recoverable writer for S3 using multipart uploads for exactly-once semantics. */ -@Experimental +@PublicEvolving public class NativeS3RecoverableWriter implements RecoverableWriter, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger(NativeS3RecoverableWriter.class);