From f28af2d27e059c3e6d4807ad43571017716fde41 Mon Sep 17 00:00:00 2001 From: dentiny Date: Sat, 30 May 2026 23:58:03 +0000 Subject: [PATCH 01/13] expose all write options --- bindings/c/include/opendal.h | 183 +++++++++ bindings/c/src/entry.rs | 2 +- bindings/c/src/lib.rs | 2 + bindings/c/src/metadata.rs | 2 +- bindings/c/src/operator.rs | 59 ++- bindings/c/src/operator_info.rs | 15 + bindings/c/src/presign.rs | 2 +- bindings/c/src/reader.rs | 2 +- bindings/c/src/types.rs | 260 ++++++++++++- bindings/go/operator_info.go | 20 + bindings/go/string_ownership_test.go | 135 +++++++ bindings/go/tests/behavior_tests/stat_test.go | 64 ++- .../go/tests/behavior_tests/write_test.go | 187 +++++++++ bindings/go/types.go | 26 ++ bindings/go/writer.go | 365 +++++++++++++++++- 15 files changed, 1308 insertions(+), 16 deletions(-) diff --git a/bindings/c/include/opendal.h b/bindings/c/include/opendal.h index 5a040ef92f21..af3f7522ca92 100644 --- a/bindings/c/include/opendal.h +++ b/bindings/c/include/opendal.h @@ -307,6 +307,81 @@ typedef struct opendal_operator_options { void *inner; } opendal_operator_options; +/** + * \brief A key-value pair for write user metadata. + */ +typedef struct opendal_write_user_metadata_pair { + /** + * The metadata key. + */ + const char *key; + /** + * The metadata value. + */ + const char *value; +} opendal_write_user_metadata_pair; + +/** + * \brief The options for write operations. + * + * Use `opendal_write_options_new()` to construct and + * `opendal_write_options_free()` to free. + */ +typedef struct opendal_write_options { + /** + * Append data to the existing file. + */ + bool append; + /** + * Cache-Control header value. + */ + const char *cache_control; + /** + * Content-Type header value. + */ + const char *content_type; + /** + * Content-Disposition header value. + */ + const char *content_disposition; + /** + * Content-Encoding header value. + */ + const char *content_encoding; + /** + * If-Match header value. + */ + const char *if_match; + /** + * If-None-Match header value. + */ + const char *if_none_match; + /** + * Only write if target does not exist. + */ + bool if_not_exists; + /** + * Concurrent write operations. + */ + uintptr_t concurrent; + /** + * Whether `chunk` has been set. + */ + bool has_chunk; + /** + * Chunk size for buffered writes. + */ + uintptr_t chunk; + /** + * User metadata pairs. + */ + const struct opendal_write_user_metadata_pair *user_metadata; + /** + * User metadata pairs length. + */ + uintptr_t user_metadata_len; +} opendal_write_options; + /** * \brief The result type returned by opendal's read operation. * @@ -560,10 +635,30 @@ typedef struct opendal_capability { * If operator supports write with content disposition. */ bool write_with_content_disposition; + /** + * If operator supports write with content encoding. + */ + bool write_with_content_encoding; /** * If operator supports write with cache control. */ bool write_with_cache_control; + /** + * If operator supports write with if match. + */ + bool write_with_if_match; + /** + * If operator supports write with if none match. + */ + bool write_with_if_none_match; + /** + * If operator supports write with if not exists. + */ + bool write_with_if_not_exists; + /** + * If operator supports write with user metadata. + */ + bool write_with_user_metadata; /** * write_multi_max_size is the max size that services support in write_multi. * @@ -1030,6 +1125,14 @@ struct opendal_error *opendal_operator_write(const struct opendal_operator *op, const char *path, const struct opendal_bytes *bytes); +/** + * \brief Blocking write raw bytes to `path` with options. + */ +struct opendal_error *opendal_operator_write_with(const struct opendal_operator *op, + const char *path, + const struct opendal_bytes *bytes, + const struct opendal_write_options *opts); + /** * \brief Blocking read the data from `path`. * @@ -1156,6 +1259,13 @@ struct opendal_result_operator_reader opendal_operator_reader(const struct opend struct opendal_result_operator_writer opendal_operator_writer(const struct opendal_operator *op, const char *path); +/** + * \brief Blocking create a writer for the specified path with options. + */ +struct opendal_result_operator_writer opendal_operator_writer_with(const struct opendal_operator *op, + const char *path, + const struct opendal_write_options *opts); + /** * \brief Blocking delete the object in `path`. * @@ -1685,6 +1795,79 @@ void opendal_list_options_set_recursive(struct opendal_list_options *opts, bool */ void opendal_list_options_free(struct opendal_list_options *opts); +/** + * \brief Construct a heap-allocated opendal_write_options with default values. + */ +struct opendal_write_options *opendal_write_options_new(void); + +/** + * \brief Free the heap memory used by opendal_write_options. + */ +void opendal_write_options_free(struct opendal_write_options *opts); + +/** + * \brief Set append mode. + */ +void opendal_write_options_set_append(struct opendal_write_options *opts, bool append); + +/** + * \brief Set Cache-Control. + */ +void opendal_write_options_set_cache_control(struct opendal_write_options *opts, + const char *cache_control); + +/** + * \brief Set Content-Type. + */ +void opendal_write_options_set_content_type(struct opendal_write_options *opts, + const char *content_type); + +/** + * \brief Set Content-Disposition. + */ +void opendal_write_options_set_content_disposition(struct opendal_write_options *opts, + const char *content_disposition); + +/** + * \brief Set Content-Encoding. + */ +void opendal_write_options_set_content_encoding(struct opendal_write_options *opts, + const char *content_encoding); + +/** + * \brief Set If-Match. + */ +void opendal_write_options_set_if_match(struct opendal_write_options *opts, const char *if_match); + +/** + * \brief Set If-None-Match. + */ +void opendal_write_options_set_if_none_match(struct opendal_write_options *opts, + const char *if_none_match); + +/** + * \brief Set if_not_exists. + */ +void opendal_write_options_set_if_not_exists(struct opendal_write_options *opts, + bool if_not_exists); + +/** + * \brief Set concurrent. + */ +void opendal_write_options_set_concurrent(struct opendal_write_options *opts, uintptr_t concurrent); + +/** + * \brief Set chunk. + */ +void opendal_write_options_set_chunk(struct opendal_write_options *opts, uintptr_t chunk); + +/** + * \brief Set user metadata. + */ +void opendal_write_options_set_user_metadata(struct opendal_write_options *opts, + const struct opendal_write_user_metadata_pair *pairs, + uintptr_t len); + /** * \brief Construct a heap-allocated opendal_operator_options * diff --git a/bindings/c/src/entry.rs b/bindings/c/src/entry.rs index 1f3eb9c1b42d..094ace09977b 100644 --- a/bindings/c/src/entry.rs +++ b/bindings/c/src/entry.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::ffi::{c_void, CString}; +use std::ffi::{CString, c_void}; use std::os::raw::c_char; use ::opendal as core; diff --git a/bindings/c/src/lib.rs b/bindings/c/src/lib.rs index 5592288bdb09..af06c9b3a94e 100644 --- a/bindings/c/src/lib.rs +++ b/bindings/c/src/lib.rs @@ -74,6 +74,8 @@ mod types; pub use types::opendal_bytes; pub use types::opendal_list_options; pub use types::opendal_operator_options; +pub use types::opendal_write_options; +pub use types::opendal_write_user_metadata_pair; mod entry; pub use entry::opendal_entry; diff --git a/bindings/c/src/metadata.rs b/bindings/c/src/metadata.rs index fe0b4254e2a5..cd3e3521bd52 100644 --- a/bindings/c/src/metadata.rs +++ b/bindings/c/src/metadata.rs @@ -17,7 +17,7 @@ use ::opendal as core; use std::collections::HashMap; -use std::ffi::{c_char, c_void, CString}; +use std::ffi::{CString, c_char, c_void}; use std::ptr; /// \brief A user metadata key-value pair. diff --git a/bindings/c/src/operator.rs b/bindings/c/src/operator.rs index 4b3a058d1648..7377836035cc 100644 --- a/bindings/c/src/operator.rs +++ b/bindings/c/src/operator.rs @@ -281,6 +281,29 @@ pub unsafe extern "C" fn opendal_operator_write( } } +/// \brief Blocking write raw bytes to `path` with options. +#[no_mangle] +pub unsafe extern "C" fn opendal_operator_write_with( + op: &opendal_operator, + path: *const c_char, + bytes: &opendal_bytes, + opts: *const opendal_write_options, +) -> *mut opendal_error { + assert!(!path.is_null()); + let path = std::ffi::CStr::from_ptr(path) + .to_str() + .expect("malformed path"); + let opts = if opts.is_null() { + core::options::WriteOptions::default() + } else { + (&*opts).into() + }; + match op.deref().write_options(path, bytes, opts) { + Ok(_) => std::ptr::null_mut(), + Err(e) => opendal_error::new(e), + } +} + /// \brief Blocking read the data from `path`. /// /// Read the data out from `path` blocking by operator. @@ -392,7 +415,7 @@ pub unsafe extern "C" fn opendal_operator_reader( return opendal_result_operator_reader { reader: std::ptr::null_mut(), error: opendal_error::new(err), - } + }; } }; @@ -459,7 +482,39 @@ pub unsafe extern "C" fn opendal_operator_writer( return opendal_result_operator_writer { writer: std::ptr::null_mut(), error: opendal_error::new(err), - } + }; + } + }; + + opendal_result_operator_writer { + writer: Box::into_raw(Box::new(opendal_writer::new(writer))), + error: std::ptr::null_mut(), + } +} + +/// \brief Blocking create a writer for the specified path with options. +#[no_mangle] +pub unsafe extern "C" fn opendal_operator_writer_with( + op: &opendal_operator, + path: *const c_char, + opts: *const opendal_write_options, +) -> opendal_result_operator_writer { + assert!(!path.is_null()); + let path = std::ffi::CStr::from_ptr(path) + .to_str() + .expect("malformed path"); + let opts = if opts.is_null() { + core::options::WriteOptions::default() + } else { + (&*opts).into() + }; + let writer = match op.deref().writer_options(path, opts) { + Ok(writer) => writer, + Err(err) => { + return opendal_result_operator_writer { + writer: std::ptr::null_mut(), + error: opendal_error::new(err), + }; } }; diff --git a/bindings/c/src/operator_info.rs b/bindings/c/src/operator_info.rs index 796ea9be2675..e7b83cb5d85a 100644 --- a/bindings/c/src/operator_info.rs +++ b/bindings/c/src/operator_info.rs @@ -75,8 +75,18 @@ pub struct opendal_capability { pub write_with_content_type: bool, /// If operator supports write with content disposition. pub write_with_content_disposition: bool, + /// If operator supports write with content encoding. + pub write_with_content_encoding: bool, /// If operator supports write with cache control. pub write_with_cache_control: bool, + /// If operator supports write with if match. + pub write_with_if_match: bool, + /// If operator supports write with if none match. + pub write_with_if_none_match: bool, + /// If operator supports write with if not exists. + pub write_with_if_not_exists: bool, + /// If operator supports write with user metadata. + pub write_with_user_metadata: bool, /// write_multi_max_size is the max size that services support in write_multi. /// /// For example, AWS S3 supports 5GiB as max in write_multi. @@ -239,7 +249,12 @@ impl From for opendal_capability { write_can_append: value.write_can_append, write_with_content_type: value.write_with_content_type, write_with_content_disposition: value.write_with_content_disposition, + write_with_content_encoding: value.write_with_content_encoding, write_with_cache_control: value.write_with_cache_control, + write_with_if_match: value.write_with_if_match, + write_with_if_none_match: value.write_with_if_none_match, + write_with_if_not_exists: value.write_with_if_not_exists, + write_with_user_metadata: value.write_with_user_metadata, write_multi_max_size: value.write_multi_max_size.unwrap_or(0), write_multi_min_size: value.write_multi_min_size.unwrap_or(0), write_total_max_size: value.write_total_max_size.unwrap_or(0), diff --git a/bindings/c/src/presign.rs b/bindings/c/src/presign.rs index 705f3334a8b9..c582c71d900d 100644 --- a/bindings/c/src/presign.rs +++ b/bindings/c/src/presign.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::ffi::{c_char, CStr, CString}; +use std::ffi::{CStr, CString, c_char}; use std::time::Duration; use opendal::raw::PresignedRequest as ocorePresignedRequest; diff --git a/bindings/c/src/reader.rs b/bindings/c/src/reader.rs index ff798f5c8490..1335de0edf41 100644 --- a/bindings/c/src/reader.rs +++ b/bindings/c/src/reader.rs @@ -96,7 +96,7 @@ impl opendal_reader { core::ErrorKind::Unexpected, "undefined whence", )), - } + }; } }; diff --git a/bindings/c/src/types.rs b/bindings/c/src/types.rs index 1b63f24ceb9d..1c656cde9613 100644 --- a/bindings/c/src/types.rs +++ b/bindings/c/src/types.rs @@ -16,10 +16,11 @@ // under the License. use std::collections::HashMap; -use std::ffi::{c_void, CString}; +use std::ffi::{CStr, CString, c_void}; use std::os::raw::c_char; use opendal::Buffer; +use opendal::options; /// \brief Frees a heap-allocated string returned by OpenDAL C APIs. /// @@ -136,6 +137,263 @@ impl opendal_list_options { } } +/// \brief A key-value pair for write user metadata. +#[repr(C)] +pub struct opendal_write_user_metadata_pair { + /// The metadata key. + pub key: *const c_char, + /// The metadata value. + pub value: *const c_char, +} + +/// \brief The options for write operations. +/// +/// Use `opendal_write_options_new()` to construct and +/// `opendal_write_options_free()` to free. +#[repr(C)] +pub struct opendal_write_options { + /// Append data to the existing file. + pub append: bool, + /// Cache-Control header value. + pub cache_control: *const c_char, + /// Content-Type header value. + pub content_type: *const c_char, + /// Content-Disposition header value. + pub content_disposition: *const c_char, + /// Content-Encoding header value. + pub content_encoding: *const c_char, + /// If-Match header value. + pub if_match: *const c_char, + /// If-None-Match header value. + pub if_none_match: *const c_char, + /// Only write if target does not exist. + pub if_not_exists: bool, + /// Concurrent write operations. + pub concurrent: usize, + /// Whether `chunk` has been set. + pub has_chunk: bool, + /// Chunk size for buffered writes. + pub chunk: usize, + /// User metadata pairs. + pub user_metadata: *const opendal_write_user_metadata_pair, + /// User metadata pairs length. + pub user_metadata_len: usize, +} + +impl opendal_write_options { + /// \brief Construct a heap-allocated opendal_write_options with default values. + #[no_mangle] + pub extern "C" fn opendal_write_options_new() -> *mut Self { + Box::into_raw(Box::new(Self::default())) + } + + /// \brief Free the heap memory used by opendal_write_options. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_free(opts: *mut opendal_write_options) { + if !opts.is_null() { + drop(Box::from_raw(opts)); + } + } + + /// \brief Set append mode. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_append( + opts: *mut opendal_write_options, + append: bool, + ) { + if !opts.is_null() { + (*opts).append = append; + } + } + + /// \brief Set Cache-Control. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_cache_control( + opts: *mut opendal_write_options, + cache_control: *const c_char, + ) { + if !opts.is_null() { + (*opts).cache_control = cache_control; + } + } + + /// \brief Set Content-Type. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_content_type( + opts: *mut opendal_write_options, + content_type: *const c_char, + ) { + if !opts.is_null() { + (*opts).content_type = content_type; + } + } + + /// \brief Set Content-Disposition. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_content_disposition( + opts: *mut opendal_write_options, + content_disposition: *const c_char, + ) { + if !opts.is_null() { + (*opts).content_disposition = content_disposition; + } + } + + /// \brief Set Content-Encoding. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_content_encoding( + opts: *mut opendal_write_options, + content_encoding: *const c_char, + ) { + if !opts.is_null() { + (*opts).content_encoding = content_encoding; + } + } + + /// \brief Set If-Match. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_if_match( + opts: *mut opendal_write_options, + if_match: *const c_char, + ) { + if !opts.is_null() { + (*opts).if_match = if_match; + } + } + + /// \brief Set If-None-Match. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_if_none_match( + opts: *mut opendal_write_options, + if_none_match: *const c_char, + ) { + if !opts.is_null() { + (*opts).if_none_match = if_none_match; + } + } + + /// \brief Set if_not_exists. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_if_not_exists( + opts: *mut opendal_write_options, + if_not_exists: bool, + ) { + if !opts.is_null() { + (*opts).if_not_exists = if_not_exists; + } + } + + /// \brief Set concurrent. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_concurrent( + opts: *mut opendal_write_options, + concurrent: usize, + ) { + if !opts.is_null() { + (*opts).concurrent = concurrent; + } + } + + /// \brief Set chunk. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_chunk( + opts: *mut opendal_write_options, + chunk: usize, + ) { + if !opts.is_null() { + (*opts).has_chunk = true; + (*opts).chunk = chunk; + } + } + + /// \brief Set user metadata. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_user_metadata( + opts: *mut opendal_write_options, + pairs: *const opendal_write_user_metadata_pair, + len: usize, + ) { + if !opts.is_null() { + (*opts).user_metadata = pairs; + (*opts).user_metadata_len = len; + } + } +} + +impl Default for opendal_write_options { + fn default() -> Self { + Self { + append: false, + cache_control: std::ptr::null(), + content_type: std::ptr::null(), + content_disposition: std::ptr::null(), + content_encoding: std::ptr::null(), + if_match: std::ptr::null(), + if_none_match: std::ptr::null(), + if_not_exists: false, + concurrent: 0, + has_chunk: false, + chunk: 0, + user_metadata: std::ptr::null(), + user_metadata_len: 0, + } + } +} + +unsafe fn optional_cstr(ptr: *const c_char) -> Option { + if ptr.is_null() { + None + } else { + Some( + CStr::from_ptr(ptr) + .to_str() + .expect("malformed option") + .to_string(), + ) + } +} + +impl From<&opendal_write_options> for options::WriteOptions { + fn from(value: &opendal_write_options) -> Self { + let user_metadata = if value.user_metadata.is_null() || value.user_metadata_len == 0 { + None + } else { + let pairs = + unsafe { std::slice::from_raw_parts(value.user_metadata, value.user_metadata_len) }; + let mut metadata = HashMap::with_capacity(pairs.len()); + for pair in pairs { + if pair.key.is_null() || pair.value.is_null() { + continue; + } + let key = unsafe { CStr::from_ptr(pair.key) } + .to_str() + .expect("malformed user metadata key") + .to_string(); + let value = unsafe { CStr::from_ptr(pair.value) } + .to_str() + .expect("malformed user metadata value") + .to_string(); + metadata.insert(key, value); + } + Some(metadata) + }; + + Self { + append: value.append, + cache_control: unsafe { optional_cstr(value.cache_control) }, + content_type: unsafe { optional_cstr(value.content_type) }, + content_disposition: unsafe { optional_cstr(value.content_disposition) }, + content_encoding: unsafe { optional_cstr(value.content_encoding) }, + user_metadata, + if_match: unsafe { optional_cstr(value.if_match) }, + if_none_match: unsafe { optional_cstr(value.if_none_match) }, + if_not_exists: value.if_not_exists, + concurrent: value.concurrent, + chunk: value.has_chunk.then_some(value.chunk), + } + } +} + impl Drop for opendal_bytes { fn drop(&mut self) { unsafe { diff --git a/bindings/go/operator_info.go b/bindings/go/operator_info.go index 210ad110d1b0..c3e5c9297f4e 100644 --- a/bindings/go/operator_info.go +++ b/bindings/go/operator_info.go @@ -160,6 +160,26 @@ func (c *Capability) WriteWithCacheControl() bool { return c.inner.writeWithCacheControl == 1 } +func (c *Capability) WriteWithContentEncoding() bool { + return c.inner.writeWithContentEncoding == 1 +} + +func (c *Capability) WriteWithIfMatch() bool { + return c.inner.writeWithIfMatch == 1 +} + +func (c *Capability) WriteWithIfNoneMatch() bool { + return c.inner.writeWithIfNoneMatch == 1 +} + +func (c *Capability) WriteWithIfNotExists() bool { + return c.inner.writeWithIfNotExists == 1 +} + +func (c *Capability) WriteWithUserMetadata() bool { + return c.inner.writeWithUserMetadata == 1 +} + func (c *Capability) WriteMultiMaxSize() uint { return c.inner.writeMultiMaxSize } diff --git a/bindings/go/string_ownership_test.go b/bindings/go/string_ownership_test.go index 400ece17c305..a4957f96a3e4 100644 --- a/bindings/go/string_ownership_test.go +++ b/bindings/go/string_ownership_test.go @@ -635,6 +635,141 @@ func assertFreedPointers(t *testing.T, got []*byte, want ...*byte) { } } +func TestWriteWithOptions(t *testing.T) { + o := &writeOptions{} + WriteWithAppend(true)(o) + WriteWithCacheControl("max-age=60")(o) + WriteWithContentType("text/plain")(o) + WriteWithContentDisposition("attachment")(o) + WriteWithContentEncoding("gzip")(o) + WriteWithUserMetadata(map[string]string{"foo": "bar"})(o) + WriteWithIfMatch("etag-a")(o) + WriteWithIfNoneMatch("etag-b")(o) + WriteWithIfNotExists(true)(o) + WriteWithConcurrent(4)(o) + WriteWithChunk(1024)(o) + + if !o.append { + t.Fatalf("append = false, want true") + } + if o.cacheControl != "max-age=60" { + t.Fatalf("cacheControl = %q, want max-age=60", o.cacheControl) + } + if o.contentType != "text/plain" { + t.Fatalf("contentType = %q, want text/plain", o.contentType) + } + if o.contentDisposition != "attachment" { + t.Fatalf("contentDisposition = %q, want attachment", o.contentDisposition) + } + if o.contentEncoding != "gzip" { + t.Fatalf("contentEncoding = %q, want gzip", o.contentEncoding) + } + assertStringMap(t, o.userMetadata, map[string]string{"foo": "bar"}) + if o.ifMatch != "etag-a" { + t.Fatalf("ifMatch = %q, want etag-a", o.ifMatch) + } + if o.ifNoneMatch != "etag-b" { + t.Fatalf("ifNoneMatch = %q, want etag-b", o.ifNoneMatch) + } + if !o.ifNotExists { + t.Fatalf("ifNotExists = false, want true") + } + if o.concurrent != 4 { + t.Fatalf("concurrent = %d, want 4", o.concurrent) + } + if o.chunk != 1024 { + t.Fatalf("chunk = %d, want 1024", o.chunk) + } +} + +func TestFfiOperatorWriteWithArgTypes(t *testing.T) { + aTypes := ffiOperatorWriteWith.opts.aTypes + if len(aTypes) != 4 { + t.Fatalf("ffiOperatorWriteWith aTypes len = %d, want 4", len(aTypes)) + } + for i, at := range aTypes { + if at != &ffi.TypePointer { + t.Fatalf("ffiOperatorWriteWith aTypes[%d] = %v, want TypePointer", i, at) + } + } +} + +func TestFfiOperatorWriterWithArgTypes(t *testing.T) { + aTypes := ffiOperatorWriterWith.opts.aTypes + if len(aTypes) != 3 { + t.Fatalf("ffiOperatorWriterWith aTypes len = %d, want 3", len(aTypes)) + } + for i, at := range aTypes { + if at != &ffi.TypePointer { + t.Fatalf("ffiOperatorWriterWith aTypes[%d] = %v, want TypePointer", i, at) + } + } +} + +func TestWriteOptionsSetterArgTypes(t *testing.T) { + stringSetters := []*FFI[func(*opendalWriteOptions, string) (*byte, error)]{ + ffiWriteOptionsSetCacheControl, + ffiWriteOptionsSetContentType, + ffiWriteOptionsSetContentDisposition, + ffiWriteOptionsSetContentEncoding, + ffiWriteOptionsSetIfMatch, + ffiWriteOptionsSetIfNoneMatch, + } + for _, setter := range stringSetters { + aTypes := setter.opts.aTypes + if len(aTypes) != 2 { + t.Fatalf("%s aTypes len = %d, want 2", setter.opts.sym, len(aTypes)) + } + for i, at := range aTypes { + if at != &ffi.TypePointer { + t.Fatalf("%s aTypes[%d] = %v, want TypePointer", setter.opts.sym, i, at) + } + } + } + + boolSetters := []*FFI[func(*opendalWriteOptions, bool)]{ + ffiWriteOptionsSetAppend, + ffiWriteOptionsSetIfNotExists, + } + for _, setter := range boolSetters { + aTypes := setter.opts.aTypes + if len(aTypes) != 2 { + t.Fatalf("%s aTypes len = %d, want 2", setter.opts.sym, len(aTypes)) + } + if aTypes[0] != &ffi.TypePointer || aTypes[1] != &ffi.TypeUint8 { + t.Fatalf("%s aTypes = %v, want TypePointer, TypeUint8", setter.opts.sym, aTypes) + } + } + + uintSetters := []*FFI[func(*opendalWriteOptions, uint)]{ + ffiWriteOptionsSetConcurrent, + ffiWriteOptionsSetChunk, + } + for _, setter := range uintSetters { + aTypes := setter.opts.aTypes + if len(aTypes) != 2 { + t.Fatalf("%s aTypes len = %d, want 2", setter.opts.sym, len(aTypes)) + } + for i, at := range aTypes { + if at != &ffi.TypePointer { + t.Fatalf("%s aTypes[%d] = %v, want TypePointer", setter.opts.sym, i, at) + } + } + } +} + +func TestWriteOptionsSetUserMetadataArgTypes(t *testing.T) { + aTypes := ffiWriteOptionsSetUserMetadata.opts.aTypes + if len(aTypes) != 3 { + t.Fatalf("ffiWriteOptionsSetUserMetadata aTypes len = %d, want 3", len(aTypes)) + } + for i, at := range aTypes { + if at != &ffi.TypePointer { + t.Fatalf("ffiWriteOptionsSetUserMetadata aTypes[%d] = %v, want TypePointer", i, at) + } + } +} + func TestListWithRecursiveDefaultNotRecursive(t *testing.T) { o := &listOptions{} if o.recursive { diff --git a/bindings/go/tests/behavior_tests/stat_test.go b/bindings/go/tests/behavior_tests/stat_test.go index 52d12e8c8848..82269d55677f 100644 --- a/bindings/go/tests/behavior_tests/stat_test.go +++ b/bindings/go/tests/behavior_tests/stat_test.go @@ -156,9 +156,35 @@ func testStatRoot(assert *require.Assertions, op *opendal.Operator, fixture *fix func testStatFileMetadata(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { path, content, size := fixture.NewFile() + cap := op.Info().GetFullCapability() + + writeOpts := make([]opendal.WithWriteFn, 0, 5) + if cap.WriteWithCacheControl() { + writeOpts = append(writeOpts, opendal.WriteWithCacheControl("max-age=60")) + } + if cap.WriteWithContentDisposition() { + writeOpts = append(writeOpts, opendal.WriteWithContentDisposition("attachment; filename=hello.txt")) + } + if cap.WriteWithContentEncoding() { + writeOpts = append(writeOpts, opendal.WriteWithContentEncoding("gzip")) + } + if cap.WriteWithContentType() { + writeOpts = append(writeOpts, opendal.WriteWithContentType("text/plain")) + } + userMetadata := map[string]string{ + "language": "go", + "project": "opendal", + } + if cap.WriteWithUserMetadata() { + writeOpts = append(writeOpts, opendal.WriteWithUserMetadata(userMetadata)) + } before := time.Now().Add(-time.Hour) - assert.Nil(op.Write(path, content), "write must succeed") + if len(writeOpts) == 0 { + assert.Nil(op.Write(path, content), "write must succeed") + } else { + assert.Nil(op.WriteWith(path, content, writeOpts...), "write with metadata must succeed") + } meta, err := op.Stat(path) assert.Nil(err, "stat must succeed") @@ -174,10 +200,34 @@ func testStatFileMetadata(assert *require.Assertions, op *opendal.Operator, fixt assert.False(lm.After(time.Now().Add(time.Minute)), "last_modified must not be in the future, got %v", lm) } - assertOptionalMetaString(assert, "cache control", meta.CacheControl) - assertOptionalMetaString(assert, "content disposition", meta.ContentDisposition) - assertOptionalMetaString(assert, "content encoding", meta.ContentEncoding) - assertOptionalMetaString(assert, "content type", meta.ContentType) + if cap.WriteWithCacheControl() { + cacheControl, ok := meta.CacheControl() + assert.True(ok, "cache control must exist") + assert.Equal("max-age=60", cacheControl) + } else { + assertOptionalMetaString(assert, "cache control", meta.CacheControl) + } + if cap.WriteWithContentDisposition() { + contentDisposition, ok := meta.ContentDisposition() + assert.True(ok, "content disposition must exist") + assert.Equal("attachment; filename=hello.txt", contentDisposition) + } else { + assertOptionalMetaString(assert, "content disposition", meta.ContentDisposition) + } + if cap.WriteWithContentEncoding() { + contentEncoding, ok := meta.ContentEncoding() + assert.True(ok, "content encoding must exist") + assert.Equal("gzip", contentEncoding) + } else { + assertOptionalMetaString(assert, "content encoding", meta.ContentEncoding) + } + if cap.WriteWithContentType() { + contentType, ok := meta.ContentType() + assert.True(ok, "content type must exist") + assert.Equal("text/plain", contentType) + } else { + assertOptionalMetaString(assert, "content type", meta.ContentType) + } assertOptionalMetaString(assert, "content md5", meta.ContentMD5) assertOptionalMetaString(assert, "etag", meta.ETag) assertOptionalMetaString(assert, "version", meta.Version) @@ -185,7 +235,9 @@ func testStatFileMetadata(assert *require.Assertions, op *opendal.Operator, fixt if isCurrent, ok := meta.IsCurrent(); ok { assert.True(isCurrent, "a live object must be reported as the current version") } - if um := meta.UserMetadata(); um != nil { + if cap.WriteWithUserMetadata() { + assert.Equal(userMetadata, meta.UserMetadata()) + } else if um := meta.UserMetadata(); um != nil { assert.Equal(um, meta.UserMetadata(), "user metadata accessor must return equal copies") } } diff --git a/bindings/go/tests/behavior_tests/write_test.go b/bindings/go/tests/behavior_tests/write_test.go index 05d626972d5f..fe343b30a8a8 100644 --- a/bindings/go/tests/behavior_tests/write_test.go +++ b/bindings/go/tests/behavior_tests/write_test.go @@ -35,7 +35,17 @@ func testsWrite(cap *opendal.Capability) []behaviorTest { testWriteWithDirPath, testWriteWithSpecialChars, testWriteOverwrite, + testWriteWithCacheControl, + testWriteWithContentType, + testWriteWithContentDisposition, + testWriteWithContentEncoding, + testWriteWithUserMetadata, + testWriteWithIfMatch, + testWriteWithIfNoneMatch, + testWriteWithIfNotExists, testWriterWrite, + testWriteWithChunkAndConcurrent, + testWriterWithAppend, } } @@ -101,6 +111,150 @@ func testWriteOverwrite(assert *require.Assertions, op *opendal.Operator, fixtur assert.Equal(contentTwo, bs, "read content_two") } +func testWriteWithCacheControl(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteWithCacheControl() { + return + } + + path := fixture.NewFilePath() + content := []byte("hello") + assert.Nil(op.WriteWith(path, content, opendal.WriteWithCacheControl("max-age=60"))) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(len(content)), meta.ContentLength()) + cacheControl, ok := meta.CacheControl() + assert.True(ok, "cache control must exist") + assert.Equal("max-age=60", cacheControl) +} + +func testWriteWithContentType(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteWithContentType() { + return + } + + path := fixture.NewFilePath() + content := []byte("hello") + assert.Nil(op.WriteWith(path, content, opendal.WriteWithContentType("text/plain"))) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(len(content)), meta.ContentLength()) + contentType, ok := meta.ContentType() + assert.True(ok, "content type must exist") + assert.Equal("text/plain", contentType) +} + +func testWriteWithContentDisposition(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteWithContentDisposition() { + return + } + + path := fixture.NewFilePath() + content := []byte("hello") + assert.Nil(op.WriteWith(path, content, opendal.WriteWithContentDisposition("attachment; filename=hello.txt"))) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(len(content)), meta.ContentLength()) + contentDisposition, ok := meta.ContentDisposition() + assert.True(ok, "content disposition must exist") + assert.Equal("attachment; filename=hello.txt", contentDisposition) +} + +func testWriteWithContentEncoding(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteWithContentEncoding() { + return + } + + path := fixture.NewFilePath() + content := []byte("hello") + assert.Nil(op.WriteWith(path, content, opendal.WriteWithContentEncoding("gzip"))) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(len(content)), meta.ContentLength()) + contentEncoding, ok := meta.ContentEncoding() + assert.True(ok, "content encoding must exist") + assert.Equal("gzip", contentEncoding) +} + +func testWriteWithUserMetadata(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteWithUserMetadata() { + return + } + + path := fixture.NewFilePath() + content := []byte("hello") + assert.Nil(op.WriteWith(path, content, opendal.WriteWithUserMetadata(map[string]string{ + "language": "go", + "project": "opendal", + }))) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(len(content)), meta.ContentLength()) + assert.Equal(map[string]string{ + "language": "go", + "project": "opendal", + }, meta.UserMetadata()) +} + +func testWriteWithIfMatch(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteWithIfMatch() { + return + } + + path := fixture.NewFilePath() + assert.Nil(op.Write(path, []byte("hello"))) + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + etag, ok := meta.ETag() + assert.True(ok, "etag must exist") + + assert.Nil(op.WriteWith(path, []byte("world"), opendal.WriteWithIfMatch(etag))) + bs, err := op.Read(path) + assert.Nil(err, "read must succeed") + assert.Equal([]byte("world"), bs) +} + +func testWriteWithIfNoneMatch(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteWithIfNoneMatch() { + return + } + + path := fixture.NewFilePath() + assert.Nil(op.Write(path, []byte("hello"))) + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + etag, ok := meta.ETag() + assert.True(ok, "etag must exist") + + err = op.WriteWith(path, []byte("world"), opendal.WriteWithIfNoneMatch(etag)) + assert.NotNil(err) + assert.Equal(opendal.CodeConditionNotMatch, assertErrorCode(err)) + + bs, err := op.Read(path) + assert.Nil(err, "read must succeed") + assert.Equal([]byte("hello"), bs) +} + +func testWriteWithIfNotExists(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteWithIfNotExists() { + return + } + + path := fixture.NewFilePath() + assert.Nil(op.WriteWith(path, []byte("hello"), opendal.WriteWithIfNotExists(true))) + err := op.WriteWith(path, []byte("world"), opendal.WriteWithIfNotExists(true)) + assert.NotNil(err) + assert.Equal(opendal.CodeAlreadyExists, assertErrorCode(err)) + + bs, err := op.Read(path) + assert.Nil(err, "read must succeed") + assert.Equal([]byte("hello"), bs) +} + func testWriterWrite(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { if !op.Info().GetFullCapability().WriteCanMulti() { return @@ -129,3 +283,36 @@ func testWriterWrite(assert *require.Assertions, op *opendal.Operator, fixture * assert.Equal(contentA, bs[:size], "read contentA") assert.Equal(contentB, bs[size:], "read contentB") } + +func testWriteWithChunkAndConcurrent(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteCanMulti() { + return + } + + path := fixture.NewFilePath() + content := genFixedBytes(1024 * 1024) + assert.Nil(op.WriteWith(path, content, opendal.WriteWithChunk(256*1024), opendal.WriteWithConcurrent(2))) + + bs, err := op.Read(path) + assert.Nil(err, "read must succeed") + assert.Equal(content, bs) +} + +func testWriterWithAppend(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !op.Info().GetFullCapability().WriteCanAppend() { + return + } + + path := fixture.NewFilePath() + assert.Nil(op.Write(path, []byte("hello"))) + + w, err := op.WriterWith(path, opendal.WriteWithAppend(true)) + assert.Nil(err) + _, err = w.Write([]byte(" world")) + assert.Nil(err) + assert.Nil(w.Close()) + + bs, err := op.Read(path) + assert.Nil(err, "read must succeed") + assert.Equal([]byte("hello world"), bs) +} diff --git a/bindings/go/types.go b/bindings/go/types.go index a19e3f9935a3..47da28182e5c 100644 --- a/bindings/go/types.go +++ b/bindings/go/types.go @@ -108,6 +108,15 @@ var ( }[0], } + typeWriteUserMetadataPair = ffi.Type{ + Type: ffi.Struct, + Elements: &[]*ffi.Type{ + &ffi.TypePointer, + &ffi.TypePointer, + nil, + }[0], + } + typeResultReaderRead = ffi.Type{ Type: ffi.Struct, Elements: &[]*ffi.Type{ @@ -162,7 +171,12 @@ var ( &ffi.TypeUint8, // write_can_append &ffi.TypeUint8, // write_with_content_type &ffi.TypeUint8, // write_with_content_disposition + &ffi.TypeUint8, // write_with_content_encoding &ffi.TypeUint8, // write_with_cache_control + &ffi.TypeUint8, // write_with_if_match + &ffi.TypeUint8, // write_with_if_none_match + &ffi.TypeUint8, // write_with_if_not_exists + &ffi.TypeUint8, // write_with_user_metadata &ffi.TypePointer, // write_multi_max_size &ffi.TypePointer, // write_multi_min_size &ffi.TypePointer, // write_total_max_size @@ -201,7 +215,12 @@ type opendalCapability struct { writeCanAppend uint8 writeWithContentType uint8 writeWithContentDisposition uint8 + writeWithContentEncoding uint8 writeWithCacheControl uint8 + writeWithIfMatch uint8 + writeWithIfNoneMatch uint8 + writeWithIfNotExists uint8 + writeWithUserMetadata uint8 writeMultiMaxSize uint writeMultiMinSize uint writeTotalMaxSize uint @@ -315,6 +334,13 @@ type opendalLister struct{} type opendalListOptions struct{} +type opendalWriteOptions struct{} + +type opendalWriteUserMetadataPair struct { + key *byte + value *byte +} + type opendalResultListerNext struct { entry *opendalEntry err *opendalError diff --git a/bindings/go/writer.go b/bindings/go/writer.go index c2d0e5088bf2..3714fa44cd41 100644 --- a/bindings/go/writer.go +++ b/bindings/go/writer.go @@ -22,6 +22,7 @@ package opendal import ( "context" "io" + "runtime" "unsafe" "github.com/jupiterrider/ffi" @@ -30,9 +31,7 @@ import ( // Write writes the given bytes to the specified path. // // Write is a wrapper around the C-binding function `opendal_operator_write`. It provides a simplified -// interface for writing data to the storage. Currently, this implementation does not support the -// `Operator::write_with` method from the original Rust library, nor does it support streaming writes -// or multipart uploads. +// interface for writing data to the storage. // // # Parameters // @@ -57,6 +56,188 @@ func (op *Operator) Write(path string, data []byte) error { return ffiOperatorWrite.symbol(op.ctx)(op.inner, path, data) } +// WithWriteFn is a functional option for write operations. +type WithWriteFn func(*writeOptions) + +// WriteWithAppend sets append mode for the write operation. +func WriteWithAppend(append bool) WithWriteFn { + return func(o *writeOptions) { + o.append = append + } +} + +// WriteWithCacheControl sets the Cache-Control header for the write operation. +func WriteWithCacheControl(cacheControl string) WithWriteFn { + return func(o *writeOptions) { + o.cacheControl = cacheControl + } +} + +// WriteWithContentType sets the Content-Type header for the write operation. +func WriteWithContentType(contentType string) WithWriteFn { + return func(o *writeOptions) { + o.contentType = contentType + } +} + +// WriteWithContentDisposition sets the Content-Disposition header for the write operation. +func WriteWithContentDisposition(contentDisposition string) WithWriteFn { + return func(o *writeOptions) { + o.contentDisposition = contentDisposition + } +} + +// WriteWithContentEncoding sets the Content-Encoding header for the write operation. +func WriteWithContentEncoding(contentEncoding string) WithWriteFn { + return func(o *writeOptions) { + o.contentEncoding = contentEncoding + } +} + +// WriteWithUserMetadata sets user metadata for the write operation. +func WriteWithUserMetadata(userMetadata map[string]string) WithWriteFn { + return func(o *writeOptions) { + o.userMetadata = userMetadata + } +} + +// WriteWithIfMatch sets the If-Match condition for the write operation. +func WriteWithIfMatch(ifMatch string) WithWriteFn { + return func(o *writeOptions) { + o.ifMatch = ifMatch + } +} + +// WriteWithIfNoneMatch sets the If-None-Match condition for the write operation. +func WriteWithIfNoneMatch(ifNoneMatch string) WithWriteFn { + return func(o *writeOptions) { + o.ifNoneMatch = ifNoneMatch + } +} + +// WriteWithIfNotExists sets whether the write operation should only succeed if the target does not exist. +func WriteWithIfNotExists(ifNotExists bool) WithWriteFn { + return func(o *writeOptions) { + o.ifNotExists = ifNotExists + } +} + +// WriteWithConcurrent sets concurrent write operations. +func WriteWithConcurrent(concurrent uint) WithWriteFn { + return func(o *writeOptions) { + o.concurrent = concurrent + } +} + +// WriteWithChunk sets the chunk size for buffered writes. +func WriteWithChunk(chunk uint) WithWriteFn { + return func(o *writeOptions) { + o.chunk = chunk + } +} + +type writeOptions struct { + append bool + cacheControl string + contentType string + contentDisposition string + contentEncoding string + userMetadata map[string]string + ifMatch string + ifNoneMatch string + ifNotExists bool + concurrent uint + chunk uint +} + +// WriteWith writes the given bytes to the specified path with options. +func (op *Operator) WriteWith(path string, data []byte, opts ...WithWriteFn) error { + o := parseWriteOptions(opts...) + cOpts, keepAlive, err := newOpendalWriteOptions(op.ctx, o) + if err != nil { + return err + } + defer ffiWriteOptionsFree.symbol(op.ctx)(cOpts) + err = ffiOperatorWriteWith.symbol(op.ctx)(op.inner, path, data, cOpts) + runtime.KeepAlive(keepAlive) + return err +} + +func parseWriteOptions(opts ...WithWriteFn) *writeOptions { + o := &writeOptions{} + for _, opt := range opts { + opt(o) + } + return o +} + +type writeOptionsKeepAlive struct { + strings []*byte + userMetadata []opendalWriteUserMetadataPair +} + +func newOpendalWriteOptions(ctx context.Context, o *writeOptions) (*opendalWriteOptions, writeOptionsKeepAlive, error) { + cOpts := ffiWriteOptionsNew.symbol(ctx)() + keepAlive := writeOptionsKeepAlive{} + ffiWriteOptionsSetAppend.symbol(ctx)(cOpts, o.append) + + setString := func(value string, set func(*opendalWriteOptions, string) (*byte, error)) error { + if value == "" { + return nil + } + ptr, err := set(cOpts, value) + if err != nil { + return err + } + keepAlive.strings = append(keepAlive.strings, ptr) + return nil + } + + if err := setString(o.cacheControl, ffiWriteOptionsSetCacheControl.symbol(ctx)); err != nil { + return nil, writeOptionsKeepAlive{}, err + } + if err := setString(o.contentType, ffiWriteOptionsSetContentType.symbol(ctx)); err != nil { + return nil, writeOptionsKeepAlive{}, err + } + if err := setString(o.contentDisposition, ffiWriteOptionsSetContentDisposition.symbol(ctx)); err != nil { + return nil, writeOptionsKeepAlive{}, err + } + if err := setString(o.contentEncoding, ffiWriteOptionsSetContentEncoding.symbol(ctx)); err != nil { + return nil, writeOptionsKeepAlive{}, err + } + if err := setString(o.ifMatch, ffiWriteOptionsSetIfMatch.symbol(ctx)); err != nil { + return nil, writeOptionsKeepAlive{}, err + } + if err := setString(o.ifNoneMatch, ffiWriteOptionsSetIfNoneMatch.symbol(ctx)); err != nil { + return nil, writeOptionsKeepAlive{}, err + } + + ffiWriteOptionsSetIfNotExists.symbol(ctx)(cOpts, o.ifNotExists) + if o.concurrent != 0 { + ffiWriteOptionsSetConcurrent.symbol(ctx)(cOpts, o.concurrent) + } + if o.chunk != 0 { + ffiWriteOptionsSetChunk.symbol(ctx)(cOpts, o.chunk) + } + if len(o.userMetadata) > 0 { + keepAlive.userMetadata = make([]opendalWriteUserMetadataPair, 0, len(o.userMetadata)) + for key, value := range o.userMetadata { + byteKey, err := BytePtrFromString(key) + if err != nil { + return nil, writeOptionsKeepAlive{}, err + } + byteValue, err := BytePtrFromString(value) + if err != nil { + return nil, writeOptionsKeepAlive{}, err + } + keepAlive.strings = append(keepAlive.strings, byteKey, byteValue) + keepAlive.userMetadata = append(keepAlive.userMetadata, opendalWriteUserMetadataPair{key: byteKey, value: byteValue}) + } + ffiWriteOptionsSetUserMetadata.symbol(ctx)(cOpts, keepAlive.userMetadata) + } + return cOpts, keepAlive, nil +} + // CreateDir creates a directory at the specified path. // // CreateDir is a wrapper around the C-binding function `opendal_operator_create_dir`. @@ -136,6 +317,26 @@ func (op *Operator) Writer(path string) (*Writer, error) { return writer, nil } +// WriterWith returns a new Writer for the specified path with options. +func (op *Operator) WriterWith(path string, opts ...WithWriteFn) (*Writer, error) { + o := parseWriteOptions(opts...) + cOpts, keepAlive, err := newOpendalWriteOptions(op.ctx, o) + if err != nil { + return nil, err + } + defer ffiWriteOptionsFree.symbol(op.ctx)(cOpts) + inner, err := ffiOperatorWriterWith.symbol(op.ctx)(op.inner, path, cOpts) + runtime.KeepAlive(keepAlive) + if err != nil { + return nil, err + } + writer := &Writer{ + inner: inner, + ctx: op.ctx, + } + return writer, nil +} + type Writer struct { inner *opendalWriter ctx context.Context @@ -204,6 +405,29 @@ var ffiOperatorWrite = newFFI(ffiOpts{ } }) +var ffiOperatorWriteWith = newFFI(ffiOpts{ + sym: "opendal_operator_write_with", + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall ffiCall) func(op *opendalOperator, path string, data []byte, opts *opendalWriteOptions) error { + return func(op *opendalOperator, path string, data []byte, opts *opendalWriteOptions) error { + bytePath, err := BytePtrFromString(path) + if err != nil { + return err + } + bytes := toOpendalBytes(data) + var e *opendalError + ffiCall( + unsafe.Pointer(&e), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + unsafe.Pointer(&bytes), + unsafe.Pointer(&opts), + ) + return parseError(ctx, e) + } +}) + var ffiOperatorCreateDir = newFFI(ffiOpts{ sym: "opendal_operator_create_dir", rType: &ffi.TypePointer, @@ -247,6 +471,141 @@ var ffiOperatorWriter = newFFI(ffiOpts{ } }) +var ffiOperatorWriterWith = newFFI(ffiOpts{ + sym: "opendal_operator_writer_with", + rType: &typeResultOperatorWriter, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall ffiCall) func(op *opendalOperator, path string, opts *opendalWriteOptions) (*opendalWriter, error) { + return func(op *opendalOperator, path string, opts *opendalWriteOptions) (*opendalWriter, error) { + bytePath, err := BytePtrFromString(path) + if err != nil { + return nil, err + } + var result resultOperatorWriter + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + unsafe.Pointer(&opts), + ) + if result.error != nil { + return nil, parseError(ctx, result.error) + } + return result.writer, nil + } +}) + +var ffiWriteOptionsNew = newFFI(ffiOpts{ + sym: "opendal_write_options_new", + rType: &ffi.TypePointer, +}, func(_ context.Context, ffiCall ffiCall) func() *opendalWriteOptions { + return func() *opendalWriteOptions { + var opts *opendalWriteOptions + ffiCall(unsafe.Pointer(&opts)) + return opts + } +}) + +var ffiWriteOptionsFree = newFFI(ffiOpts{ + sym: "opendal_write_options_free", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions) { + return func(opts *opendalWriteOptions) { + ffiCall( + nil, + unsafe.Pointer(&opts), + ) + } +}) + +var ffiWriteOptionsSetAppend = newFFI(ffiOpts{ + sym: "opendal_write_options_set_append", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypeUint8}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions, append bool) { + return func(opts *opendalWriteOptions, append bool) { + var v uint8 + if append { + v = 1 + } + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&v)) + } +}) + +func newWriteOptionsSetStringFFI(sym string) *FFI[func(*opendalWriteOptions, string) (*byte, error)] { + return newFFI(ffiOpts{ + sym: contextKey(sym), + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, + }, func(_ context.Context, ffiCall ffiCall) func(*opendalWriteOptions, string) (*byte, error) { + return func(opts *opendalWriteOptions, value string) (*byte, error) { + byteValue, err := BytePtrFromString(value) + if err != nil { + return nil, err + } + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&byteValue)) + return byteValue, nil + } + }) +} + +var ffiWriteOptionsSetCacheControl = newWriteOptionsSetStringFFI("opendal_write_options_set_cache_control") +var ffiWriteOptionsSetContentType = newWriteOptionsSetStringFFI("opendal_write_options_set_content_type") +var ffiWriteOptionsSetContentDisposition = newWriteOptionsSetStringFFI("opendal_write_options_set_content_disposition") +var ffiWriteOptionsSetContentEncoding = newWriteOptionsSetStringFFI("opendal_write_options_set_content_encoding") +var ffiWriteOptionsSetIfMatch = newWriteOptionsSetStringFFI("opendal_write_options_set_if_match") +var ffiWriteOptionsSetIfNoneMatch = newWriteOptionsSetStringFFI("opendal_write_options_set_if_none_match") + +var ffiWriteOptionsSetIfNotExists = newFFI(ffiOpts{ + sym: "opendal_write_options_set_if_not_exists", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypeUint8}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions, ifNotExists bool) { + return func(opts *opendalWriteOptions, ifNotExists bool) { + var v uint8 + if ifNotExists { + v = 1 + } + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&v)) + } +}) + +var ffiWriteOptionsSetConcurrent = newFFI(ffiOpts{ + sym: "opendal_write_options_set_concurrent", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions, concurrent uint) { + return func(opts *opendalWriteOptions, concurrent uint) { + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&concurrent)) + } +}) + +var ffiWriteOptionsSetChunk = newFFI(ffiOpts{ + sym: "opendal_write_options_set_chunk", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions, chunk uint) { + return func(opts *opendalWriteOptions, chunk uint) { + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&chunk)) + } +}) + +var ffiWriteOptionsSetUserMetadata = newFFI(ffiOpts{ + sym: "opendal_write_options_set_user_metadata", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions, userMetadata []opendalWriteUserMetadataPair) { + return func(opts *opendalWriteOptions, userMetadata []opendalWriteUserMetadataPair) { + var ptr *opendalWriteUserMetadataPair + if len(userMetadata) > 0 { + ptr = &userMetadata[0] + } + length := uint(len(userMetadata)) + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&ptr), unsafe.Pointer(&length)) + } +}) + var ffiWriterFree = newFFI(ffiOpts{ sym: "opendal_writer_free", rType: &ffi.TypeVoid, From 673ac1435b954cf46e6f3b14839d2db038f14e7b Mon Sep 17 00:00:00 2001 From: dentiny Date: Sat, 30 May 2026 23:59:42 +0000 Subject: [PATCH 02/13] fix memleak on error path --- bindings/go/writer.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/bindings/go/writer.go b/bindings/go/writer.go index 3714fa44cd41..46d445d29d2a 100644 --- a/bindings/go/writer.go +++ b/bindings/go/writer.go @@ -181,6 +181,12 @@ func newOpendalWriteOptions(ctx context.Context, o *writeOptions) (*opendalWrite keepAlive := writeOptionsKeepAlive{} ffiWriteOptionsSetAppend.symbol(ctx)(cOpts, o.append) + // fail frees the C-allocated options before returning + fail := func(err error) (*opendalWriteOptions, writeOptionsKeepAlive, error) { + ffiWriteOptionsFree.symbol(ctx)(cOpts) + return nil, writeOptionsKeepAlive{}, err + } + setString := func(value string, set func(*opendalWriteOptions, string) (*byte, error)) error { if value == "" { return nil @@ -194,22 +200,22 @@ func newOpendalWriteOptions(ctx context.Context, o *writeOptions) (*opendalWrite } if err := setString(o.cacheControl, ffiWriteOptionsSetCacheControl.symbol(ctx)); err != nil { - return nil, writeOptionsKeepAlive{}, err + return fail(err) } if err := setString(o.contentType, ffiWriteOptionsSetContentType.symbol(ctx)); err != nil { - return nil, writeOptionsKeepAlive{}, err + return fail(err) } if err := setString(o.contentDisposition, ffiWriteOptionsSetContentDisposition.symbol(ctx)); err != nil { - return nil, writeOptionsKeepAlive{}, err + return fail(err) } if err := setString(o.contentEncoding, ffiWriteOptionsSetContentEncoding.symbol(ctx)); err != nil { - return nil, writeOptionsKeepAlive{}, err + return fail(err) } if err := setString(o.ifMatch, ffiWriteOptionsSetIfMatch.symbol(ctx)); err != nil { - return nil, writeOptionsKeepAlive{}, err + return fail(err) } if err := setString(o.ifNoneMatch, ffiWriteOptionsSetIfNoneMatch.symbol(ctx)); err != nil { - return nil, writeOptionsKeepAlive{}, err + return fail(err) } ffiWriteOptionsSetIfNotExists.symbol(ctx)(cOpts, o.ifNotExists) @@ -224,11 +230,11 @@ func newOpendalWriteOptions(ctx context.Context, o *writeOptions) (*opendalWrite for key, value := range o.userMetadata { byteKey, err := BytePtrFromString(key) if err != nil { - return nil, writeOptionsKeepAlive{}, err + return fail(err) } byteValue, err := BytePtrFromString(value) if err != nil { - return nil, writeOptionsKeepAlive{}, err + return fail(err) } keepAlive.strings = append(keepAlive.strings, byteKey, byteValue) keepAlive.userMetadata = append(keepAlive.userMetadata, opendalWriteUserMetadataPair{key: byteKey, value: byteValue}) From dbc3310a775687ba56a8be99db997baffa46b6bd Mon Sep 17 00:00:00 2001 From: dentiny Date: Sun, 31 May 2026 00:00:53 +0000 Subject: [PATCH 03/13] remove dead code --- bindings/go/types.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/bindings/go/types.go b/bindings/go/types.go index 47da28182e5c..7d07268b6ffc 100644 --- a/bindings/go/types.go +++ b/bindings/go/types.go @@ -108,15 +108,6 @@ var ( }[0], } - typeWriteUserMetadataPair = ffi.Type{ - Type: ffi.Struct, - Elements: &[]*ffi.Type{ - &ffi.TypePointer, - &ffi.TypePointer, - nil, - }[0], - } - typeResultReaderRead = ffi.Type{ Type: ffi.Struct, Elements: &[]*ffi.Type{ From fd535e636a6eca759c4b1b3b1ba258ca00573dcd Mon Sep 17 00:00:00 2001 From: dentiny Date: Sun, 31 May 2026 00:02:42 +0000 Subject: [PATCH 04/13] comment on concurrent value --- bindings/c/include/opendal.h | 3 ++- bindings/c/src/types.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bindings/c/include/opendal.h b/bindings/c/include/opendal.h index af3f7522ca92..5068388f9899 100644 --- a/bindings/c/include/opendal.h +++ b/bindings/c/include/opendal.h @@ -361,7 +361,8 @@ typedef struct opendal_write_options { */ bool if_not_exists; /** - * Concurrent write operations. + * Concurrent write operations. `0` means sequential writes, so no + * separate "has_concurrent" flag is needed (unlike `chunk`). */ uintptr_t concurrent; /** diff --git a/bindings/c/src/types.rs b/bindings/c/src/types.rs index 1c656cde9613..f733f6fe5e87 100644 --- a/bindings/c/src/types.rs +++ b/bindings/c/src/types.rs @@ -168,7 +168,7 @@ pub struct opendal_write_options { pub if_none_match: *const c_char, /// Only write if target does not exist. pub if_not_exists: bool, - /// Concurrent write operations. + /// Concurrent write operations. `0` means sequential writes pub concurrent: usize, /// Whether `chunk` has been set. pub has_chunk: bool, From 7383d4fce19d124fd18cd7a0b351f8149fa3199d Mon Sep 17 00:00:00 2001 From: dentiny Date: Sun, 31 May 2026 00:32:54 +0000 Subject: [PATCH 05/13] consider test backend capability --- bindings/c/include/opendal.h | 3 +- .../go/tests/behavior_tests/opendal_test.go | 44 +++++++++++++++++++ .../go/tests/behavior_tests/write_test.go | 20 ++++----- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/bindings/c/include/opendal.h b/bindings/c/include/opendal.h index 5068388f9899..e707b98faa6d 100644 --- a/bindings/c/include/opendal.h +++ b/bindings/c/include/opendal.h @@ -361,8 +361,7 @@ typedef struct opendal_write_options { */ bool if_not_exists; /** - * Concurrent write operations. `0` means sequential writes, so no - * separate "has_concurrent" flag is needed (unlike `chunk`). + * Concurrent write operations. `0` means sequential writes */ uintptr_t concurrent; /** diff --git a/bindings/go/tests/behavior_tests/opendal_test.go b/bindings/go/tests/behavior_tests/opendal_test.go index 75c32a07c500..3c7cafab7b96 100644 --- a/bindings/go/tests/behavior_tests/opendal_test.go +++ b/bindings/go/tests/behavior_tests/opendal_test.go @@ -26,6 +26,7 @@ import ( "os" "reflect" "runtime" + "strconv" "strings" "sync" "testing" @@ -154,6 +155,49 @@ func assertErrorCode(err error) opendal.ErrorCode { return err.(*opendal.Error).Code() } +// parsedOverrides is lazily parsed from OPENDAL_TEST_CAPABILITY_OVERRIDES. +var parsedOverrides map[string]bool + +func getCapOverrides() map[string]bool { + if parsedOverrides != nil { + return parsedOverrides + } + raw := os.Getenv("OPENDAL_TEST_CAPABILITY_OVERRIDES") + parsedOverrides = make(map[string]bool) + if raw == "" { + return parsedOverrides + } + for _, pair := range strings.Split(raw, ",") { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + kv := strings.SplitN(pair, "=", 2) + if len(kv) != 2 { + continue + } + v, err := strconv.ParseBool(strings.TrimSpace(kv[1])) + if err != nil { + continue + } + parsedOverrides[strings.TrimSpace(kv[0])] = v + } + return parsedOverrides +} + +// isCapEnabled checks both the capability reported by the operator and any +// test-level overrides set via OPENDAL_TEST_CAPABILITY_OVERRIDES. +func isCapEnabled(check func() bool, name string) bool { + if !check() { + return false + } + overrides := getCapOverrides() + if v, ok := overrides[name]; ok { + return v + } + return true +} + func genBytesWithRange(min, max uint) ([]byte, uint) { diff := max - min n, _ := rand.Int(rand.Reader, big.NewInt(int64(diff+1))) diff --git a/bindings/go/tests/behavior_tests/write_test.go b/bindings/go/tests/behavior_tests/write_test.go index fe343b30a8a8..638f384ee818 100644 --- a/bindings/go/tests/behavior_tests/write_test.go +++ b/bindings/go/tests/behavior_tests/write_test.go @@ -112,7 +112,7 @@ func testWriteOverwrite(assert *require.Assertions, op *opendal.Operator, fixtur } func testWriteWithCacheControl(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { - if !op.Info().GetFullCapability().WriteWithCacheControl() { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithCacheControl, "write_with_cache_control") { return } @@ -129,7 +129,7 @@ func testWriteWithCacheControl(assert *require.Assertions, op *opendal.Operator, } func testWriteWithContentType(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { - if !op.Info().GetFullCapability().WriteWithContentType() { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithContentType, "write_with_content_type") { return } @@ -146,7 +146,7 @@ func testWriteWithContentType(assert *require.Assertions, op *opendal.Operator, } func testWriteWithContentDisposition(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { - if !op.Info().GetFullCapability().WriteWithContentDisposition() { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithContentDisposition, "write_with_content_disposition") { return } @@ -163,7 +163,7 @@ func testWriteWithContentDisposition(assert *require.Assertions, op *opendal.Ope } func testWriteWithContentEncoding(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { - if !op.Info().GetFullCapability().WriteWithContentEncoding() { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithContentEncoding, "write_with_content_encoding") { return } @@ -180,7 +180,7 @@ func testWriteWithContentEncoding(assert *require.Assertions, op *opendal.Operat } func testWriteWithUserMetadata(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { - if !op.Info().GetFullCapability().WriteWithUserMetadata() { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithUserMetadata, "write_with_user_metadata") { return } @@ -201,7 +201,7 @@ func testWriteWithUserMetadata(assert *require.Assertions, op *opendal.Operator, } func testWriteWithIfMatch(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { - if !op.Info().GetFullCapability().WriteWithIfMatch() { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithIfMatch, "write_with_if_match") { return } @@ -219,7 +219,7 @@ func testWriteWithIfMatch(assert *require.Assertions, op *opendal.Operator, fixt } func testWriteWithIfNoneMatch(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { - if !op.Info().GetFullCapability().WriteWithIfNoneMatch() { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithIfNoneMatch, "write_with_if_none_match") { return } @@ -240,7 +240,7 @@ func testWriteWithIfNoneMatch(assert *require.Assertions, op *opendal.Operator, } func testWriteWithIfNotExists(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { - if !op.Info().GetFullCapability().WriteWithIfNotExists() { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithIfNotExists, "write_with_if_not_exists") { return } @@ -285,7 +285,7 @@ func testWriterWrite(assert *require.Assertions, op *opendal.Operator, fixture * } func testWriteWithChunkAndConcurrent(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { - if !op.Info().GetFullCapability().WriteCanMulti() { + if !isCapEnabled(op.Info().GetFullCapability().WriteCanMulti, "write_can_multi") { return } @@ -299,7 +299,7 @@ func testWriteWithChunkAndConcurrent(assert *require.Assertions, op *opendal.Ope } func testWriterWithAppend(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { - if !op.Info().GetFullCapability().WriteCanAppend() { + if !isCapEnabled(op.Info().GetFullCapability().WriteCanAppend, "write_can_append") { return } From e76d1467e3ea3e1e23e4e35b9d4124978c5419a9 Mon Sep 17 00:00:00 2001 From: dentiny Date: Sun, 31 May 2026 00:51:07 +0000 Subject: [PATCH 06/13] format and fix behavior --- bindings/c/src/entry.rs | 2 +- bindings/c/src/metadata.rs | 2 +- bindings/c/src/presign.rs | 2 +- bindings/c/src/types.rs | 4 +-- bindings/go/string_ownership_test.go | 2 +- bindings/go/tests/behavior_tests/go.mod | 1 - bindings/go/tests/behavior_tests/go.sum | 2 ++ .../go/tests/behavior_tests/write_test.go | 2 +- bindings/go/writer.go | 36 ++++++++++++------- 9 files changed, 33 insertions(+), 20 deletions(-) diff --git a/bindings/c/src/entry.rs b/bindings/c/src/entry.rs index 094ace09977b..1f3eb9c1b42d 100644 --- a/bindings/c/src/entry.rs +++ b/bindings/c/src/entry.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::ffi::{CString, c_void}; +use std::ffi::{c_void, CString}; use std::os::raw::c_char; use ::opendal as core; diff --git a/bindings/c/src/metadata.rs b/bindings/c/src/metadata.rs index cd3e3521bd52..fe0b4254e2a5 100644 --- a/bindings/c/src/metadata.rs +++ b/bindings/c/src/metadata.rs @@ -17,7 +17,7 @@ use ::opendal as core; use std::collections::HashMap; -use std::ffi::{CString, c_char, c_void}; +use std::ffi::{c_char, c_void, CString}; use std::ptr; /// \brief A user metadata key-value pair. diff --git a/bindings/c/src/presign.rs b/bindings/c/src/presign.rs index c582c71d900d..705f3334a8b9 100644 --- a/bindings/c/src/presign.rs +++ b/bindings/c/src/presign.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::ffi::{CStr, CString, c_char}; +use std::ffi::{c_char, CStr, CString}; use std::time::Duration; use opendal::raw::PresignedRequest as ocorePresignedRequest; diff --git a/bindings/c/src/types.rs b/bindings/c/src/types.rs index f733f6fe5e87..7a368d552e9e 100644 --- a/bindings/c/src/types.rs +++ b/bindings/c/src/types.rs @@ -16,11 +16,11 @@ // under the License. use std::collections::HashMap; -use std::ffi::{CStr, CString, c_void}; +use std::ffi::{c_void, CStr, CString}; use std::os::raw::c_char; -use opendal::Buffer; use opendal::options; +use opendal::Buffer; /// \brief Frees a heap-allocated string returned by OpenDAL C APIs. /// diff --git a/bindings/go/string_ownership_test.go b/bindings/go/string_ownership_test.go index a4957f96a3e4..c348f2f76643 100644 --- a/bindings/go/string_ownership_test.go +++ b/bindings/go/string_ownership_test.go @@ -707,7 +707,7 @@ func TestFfiOperatorWriterWithArgTypes(t *testing.T) { } func TestWriteOptionsSetterArgTypes(t *testing.T) { - stringSetters := []*FFI[func(*opendalWriteOptions, string) (*byte, error)]{ + stringSetters := []*FFI[func(*opendalWriteOptions, string) ([]byte, error)]{ ffiWriteOptionsSetCacheControl, ffiWriteOptionsSetContentType, ffiWriteOptionsSetContentDisposition, diff --git a/bindings/go/tests/behavior_tests/go.mod b/bindings/go/tests/behavior_tests/go.mod index 1a32049bf9d2..87cfbb7e32d8 100644 --- a/bindings/go/tests/behavior_tests/go.mod +++ b/bindings/go/tests/behavior_tests/go.mod @@ -37,4 +37,3 @@ require ( golang.org/x/sys v0.22.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - diff --git a/bindings/go/tests/behavior_tests/go.sum b/bindings/go/tests/behavior_tests/go.sum index c1473d33cab7..8c30f56105f7 100644 --- a/bindings/go/tests/behavior_tests/go.sum +++ b/bindings/go/tests/behavior_tests/go.sum @@ -1,3 +1,5 @@ +github.com/apache/opendal-go-services/fs v0.1.3 h1:k5pA73gKbQ3MHH2envsKhr1cec2spLm2tl/bCyU53j8= +github.com/apache/opendal-go-services/fs v0.1.3/go.mod h1:7EnuyeXRuQh+L47rZ7y2OrhYJLlUYvgvFPItM98XJ5s= github.com/apache/opendal/bindings/go v0.0.0-20240719044908-d9d4279b3a24 h1:2fAl+WS/lZMTtP6onlrmDbb3pltf+5xNTc0Aeu9nYWE= github.com/apache/opendal/bindings/go v0.0.0-20240719044908-d9d4279b3a24/go.mod h1:jyMN6M6h0jMDZitnjvB3KPobM+oZiESrFb3XUplLxhI= github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= diff --git a/bindings/go/tests/behavior_tests/write_test.go b/bindings/go/tests/behavior_tests/write_test.go index 638f384ee818..19edab3d5783 100644 --- a/bindings/go/tests/behavior_tests/write_test.go +++ b/bindings/go/tests/behavior_tests/write_test.go @@ -248,7 +248,7 @@ func testWriteWithIfNotExists(assert *require.Assertions, op *opendal.Operator, assert.Nil(op.WriteWith(path, []byte("hello"), opendal.WriteWithIfNotExists(true))) err := op.WriteWith(path, []byte("world"), opendal.WriteWithIfNotExists(true)) assert.NotNil(err) - assert.Equal(opendal.CodeAlreadyExists, assertErrorCode(err)) + assert.Equal(opendal.CodeConditionNotMatch, assertErrorCode(err)) bs, err := op.Read(path) assert.Nil(err, "read must succeed") diff --git a/bindings/go/writer.go b/bindings/go/writer.go index 46d445d29d2a..cf54fdfdf363 100644 --- a/bindings/go/writer.go +++ b/bindings/go/writer.go @@ -21,8 +21,10 @@ package opendal import ( "context" + "errors" "io" "runtime" + "strings" "unsafe" "github.com/jupiterrider/ffi" @@ -172,7 +174,7 @@ func parseWriteOptions(opts ...WithWriteFn) *writeOptions { } type writeOptionsKeepAlive struct { - strings []*byte + strings [][]byte userMetadata []opendalWriteUserMetadataPair } @@ -187,15 +189,15 @@ func newOpendalWriteOptions(ctx context.Context, o *writeOptions) (*opendalWrite return nil, writeOptionsKeepAlive{}, err } - setString := func(value string, set func(*opendalWriteOptions, string) (*byte, error)) error { + setString := func(value string, set func(*opendalWriteOptions, string) ([]byte, error)) error { if value == "" { return nil } - ptr, err := set(cOpts, value) + data, err := set(cOpts, value) if err != nil { return err } - keepAlive.strings = append(keepAlive.strings, ptr) + keepAlive.strings = append(keepAlive.strings, data) return nil } @@ -228,15 +230,17 @@ func newOpendalWriteOptions(ctx context.Context, o *writeOptions) (*opendalWrite if len(o.userMetadata) > 0 { keepAlive.userMetadata = make([]opendalWriteUserMetadataPair, 0, len(o.userMetadata)) for key, value := range o.userMetadata { - byteKey, err := BytePtrFromString(key) + keyData, err := byteSliceFromString(key) if err != nil { return fail(err) } - byteValue, err := BytePtrFromString(value) + valueData, err := byteSliceFromString(value) if err != nil { return fail(err) } - keepAlive.strings = append(keepAlive.strings, byteKey, byteValue) + keepAlive.strings = append(keepAlive.strings, keyData, valueData) + byteKey := &keyData[0] + byteValue := &valueData[0] keepAlive.userMetadata = append(keepAlive.userMetadata, opendalWriteUserMetadataPair{key: byteKey, value: byteValue}) } ffiWriteOptionsSetUserMetadata.symbol(ctx)(cOpts, keepAlive.userMetadata) @@ -539,19 +543,27 @@ var ffiWriteOptionsSetAppend = newFFI(ffiOpts{ } }) -func newWriteOptionsSetStringFFI(sym string) *FFI[func(*opendalWriteOptions, string) (*byte, error)] { +func byteSliceFromString(value string) ([]byte, error) { + if strings.IndexByte(value, 0) >= 0 { + return nil, errors.New("string contains nul") + } + return append([]byte(value), 0), nil +} + +func newWriteOptionsSetStringFFI(sym string) *FFI[func(*opendalWriteOptions, string) ([]byte, error)] { return newFFI(ffiOpts{ sym: contextKey(sym), rType: &ffi.TypeVoid, aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, - }, func(_ context.Context, ffiCall ffiCall) func(*opendalWriteOptions, string) (*byte, error) { - return func(opts *opendalWriteOptions, value string) (*byte, error) { - byteValue, err := BytePtrFromString(value) + }, func(_ context.Context, ffiCall ffiCall) func(*opendalWriteOptions, string) ([]byte, error) { + return func(opts *opendalWriteOptions, value string) ([]byte, error) { + data, err := byteSliceFromString(value) if err != nil { return nil, err } + byteValue := &data[0] ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&byteValue)) - return byteValue, nil + return data, nil } }) } From 085bbdd088198fc19081666502b601c803beef58 Mon Sep 17 00:00:00 2001 From: dentiny Date: Sun, 31 May 2026 02:42:49 +0000 Subject: [PATCH 07/13] fix string --- bindings/go/string.go | 7 ++++++- bindings/go/string_ownership_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/bindings/go/string.go b/bindings/go/string.go index 1686aeb86ff4..0a046d75d388 100644 --- a/bindings/go/string.go +++ b/bindings/go/string.go @@ -32,7 +32,12 @@ func copyCStringAndFree(ptr *byte, free func(*byte)) string { } defer free(ptr) - return BytePtrToString(ptr) + n := 0 + for p := unsafe.Pointer(ptr); *(*byte)(p) != 0; n++ { + p = unsafe.Pointer(uintptr(p) + 1) + } + // Copy into Go-owned memory before freeing Rust's CString. + return string(append([]byte(nil), unsafe.Slice(ptr, n)...)) } var ffiStringFree = newFFI(ffiOpts{ diff --git a/bindings/go/string_ownership_test.go b/bindings/go/string_ownership_test.go index c348f2f76643..810c997026b7 100644 --- a/bindings/go/string_ownership_test.go +++ b/bindings/go/string_ownership_test.go @@ -48,6 +48,31 @@ func TestCopyCStringAndFreeNil(t *testing.T) { } } +func TestCopyCStringAndFreeOwnsReturnedString(t *testing.T) { + raw := []byte("text/plain\x00") + ptr := &raw[0] + var freed bool + + got := copyCStringAndFree(ptr, func(ptr *byte) { + freed = true + buf := unsafe.Slice(ptr, len(raw)-1) + for _, b := range buf { + if b == 0 { + t.Fatal("unexpected nul before terminator") + } + } + for i := range buf { + buf[i] = 0 + } + }) + if !freed { + t.Fatal("copyCStringAndFree did not free pointer") + } + if got != "text/plain" { + t.Fatalf("copyCStringAndFree returned %q after free mutation, want text/plain", got) + } +} + func TestOperatorInfoCopiesAndFreesOwnedStrings(t *testing.T) { var freed []*byte freeCString := func(ptr *byte) { From cc13dad01d518b6315d0d87bc99e96949055fe5f Mon Sep 17 00:00:00 2001 From: dentiny Date: Sun, 31 May 2026 03:25:52 +0000 Subject: [PATCH 08/13] append write --- bindings/go/tests/behavior_tests/write_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bindings/go/tests/behavior_tests/write_test.go b/bindings/go/tests/behavior_tests/write_test.go index 19edab3d5783..185cec1615bb 100644 --- a/bindings/go/tests/behavior_tests/write_test.go +++ b/bindings/go/tests/behavior_tests/write_test.go @@ -20,7 +20,7 @@ package opendal_test import ( - "github.com/apache/opendal/bindings/go" + opendal "github.com/apache/opendal/bindings/go" "github.com/google/uuid" "github.com/stretchr/testify/require" ) @@ -304,10 +304,15 @@ func testWriterWithAppend(assert *require.Assertions, op *opendal.Operator, fixt } path := fixture.NewFilePath() - assert.Nil(op.Write(path, []byte("hello"))) w, err := op.WriterWith(path, opendal.WriteWithAppend(true)) assert.Nil(err) + _, err = w.Write([]byte("hello")) + assert.Nil(err) + assert.Nil(w.Close()) + + w, err = op.WriterWith(path, opendal.WriteWithAppend(true)) + assert.Nil(err) _, err = w.Write([]byte(" world")) assert.Nil(err) assert.Nil(w.Close()) From f2e39ec33acd56ee92295f8db3e062896536ebb0 Mon Sep 17 00:00:00 2001 From: dentiny Date: Sun, 31 May 2026 05:28:08 +0000 Subject: [PATCH 09/13] consider test backend --- bindings/go/tests/behavior_tests/stat_test.go | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/bindings/go/tests/behavior_tests/stat_test.go b/bindings/go/tests/behavior_tests/stat_test.go index 82269d55677f..9d282892f0ef 100644 --- a/bindings/go/tests/behavior_tests/stat_test.go +++ b/bindings/go/tests/behavior_tests/stat_test.go @@ -159,23 +159,29 @@ func testStatFileMetadata(assert *require.Assertions, op *opendal.Operator, fixt cap := op.Info().GetFullCapability() writeOpts := make([]opendal.WithWriteFn, 0, 5) - if cap.WriteWithCacheControl() { + writeWithCacheControl := isCapEnabled(cap.WriteWithCacheControl, "write_with_cache_control") + writeWithContentDisposition := isCapEnabled(cap.WriteWithContentDisposition, "write_with_content_disposition") + writeWithContentEncoding := isCapEnabled(cap.WriteWithContentEncoding, "write_with_content_encoding") + writeWithContentType := isCapEnabled(cap.WriteWithContentType, "write_with_content_type") + writeWithUserMetadata := isCapEnabled(cap.WriteWithUserMetadata, "write_with_user_metadata") + + if writeWithCacheControl { writeOpts = append(writeOpts, opendal.WriteWithCacheControl("max-age=60")) } - if cap.WriteWithContentDisposition() { + if writeWithContentDisposition { writeOpts = append(writeOpts, opendal.WriteWithContentDisposition("attachment; filename=hello.txt")) } - if cap.WriteWithContentEncoding() { + if writeWithContentEncoding { writeOpts = append(writeOpts, opendal.WriteWithContentEncoding("gzip")) } - if cap.WriteWithContentType() { + if writeWithContentType { writeOpts = append(writeOpts, opendal.WriteWithContentType("text/plain")) } userMetadata := map[string]string{ "language": "go", "project": "opendal", } - if cap.WriteWithUserMetadata() { + if writeWithUserMetadata { writeOpts = append(writeOpts, opendal.WriteWithUserMetadata(userMetadata)) } @@ -200,28 +206,28 @@ func testStatFileMetadata(assert *require.Assertions, op *opendal.Operator, fixt assert.False(lm.After(time.Now().Add(time.Minute)), "last_modified must not be in the future, got %v", lm) } - if cap.WriteWithCacheControl() { + if writeWithCacheControl { cacheControl, ok := meta.CacheControl() assert.True(ok, "cache control must exist") assert.Equal("max-age=60", cacheControl) } else { assertOptionalMetaString(assert, "cache control", meta.CacheControl) } - if cap.WriteWithContentDisposition() { + if writeWithContentDisposition { contentDisposition, ok := meta.ContentDisposition() assert.True(ok, "content disposition must exist") assert.Equal("attachment; filename=hello.txt", contentDisposition) } else { assertOptionalMetaString(assert, "content disposition", meta.ContentDisposition) } - if cap.WriteWithContentEncoding() { + if writeWithContentEncoding { contentEncoding, ok := meta.ContentEncoding() assert.True(ok, "content encoding must exist") assert.Equal("gzip", contentEncoding) } else { assertOptionalMetaString(assert, "content encoding", meta.ContentEncoding) } - if cap.WriteWithContentType() { + if writeWithContentType { contentType, ok := meta.ContentType() assert.True(ok, "content type must exist") assert.Equal("text/plain", contentType) @@ -235,7 +241,7 @@ func testStatFileMetadata(assert *require.Assertions, op *opendal.Operator, fixt if isCurrent, ok := meta.IsCurrent(); ok { assert.True(isCurrent, "a live object must be reported as the current version") } - if cap.WriteWithUserMetadata() { + if writeWithUserMetadata { assert.Equal(userMetadata, meta.UserMetadata()) } else if um := meta.UserMetadata(); um != nil { assert.Equal(um, meta.UserMetadata(), "user metadata accessor must return equal copies") From bc1fdd9b67dfe1dd15413a228e19d45516a04bfd Mon Sep 17 00:00:00 2001 From: dentiny Date: Mon, 1 Jun 2026 04:46:20 +0000 Subject: [PATCH 10/13] revert unnecessary --- bindings/go/string.go | 7 +------ bindings/go/string_ownership_test.go | 25 ------------------------- 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/bindings/go/string.go b/bindings/go/string.go index 0a046d75d388..1686aeb86ff4 100644 --- a/bindings/go/string.go +++ b/bindings/go/string.go @@ -32,12 +32,7 @@ func copyCStringAndFree(ptr *byte, free func(*byte)) string { } defer free(ptr) - n := 0 - for p := unsafe.Pointer(ptr); *(*byte)(p) != 0; n++ { - p = unsafe.Pointer(uintptr(p) + 1) - } - // Copy into Go-owned memory before freeing Rust's CString. - return string(append([]byte(nil), unsafe.Slice(ptr, n)...)) + return BytePtrToString(ptr) } var ffiStringFree = newFFI(ffiOpts{ diff --git a/bindings/go/string_ownership_test.go b/bindings/go/string_ownership_test.go index 810c997026b7..c348f2f76643 100644 --- a/bindings/go/string_ownership_test.go +++ b/bindings/go/string_ownership_test.go @@ -48,31 +48,6 @@ func TestCopyCStringAndFreeNil(t *testing.T) { } } -func TestCopyCStringAndFreeOwnsReturnedString(t *testing.T) { - raw := []byte("text/plain\x00") - ptr := &raw[0] - var freed bool - - got := copyCStringAndFree(ptr, func(ptr *byte) { - freed = true - buf := unsafe.Slice(ptr, len(raw)-1) - for _, b := range buf { - if b == 0 { - t.Fatal("unexpected nul before terminator") - } - } - for i := range buf { - buf[i] = 0 - } - }) - if !freed { - t.Fatal("copyCStringAndFree did not free pointer") - } - if got != "text/plain" { - t.Fatalf("copyCStringAndFree returned %q after free mutation, want text/plain", got) - } -} - func TestOperatorInfoCopiesAndFreesOwnedStrings(t *testing.T) { var freed []*byte freeCString := func(ptr *byte) { From 7f0e95a04e08a995037b70b84f5ff0e182c70f60 Mon Sep 17 00:00:00 2001 From: dentiny Date: Mon, 1 Jun 2026 05:15:11 +0000 Subject: [PATCH 11/13] fix memory corruption --- bindings/go/metadata.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bindings/go/metadata.go b/bindings/go/metadata.go index 057815391327..d18a94af0adf 100644 --- a/bindings/go/metadata.go +++ b/bindings/go/metadata.go @@ -207,12 +207,12 @@ var ffiMetaMode = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) EntryMode { return func(m *opendalMetadata) EntryMode { - var mode uint8 + var mode uint64 ffiCall( unsafe.Pointer(&mode), unsafe.Pointer(&m), ) - return EntryMode(mode) + return EntryMode(uint8(mode)) } }) @@ -251,12 +251,12 @@ var ffiMetaIsFile = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) bool { return func(m *opendalMetadata) bool { - var result uint8 + var result uint64 ffiCall( unsafe.Pointer(&result), unsafe.Pointer(&m), ) - return result == 1 + return uint8(result) == 1 } }) @@ -266,12 +266,12 @@ var ffiMetaIsDir = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) bool { return func(m *opendalMetadata) bool { - var result uint8 + var result uint64 ffiCall( unsafe.Pointer(&result), unsafe.Pointer(&m), ) - return result == 1 + return uint8(result) == 1 } }) @@ -281,12 +281,12 @@ var ffiMetaIsCurrent = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) uint8 { return func(m *opendalMetadata) uint8 { - var result uint8 + var result uint64 ffiCall( unsafe.Pointer(&result), unsafe.Pointer(&m), ) - return result + return uint8(result) } }) @@ -296,12 +296,12 @@ var ffiMetaIsDeleted = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) bool { return func(m *opendalMetadata) bool { - var result uint8 + var result uint64 ffiCall( unsafe.Pointer(&result), unsafe.Pointer(&m), ) - return result == 1 + return uint8(result) == 1 } }) From 48c124614ce0977b272e6de1ce033c23abccaeb6 Mon Sep 17 00:00:00 2001 From: dentiny Date: Mon, 1 Jun 2026 05:16:40 +0000 Subject: [PATCH 12/13] doc --- bindings/go/metadata.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bindings/go/metadata.go b/bindings/go/metadata.go index d18a94af0adf..029d543a42fb 100644 --- a/bindings/go/metadata.go +++ b/bindings/go/metadata.go @@ -207,6 +207,10 @@ var ffiMetaMode = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) EntryMode { return func(m *opendalMetadata) EntryMode { + // libffi may write integer return values using a full register-sized slot + // even when the declared C return type is u8/bool. Use word-sized + // storage here, then narrow after the call, to avoid overwriting nearby + // Go stack memory. var mode uint64 ffiCall( unsafe.Pointer(&mode), @@ -251,6 +255,7 @@ var ffiMetaIsFile = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) bool { return func(m *opendalMetadata) bool { + // See ffiMetaMode for why this uses word-sized return storage. var result uint64 ffiCall( unsafe.Pointer(&result), @@ -266,6 +271,7 @@ var ffiMetaIsDir = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) bool { return func(m *opendalMetadata) bool { + // See ffiMetaMode for why this uses word-sized return storage. var result uint64 ffiCall( unsafe.Pointer(&result), @@ -281,6 +287,7 @@ var ffiMetaIsCurrent = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) uint8 { return func(m *opendalMetadata) uint8 { + // See ffiMetaMode for why this uses word-sized return storage. var result uint64 ffiCall( unsafe.Pointer(&result), @@ -296,6 +303,7 @@ var ffiMetaIsDeleted = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) bool { return func(m *opendalMetadata) bool { + // See ffiMetaMode for why this uses word-sized return storage. var result uint64 ffiCall( unsafe.Pointer(&result), From 6730eaa48e7729726d366b8aee54555e74ddf0da Mon Sep 17 00:00:00 2001 From: dentiny Date: Mon, 1 Jun 2026 06:15:56 +0000 Subject: [PATCH 13/13] consolidate write APIs --- .../go/tests/behavior_tests/benchmark_test.go | 4 ++ bindings/go/tests/behavior_tests/stat_test.go | 2 +- .../go/tests/behavior_tests/write_test.go | 24 ++++---- bindings/go/writer.go | 59 ++++++++++--------- 4 files changed, 47 insertions(+), 42 deletions(-) diff --git a/bindings/go/tests/behavior_tests/benchmark_test.go b/bindings/go/tests/behavior_tests/benchmark_test.go index f53e6566c869..c46e78bd848d 100644 --- a/bindings/go/tests/behavior_tests/benchmark_test.go +++ b/bindings/go/tests/behavior_tests/benchmark_test.go @@ -136,6 +136,10 @@ func NewOpenDALReadWriter(op *opendal.Operator) ReadWriter { } } +func (rw *OpenDALReadWriter) Write(path string, data []byte) error { + return rw.Operator.Write(path, data) +} + func (rw *OpenDALReadWriter) Name() string { return "OpenDAL" } diff --git a/bindings/go/tests/behavior_tests/stat_test.go b/bindings/go/tests/behavior_tests/stat_test.go index 9d282892f0ef..5e9e31cce2a5 100644 --- a/bindings/go/tests/behavior_tests/stat_test.go +++ b/bindings/go/tests/behavior_tests/stat_test.go @@ -189,7 +189,7 @@ func testStatFileMetadata(assert *require.Assertions, op *opendal.Operator, fixt if len(writeOpts) == 0 { assert.Nil(op.Write(path, content), "write must succeed") } else { - assert.Nil(op.WriteWith(path, content, writeOpts...), "write with metadata must succeed") + assert.Nil(op.Write(path, content, writeOpts...), "write with metadata must succeed") } meta, err := op.Stat(path) diff --git a/bindings/go/tests/behavior_tests/write_test.go b/bindings/go/tests/behavior_tests/write_test.go index 185cec1615bb..9e866dfe4e20 100644 --- a/bindings/go/tests/behavior_tests/write_test.go +++ b/bindings/go/tests/behavior_tests/write_test.go @@ -118,7 +118,7 @@ func testWriteWithCacheControl(assert *require.Assertions, op *opendal.Operator, path := fixture.NewFilePath() content := []byte("hello") - assert.Nil(op.WriteWith(path, content, opendal.WriteWithCacheControl("max-age=60"))) + assert.Nil(op.Write(path, content, opendal.WriteWithCacheControl("max-age=60"))) meta, err := op.Stat(path) assert.Nil(err, "stat must succeed") @@ -135,7 +135,7 @@ func testWriteWithContentType(assert *require.Assertions, op *opendal.Operator, path := fixture.NewFilePath() content := []byte("hello") - assert.Nil(op.WriteWith(path, content, opendal.WriteWithContentType("text/plain"))) + assert.Nil(op.Write(path, content, opendal.WriteWithContentType("text/plain"))) meta, err := op.Stat(path) assert.Nil(err, "stat must succeed") @@ -152,7 +152,7 @@ func testWriteWithContentDisposition(assert *require.Assertions, op *opendal.Ope path := fixture.NewFilePath() content := []byte("hello") - assert.Nil(op.WriteWith(path, content, opendal.WriteWithContentDisposition("attachment; filename=hello.txt"))) + assert.Nil(op.Write(path, content, opendal.WriteWithContentDisposition("attachment; filename=hello.txt"))) meta, err := op.Stat(path) assert.Nil(err, "stat must succeed") @@ -169,7 +169,7 @@ func testWriteWithContentEncoding(assert *require.Assertions, op *opendal.Operat path := fixture.NewFilePath() content := []byte("hello") - assert.Nil(op.WriteWith(path, content, opendal.WriteWithContentEncoding("gzip"))) + assert.Nil(op.Write(path, content, opendal.WriteWithContentEncoding("gzip"))) meta, err := op.Stat(path) assert.Nil(err, "stat must succeed") @@ -186,7 +186,7 @@ func testWriteWithUserMetadata(assert *require.Assertions, op *opendal.Operator, path := fixture.NewFilePath() content := []byte("hello") - assert.Nil(op.WriteWith(path, content, opendal.WriteWithUserMetadata(map[string]string{ + assert.Nil(op.Write(path, content, opendal.WriteWithUserMetadata(map[string]string{ "language": "go", "project": "opendal", }))) @@ -212,7 +212,7 @@ func testWriteWithIfMatch(assert *require.Assertions, op *opendal.Operator, fixt etag, ok := meta.ETag() assert.True(ok, "etag must exist") - assert.Nil(op.WriteWith(path, []byte("world"), opendal.WriteWithIfMatch(etag))) + assert.Nil(op.Write(path, []byte("world"), opendal.WriteWithIfMatch(etag))) bs, err := op.Read(path) assert.Nil(err, "read must succeed") assert.Equal([]byte("world"), bs) @@ -230,7 +230,7 @@ func testWriteWithIfNoneMatch(assert *require.Assertions, op *opendal.Operator, etag, ok := meta.ETag() assert.True(ok, "etag must exist") - err = op.WriteWith(path, []byte("world"), opendal.WriteWithIfNoneMatch(etag)) + err = op.Write(path, []byte("world"), opendal.WriteWithIfNoneMatch(etag)) assert.NotNil(err) assert.Equal(opendal.CodeConditionNotMatch, assertErrorCode(err)) @@ -245,8 +245,8 @@ func testWriteWithIfNotExists(assert *require.Assertions, op *opendal.Operator, } path := fixture.NewFilePath() - assert.Nil(op.WriteWith(path, []byte("hello"), opendal.WriteWithIfNotExists(true))) - err := op.WriteWith(path, []byte("world"), opendal.WriteWithIfNotExists(true)) + assert.Nil(op.Write(path, []byte("hello"), opendal.WriteWithIfNotExists(true))) + err := op.Write(path, []byte("world"), opendal.WriteWithIfNotExists(true)) assert.NotNil(err) assert.Equal(opendal.CodeConditionNotMatch, assertErrorCode(err)) @@ -291,7 +291,7 @@ func testWriteWithChunkAndConcurrent(assert *require.Assertions, op *opendal.Ope path := fixture.NewFilePath() content := genFixedBytes(1024 * 1024) - assert.Nil(op.WriteWith(path, content, opendal.WriteWithChunk(256*1024), opendal.WriteWithConcurrent(2))) + assert.Nil(op.Write(path, content, opendal.WriteWithChunk(256*1024), opendal.WriteWithConcurrent(2))) bs, err := op.Read(path) assert.Nil(err, "read must succeed") @@ -305,13 +305,13 @@ func testWriterWithAppend(assert *require.Assertions, op *opendal.Operator, fixt path := fixture.NewFilePath() - w, err := op.WriterWith(path, opendal.WriteWithAppend(true)) + w, err := op.Writer(path, opendal.WriteWithAppend(true)) assert.Nil(err) _, err = w.Write([]byte("hello")) assert.Nil(err) assert.Nil(w.Close()) - w, err = op.WriterWith(path, opendal.WriteWithAppend(true)) + w, err = op.Writer(path, opendal.WriteWithAppend(true)) assert.Nil(err) _, err = w.Write([]byte(" world")) assert.Nil(err) diff --git a/bindings/go/writer.go b/bindings/go/writer.go index cf54fdfdf363..9c4415fe0829 100644 --- a/bindings/go/writer.go +++ b/bindings/go/writer.go @@ -32,13 +32,14 @@ import ( // Write writes the given bytes to the specified path. // -// Write is a wrapper around the C-binding function `opendal_operator_write`. It provides a simplified -// interface for writing data to the storage. +// Write is a wrapper around the C-binding function `opendal_operator_write`. +// When options are provided, it uses `opendal_operator_write_with`. // // # Parameters // // - path: The destination path where the bytes will be written. // - data: The byte slice containing the data to be written. +// - opts: Optional write options. // // # Returns // @@ -54,8 +55,20 @@ import ( // } // // Note: This example assumes proper error handling and import statements. -func (op *Operator) Write(path string, data []byte) error { - return ffiOperatorWrite.symbol(op.ctx)(op.inner, path, data) +func (op *Operator) Write(path string, data []byte, opts ...WithWriteFn) error { + if len(opts) == 0 { + return ffiOperatorWrite.symbol(op.ctx)(op.inner, path, data) + } + + o := parseWriteOptions(opts...) + cOpts, keepAlive, err := newOpendalWriteOptions(op.ctx, o) + if err != nil { + return err + } + defer ffiWriteOptionsFree.symbol(op.ctx)(cOpts) + err = ffiOperatorWriteWith.symbol(op.ctx)(op.inner, path, data, cOpts) + runtime.KeepAlive(keepAlive) + return err } // WithWriteFn is a functional option for write operations. @@ -152,19 +165,6 @@ type writeOptions struct { chunk uint } -// WriteWith writes the given bytes to the specified path with options. -func (op *Operator) WriteWith(path string, data []byte, opts ...WithWriteFn) error { - o := parseWriteOptions(opts...) - cOpts, keepAlive, err := newOpendalWriteOptions(op.ctx, o) - if err != nil { - return err - } - defer ffiWriteOptionsFree.symbol(op.ctx)(cOpts) - err = ffiOperatorWriteWith.symbol(op.ctx)(op.inner, path, data, cOpts) - runtime.KeepAlive(keepAlive) - return err -} - func parseWriteOptions(opts ...WithWriteFn) *writeOptions { o := &writeOptions{} for _, opt := range opts { @@ -290,11 +290,13 @@ func (op *Operator) CreateDir(path string) error { // Writer returns a new Writer for the specified path. // // Writer is a wrapper around the C-binding function `opendal_operator_writer`. +// When options are provided, it uses `opendal_operator_writer_with`. // It provides a way to obtain a writer for writing data to the storage system. // // # Parameters // // - path: The destination path where data will be written. +// - opts: Optional write options. // // # Returns // @@ -315,20 +317,19 @@ func (op *Operator) CreateDir(path string) error { // } // // Note: This example assumes proper error handling and import statements. -func (op *Operator) Writer(path string) (*Writer, error) { - inner, err := ffiOperatorWriter.symbol(op.ctx)(op.inner, path) - if err != nil { - return nil, err - } - writer := &Writer{ - inner: inner, - ctx: op.ctx, +func (op *Operator) Writer(path string, opts ...WithWriteFn) (*Writer, error) { + if len(opts) == 0 { + inner, err := ffiOperatorWriter.symbol(op.ctx)(op.inner, path) + if err != nil { + return nil, err + } + writer := &Writer{ + inner: inner, + ctx: op.ctx, + } + return writer, nil } - return writer, nil -} -// WriterWith returns a new Writer for the specified path with options. -func (op *Operator) WriterWith(path string, opts ...WithWriteFn) (*Writer, error) { o := parseWriteOptions(opts...) cOpts, keepAlive, err := newOpendalWriteOptions(op.ctx, o) if err != nil {