diff --git a/bindings/c/include/opendal.h b/bindings/c/include/opendal.h index 5a040ef92f21..e707b98faa6d 100644 --- a/bindings/c/include/opendal.h +++ b/bindings/c/include/opendal.h @@ -307,6 +307,81 @@ typedef struct opendal_operator_options { void *inner; } opendal_operator_options; +/** + * \brief A key-value pair for write user metadata. + */ +typedef struct opendal_write_user_metadata_pair { + /** + * The metadata key. + */ + const char *key; + /** + * The metadata value. + */ + const char *value; +} opendal_write_user_metadata_pair; + +/** + * \brief The options for write operations. + * + * Use `opendal_write_options_new()` to construct and + * `opendal_write_options_free()` to free. + */ +typedef struct opendal_write_options { + /** + * Append data to the existing file. + */ + bool append; + /** + * Cache-Control header value. + */ + const char *cache_control; + /** + * Content-Type header value. + */ + const char *content_type; + /** + * Content-Disposition header value. + */ + const char *content_disposition; + /** + * Content-Encoding header value. + */ + const char *content_encoding; + /** + * If-Match header value. + */ + const char *if_match; + /** + * If-None-Match header value. + */ + const char *if_none_match; + /** + * Only write if target does not exist. + */ + bool if_not_exists; + /** + * Concurrent write operations. `0` means sequential writes + */ + uintptr_t concurrent; + /** + * Whether `chunk` has been set. + */ + bool has_chunk; + /** + * Chunk size for buffered writes. + */ + uintptr_t chunk; + /** + * User metadata pairs. + */ + const struct opendal_write_user_metadata_pair *user_metadata; + /** + * User metadata pairs length. + */ + uintptr_t user_metadata_len; +} opendal_write_options; + /** * \brief The result type returned by opendal's read operation. * @@ -560,10 +635,30 @@ typedef struct opendal_capability { * If operator supports write with content disposition. */ bool write_with_content_disposition; + /** + * If operator supports write with content encoding. + */ + bool write_with_content_encoding; /** * If operator supports write with cache control. */ bool write_with_cache_control; + /** + * If operator supports write with if match. + */ + bool write_with_if_match; + /** + * If operator supports write with if none match. + */ + bool write_with_if_none_match; + /** + * If operator supports write with if not exists. + */ + bool write_with_if_not_exists; + /** + * If operator supports write with user metadata. + */ + bool write_with_user_metadata; /** * write_multi_max_size is the max size that services support in write_multi. * @@ -1030,6 +1125,14 @@ struct opendal_error *opendal_operator_write(const struct opendal_operator *op, const char *path, const struct opendal_bytes *bytes); +/** + * \brief Blocking write raw bytes to `path` with options. + */ +struct opendal_error *opendal_operator_write_with(const struct opendal_operator *op, + const char *path, + const struct opendal_bytes *bytes, + const struct opendal_write_options *opts); + /** * \brief Blocking read the data from `path`. * @@ -1156,6 +1259,13 @@ struct opendal_result_operator_reader opendal_operator_reader(const struct opend struct opendal_result_operator_writer opendal_operator_writer(const struct opendal_operator *op, const char *path); +/** + * \brief Blocking create a writer for the specified path with options. + */ +struct opendal_result_operator_writer opendal_operator_writer_with(const struct opendal_operator *op, + const char *path, + const struct opendal_write_options *opts); + /** * \brief Blocking delete the object in `path`. * @@ -1685,6 +1795,79 @@ void opendal_list_options_set_recursive(struct opendal_list_options *opts, bool */ void opendal_list_options_free(struct opendal_list_options *opts); +/** + * \brief Construct a heap-allocated opendal_write_options with default values. + */ +struct opendal_write_options *opendal_write_options_new(void); + +/** + * \brief Free the heap memory used by opendal_write_options. + */ +void opendal_write_options_free(struct opendal_write_options *opts); + +/** + * \brief Set append mode. + */ +void opendal_write_options_set_append(struct opendal_write_options *opts, bool append); + +/** + * \brief Set Cache-Control. + */ +void opendal_write_options_set_cache_control(struct opendal_write_options *opts, + const char *cache_control); + +/** + * \brief Set Content-Type. + */ +void opendal_write_options_set_content_type(struct opendal_write_options *opts, + const char *content_type); + +/** + * \brief Set Content-Disposition. + */ +void opendal_write_options_set_content_disposition(struct opendal_write_options *opts, + const char *content_disposition); + +/** + * \brief Set Content-Encoding. + */ +void opendal_write_options_set_content_encoding(struct opendal_write_options *opts, + const char *content_encoding); + +/** + * \brief Set If-Match. + */ +void opendal_write_options_set_if_match(struct opendal_write_options *opts, const char *if_match); + +/** + * \brief Set If-None-Match. + */ +void opendal_write_options_set_if_none_match(struct opendal_write_options *opts, + const char *if_none_match); + +/** + * \brief Set if_not_exists. + */ +void opendal_write_options_set_if_not_exists(struct opendal_write_options *opts, + bool if_not_exists); + +/** + * \brief Set concurrent. + */ +void opendal_write_options_set_concurrent(struct opendal_write_options *opts, uintptr_t concurrent); + +/** + * \brief Set chunk. + */ +void opendal_write_options_set_chunk(struct opendal_write_options *opts, uintptr_t chunk); + +/** + * \brief Set user metadata. + */ +void opendal_write_options_set_user_metadata(struct opendal_write_options *opts, + const struct opendal_write_user_metadata_pair *pairs, + uintptr_t len); + /** * \brief Construct a heap-allocated opendal_operator_options * diff --git a/bindings/c/src/lib.rs b/bindings/c/src/lib.rs index 5592288bdb09..af06c9b3a94e 100644 --- a/bindings/c/src/lib.rs +++ b/bindings/c/src/lib.rs @@ -74,6 +74,8 @@ mod types; pub use types::opendal_bytes; pub use types::opendal_list_options; pub use types::opendal_operator_options; +pub use types::opendal_write_options; +pub use types::opendal_write_user_metadata_pair; mod entry; pub use entry::opendal_entry; diff --git a/bindings/c/src/operator.rs b/bindings/c/src/operator.rs index 4b3a058d1648..7377836035cc 100644 --- a/bindings/c/src/operator.rs +++ b/bindings/c/src/operator.rs @@ -281,6 +281,29 @@ pub unsafe extern "C" fn opendal_operator_write( } } +/// \brief Blocking write raw bytes to `path` with options. +#[no_mangle] +pub unsafe extern "C" fn opendal_operator_write_with( + op: &opendal_operator, + path: *const c_char, + bytes: &opendal_bytes, + opts: *const opendal_write_options, +) -> *mut opendal_error { + assert!(!path.is_null()); + let path = std::ffi::CStr::from_ptr(path) + .to_str() + .expect("malformed path"); + let opts = if opts.is_null() { + core::options::WriteOptions::default() + } else { + (&*opts).into() + }; + match op.deref().write_options(path, bytes, opts) { + Ok(_) => std::ptr::null_mut(), + Err(e) => opendal_error::new(e), + } +} + /// \brief Blocking read the data from `path`. /// /// Read the data out from `path` blocking by operator. @@ -392,7 +415,7 @@ pub unsafe extern "C" fn opendal_operator_reader( return opendal_result_operator_reader { reader: std::ptr::null_mut(), error: opendal_error::new(err), - } + }; } }; @@ -459,7 +482,39 @@ pub unsafe extern "C" fn opendal_operator_writer( return opendal_result_operator_writer { writer: std::ptr::null_mut(), error: opendal_error::new(err), - } + }; + } + }; + + opendal_result_operator_writer { + writer: Box::into_raw(Box::new(opendal_writer::new(writer))), + error: std::ptr::null_mut(), + } +} + +/// \brief Blocking create a writer for the specified path with options. +#[no_mangle] +pub unsafe extern "C" fn opendal_operator_writer_with( + op: &opendal_operator, + path: *const c_char, + opts: *const opendal_write_options, +) -> opendal_result_operator_writer { + assert!(!path.is_null()); + let path = std::ffi::CStr::from_ptr(path) + .to_str() + .expect("malformed path"); + let opts = if opts.is_null() { + core::options::WriteOptions::default() + } else { + (&*opts).into() + }; + let writer = match op.deref().writer_options(path, opts) { + Ok(writer) => writer, + Err(err) => { + return opendal_result_operator_writer { + writer: std::ptr::null_mut(), + error: opendal_error::new(err), + }; } }; diff --git a/bindings/c/src/operator_info.rs b/bindings/c/src/operator_info.rs index 796ea9be2675..e7b83cb5d85a 100644 --- a/bindings/c/src/operator_info.rs +++ b/bindings/c/src/operator_info.rs @@ -75,8 +75,18 @@ pub struct opendal_capability { pub write_with_content_type: bool, /// If operator supports write with content disposition. pub write_with_content_disposition: bool, + /// If operator supports write with content encoding. + pub write_with_content_encoding: bool, /// If operator supports write with cache control. pub write_with_cache_control: bool, + /// If operator supports write with if match. + pub write_with_if_match: bool, + /// If operator supports write with if none match. + pub write_with_if_none_match: bool, + /// If operator supports write with if not exists. + pub write_with_if_not_exists: bool, + /// If operator supports write with user metadata. + pub write_with_user_metadata: bool, /// write_multi_max_size is the max size that services support in write_multi. /// /// For example, AWS S3 supports 5GiB as max in write_multi. @@ -239,7 +249,12 @@ impl From for opendal_capability { write_can_append: value.write_can_append, write_with_content_type: value.write_with_content_type, write_with_content_disposition: value.write_with_content_disposition, + write_with_content_encoding: value.write_with_content_encoding, write_with_cache_control: value.write_with_cache_control, + write_with_if_match: value.write_with_if_match, + write_with_if_none_match: value.write_with_if_none_match, + write_with_if_not_exists: value.write_with_if_not_exists, + write_with_user_metadata: value.write_with_user_metadata, write_multi_max_size: value.write_multi_max_size.unwrap_or(0), write_multi_min_size: value.write_multi_min_size.unwrap_or(0), write_total_max_size: value.write_total_max_size.unwrap_or(0), diff --git a/bindings/c/src/reader.rs b/bindings/c/src/reader.rs index ff798f5c8490..1335de0edf41 100644 --- a/bindings/c/src/reader.rs +++ b/bindings/c/src/reader.rs @@ -96,7 +96,7 @@ impl opendal_reader { core::ErrorKind::Unexpected, "undefined whence", )), - } + }; } }; diff --git a/bindings/c/src/types.rs b/bindings/c/src/types.rs index 1b63f24ceb9d..7a368d552e9e 100644 --- a/bindings/c/src/types.rs +++ b/bindings/c/src/types.rs @@ -16,9 +16,10 @@ // under the License. use std::collections::HashMap; -use std::ffi::{c_void, CString}; +use std::ffi::{c_void, CStr, CString}; use std::os::raw::c_char; +use opendal::options; use opendal::Buffer; /// \brief Frees a heap-allocated string returned by OpenDAL C APIs. @@ -136,6 +137,263 @@ impl opendal_list_options { } } +/// \brief A key-value pair for write user metadata. +#[repr(C)] +pub struct opendal_write_user_metadata_pair { + /// The metadata key. + pub key: *const c_char, + /// The metadata value. + pub value: *const c_char, +} + +/// \brief The options for write operations. +/// +/// Use `opendal_write_options_new()` to construct and +/// `opendal_write_options_free()` to free. +#[repr(C)] +pub struct opendal_write_options { + /// Append data to the existing file. + pub append: bool, + /// Cache-Control header value. + pub cache_control: *const c_char, + /// Content-Type header value. + pub content_type: *const c_char, + /// Content-Disposition header value. + pub content_disposition: *const c_char, + /// Content-Encoding header value. + pub content_encoding: *const c_char, + /// If-Match header value. + pub if_match: *const c_char, + /// If-None-Match header value. + pub if_none_match: *const c_char, + /// Only write if target does not exist. + pub if_not_exists: bool, + /// Concurrent write operations. `0` means sequential writes + pub concurrent: usize, + /// Whether `chunk` has been set. + pub has_chunk: bool, + /// Chunk size for buffered writes. + pub chunk: usize, + /// User metadata pairs. + pub user_metadata: *const opendal_write_user_metadata_pair, + /// User metadata pairs length. + pub user_metadata_len: usize, +} + +impl opendal_write_options { + /// \brief Construct a heap-allocated opendal_write_options with default values. + #[no_mangle] + pub extern "C" fn opendal_write_options_new() -> *mut Self { + Box::into_raw(Box::new(Self::default())) + } + + /// \brief Free the heap memory used by opendal_write_options. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_free(opts: *mut opendal_write_options) { + if !opts.is_null() { + drop(Box::from_raw(opts)); + } + } + + /// \brief Set append mode. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_append( + opts: *mut opendal_write_options, + append: bool, + ) { + if !opts.is_null() { + (*opts).append = append; + } + } + + /// \brief Set Cache-Control. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_cache_control( + opts: *mut opendal_write_options, + cache_control: *const c_char, + ) { + if !opts.is_null() { + (*opts).cache_control = cache_control; + } + } + + /// \brief Set Content-Type. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_content_type( + opts: *mut opendal_write_options, + content_type: *const c_char, + ) { + if !opts.is_null() { + (*opts).content_type = content_type; + } + } + + /// \brief Set Content-Disposition. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_content_disposition( + opts: *mut opendal_write_options, + content_disposition: *const c_char, + ) { + if !opts.is_null() { + (*opts).content_disposition = content_disposition; + } + } + + /// \brief Set Content-Encoding. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_content_encoding( + opts: *mut opendal_write_options, + content_encoding: *const c_char, + ) { + if !opts.is_null() { + (*opts).content_encoding = content_encoding; + } + } + + /// \brief Set If-Match. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_if_match( + opts: *mut opendal_write_options, + if_match: *const c_char, + ) { + if !opts.is_null() { + (*opts).if_match = if_match; + } + } + + /// \brief Set If-None-Match. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_if_none_match( + opts: *mut opendal_write_options, + if_none_match: *const c_char, + ) { + if !opts.is_null() { + (*opts).if_none_match = if_none_match; + } + } + + /// \brief Set if_not_exists. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_if_not_exists( + opts: *mut opendal_write_options, + if_not_exists: bool, + ) { + if !opts.is_null() { + (*opts).if_not_exists = if_not_exists; + } + } + + /// \brief Set concurrent. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_concurrent( + opts: *mut opendal_write_options, + concurrent: usize, + ) { + if !opts.is_null() { + (*opts).concurrent = concurrent; + } + } + + /// \brief Set chunk. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_chunk( + opts: *mut opendal_write_options, + chunk: usize, + ) { + if !opts.is_null() { + (*opts).has_chunk = true; + (*opts).chunk = chunk; + } + } + + /// \brief Set user metadata. + #[no_mangle] + pub unsafe extern "C" fn opendal_write_options_set_user_metadata( + opts: *mut opendal_write_options, + pairs: *const opendal_write_user_metadata_pair, + len: usize, + ) { + if !opts.is_null() { + (*opts).user_metadata = pairs; + (*opts).user_metadata_len = len; + } + } +} + +impl Default for opendal_write_options { + fn default() -> Self { + Self { + append: false, + cache_control: std::ptr::null(), + content_type: std::ptr::null(), + content_disposition: std::ptr::null(), + content_encoding: std::ptr::null(), + if_match: std::ptr::null(), + if_none_match: std::ptr::null(), + if_not_exists: false, + concurrent: 0, + has_chunk: false, + chunk: 0, + user_metadata: std::ptr::null(), + user_metadata_len: 0, + } + } +} + +unsafe fn optional_cstr(ptr: *const c_char) -> Option { + if ptr.is_null() { + None + } else { + Some( + CStr::from_ptr(ptr) + .to_str() + .expect("malformed option") + .to_string(), + ) + } +} + +impl From<&opendal_write_options> for options::WriteOptions { + fn from(value: &opendal_write_options) -> Self { + let user_metadata = if value.user_metadata.is_null() || value.user_metadata_len == 0 { + None + } else { + let pairs = + unsafe { std::slice::from_raw_parts(value.user_metadata, value.user_metadata_len) }; + let mut metadata = HashMap::with_capacity(pairs.len()); + for pair in pairs { + if pair.key.is_null() || pair.value.is_null() { + continue; + } + let key = unsafe { CStr::from_ptr(pair.key) } + .to_str() + .expect("malformed user metadata key") + .to_string(); + let value = unsafe { CStr::from_ptr(pair.value) } + .to_str() + .expect("malformed user metadata value") + .to_string(); + metadata.insert(key, value); + } + Some(metadata) + }; + + Self { + append: value.append, + cache_control: unsafe { optional_cstr(value.cache_control) }, + content_type: unsafe { optional_cstr(value.content_type) }, + content_disposition: unsafe { optional_cstr(value.content_disposition) }, + content_encoding: unsafe { optional_cstr(value.content_encoding) }, + user_metadata, + if_match: unsafe { optional_cstr(value.if_match) }, + if_none_match: unsafe { optional_cstr(value.if_none_match) }, + if_not_exists: value.if_not_exists, + concurrent: value.concurrent, + chunk: value.has_chunk.then_some(value.chunk), + } + } +} + impl Drop for opendal_bytes { fn drop(&mut self) { unsafe { diff --git a/bindings/go/metadata.go b/bindings/go/metadata.go index 057815391327..029d543a42fb 100644 --- a/bindings/go/metadata.go +++ b/bindings/go/metadata.go @@ -207,12 +207,16 @@ var ffiMetaMode = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) EntryMode { return func(m *opendalMetadata) EntryMode { - var mode uint8 + // libffi may write integer return values using a full register-sized slot + // even when the declared C return type is u8/bool. Use word-sized + // storage here, then narrow after the call, to avoid overwriting nearby + // Go stack memory. + var mode uint64 ffiCall( unsafe.Pointer(&mode), unsafe.Pointer(&m), ) - return EntryMode(mode) + return EntryMode(uint8(mode)) } }) @@ -251,12 +255,13 @@ var ffiMetaIsFile = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) bool { return func(m *opendalMetadata) bool { - var result uint8 + // See ffiMetaMode for why this uses word-sized return storage. + var result uint64 ffiCall( unsafe.Pointer(&result), unsafe.Pointer(&m), ) - return result == 1 + return uint8(result) == 1 } }) @@ -266,12 +271,13 @@ var ffiMetaIsDir = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) bool { return func(m *opendalMetadata) bool { - var result uint8 + // See ffiMetaMode for why this uses word-sized return storage. + var result uint64 ffiCall( unsafe.Pointer(&result), unsafe.Pointer(&m), ) - return result == 1 + return uint8(result) == 1 } }) @@ -281,12 +287,13 @@ var ffiMetaIsCurrent = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) uint8 { return func(m *opendalMetadata) uint8 { - var result uint8 + // See ffiMetaMode for why this uses word-sized return storage. + var result uint64 ffiCall( unsafe.Pointer(&result), unsafe.Pointer(&m), ) - return result + return uint8(result) } }) @@ -296,12 +303,13 @@ var ffiMetaIsDeleted = newFFI(ffiOpts{ aTypes: []*ffi.Type{&ffi.TypePointer}, }, func(ctx context.Context, ffiCall ffiCall) func(m *opendalMetadata) bool { return func(m *opendalMetadata) bool { - var result uint8 + // See ffiMetaMode for why this uses word-sized return storage. + var result uint64 ffiCall( unsafe.Pointer(&result), unsafe.Pointer(&m), ) - return result == 1 + return uint8(result) == 1 } }) diff --git a/bindings/go/operator_info.go b/bindings/go/operator_info.go index 210ad110d1b0..c3e5c9297f4e 100644 --- a/bindings/go/operator_info.go +++ b/bindings/go/operator_info.go @@ -160,6 +160,26 @@ func (c *Capability) WriteWithCacheControl() bool { return c.inner.writeWithCacheControl == 1 } +func (c *Capability) WriteWithContentEncoding() bool { + return c.inner.writeWithContentEncoding == 1 +} + +func (c *Capability) WriteWithIfMatch() bool { + return c.inner.writeWithIfMatch == 1 +} + +func (c *Capability) WriteWithIfNoneMatch() bool { + return c.inner.writeWithIfNoneMatch == 1 +} + +func (c *Capability) WriteWithIfNotExists() bool { + return c.inner.writeWithIfNotExists == 1 +} + +func (c *Capability) WriteWithUserMetadata() bool { + return c.inner.writeWithUserMetadata == 1 +} + func (c *Capability) WriteMultiMaxSize() uint { return c.inner.writeMultiMaxSize } diff --git a/bindings/go/string_ownership_test.go b/bindings/go/string_ownership_test.go index 400ece17c305..c348f2f76643 100644 --- a/bindings/go/string_ownership_test.go +++ b/bindings/go/string_ownership_test.go @@ -635,6 +635,141 @@ func assertFreedPointers(t *testing.T, got []*byte, want ...*byte) { } } +func TestWriteWithOptions(t *testing.T) { + o := &writeOptions{} + WriteWithAppend(true)(o) + WriteWithCacheControl("max-age=60")(o) + WriteWithContentType("text/plain")(o) + WriteWithContentDisposition("attachment")(o) + WriteWithContentEncoding("gzip")(o) + WriteWithUserMetadata(map[string]string{"foo": "bar"})(o) + WriteWithIfMatch("etag-a")(o) + WriteWithIfNoneMatch("etag-b")(o) + WriteWithIfNotExists(true)(o) + WriteWithConcurrent(4)(o) + WriteWithChunk(1024)(o) + + if !o.append { + t.Fatalf("append = false, want true") + } + if o.cacheControl != "max-age=60" { + t.Fatalf("cacheControl = %q, want max-age=60", o.cacheControl) + } + if o.contentType != "text/plain" { + t.Fatalf("contentType = %q, want text/plain", o.contentType) + } + if o.contentDisposition != "attachment" { + t.Fatalf("contentDisposition = %q, want attachment", o.contentDisposition) + } + if o.contentEncoding != "gzip" { + t.Fatalf("contentEncoding = %q, want gzip", o.contentEncoding) + } + assertStringMap(t, o.userMetadata, map[string]string{"foo": "bar"}) + if o.ifMatch != "etag-a" { + t.Fatalf("ifMatch = %q, want etag-a", o.ifMatch) + } + if o.ifNoneMatch != "etag-b" { + t.Fatalf("ifNoneMatch = %q, want etag-b", o.ifNoneMatch) + } + if !o.ifNotExists { + t.Fatalf("ifNotExists = false, want true") + } + if o.concurrent != 4 { + t.Fatalf("concurrent = %d, want 4", o.concurrent) + } + if o.chunk != 1024 { + t.Fatalf("chunk = %d, want 1024", o.chunk) + } +} + +func TestFfiOperatorWriteWithArgTypes(t *testing.T) { + aTypes := ffiOperatorWriteWith.opts.aTypes + if len(aTypes) != 4 { + t.Fatalf("ffiOperatorWriteWith aTypes len = %d, want 4", len(aTypes)) + } + for i, at := range aTypes { + if at != &ffi.TypePointer { + t.Fatalf("ffiOperatorWriteWith aTypes[%d] = %v, want TypePointer", i, at) + } + } +} + +func TestFfiOperatorWriterWithArgTypes(t *testing.T) { + aTypes := ffiOperatorWriterWith.opts.aTypes + if len(aTypes) != 3 { + t.Fatalf("ffiOperatorWriterWith aTypes len = %d, want 3", len(aTypes)) + } + for i, at := range aTypes { + if at != &ffi.TypePointer { + t.Fatalf("ffiOperatorWriterWith aTypes[%d] = %v, want TypePointer", i, at) + } + } +} + +func TestWriteOptionsSetterArgTypes(t *testing.T) { + stringSetters := []*FFI[func(*opendalWriteOptions, string) ([]byte, error)]{ + ffiWriteOptionsSetCacheControl, + ffiWriteOptionsSetContentType, + ffiWriteOptionsSetContentDisposition, + ffiWriteOptionsSetContentEncoding, + ffiWriteOptionsSetIfMatch, + ffiWriteOptionsSetIfNoneMatch, + } + for _, setter := range stringSetters { + aTypes := setter.opts.aTypes + if len(aTypes) != 2 { + t.Fatalf("%s aTypes len = %d, want 2", setter.opts.sym, len(aTypes)) + } + for i, at := range aTypes { + if at != &ffi.TypePointer { + t.Fatalf("%s aTypes[%d] = %v, want TypePointer", setter.opts.sym, i, at) + } + } + } + + boolSetters := []*FFI[func(*opendalWriteOptions, bool)]{ + ffiWriteOptionsSetAppend, + ffiWriteOptionsSetIfNotExists, + } + for _, setter := range boolSetters { + aTypes := setter.opts.aTypes + if len(aTypes) != 2 { + t.Fatalf("%s aTypes len = %d, want 2", setter.opts.sym, len(aTypes)) + } + if aTypes[0] != &ffi.TypePointer || aTypes[1] != &ffi.TypeUint8 { + t.Fatalf("%s aTypes = %v, want TypePointer, TypeUint8", setter.opts.sym, aTypes) + } + } + + uintSetters := []*FFI[func(*opendalWriteOptions, uint)]{ + ffiWriteOptionsSetConcurrent, + ffiWriteOptionsSetChunk, + } + for _, setter := range uintSetters { + aTypes := setter.opts.aTypes + if len(aTypes) != 2 { + t.Fatalf("%s aTypes len = %d, want 2", setter.opts.sym, len(aTypes)) + } + for i, at := range aTypes { + if at != &ffi.TypePointer { + t.Fatalf("%s aTypes[%d] = %v, want TypePointer", setter.opts.sym, i, at) + } + } + } +} + +func TestWriteOptionsSetUserMetadataArgTypes(t *testing.T) { + aTypes := ffiWriteOptionsSetUserMetadata.opts.aTypes + if len(aTypes) != 3 { + t.Fatalf("ffiWriteOptionsSetUserMetadata aTypes len = %d, want 3", len(aTypes)) + } + for i, at := range aTypes { + if at != &ffi.TypePointer { + t.Fatalf("ffiWriteOptionsSetUserMetadata aTypes[%d] = %v, want TypePointer", i, at) + } + } +} + func TestListWithRecursiveDefaultNotRecursive(t *testing.T) { o := &listOptions{} if o.recursive { diff --git a/bindings/go/tests/behavior_tests/benchmark_test.go b/bindings/go/tests/behavior_tests/benchmark_test.go index f53e6566c869..c46e78bd848d 100644 --- a/bindings/go/tests/behavior_tests/benchmark_test.go +++ b/bindings/go/tests/behavior_tests/benchmark_test.go @@ -136,6 +136,10 @@ func NewOpenDALReadWriter(op *opendal.Operator) ReadWriter { } } +func (rw *OpenDALReadWriter) Write(path string, data []byte) error { + return rw.Operator.Write(path, data) +} + func (rw *OpenDALReadWriter) Name() string { return "OpenDAL" } diff --git a/bindings/go/tests/behavior_tests/go.mod b/bindings/go/tests/behavior_tests/go.mod index 1a32049bf9d2..87cfbb7e32d8 100644 --- a/bindings/go/tests/behavior_tests/go.mod +++ b/bindings/go/tests/behavior_tests/go.mod @@ -37,4 +37,3 @@ require ( golang.org/x/sys v0.22.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - diff --git a/bindings/go/tests/behavior_tests/go.sum b/bindings/go/tests/behavior_tests/go.sum index c1473d33cab7..8c30f56105f7 100644 --- a/bindings/go/tests/behavior_tests/go.sum +++ b/bindings/go/tests/behavior_tests/go.sum @@ -1,3 +1,5 @@ +github.com/apache/opendal-go-services/fs v0.1.3 h1:k5pA73gKbQ3MHH2envsKhr1cec2spLm2tl/bCyU53j8= +github.com/apache/opendal-go-services/fs v0.1.3/go.mod h1:7EnuyeXRuQh+L47rZ7y2OrhYJLlUYvgvFPItM98XJ5s= github.com/apache/opendal/bindings/go v0.0.0-20240719044908-d9d4279b3a24 h1:2fAl+WS/lZMTtP6onlrmDbb3pltf+5xNTc0Aeu9nYWE= github.com/apache/opendal/bindings/go v0.0.0-20240719044908-d9d4279b3a24/go.mod h1:jyMN6M6h0jMDZitnjvB3KPobM+oZiESrFb3XUplLxhI= github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= diff --git a/bindings/go/tests/behavior_tests/opendal_test.go b/bindings/go/tests/behavior_tests/opendal_test.go index 75c32a07c500..3c7cafab7b96 100644 --- a/bindings/go/tests/behavior_tests/opendal_test.go +++ b/bindings/go/tests/behavior_tests/opendal_test.go @@ -26,6 +26,7 @@ import ( "os" "reflect" "runtime" + "strconv" "strings" "sync" "testing" @@ -154,6 +155,49 @@ func assertErrorCode(err error) opendal.ErrorCode { return err.(*opendal.Error).Code() } +// parsedOverrides is lazily parsed from OPENDAL_TEST_CAPABILITY_OVERRIDES. +var parsedOverrides map[string]bool + +func getCapOverrides() map[string]bool { + if parsedOverrides != nil { + return parsedOverrides + } + raw := os.Getenv("OPENDAL_TEST_CAPABILITY_OVERRIDES") + parsedOverrides = make(map[string]bool) + if raw == "" { + return parsedOverrides + } + for _, pair := range strings.Split(raw, ",") { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + kv := strings.SplitN(pair, "=", 2) + if len(kv) != 2 { + continue + } + v, err := strconv.ParseBool(strings.TrimSpace(kv[1])) + if err != nil { + continue + } + parsedOverrides[strings.TrimSpace(kv[0])] = v + } + return parsedOverrides +} + +// isCapEnabled checks both the capability reported by the operator and any +// test-level overrides set via OPENDAL_TEST_CAPABILITY_OVERRIDES. +func isCapEnabled(check func() bool, name string) bool { + if !check() { + return false + } + overrides := getCapOverrides() + if v, ok := overrides[name]; ok { + return v + } + return true +} + func genBytesWithRange(min, max uint) ([]byte, uint) { diff := max - min n, _ := rand.Int(rand.Reader, big.NewInt(int64(diff+1))) diff --git a/bindings/go/tests/behavior_tests/stat_test.go b/bindings/go/tests/behavior_tests/stat_test.go index 52d12e8c8848..5e9e31cce2a5 100644 --- a/bindings/go/tests/behavior_tests/stat_test.go +++ b/bindings/go/tests/behavior_tests/stat_test.go @@ -156,9 +156,41 @@ func testStatRoot(assert *require.Assertions, op *opendal.Operator, fixture *fix func testStatFileMetadata(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { path, content, size := fixture.NewFile() + cap := op.Info().GetFullCapability() + + writeOpts := make([]opendal.WithWriteFn, 0, 5) + writeWithCacheControl := isCapEnabled(cap.WriteWithCacheControl, "write_with_cache_control") + writeWithContentDisposition := isCapEnabled(cap.WriteWithContentDisposition, "write_with_content_disposition") + writeWithContentEncoding := isCapEnabled(cap.WriteWithContentEncoding, "write_with_content_encoding") + writeWithContentType := isCapEnabled(cap.WriteWithContentType, "write_with_content_type") + writeWithUserMetadata := isCapEnabled(cap.WriteWithUserMetadata, "write_with_user_metadata") + + if writeWithCacheControl { + writeOpts = append(writeOpts, opendal.WriteWithCacheControl("max-age=60")) + } + if writeWithContentDisposition { + writeOpts = append(writeOpts, opendal.WriteWithContentDisposition("attachment; filename=hello.txt")) + } + if writeWithContentEncoding { + writeOpts = append(writeOpts, opendal.WriteWithContentEncoding("gzip")) + } + if writeWithContentType { + writeOpts = append(writeOpts, opendal.WriteWithContentType("text/plain")) + } + userMetadata := map[string]string{ + "language": "go", + "project": "opendal", + } + if writeWithUserMetadata { + writeOpts = append(writeOpts, opendal.WriteWithUserMetadata(userMetadata)) + } before := time.Now().Add(-time.Hour) - assert.Nil(op.Write(path, content), "write must succeed") + if len(writeOpts) == 0 { + assert.Nil(op.Write(path, content), "write must succeed") + } else { + assert.Nil(op.Write(path, content, writeOpts...), "write with metadata must succeed") + } meta, err := op.Stat(path) assert.Nil(err, "stat must succeed") @@ -174,10 +206,34 @@ func testStatFileMetadata(assert *require.Assertions, op *opendal.Operator, fixt assert.False(lm.After(time.Now().Add(time.Minute)), "last_modified must not be in the future, got %v", lm) } - assertOptionalMetaString(assert, "cache control", meta.CacheControl) - assertOptionalMetaString(assert, "content disposition", meta.ContentDisposition) - assertOptionalMetaString(assert, "content encoding", meta.ContentEncoding) - assertOptionalMetaString(assert, "content type", meta.ContentType) + if writeWithCacheControl { + cacheControl, ok := meta.CacheControl() + assert.True(ok, "cache control must exist") + assert.Equal("max-age=60", cacheControl) + } else { + assertOptionalMetaString(assert, "cache control", meta.CacheControl) + } + if writeWithContentDisposition { + contentDisposition, ok := meta.ContentDisposition() + assert.True(ok, "content disposition must exist") + assert.Equal("attachment; filename=hello.txt", contentDisposition) + } else { + assertOptionalMetaString(assert, "content disposition", meta.ContentDisposition) + } + if writeWithContentEncoding { + contentEncoding, ok := meta.ContentEncoding() + assert.True(ok, "content encoding must exist") + assert.Equal("gzip", contentEncoding) + } else { + assertOptionalMetaString(assert, "content encoding", meta.ContentEncoding) + } + if writeWithContentType { + contentType, ok := meta.ContentType() + assert.True(ok, "content type must exist") + assert.Equal("text/plain", contentType) + } else { + assertOptionalMetaString(assert, "content type", meta.ContentType) + } assertOptionalMetaString(assert, "content md5", meta.ContentMD5) assertOptionalMetaString(assert, "etag", meta.ETag) assertOptionalMetaString(assert, "version", meta.Version) @@ -185,7 +241,9 @@ func testStatFileMetadata(assert *require.Assertions, op *opendal.Operator, fixt if isCurrent, ok := meta.IsCurrent(); ok { assert.True(isCurrent, "a live object must be reported as the current version") } - if um := meta.UserMetadata(); um != nil { + if writeWithUserMetadata { + assert.Equal(userMetadata, meta.UserMetadata()) + } else if um := meta.UserMetadata(); um != nil { assert.Equal(um, meta.UserMetadata(), "user metadata accessor must return equal copies") } } diff --git a/bindings/go/tests/behavior_tests/write_test.go b/bindings/go/tests/behavior_tests/write_test.go index 05d626972d5f..9e866dfe4e20 100644 --- a/bindings/go/tests/behavior_tests/write_test.go +++ b/bindings/go/tests/behavior_tests/write_test.go @@ -20,7 +20,7 @@ package opendal_test import ( - "github.com/apache/opendal/bindings/go" + opendal "github.com/apache/opendal/bindings/go" "github.com/google/uuid" "github.com/stretchr/testify/require" ) @@ -35,7 +35,17 @@ func testsWrite(cap *opendal.Capability) []behaviorTest { testWriteWithDirPath, testWriteWithSpecialChars, testWriteOverwrite, + testWriteWithCacheControl, + testWriteWithContentType, + testWriteWithContentDisposition, + testWriteWithContentEncoding, + testWriteWithUserMetadata, + testWriteWithIfMatch, + testWriteWithIfNoneMatch, + testWriteWithIfNotExists, testWriterWrite, + testWriteWithChunkAndConcurrent, + testWriterWithAppend, } } @@ -101,6 +111,150 @@ func testWriteOverwrite(assert *require.Assertions, op *opendal.Operator, fixtur assert.Equal(contentTwo, bs, "read content_two") } +func testWriteWithCacheControl(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithCacheControl, "write_with_cache_control") { + return + } + + path := fixture.NewFilePath() + content := []byte("hello") + assert.Nil(op.Write(path, content, opendal.WriteWithCacheControl("max-age=60"))) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(len(content)), meta.ContentLength()) + cacheControl, ok := meta.CacheControl() + assert.True(ok, "cache control must exist") + assert.Equal("max-age=60", cacheControl) +} + +func testWriteWithContentType(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithContentType, "write_with_content_type") { + return + } + + path := fixture.NewFilePath() + content := []byte("hello") + assert.Nil(op.Write(path, content, opendal.WriteWithContentType("text/plain"))) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(len(content)), meta.ContentLength()) + contentType, ok := meta.ContentType() + assert.True(ok, "content type must exist") + assert.Equal("text/plain", contentType) +} + +func testWriteWithContentDisposition(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithContentDisposition, "write_with_content_disposition") { + return + } + + path := fixture.NewFilePath() + content := []byte("hello") + assert.Nil(op.Write(path, content, opendal.WriteWithContentDisposition("attachment; filename=hello.txt"))) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(len(content)), meta.ContentLength()) + contentDisposition, ok := meta.ContentDisposition() + assert.True(ok, "content disposition must exist") + assert.Equal("attachment; filename=hello.txt", contentDisposition) +} + +func testWriteWithContentEncoding(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithContentEncoding, "write_with_content_encoding") { + return + } + + path := fixture.NewFilePath() + content := []byte("hello") + assert.Nil(op.Write(path, content, opendal.WriteWithContentEncoding("gzip"))) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(len(content)), meta.ContentLength()) + contentEncoding, ok := meta.ContentEncoding() + assert.True(ok, "content encoding must exist") + assert.Equal("gzip", contentEncoding) +} + +func testWriteWithUserMetadata(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithUserMetadata, "write_with_user_metadata") { + return + } + + path := fixture.NewFilePath() + content := []byte("hello") + assert.Nil(op.Write(path, content, opendal.WriteWithUserMetadata(map[string]string{ + "language": "go", + "project": "opendal", + }))) + + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + assert.Equal(uint64(len(content)), meta.ContentLength()) + assert.Equal(map[string]string{ + "language": "go", + "project": "opendal", + }, meta.UserMetadata()) +} + +func testWriteWithIfMatch(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithIfMatch, "write_with_if_match") { + return + } + + path := fixture.NewFilePath() + assert.Nil(op.Write(path, []byte("hello"))) + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + etag, ok := meta.ETag() + assert.True(ok, "etag must exist") + + assert.Nil(op.Write(path, []byte("world"), opendal.WriteWithIfMatch(etag))) + bs, err := op.Read(path) + assert.Nil(err, "read must succeed") + assert.Equal([]byte("world"), bs) +} + +func testWriteWithIfNoneMatch(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithIfNoneMatch, "write_with_if_none_match") { + return + } + + path := fixture.NewFilePath() + assert.Nil(op.Write(path, []byte("hello"))) + meta, err := op.Stat(path) + assert.Nil(err, "stat must succeed") + etag, ok := meta.ETag() + assert.True(ok, "etag must exist") + + err = op.Write(path, []byte("world"), opendal.WriteWithIfNoneMatch(etag)) + assert.NotNil(err) + assert.Equal(opendal.CodeConditionNotMatch, assertErrorCode(err)) + + bs, err := op.Read(path) + assert.Nil(err, "read must succeed") + assert.Equal([]byte("hello"), bs) +} + +func testWriteWithIfNotExists(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !isCapEnabled(op.Info().GetFullCapability().WriteWithIfNotExists, "write_with_if_not_exists") { + return + } + + path := fixture.NewFilePath() + assert.Nil(op.Write(path, []byte("hello"), opendal.WriteWithIfNotExists(true))) + err := op.Write(path, []byte("world"), opendal.WriteWithIfNotExists(true)) + assert.NotNil(err) + assert.Equal(opendal.CodeConditionNotMatch, assertErrorCode(err)) + + bs, err := op.Read(path) + assert.Nil(err, "read must succeed") + assert.Equal([]byte("hello"), bs) +} + func testWriterWrite(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { if !op.Info().GetFullCapability().WriteCanMulti() { return @@ -129,3 +283,41 @@ func testWriterWrite(assert *require.Assertions, op *opendal.Operator, fixture * assert.Equal(contentA, bs[:size], "read contentA") assert.Equal(contentB, bs[size:], "read contentB") } + +func testWriteWithChunkAndConcurrent(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !isCapEnabled(op.Info().GetFullCapability().WriteCanMulti, "write_can_multi") { + return + } + + path := fixture.NewFilePath() + content := genFixedBytes(1024 * 1024) + assert.Nil(op.Write(path, content, opendal.WriteWithChunk(256*1024), opendal.WriteWithConcurrent(2))) + + bs, err := op.Read(path) + assert.Nil(err, "read must succeed") + assert.Equal(content, bs) +} + +func testWriterWithAppend(assert *require.Assertions, op *opendal.Operator, fixture *fixture) { + if !isCapEnabled(op.Info().GetFullCapability().WriteCanAppend, "write_can_append") { + return + } + + path := fixture.NewFilePath() + + w, err := op.Writer(path, opendal.WriteWithAppend(true)) + assert.Nil(err) + _, err = w.Write([]byte("hello")) + assert.Nil(err) + assert.Nil(w.Close()) + + w, err = op.Writer(path, opendal.WriteWithAppend(true)) + assert.Nil(err) + _, err = w.Write([]byte(" world")) + assert.Nil(err) + assert.Nil(w.Close()) + + bs, err := op.Read(path) + assert.Nil(err, "read must succeed") + assert.Equal([]byte("hello world"), bs) +} diff --git a/bindings/go/types.go b/bindings/go/types.go index a19e3f9935a3..7d07268b6ffc 100644 --- a/bindings/go/types.go +++ b/bindings/go/types.go @@ -162,7 +162,12 @@ var ( &ffi.TypeUint8, // write_can_append &ffi.TypeUint8, // write_with_content_type &ffi.TypeUint8, // write_with_content_disposition + &ffi.TypeUint8, // write_with_content_encoding &ffi.TypeUint8, // write_with_cache_control + &ffi.TypeUint8, // write_with_if_match + &ffi.TypeUint8, // write_with_if_none_match + &ffi.TypeUint8, // write_with_if_not_exists + &ffi.TypeUint8, // write_with_user_metadata &ffi.TypePointer, // write_multi_max_size &ffi.TypePointer, // write_multi_min_size &ffi.TypePointer, // write_total_max_size @@ -201,7 +206,12 @@ type opendalCapability struct { writeCanAppend uint8 writeWithContentType uint8 writeWithContentDisposition uint8 + writeWithContentEncoding uint8 writeWithCacheControl uint8 + writeWithIfMatch uint8 + writeWithIfNoneMatch uint8 + writeWithIfNotExists uint8 + writeWithUserMetadata uint8 writeMultiMaxSize uint writeMultiMinSize uint writeTotalMaxSize uint @@ -315,6 +325,13 @@ type opendalLister struct{} type opendalListOptions struct{} +type opendalWriteOptions struct{} + +type opendalWriteUserMetadataPair struct { + key *byte + value *byte +} + type opendalResultListerNext struct { entry *opendalEntry err *opendalError diff --git a/bindings/go/writer.go b/bindings/go/writer.go index c2d0e5088bf2..9c4415fe0829 100644 --- a/bindings/go/writer.go +++ b/bindings/go/writer.go @@ -21,7 +21,10 @@ package opendal import ( "context" + "errors" "io" + "runtime" + "strings" "unsafe" "github.com/jupiterrider/ffi" @@ -29,15 +32,14 @@ import ( // Write writes the given bytes to the specified path. // -// Write is a wrapper around the C-binding function `opendal_operator_write`. It provides a simplified -// interface for writing data to the storage. Currently, this implementation does not support the -// `Operator::write_with` method from the original Rust library, nor does it support streaming writes -// or multipart uploads. +// Write is a wrapper around the C-binding function `opendal_operator_write`. +// When options are provided, it uses `opendal_operator_write_with`. // // # Parameters // // - path: The destination path where the bytes will be written. // - data: The byte slice containing the data to be written. +// - opts: Optional write options. // // # Returns // @@ -53,8 +55,197 @@ import ( // } // // Note: This example assumes proper error handling and import statements. -func (op *Operator) Write(path string, data []byte) error { - return ffiOperatorWrite.symbol(op.ctx)(op.inner, path, data) +func (op *Operator) Write(path string, data []byte, opts ...WithWriteFn) error { + if len(opts) == 0 { + return ffiOperatorWrite.symbol(op.ctx)(op.inner, path, data) + } + + o := parseWriteOptions(opts...) + cOpts, keepAlive, err := newOpendalWriteOptions(op.ctx, o) + if err != nil { + return err + } + defer ffiWriteOptionsFree.symbol(op.ctx)(cOpts) + err = ffiOperatorWriteWith.symbol(op.ctx)(op.inner, path, data, cOpts) + runtime.KeepAlive(keepAlive) + return err +} + +// WithWriteFn is a functional option for write operations. +type WithWriteFn func(*writeOptions) + +// WriteWithAppend sets append mode for the write operation. +func WriteWithAppend(append bool) WithWriteFn { + return func(o *writeOptions) { + o.append = append + } +} + +// WriteWithCacheControl sets the Cache-Control header for the write operation. +func WriteWithCacheControl(cacheControl string) WithWriteFn { + return func(o *writeOptions) { + o.cacheControl = cacheControl + } +} + +// WriteWithContentType sets the Content-Type header for the write operation. +func WriteWithContentType(contentType string) WithWriteFn { + return func(o *writeOptions) { + o.contentType = contentType + } +} + +// WriteWithContentDisposition sets the Content-Disposition header for the write operation. +func WriteWithContentDisposition(contentDisposition string) WithWriteFn { + return func(o *writeOptions) { + o.contentDisposition = contentDisposition + } +} + +// WriteWithContentEncoding sets the Content-Encoding header for the write operation. +func WriteWithContentEncoding(contentEncoding string) WithWriteFn { + return func(o *writeOptions) { + o.contentEncoding = contentEncoding + } +} + +// WriteWithUserMetadata sets user metadata for the write operation. +func WriteWithUserMetadata(userMetadata map[string]string) WithWriteFn { + return func(o *writeOptions) { + o.userMetadata = userMetadata + } +} + +// WriteWithIfMatch sets the If-Match condition for the write operation. +func WriteWithIfMatch(ifMatch string) WithWriteFn { + return func(o *writeOptions) { + o.ifMatch = ifMatch + } +} + +// WriteWithIfNoneMatch sets the If-None-Match condition for the write operation. +func WriteWithIfNoneMatch(ifNoneMatch string) WithWriteFn { + return func(o *writeOptions) { + o.ifNoneMatch = ifNoneMatch + } +} + +// WriteWithIfNotExists sets whether the write operation should only succeed if the target does not exist. +func WriteWithIfNotExists(ifNotExists bool) WithWriteFn { + return func(o *writeOptions) { + o.ifNotExists = ifNotExists + } +} + +// WriteWithConcurrent sets concurrent write operations. +func WriteWithConcurrent(concurrent uint) WithWriteFn { + return func(o *writeOptions) { + o.concurrent = concurrent + } +} + +// WriteWithChunk sets the chunk size for buffered writes. +func WriteWithChunk(chunk uint) WithWriteFn { + return func(o *writeOptions) { + o.chunk = chunk + } +} + +type writeOptions struct { + append bool + cacheControl string + contentType string + contentDisposition string + contentEncoding string + userMetadata map[string]string + ifMatch string + ifNoneMatch string + ifNotExists bool + concurrent uint + chunk uint +} + +func parseWriteOptions(opts ...WithWriteFn) *writeOptions { + o := &writeOptions{} + for _, opt := range opts { + opt(o) + } + return o +} + +type writeOptionsKeepAlive struct { + strings [][]byte + userMetadata []opendalWriteUserMetadataPair +} + +func newOpendalWriteOptions(ctx context.Context, o *writeOptions) (*opendalWriteOptions, writeOptionsKeepAlive, error) { + cOpts := ffiWriteOptionsNew.symbol(ctx)() + keepAlive := writeOptionsKeepAlive{} + ffiWriteOptionsSetAppend.symbol(ctx)(cOpts, o.append) + + // fail frees the C-allocated options before returning + fail := func(err error) (*opendalWriteOptions, writeOptionsKeepAlive, error) { + ffiWriteOptionsFree.symbol(ctx)(cOpts) + return nil, writeOptionsKeepAlive{}, err + } + + setString := func(value string, set func(*opendalWriteOptions, string) ([]byte, error)) error { + if value == "" { + return nil + } + data, err := set(cOpts, value) + if err != nil { + return err + } + keepAlive.strings = append(keepAlive.strings, data) + return nil + } + + if err := setString(o.cacheControl, ffiWriteOptionsSetCacheControl.symbol(ctx)); err != nil { + return fail(err) + } + if err := setString(o.contentType, ffiWriteOptionsSetContentType.symbol(ctx)); err != nil { + return fail(err) + } + if err := setString(o.contentDisposition, ffiWriteOptionsSetContentDisposition.symbol(ctx)); err != nil { + return fail(err) + } + if err := setString(o.contentEncoding, ffiWriteOptionsSetContentEncoding.symbol(ctx)); err != nil { + return fail(err) + } + if err := setString(o.ifMatch, ffiWriteOptionsSetIfMatch.symbol(ctx)); err != nil { + return fail(err) + } + if err := setString(o.ifNoneMatch, ffiWriteOptionsSetIfNoneMatch.symbol(ctx)); err != nil { + return fail(err) + } + + ffiWriteOptionsSetIfNotExists.symbol(ctx)(cOpts, o.ifNotExists) + if o.concurrent != 0 { + ffiWriteOptionsSetConcurrent.symbol(ctx)(cOpts, o.concurrent) + } + if o.chunk != 0 { + ffiWriteOptionsSetChunk.symbol(ctx)(cOpts, o.chunk) + } + if len(o.userMetadata) > 0 { + keepAlive.userMetadata = make([]opendalWriteUserMetadataPair, 0, len(o.userMetadata)) + for key, value := range o.userMetadata { + keyData, err := byteSliceFromString(key) + if err != nil { + return fail(err) + } + valueData, err := byteSliceFromString(value) + if err != nil { + return fail(err) + } + keepAlive.strings = append(keepAlive.strings, keyData, valueData) + byteKey := &keyData[0] + byteValue := &valueData[0] + keepAlive.userMetadata = append(keepAlive.userMetadata, opendalWriteUserMetadataPair{key: byteKey, value: byteValue}) + } + ffiWriteOptionsSetUserMetadata.symbol(ctx)(cOpts, keepAlive.userMetadata) + } + return cOpts, keepAlive, nil } // CreateDir creates a directory at the specified path. @@ -99,11 +290,13 @@ func (op *Operator) CreateDir(path string) error { // Writer returns a new Writer for the specified path. // // Writer is a wrapper around the C-binding function `opendal_operator_writer`. +// When options are provided, it uses `opendal_operator_writer_with`. // It provides a way to obtain a writer for writing data to the storage system. // // # Parameters // // - path: The destination path where data will be written. +// - opts: Optional write options. // // # Returns // @@ -124,8 +317,27 @@ func (op *Operator) CreateDir(path string) error { // } // // Note: This example assumes proper error handling and import statements. -func (op *Operator) Writer(path string) (*Writer, error) { - inner, err := ffiOperatorWriter.symbol(op.ctx)(op.inner, path) +func (op *Operator) Writer(path string, opts ...WithWriteFn) (*Writer, error) { + if len(opts) == 0 { + inner, err := ffiOperatorWriter.symbol(op.ctx)(op.inner, path) + if err != nil { + return nil, err + } + writer := &Writer{ + inner: inner, + ctx: op.ctx, + } + return writer, nil + } + + o := parseWriteOptions(opts...) + cOpts, keepAlive, err := newOpendalWriteOptions(op.ctx, o) + if err != nil { + return nil, err + } + defer ffiWriteOptionsFree.symbol(op.ctx)(cOpts) + inner, err := ffiOperatorWriterWith.symbol(op.ctx)(op.inner, path, cOpts) + runtime.KeepAlive(keepAlive) if err != nil { return nil, err } @@ -204,6 +416,29 @@ var ffiOperatorWrite = newFFI(ffiOpts{ } }) +var ffiOperatorWriteWith = newFFI(ffiOpts{ + sym: "opendal_operator_write_with", + rType: &ffi.TypePointer, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall ffiCall) func(op *opendalOperator, path string, data []byte, opts *opendalWriteOptions) error { + return func(op *opendalOperator, path string, data []byte, opts *opendalWriteOptions) error { + bytePath, err := BytePtrFromString(path) + if err != nil { + return err + } + bytes := toOpendalBytes(data) + var e *opendalError + ffiCall( + unsafe.Pointer(&e), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + unsafe.Pointer(&bytes), + unsafe.Pointer(&opts), + ) + return parseError(ctx, e) + } +}) + var ffiOperatorCreateDir = newFFI(ffiOpts{ sym: "opendal_operator_create_dir", rType: &ffi.TypePointer, @@ -247,6 +482,149 @@ var ffiOperatorWriter = newFFI(ffiOpts{ } }) +var ffiOperatorWriterWith = newFFI(ffiOpts{ + sym: "opendal_operator_writer_with", + rType: &typeResultOperatorWriter, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer}, +}, func(ctx context.Context, ffiCall ffiCall) func(op *opendalOperator, path string, opts *opendalWriteOptions) (*opendalWriter, error) { + return func(op *opendalOperator, path string, opts *opendalWriteOptions) (*opendalWriter, error) { + bytePath, err := BytePtrFromString(path) + if err != nil { + return nil, err + } + var result resultOperatorWriter + ffiCall( + unsafe.Pointer(&result), + unsafe.Pointer(&op), + unsafe.Pointer(&bytePath), + unsafe.Pointer(&opts), + ) + if result.error != nil { + return nil, parseError(ctx, result.error) + } + return result.writer, nil + } +}) + +var ffiWriteOptionsNew = newFFI(ffiOpts{ + sym: "opendal_write_options_new", + rType: &ffi.TypePointer, +}, func(_ context.Context, ffiCall ffiCall) func() *opendalWriteOptions { + return func() *opendalWriteOptions { + var opts *opendalWriteOptions + ffiCall(unsafe.Pointer(&opts)) + return opts + } +}) + +var ffiWriteOptionsFree = newFFI(ffiOpts{ + sym: "opendal_write_options_free", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions) { + return func(opts *opendalWriteOptions) { + ffiCall( + nil, + unsafe.Pointer(&opts), + ) + } +}) + +var ffiWriteOptionsSetAppend = newFFI(ffiOpts{ + sym: "opendal_write_options_set_append", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypeUint8}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions, append bool) { + return func(opts *opendalWriteOptions, append bool) { + var v uint8 + if append { + v = 1 + } + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&v)) + } +}) + +func byteSliceFromString(value string) ([]byte, error) { + if strings.IndexByte(value, 0) >= 0 { + return nil, errors.New("string contains nul") + } + return append([]byte(value), 0), nil +} + +func newWriteOptionsSetStringFFI(sym string) *FFI[func(*opendalWriteOptions, string) ([]byte, error)] { + return newFFI(ffiOpts{ + sym: contextKey(sym), + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, + }, func(_ context.Context, ffiCall ffiCall) func(*opendalWriteOptions, string) ([]byte, error) { + return func(opts *opendalWriteOptions, value string) ([]byte, error) { + data, err := byteSliceFromString(value) + if err != nil { + return nil, err + } + byteValue := &data[0] + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&byteValue)) + return data, nil + } + }) +} + +var ffiWriteOptionsSetCacheControl = newWriteOptionsSetStringFFI("opendal_write_options_set_cache_control") +var ffiWriteOptionsSetContentType = newWriteOptionsSetStringFFI("opendal_write_options_set_content_type") +var ffiWriteOptionsSetContentDisposition = newWriteOptionsSetStringFFI("opendal_write_options_set_content_disposition") +var ffiWriteOptionsSetContentEncoding = newWriteOptionsSetStringFFI("opendal_write_options_set_content_encoding") +var ffiWriteOptionsSetIfMatch = newWriteOptionsSetStringFFI("opendal_write_options_set_if_match") +var ffiWriteOptionsSetIfNoneMatch = newWriteOptionsSetStringFFI("opendal_write_options_set_if_none_match") + +var ffiWriteOptionsSetIfNotExists = newFFI(ffiOpts{ + sym: "opendal_write_options_set_if_not_exists", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypeUint8}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions, ifNotExists bool) { + return func(opts *opendalWriteOptions, ifNotExists bool) { + var v uint8 + if ifNotExists { + v = 1 + } + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&v)) + } +}) + +var ffiWriteOptionsSetConcurrent = newFFI(ffiOpts{ + sym: "opendal_write_options_set_concurrent", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions, concurrent uint) { + return func(opts *opendalWriteOptions, concurrent uint) { + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&concurrent)) + } +}) + +var ffiWriteOptionsSetChunk = newFFI(ffiOpts{ + sym: "opendal_write_options_set_chunk", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions, chunk uint) { + return func(opts *opendalWriteOptions, chunk uint) { + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&chunk)) + } +}) + +var ffiWriteOptionsSetUserMetadata = newFFI(ffiOpts{ + sym: "opendal_write_options_set_user_metadata", + rType: &ffi.TypeVoid, + aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer, &ffi.TypePointer}, +}, func(_ context.Context, ffiCall ffiCall) func(opts *opendalWriteOptions, userMetadata []opendalWriteUserMetadataPair) { + return func(opts *opendalWriteOptions, userMetadata []opendalWriteUserMetadataPair) { + var ptr *opendalWriteUserMetadataPair + if len(userMetadata) > 0 { + ptr = &userMetadata[0] + } + length := uint(len(userMetadata)) + ffiCall(nil, unsafe.Pointer(&opts), unsafe.Pointer(&ptr), unsafe.Pointer(&length)) + } +}) + var ffiWriterFree = newFFI(ffiOpts{ sym: "opendal_writer_free", rType: &ffi.TypeVoid,