Skip to content

fix(bindings/python): support foyer wheels on armv7 #7685

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
suyanhanx wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(bindings/python): support foyer wheels on armv7 #7685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions bindings/python/.cargo/config.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[env]
# Keep io-uring bindgen independent from manylinux/musllinux kernel headers.
BUILD_IO_URING_INCLUDE_FILE = { value = "uapi/io_uring_bindgen.h", relative = true }
12 changes: 9 additions & 3 deletions bindings/python/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ default = [
"services-webhdfs",
]

# NOTE: this is the feature we used to build pypi wheels.
# When enable or disable some features,
# Also, you need to update the `enabled_service` function in dev/src/generate/python.rs to match it.
# NOTE: this is the feature we use to build PyPI wheels.
# When enabling or disabling services here, also update `enabled_service` in
# dev/src/generate/python.rs to match it.
services-all = [
'default',
'services-aliyun-drive',
Expand All @@ -57,6 +57,7 @@ services-all = [
'services-cacache',
'services-dashmap',
'services-dropbox',
'services-foyer',
'services-ftp',
'services-gdrive',
'services-goosefs',
Expand Down Expand Up @@ -217,6 +218,11 @@ pyo3-async-runtimes = { version = "0.27.0", features = ["tokio-runtime"] }
pyo3-stub-gen = { version = "0.17" }
tokio = "1"

# Python armv7 release wheels include foyer, which pulls io-uring on Linux.
# io-uring has no prebuilt arm bindings, so generate them only for arm targets.
[target.'cfg(target_arch = "arm")'.dependencies]
io-uring = { version = "0.7", features = ["bindgen"] }

[profile.release]
codegen-units = 1
debug = false
Expand Down
132 changes: 132 additions & 0 deletions bindings/python/src/services.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ pub enum PyScheme {
Dashmap,
#[cfg(feature = "services-dropbox")]
Dropbox,
#[cfg(feature = "services-foyer")]
Foyer,
#[cfg(feature = "services-fs")]
Fs,
#[cfg(feature = "services-ftp")]
Expand Down Expand Up @@ -724,6 +726,70 @@ submit! {
}
}

submit! {
gen_methods_from_python! {
r#"
import builtins
import typing
import opendal.services
class Operator:
@overload
def __new__(cls,
scheme: typing.Literal[opendal.services.Scheme.Foyer, "foyer"],
/,
*,
disk_capacity: builtins.int = ...,
disk_file_size: builtins.int = ...,
disk_path: builtins.str = ...,
memory: builtins.int = ...,
name: builtins.str = ...,
recover_mode: builtins.str = ...,
root: builtins.str = ...,
shards: builtins.int = ...,
) -> typing.Self:
r"""
Create a new `Operator` for `foyer` service.

Parameters
----------
disk_capacity : builtins.int, optional
Disk cache total capacity in bytes.
Only used when `disk_path` is set.
disk_file_size : builtins.int, optional
Individual cache file size in bytes.
Default is 1 MiB.
Only used when `disk_path` is set.
disk_path : builtins.str, optional
Disk cache directory path.
If set, enables hybrid cache with disk storage.
Data will be persisted to this directory when memory
cache is full.
memory : builtins.int, optional
Memory capacity in bytes for the cache.
name : builtins.str, optional
Name for this cache instance.
recover_mode : builtins.str, optional
Recovery mode when starting the cache.
Valid values: "none" (default), "quiet", "strict".
- "none": Don't recover from disk - "quiet": Recover
and skip errors - "strict": Recover and panic on
errors
root : builtins.str, optional
Root path of this backend.
shards : builtins.int, optional
Number of shards for concurrent access.
Default is 1.
Higher values improve concurrency but increase
overhead.
Returns
-------
Operator
The new `Operator` for `foyer` service
"""
"#
}
}

submit! {
gen_methods_from_python! {
r#"
Expand Down Expand Up @@ -3368,6 +3434,70 @@ submit! {
}
}

submit! {
gen_methods_from_python! {
r#"
import builtins
import typing
import opendal.services
class AsyncOperator:
@overload
def __new__(cls,
scheme: typing.Literal[opendal.services.Scheme.Foyer, "foyer"],
/,
*,
disk_capacity: builtins.int = ...,
disk_file_size: builtins.int = ...,
disk_path: builtins.str = ...,
memory: builtins.int = ...,
name: builtins.str = ...,
recover_mode: builtins.str = ...,
root: builtins.str = ...,
shards: builtins.int = ...,
) -> typing.Self:
r"""
Create a new `AsyncOperator` for `foyer` service.

Parameters
----------
disk_capacity : builtins.int, optional
Disk cache total capacity in bytes.
Only used when `disk_path` is set.
disk_file_size : builtins.int, optional
Individual cache file size in bytes.
Default is 1 MiB.
Only used when `disk_path` is set.
disk_path : builtins.str, optional
Disk cache directory path.
If set, enables hybrid cache with disk storage.
Data will be persisted to this directory when memory
cache is full.
memory : builtins.int, optional
Memory capacity in bytes for the cache.
name : builtins.str, optional
Name for this cache instance.
recover_mode : builtins.str, optional
Recovery mode when starting the cache.
Valid values: "none" (default), "quiet", "strict".
- "none": Don't recover from disk - "quiet": Recover
and skip errors - "strict": Recover and panic on
errors
root : builtins.str, optional
Root path of this backend.
shards : builtins.int, optional
Number of shards for concurrent access.
Default is 1.
Higher values improve concurrency but increase
overhead.
Returns
-------
AsyncOperator
The new `AsyncOperator` for `foyer` service
"""
"#
}
}

submit! {
gen_methods_from_python! {
r#"
Expand Down Expand Up @@ -5498,6 +5628,8 @@ impl_enum_to_str!(
Dashmap => "dashmap",
#[cfg(feature = "services-dropbox")]
Dropbox => "dropbox",
#[cfg(feature = "services-foyer")]
Foyer => "foyer",
#[cfg(feature = "services-fs")]
Fs => "fs",
#[cfg(feature = "services-ftp")]
Expand Down
42 changes: 42 additions & 0 deletions bindings/python/uapi/io_uring_bindgen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#define __NR_io_uring_setup 425
#define __NR_io_uring_enter 426
#define __NR_io_uring_register 427

typedef signed int __s32;
typedef signed long long __s64;
typedef unsigned char __u8;
typedef unsigned short __u16;
typedef unsigned int __u32;
typedef unsigned long long __u64;
typedef int __kernel_rwf_t;

#define __aligned_u64 __u64 __attribute__((aligned(8)))
#define __DECLARE_FLEX_ARRAY(TYPE, NAME) \
struct { \
struct { } __empty_ ## NAME; \
TYPE NAME[]; \
}
#define SPLICE_F_FD_IN_FIXED (1U << 31)

struct __kernel_timespec {
__s64 tv_sec;
__s64 tv_nsec;
};

struct open_how {
__u64 flags;
__u64 mode;
__u64 resolve;
};

#define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H
#define OPENDAL_IO_URING_BINDGEN_TYPES
#include "linux/io_uring.h"

#define FUTEX_WAITV_MAX 128
struct futex_waitv {
__u64 val;
__u64 uaddr;
__u32 flags;
__u32 __reserved;
};
Loading
Loading