diff --git a/.github/musl_build.sh b/.github/musl_build.sh index 86ac2521..eb439b08 100644 --- a/.github/musl_build.sh +++ b/.github/musl_build.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + if [ -z "$1" ]; then echo "Usage: $0 [maturin_args]" exit 1 @@ -9,8 +11,15 @@ TARGET=$1 ARGS=$2 IMAGE="ghcr.io/0x676e67/rust-musl-cross" -VOLUME_MAPPING="-v $(pwd):/home/rust/src" +VOLUME_MAPPING=("-v" "$(pwd):/home/rust/src") MATURIN_CMD="maturin build --release --out dist $ARGS" +EXTRA_ENV=() + +for var in BORING_BSSL_RUST_CPPLIB MATURIN_VERSION CFLAGS CXXFLAGS LDFLAGS RUSTFLAGS; do + if [ -n "${!var}" ]; then + EXTRA_ENV+=("-e" "$var=${!var}") + fi +done case $TARGET in x86_64-unknown-linux-musl | \ @@ -26,6 +35,6 @@ esac echo "Building for $TARGET..." docker pull $IMAGE:$TARGET -docker run --rm $VOLUME_MAPPING $IMAGE:$TARGET /bin/bash -c "$MATURIN_CMD" +docker run --rm "${VOLUME_MAPPING[@]}" "${EXTRA_ENV[@]}" $IMAGE:$TARGET /bin/bash -c "$MATURIN_CMD" echo "Build completed for target: $TARGET" \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25dad5ae..aa2da1d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,6 +105,10 @@ jobs: custom_env: CC: i686-linux-gnu-gcc CXX: i686-linux-gnu-g++ + CFLAGS: -msse2 + CXXFLAGS: -msse2 + CFLAGS_i686_unknown_linux_gnu: -msse2 + CXXFLAGS_i686_unknown_linux_gnu: -msse2 CARGO_TARGET_I686_UNKNOWN_LINUX_GNU_LINKER: i686-linux-gnu-g++ - runner: ubuntu-latest target: aarch64 @@ -127,10 +131,6 @@ jobs: - uses: actions/setup-python@v6 with: python-version: 3.x - - name: Install LLVM and Clang - uses: KyleMayes/install-llvm-action@v2 - with: - version: "13.0" - name: Install target-specific APT dependencies if: "matrix.platform.apt_packages != ''" run: sudo apt-get update && sudo apt-get install -y ${{ matrix.platform.apt_packages }} @@ -177,9 +177,16 @@ jobs: steps: - uses: actions/checkout@v6 - name: Build wheels - run: bash .github/musl_build.sh ${{ matrix.platform.target }} "${{ matrix.build_type.maturin_args }}" --features "${{ matrix.platform.allocator }}" env: MATURIN_VERSION: ${{ env.MATURIN_VERSION }} + BORING_BSSL_RUST_CPPLIB: static=stdc++ + run: | + export RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-L native=/usr/local/musl/${{ matrix.platform.target }}/lib -L native=/usr/local/musl/lib/gcc/${{ matrix.platform.target }}/11.4.0 -C link-arg=-static-libgcc -l static=gcc" + if [ "${{ matrix.platform.target }}" = "i686-unknown-linux-musl" ]; then + export CFLAGS="${CFLAGS} -msse2" + export CXXFLAGS="${CXXFLAGS} -msse2" + fi + bash .github/musl_build.sh ${{ matrix.platform.target }} "${{ matrix.build_type.maturin_args }}" --features "${{ matrix.platform.allocator }}" - name: Upload wheels uses: actions/upload-artifact@v7 with: @@ -242,10 +249,10 @@ jobs: - name: free-threaded maturin_args: "--interpreter 3.13t 3.14t" platform: - - runner: macos-14 + - runner: macos-latest target: x86_64 allocator: jemalloc - - runner: macos-14 + - runner: macos-latest target: aarch64 allocator: jemalloc steps: diff --git a/Cargo.toml b/Cargo.toml index bc281847..c3660f73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,8 +65,9 @@ wreq = { version = "6.0.0-rc.27", features = [ "deflate", "zstd", "ws", + "parking_lot", ] } -wreq-util = { version = "3.0.0-rc.10", features = ["emulation-rand", "emulation-compression"] } +wreq-util = { version = "3.0.0-rc.10", features = ["emulation-compression"] } hickory-resolver = "0.25.2" cookie = "0.18" mimalloc = { version = "0.1.43", default-features = false, features = [ @@ -86,4 +87,5 @@ panic = "abort" strip = true [patch.crates-io] -wreq = { git = "https://github.com/0x676e67/wreq", rev = "240d84eab5eb4df8548933ee2c13337d86e1afe1" } +wreq = { git = "https://github.com/0x676e67/wreq" } +wreq-util = { git = "https://github.com/0x676e67/wreq-util" } \ No newline at end of file diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index ff3386aa..209f5e2d 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -86,7 +86,9 @@ markdown_extensions: - pymdownx.details - attr_list - md_in_html - - pymdownx.emoji + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.tasklist - pymdownx.caret - pymdownx.mark diff --git a/docs/source/getting-started/introduction.md b/docs/source/getting-started/introduction.md index a30236f9..a56a86d2 100644 --- a/docs/source/getting-started/introduction.md +++ b/docs/source/getting-started/introduction.md @@ -55,26 +55,83 @@ Due to the complexity of **TLS** encryption and the widespread adoption of **HTT **TLS** and **HTTP/2** fingerprints are often identical across various browser models because these underlying protocols evolve slower than browser release cycles. **100+ browser device emulation profiles** are maintained in **[wreq-python]**. -??? note "Available OS emulations" - - | **OS** | **Description** | - | ----------- | ------------------------------ | - | **Windows** | Windows (any version) | - | **MacOS** | macOS (any version) | - | **Linux** | Linux (any distribution) | - | **Android** | Android (mobile) | - | **iOS** | iOS (iPhone/iPad) | - -??? note "Available browser emulations" - - | **Browser** | **Versions** | - | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | - | **Chrome** | `Chrome100`, `Chrome101`, `Chrome104`, `Chrome105`, `Chrome106`, `Chrome107`, `Chrome108`, `Chrome109`, `Chrome110`, `Chrome114`, `Chrome116`, `Chrome117`, `Chrome118`, `Chrome119`, `Chrome120`, `Chrome123`, `Chrome124`, `Chrome126`, `Chrome127`, `Chrome128`, `Chrome129`, `Chrome130`, `Chrome131`, `Chrome132`, `Chrome133`, `Chrome134`, `Chrome135`, `Chrome136`, `Chrome137`, `Chrome138`, `Chrome139`, `Chrome140`, `Chrome141`, `Chrome142`, `Chrome143`, `Chrome144`, `Chrome145` | - | **Safari** | `SafariIos17_2`, `SafariIos17_4_1`, `SafariIos16_5`, `Safari15_3`, `Safari15_5`, `Safari15_6_1`, `Safari16`, `Safari16_5`, `Safari17_0`, `Safari17_2_1`, `Safari17_4_1`, `Safari17_5`, `Safari18`, `SafariIPad18`, `Safari18_2`, `SafariIos18_1_1`, `Safari18_3`, `Safari18_3_1`, `Safari18_5`, `Safari26`, `Safari26_1`, `Safari26_2`, `SafariIos26`, `SafariIos26_2`, `SafariIPad26`, `SafariIPad26_2` | - | **Firefox** | `Firefox109`, `Firefox117`, `Firefox128`, `Firefox133`, `Firefox135`, `FirefoxPrivate135`, `FirefoxAndroid135`, `Firefox136`, `FirefoxPrivate136`, `Firefox139`, `Firefox142`, `Firefox143`, `Firefox144`, `Firefox145`, `Firefox146`, `Firefox147` | - | **OkHttp** | `OkHttp3_9`, `OkHttp3_11`, `OkHttp3_13`, `OkHttp3_14`, `OkHttp4_9`, `OkHttp4_10`, `OkHttp4_12`, `OkHttp5` | - | **Edge** | `Edge101`, `Edge122`, `Edge127`, `Edge131`, `Edge134`, `Edge135`, `Edge136`, `Edge137`, `Edge138`, `Edge139`, `Edge140`, `Edge141`, `Edge142`, `Edge143`, `Edge144`, `Edge145` | - | **Opera** | `Opera116`, `Opera117`, `Opera118`, `Opera119` +???+ note "Available OS emulations" + +
+ + - :fontawesome-brands-windows: **Windows** + + --- + + Windows (any version) + + - :fontawesome-brands-apple: **macOS** + + --- + + macOS (any version) + + - :fontawesome-brands-linux: **Linux** + + --- + + Linux (any distribution) + + - :fontawesome-brands-android: **Android** + + --- + + Android (mobile) + + - :fontawesome-brands-apple: **iOS** + + --- + + iOS (iPhone / iPad) + +
+ +???+ note "Available browser emulations" + +
+ + - :fontawesome-brands-chrome: **Chrome** + + --- + + `Chrome100` `Chrome101` `Chrome104` `Chrome105` `Chrome106` `Chrome107` `Chrome108` `Chrome109` `Chrome110` `Chrome114` `Chrome116` `Chrome117` `Chrome118` `Chrome119` `Chrome120` `Chrome123` `Chrome124` `Chrome126` `Chrome127` `Chrome128` `Chrome129` `Chrome130` `Chrome131` `Chrome132` `Chrome133` `Chrome134` `Chrome135` `Chrome136` `Chrome137` `Chrome138` `Chrome139` `Chrome140` `Chrome141` `Chrome142` `Chrome143` `Chrome144` `Chrome145` `Chrome146` `Chrome147` + + - :fontawesome-brands-edge: **Edge** + + --- + + `Edge101` `Edge122` `Edge127` `Edge131` `Edge134` `Edge135` `Edge136` `Edge137` `Edge138` `Edge139` `Edge140` `Edge141` `Edge142` `Edge143` `Edge144` `Edge145` + + - :fontawesome-brands-firefox: **Firefox** + + --- + + `Firefox109` `Firefox117` `Firefox128` `Firefox133` `Firefox135` `FirefoxPrivate135` `FirefoxAndroid135` `Firefox136` `FirefoxPrivate136` `Firefox139` `Firefox142` `Firefox143` `Firefox144` `Firefox145` `Firefox146` `Firefox147` `Firefox148` `Firefox149` + + - :fontawesome-brands-safari: **Safari** + + --- + + `SafariIos17_2` `SafariIos17_4_1` `SafariIos16_5` `Safari15_3` `Safari15_5` `Safari15_6_1` `Safari16` `Safari16_5` `Safari17_0` `Safari17_2_1` `Safari17_4_1` `Safari17_5` `Safari18` `SafariIPad18` `Safari18_2` `Safari18_3` `Safari18_3_1` `SafariIos18_1_1` `Safari18_5` `Safari26` `Safari26_1` `Safari26_2` `SafariIos26` `SafariIos26_2` `SafariIPad26` `SafariIpad26_2` + + - :fontawesome-brands-opera: **Opera** + + --- + + `Opera116` `Opera117` `Opera118` `Opera119` `Opera120` `Opera121` `Opera122` `Opera123` `Opera124` `Opera125` `Opera126` `Opera127` `Opera128` `Opera129` `Opera130` + + - :material-web: **OkHttp** + + --- + + `OkHttp3_9` `OkHttp3_11` `OkHttp3_13` `OkHttp3_14` `OkHttp4_9` `OkHttp4_10` `OkHttp4_12` `OkHttp5` + +
## Performance diff --git a/docs/source/guide/emulation.md b/docs/source/guide/emulation.md index 017b2494..ccb36084 100644 --- a/docs/source/guide/emulation.md +++ b/docs/source/guide/emulation.md @@ -30,16 +30,16 @@ if __name__ == "__main__": ```python import asyncio from wreq import Client -from wreq.emulation import Emulation, EmulationOS, EmulationOption +from wreq.emulation import Emulation, Profile, Platform async def main(): client = Client() resp = await client.get( "https://tls.peet.ws/api/all", - emulation=EmulationOption( - emulation=Emulation.Chrome134, - emulation_os=EmulationOS.Android, + emulation=Emulation( + profile=Profile.Chrome134, + platform=Platform.Android, ), # Disable client default headers default_headers=False, diff --git a/docs/source/guide/websocket.md b/docs/source/guide/websocket.md index 264fd092..2b21d910 100644 --- a/docs/source/guide/websocket.md +++ b/docs/source/guide/websocket.md @@ -10,7 +10,7 @@ import asyncio import datetime import wreq -from wreq import Message, WebSocket +from wreq import Message, WebSocket, Version from wreq import exceptions @@ -95,11 +95,11 @@ async def recv_message(ws): async def main(): - # Connect to HTTP/2 WebSocket server with force_http2=True + # Connect to HTTP/2 WebSocket server client = wreq.Client(verify=False) ws: WebSocket = await client.websocket( "wss://127.0.0.1:3000/ws", - force_http2=True + version=Version.HTTP_2 ) async with ws: print("Status Code: ", ws.status) diff --git a/examples/emulation.py b/examples/emulation.py index fb1e822c..8e1eed27 100644 --- a/examples/emulation.py +++ b/examples/emulation.py @@ -1,6 +1,6 @@ import asyncio from wreq import Client, Response -from wreq.emulation import Emulation, EmulationOS, EmulationOption +from wreq.emulation import Emulation, Profile, Platform from wreq.tls import TlsOptions, TlsVersion, AlpnProtocol from wreq.http2 import Http2Options, PseudoId, PseudoOrder from wreq.header import HeaderMap, OrigHeaderMap @@ -49,9 +49,9 @@ async def request_chrome_android(client: Client): print("\n[Testing Chrome on Android Emulation]") resp = await client.get( "https://tls.peet.ws/api/all", - emulation=EmulationOption( - emulation=Emulation.Chrome134, - emulation_os=EmulationOS.Android, + emulation=Emulation( + profile=Profile.Chrome134, + platform=Platform.Android, ), # Disable client default headers default_headers=False, diff --git a/examples/http2_websocket.py b/examples/http2_websocket.py index a7072fcc..7d510989 100644 --- a/examples/http2_websocket.py +++ b/examples/http2_websocket.py @@ -2,7 +2,7 @@ import datetime import signal import wreq -from wreq import Message, WebSocket +from wreq import Message, WebSocket, Version from wreq import exceptions @@ -45,7 +45,9 @@ async def recv_message(ws): async def main(): client = wreq.Client(tls_verify=False) - ws: WebSocket = await client.websocket("wss://127.0.0.1:3000/ws", force_http2=True) + ws: WebSocket = await client.websocket( + "wss://127.0.0.1:3000/ws", version=Version.HTTP_2 + ) async with ws: print("Status Code: ", ws.status) print("Version: ", ws.version) diff --git a/python/wreq/blocking.py b/python/wreq/blocking.py index 3ad9a7c2..4a30eb75 100644 --- a/python/wreq/blocking.py +++ b/python/wreq/blocking.py @@ -114,22 +114,9 @@ def close(self) -> None: r""" Close the response. - **Current behavior:** - - - When connection pooling is **disabled**: This method closes the network connection. - - When connection pooling is **enabled**: This method closes the response, prevents further body reads, - and returns the connection to the pool for reuse. - - **Future changes:** - - In future versions, this method will be changed to always close the network connection regardless of - whether connection pooling is enabled or not. - - **Recommendation:** - - It is **not recommended** to manually call this method at present. Instead, use context managers - (with statement) to properly manage response lifecycle. Wait for the improved implementation - in future versions. + This method closes the network connection regardless of whether connection pooling is + enabled or not. It is recommended to use context managers (`with` statement) to properly + manage response lifecycle instead of calling this method manually. """ def __enter__(self) -> Any: ... diff --git a/python/wreq/cookie.py b/python/wreq/cookie.py index 3dd1753c..f3325cdb 100644 --- a/python/wreq/cookie.py +++ b/python/wreq/cookie.py @@ -94,57 +94,72 @@ class Jar: r""" A thread-safe cookie jar for storing and managing HTTP cookies. - This cookie jar can be safely shared across multiple threads and is used - to automatically handle cookies during HTTP requests and responses. + `Jar` can be shared across multiple threads and tasks. When passed to a + client, it is used to automatically persist and send cookies across + requests and responses. - By default, cookie compression is enabled to reduce storage overhead. - Use `uncompressed()` to create a variant without compression if needed. - """ + **Protocol-level behaviour** - def __init__(self, compression: bool | None = None) -> None: - r""" - Create a new cookie jar with compression enabled by default. - """ - ... + - **HTTP/1.1** — all cookies are folded into a single `Cookie` header, + as required by [RFC 9112 §5.6.3]. + - **HTTP/2 and above** — each cookie is sent as an individual header + field per [RFC 9113 §8.1.2.5]. - def compressed(self) -> "Jar": - r""" - Clone this Jar, sharing storage but enabling compression. - """ - ... + [RFC 9112 §5.6.3]: https://www.rfc-editor.org/rfc/rfc9112#section-5.6.3 + [RFC 9113 §8.1.2.5]: https://www.rfc-editor.org/rfc/rfc9113#section-8.1.2.5 + """ - def uncompressed(self) -> "Jar": + def __init__(self) -> None: r""" - Clone this Jar, sharing storage but disabling compression. + Create a new empty cookie jar. """ ... def get(self, name: str, url: str) -> Cookie | None: r""" - Get a cookie by name and URL. + Look up a cookie by name scoped to the given URL. + + Returns `None` if no matching cookie is found. + + Args: + name: The cookie name to look up. + url: The URL the cookie is scoped to (used for domain / path matching). """ ... def get_all(self) -> Sequence[Cookie]: r""" - Get all cookies. + Return all cookies currently stored in the jar. """ ... def add(self, cookie: Cookie | str, url: str) -> None: r""" - Add a cookie or cookie string to this jar. + Insert a cookie into the jar, scoped to the given URL. + + Args: + cookie: A `Cookie` object or a raw `Set-Cookie` header string. + url: The URL the cookie originates from (used for domain / path scoping). + + Example: + ```python + jar.add("session=abc123; Path=/; HttpOnly", "https://example.com") + ``` """ ... def remove(self, name: str, url: str) -> None: r""" - Remove a cookie from this jar by name and URL. + Remove a cookie by name, scoped to the given URL. + + Args: + name: The cookie name to remove. + url: The URL the cookie is scoped to. """ ... def clear(self) -> None: r""" - Clear all cookies in this jar. + Remove all cookies from the jar. """ ... diff --git a/python/wreq/emulation.py b/python/wreq/emulation.py index 82efd45a..6a50dada 100644 --- a/python/wreq/emulation.py +++ b/python/wreq/emulation.py @@ -9,15 +9,19 @@ """ from enum import Enum, auto -from typing import final +from typing import ClassVar, final -__all__ = ["Emulation", "EmulationOS", "EmulationOption"] +__all__ = ["Emulation", "Profile", "Platform"] @final -class Emulation(Enum): +class Profile(Enum): r""" - An emulation. + Selects which client profile the request should look like. + + This controls the built-in TLS, HTTP/2, and header presets used for the + request. Variants cover browser-style profiles as well as other clients, + such as OkHttp. """ # Chrome versions @@ -58,6 +62,8 @@ class Emulation(Enum): Chrome143 = auto() Chrome144 = auto() Chrome145 = auto() + Chrome146 = auto() + Chrome147 = auto() # Microsoft Edge versions Edge101 = auto() @@ -76,6 +82,8 @@ class Emulation(Enum): Edge143 = auto() Edge144 = auto() Edge145 = auto() + Edge146 = auto() + Edge147 = auto() # Firefox versions Firefox109 = auto() @@ -94,6 +102,8 @@ class Emulation(Enum): Firefox145 = auto() Firefox146 = auto() Firefox147 = auto() + Firefox148 = auto() + Firefox149 = auto() # Safari versions SafariIos17_2 = auto() @@ -138,15 +148,27 @@ class Emulation(Enum): Opera117 = auto() Opera118 = auto() Opera119 = auto() + Opera120 = auto() + Opera121 = auto() + Opera122 = auto() + Opera123 = auto() + Opera124 = auto() + Opera125 = auto() + Opera126 = auto() + Opera127 = auto() + Opera128 = auto() + Opera129 = auto() + Opera130 = auto() @final -class EmulationOS(Enum): +class Platform(Enum): """ - Operating systems that can be emulated. + Selects which platform the client should look like. - This enum defines the operating systems that can be combined with - browser emulations to create more specific fingerprints. + This mainly affects platform-specific headers and user-agent details. + In most cases you can keep the default unless you need to match a + specific Windows, macOS, Linux, Android, or iOS profile. """ Windows = auto() # Windows (any version) @@ -157,66 +179,192 @@ class EmulationOS(Enum): @final -class EmulationOption: +class Emulation: """ - Configuration options for browser and client emulation. + Represents the configuration options for emulating a client profile and platform. - This class allows fine-grained control over emulation behavior, - including the ability to disable specific features or combine - browser types with specific operating systems. + The `Emulation` struct allows you to configure various aspects of profile and platform + emulation, including the profile, platform, and whether to enable certain features + like HTTP/2 or headers. """ + # Chrome versions + Chrome100: ClassVar[Profile] = Profile.Chrome100 + Chrome101: ClassVar[Profile] = Profile.Chrome101 + Chrome104: ClassVar[Profile] = Profile.Chrome104 + Chrome105: ClassVar[Profile] = Profile.Chrome105 + Chrome106: ClassVar[Profile] = Profile.Chrome106 + Chrome107: ClassVar[Profile] = Profile.Chrome107 + Chrome108: ClassVar[Profile] = Profile.Chrome108 + Chrome109: ClassVar[Profile] = Profile.Chrome109 + Chrome110: ClassVar[Profile] = Profile.Chrome110 + Chrome114: ClassVar[Profile] = Profile.Chrome114 + Chrome116: ClassVar[Profile] = Profile.Chrome116 + Chrome117: ClassVar[Profile] = Profile.Chrome117 + Chrome118: ClassVar[Profile] = Profile.Chrome118 + Chrome119: ClassVar[Profile] = Profile.Chrome119 + Chrome120: ClassVar[Profile] = Profile.Chrome120 + Chrome123: ClassVar[Profile] = Profile.Chrome123 + Chrome124: ClassVar[Profile] = Profile.Chrome124 + Chrome126: ClassVar[Profile] = Profile.Chrome126 + Chrome127: ClassVar[Profile] = Profile.Chrome127 + Chrome128: ClassVar[Profile] = Profile.Chrome128 + Chrome129: ClassVar[Profile] = Profile.Chrome129 + Chrome130: ClassVar[Profile] = Profile.Chrome130 + Chrome131: ClassVar[Profile] = Profile.Chrome131 + Chrome132: ClassVar[Profile] = Profile.Chrome132 + Chrome133: ClassVar[Profile] = Profile.Chrome133 + Chrome134: ClassVar[Profile] = Profile.Chrome134 + Chrome135: ClassVar[Profile] = Profile.Chrome135 + Chrome136: ClassVar[Profile] = Profile.Chrome136 + Chrome137: ClassVar[Profile] = Profile.Chrome137 + Chrome138: ClassVar[Profile] = Profile.Chrome138 + Chrome139: ClassVar[Profile] = Profile.Chrome139 + Chrome140: ClassVar[Profile] = Profile.Chrome140 + Chrome141: ClassVar[Profile] = Profile.Chrome141 + Chrome142: ClassVar[Profile] = Profile.Chrome142 + Chrome143: ClassVar[Profile] = Profile.Chrome143 + Chrome144: ClassVar[Profile] = Profile.Chrome144 + Chrome145: ClassVar[Profile] = Profile.Chrome145 + Chrome146: ClassVar[Profile] = Profile.Chrome146 + Chrome147: ClassVar[Profile] = Profile.Chrome147 + + # Microsoft Edge versions + Edge101: ClassVar[Profile] = Profile.Edge101 + Edge122: ClassVar[Profile] = Profile.Edge122 + Edge127: ClassVar[Profile] = Profile.Edge127 + Edge131: ClassVar[Profile] = Profile.Edge131 + Edge134: ClassVar[Profile] = Profile.Edge134 + Edge135: ClassVar[Profile] = Profile.Edge135 + Edge136: ClassVar[Profile] = Profile.Edge136 + Edge137: ClassVar[Profile] = Profile.Edge137 + Edge138: ClassVar[Profile] = Profile.Edge138 + Edge139: ClassVar[Profile] = Profile.Edge139 + Edge140: ClassVar[Profile] = Profile.Edge140 + Edge141: ClassVar[Profile] = Profile.Edge141 + Edge142: ClassVar[Profile] = Profile.Edge142 + Edge143: ClassVar[Profile] = Profile.Edge143 + Edge144: ClassVar[Profile] = Profile.Edge144 + Edge145: ClassVar[Profile] = Profile.Edge145 + Edge146: ClassVar[Profile] = Profile.Edge146 + Edge147: ClassVar[Profile] = Profile.Edge147 + + # Firefox versions + Firefox109: ClassVar[Profile] = Profile.Firefox109 + Firefox117: ClassVar[Profile] = Profile.Firefox117 + Firefox128: ClassVar[Profile] = Profile.Firefox128 + Firefox133: ClassVar[Profile] = Profile.Firefox133 + Firefox135: ClassVar[Profile] = Profile.Firefox135 + FirefoxPrivate135: ClassVar[Profile] = Profile.FirefoxPrivate135 + FirefoxAndroid135: ClassVar[Profile] = Profile.FirefoxAndroid135 + Firefox136: ClassVar[Profile] = Profile.Firefox136 + FirefoxPrivate136: ClassVar[Profile] = Profile.FirefoxPrivate136 + Firefox139: ClassVar[Profile] = Profile.Firefox139 + Firefox142: ClassVar[Profile] = Profile.Firefox142 + Firefox143: ClassVar[Profile] = Profile.Firefox143 + Firefox144: ClassVar[Profile] = Profile.Firefox144 + Firefox145: ClassVar[Profile] = Profile.Firefox145 + Firefox146: ClassVar[Profile] = Profile.Firefox146 + Firefox147: ClassVar[Profile] = Profile.Firefox147 + Firefox148: ClassVar[Profile] = Profile.Firefox148 + Firefox149: ClassVar[Profile] = Profile.Firefox149 + + # Safari versions + SafariIos17_2: ClassVar[Profile] = Profile.SafariIos17_2 + SafariIos17_4_1: ClassVar[Profile] = Profile.SafariIos17_4_1 + SafariIos16_5: ClassVar[Profile] = Profile.SafariIos16_5 + Safari15_3: ClassVar[Profile] = Profile.Safari15_3 + Safari15_5: ClassVar[Profile] = Profile.Safari15_5 + Safari15_6_1: ClassVar[Profile] = Profile.Safari15_6_1 + Safari16: ClassVar[Profile] = Profile.Safari16 + Safari16_5: ClassVar[Profile] = Profile.Safari16_5 + Safari17_0: ClassVar[Profile] = Profile.Safari17_0 + Safari17_2_1: ClassVar[Profile] = Profile.Safari17_2_1 + Safari17_4_1: ClassVar[Profile] = Profile.Safari17_4_1 + Safari17_5: ClassVar[Profile] = Profile.Safari17_5 + Safari18: ClassVar[Profile] = Profile.Safari18 + SafariIPad18: ClassVar[Profile] = Profile.SafariIPad18 + Safari18_2: ClassVar[Profile] = Profile.Safari18_2 + Safari18_3: ClassVar[Profile] = Profile.Safari18_3 + Safari18_3_1: ClassVar[Profile] = Profile.Safari18_3_1 + SafariIos18_1_1: ClassVar[Profile] = Profile.SafariIos18_1_1 + Safari18_5: ClassVar[Profile] = Profile.Safari18_5 + Safari26: ClassVar[Profile] = Profile.Safari26 + Safari26_1: ClassVar[Profile] = Profile.Safari26_1 + Safari26_2: ClassVar[Profile] = Profile.Safari26_2 + SafariIos26: ClassVar[Profile] = Profile.SafariIos26 + SafariIos26_2: ClassVar[Profile] = Profile.SafariIos26_2 + SafariIPad26: ClassVar[Profile] = Profile.SafariIPad26 + SafariIpad26_2: ClassVar[Profile] = Profile.SafariIpad26_2 + + # OkHttp versions + OkHttp3_9: ClassVar[Profile] = Profile.OkHttp3_9 + OkHttp3_11: ClassVar[Profile] = Profile.OkHttp3_11 + OkHttp3_13: ClassVar[Profile] = Profile.OkHttp3_13 + OkHttp3_14: ClassVar[Profile] = Profile.OkHttp3_14 + OkHttp4_9: ClassVar[Profile] = Profile.OkHttp4_9 + OkHttp4_10: ClassVar[Profile] = Profile.OkHttp4_10 + OkHttp4_12: ClassVar[Profile] = Profile.OkHttp4_12 + OkHttp5: ClassVar[Profile] = Profile.OkHttp5 + + # Opera versions + Opera116: ClassVar[Profile] = Profile.Opera116 + Opera117: ClassVar[Profile] = Profile.Opera117 + Opera118: ClassVar[Profile] = Profile.Opera118 + Opera119: ClassVar[Profile] = Profile.Opera119 + Opera120: ClassVar[Profile] = Profile.Opera120 + Opera121: ClassVar[Profile] = Profile.Opera121 + Opera122: ClassVar[Profile] = Profile.Opera122 + Opera123: ClassVar[Profile] = Profile.Opera123 + Opera124: ClassVar[Profile] = Profile.Opera124 + Opera125: ClassVar[Profile] = Profile.Opera125 + Opera126: ClassVar[Profile] = Profile.Opera126 + Opera127: ClassVar[Profile] = Profile.Opera127 + Opera128: ClassVar[Profile] = Profile.Opera128 + Opera129: ClassVar[Profile] = Profile.Opera129 + Opera130: ClassVar[Profile] = Profile.Opera130 + def __init__( self, - emulation: Emulation, - emulation_os: EmulationOS | None = None, - skip_http2: bool | None = None, - skip_headers: bool | None = None, + profile: Profile = Profile.Chrome100, + platform: Platform = Platform.MacOS, + http2: bool = True, + headers: bool = True, ) -> None: """ Create a new emulation configuration. Args: - emulation: The browser/client type to emulate - emulation_os: The operating system to emulate (optional) - skip_http2: Whether to disable HTTP/2 emulation (default: False) - skip_headers: Whether to skip default browser headers (default: False) + profile: Whether to change the profile (browser/okhttp) information. + platform: Whether to change the platform (Windows/macOS/Linux/Android/iOS) information. + http2: Whether to enable HTTP/2. + headers: Whether to include default headers. Returns: - A configured EmulationOption instance + A configured Emulation instance Example: ```python - # Basic Chrome emulation - option = EmulationOption(Emulation.Chrome137) - # Chrome on Windows with HTTP/2 disabled - option = EmulationOption( - emulation=Emulation.Chrome137, - emulation_os=EmulationOS.Windows, - skip_http2=True + option = Emulation( + profile=Profile.Chrome137, + platform=Platform.Windows, + http2=False, + headers=True ) ``` """ ... @staticmethod - def random() -> "EmulationOption": + def random() -> "Emulation": """ Generate a random emulation configuration. - This method creates a randomized emulation setup using a random - browser/client type and operating system combination. Useful for - scenarios where you want to vary your fingerprint across requests. - - Returns: - A randomly configured EmulationOption instance - Example: ```python # Use different random emulation for each client - client1 = wreq.Client(emulation=EmulationOption.random()) - client2 = wreq.Client(emulation=EmulationOption.random()) + client = wreq.Client(emulation=Emulation.random()) ``` """ ... diff --git a/python/wreq/tls.py b/python/wreq/tls.py index 6a6ece8d..fa7b72c1 100644 --- a/python/wreq/tls.py +++ b/python/wreq/tls.py @@ -20,6 +20,7 @@ "TlsOptions", "TlsInfo", "Params", + "KeyShare", ] @@ -57,6 +58,24 @@ class AlpsProtocol(Enum): HTTP3 = auto() +@final +class KeyShare(Enum): + """ + Key exchange groups (elliptic curves) for TLS 1.3. + """ + + P256 = auto() + P384 = auto() + P521 = auto() + X25519 = auto() + X25519_MLKEM768 = auto() + X25519_KYBER768_DRAFT00 = auto() + P256_KYBER768_DRAFT00 = auto() + MLKEM1024 = auto() + FFDHE2048 = auto() + FFDHE3072 = auto() + + @final class CertificateCompressionAlgorithm(Enum): """ @@ -98,8 +117,8 @@ class ExtensionType(Enum): KEY_SHARE = auto() RENEGOTIATE = auto() DELEGATED_CREDENTIAL = auto() + APPLICATION_SETTINGS_OLD = auto() APPLICATION_SETTINGS = auto() - APPLICATION_SETTINGS_NEW = auto() ENCRYPTED_CLIENT_HELLO = auto() CERTIFICATE_TIMESTAMP = auto() NEXT_PROTO_NEG = auto() @@ -321,9 +340,9 @@ class Params(TypedDict): Whether to skip session tickets when using PSK. """ - key_shares_limit: NotRequired[int] + key_shares: NotRequired[Sequence[KeyShare]] """ - Maximum number of key shares to include in ClientHello. + Whether to set specific key shares for TLS 1.3 handshakes. """ psk_dhe_ke: NotRequired[bool] @@ -349,6 +368,11 @@ class Params(TypedDict): List of supported elliptic curves. """ + sigalgs_list: NotRequired[str] + """ + List of supported signature algorithms. + """ + cipher_list: NotRequired[str] """ Cipher suite configuration string. @@ -356,9 +380,19 @@ class Params(TypedDict): Uses BoringSSL's mini-language to select, enable, and prioritize ciphers. """ - sigalgs_list: NotRequired[str] + preserve_tls13_cipher_list: NotRequired[bool] """ - List of supported signature algorithms. + Sets whether to preserve the TLS 1.3 cipher list as configured by cipher_list. + + By default, BoringSSL does not preserve the TLS 1.3 cipher list. When this option is disabled + (the default), BoringSSL uses its internal default TLS 1.3 cipher suites in its default order, + regardless of what is set via cipher_list. + + When enabled, this option ensures that the TLS 1.3 cipher suites explicitly set via + cipher_list are retained in their original order, without being reordered or + modified by BoringSSL's internal logic. This is useful for maintaining specific cipher suite + priorities for TLS 1.3. Note that if cipher_list does not include any TLS 1.3 + cipher suites, BoringSSL will still fall back to its default TLS 1.3 cipher suites and order. """ certificate_compression_algorithms: NotRequired[ @@ -383,21 +417,6 @@ class Params(TypedDict): Overrides the random AES hardware acceleration. """ - preserve_tls13_cipher_list: NotRequired[bool] - """ - Sets whether to preserve the TLS 1.3 cipher list as configured by cipher_list. - - By default, BoringSSL does not preserve the TLS 1.3 cipher list. When this option is disabled - (the default), BoringSSL uses its internal default TLS 1.3 cipher suites in its default order, - regardless of what is set via cipher_list. - - When enabled, this option ensures that the TLS 1.3 cipher suites explicitly set via - cipher_list are retained in their original order, without being reordered or - modified by BoringSSL's internal logic. This is useful for maintaining specific cipher suite - priorities for TLS 1.3. Note that if cipher_list does not include any TLS 1.3 - cipher suites, BoringSSL will still fall back to its default TLS 1.3 cipher suites and order. - """ - @final class TlsOptions: diff --git a/python/wreq/wreq.py b/python/wreq/wreq.py index b8ac93d4..8603e131 100644 --- a/python/wreq/wreq.py +++ b/python/wreq/wreq.py @@ -17,6 +17,7 @@ ) from . import redirect +from . import emulation from .cookie import * from .dns import ResolverOptions from .emulation import * @@ -425,22 +426,9 @@ async def close(self) -> None: r""" Close the response. - **Current behavior:** - - - When connection pooling is **disabled**: This method closes the network connection. - - When connection pooling is **enabled**: This method closes the response, prevents further body reads, - and returns the connection to the pool for reuse. - - **Future changes:** - - In future versions, this method will be changed to always close the network connection regardless of - whether connection pooling is enabled or not. - - **Recommendation:** - - It is **not recommended** to manually call this method at present. Instead, use context managers - (async with statement) to properly manage response lifecycle. Wait for the improved implementation - in future versions. + This method closes the network connection regardless of whether connection pooling is + enabled or not. It is recommended to use async context managers (`async with` statement) + to properly manage response lifecycle instead of calling this method manually. """ async def __aenter__(self) -> Any: ... @@ -520,7 +508,7 @@ def __str__(self) -> str: ... class ClientConfig(TypedDict): - emulation: NotRequired[Emulation | EmulationOption] + emulation: NotRequired[emulation.Emulation | emulation.Profile] """Emulation config.""" user_agent: NotRequired[str] @@ -791,7 +779,7 @@ class ClientConfig(TypedDict): class Request(TypedDict): - emulation: NotRequired[Emulation | EmulationOption] + emulation: NotRequired[emulation.Emulation | emulation.Profile] """ The Emulation settings for the request. """ @@ -957,7 +945,7 @@ class Request(TypedDict): class WebSocketRequest(TypedDict): - emulation: NotRequired[Emulation | EmulationOption] + emulation: NotRequired[emulation.Emulation | emulation.Profile] """ The Emulation settings for the request. """ @@ -1007,9 +995,9 @@ class WebSocketRequest(TypedDict): The protocols to use for the request. """ - force_http2: NotRequired[bool] + version: NotRequired[Version] """ - Whether to use HTTP/2 for the websocket. + The HTTP version to use for the request. """ auth: NotRequired[str] diff --git a/src/client.rs b/src/client.rs index 4d96cb0f..6bb709f3 100644 --- a/src/client.rs +++ b/src/client.rs @@ -15,8 +15,7 @@ use std::{ use pyo3::{IntoPyObjectExt, coroutine::CancelHandle, prelude::*, pybacked::PyBackedStr}; use req::{Request, WebSocketRequest}; use tokio_util::sync::CancellationToken; -use wreq::{Proxy, tls::CertStore}; -use wreq_util::EmulationOption; +use wreq::{Proxy, tls::trust::CertStore}; use self::{ nogil::NoGIL, @@ -26,6 +25,7 @@ use self::{ use crate::{ cookie::Jar, dns::{HickoryDnsResolver, LookupIpStrategy, ResolverOptions}, + emulate::EmulationLike, error::Error, extractor::Extractor, header::{HeaderMap, OrigHeaderMap}, @@ -60,7 +60,7 @@ define_display!(SocketAddr); #[derive(Default)] struct Builder { /// The Emulation settings for the client. - emulation: Option>, + emulation: Option, /// The user agent to use for the client. user_agent: Option, /// The headers to use for the client. @@ -257,7 +257,7 @@ impl Client { if let Some(mut config) = kwds { // Emulation options. - apply_option!(set_if_some_inner, builder, config.emulation, emulation); + apply_option!(set_if_some, builder, config.emulation, emulation); // User agent options. apply_option!( @@ -288,7 +288,7 @@ impl Client { } else if config.cookie_store.unwrap_or_default() { // `cookie_store` is true and no provider was given, so create a default jar to // be accessed later through the client interface. - let jar = Jar::new(None); + let jar = Jar::new(); builder = builder.cookie_provider(jar.clone().0); cookie_jar = Some(jar); } @@ -369,14 +369,14 @@ impl Client { set_if_some_map, builder, config.tls_min_version, - min_tls_version, + tls_min_version, TlsVersion::into_ffi ); apply_option!( set_if_some_map, builder, config.tls_max_version, - max_tls_version, + tls_max_version, TlsVersion::into_ffi ); apply_option!(set_if_some, builder, config.tls_info, tls_info); @@ -384,21 +384,28 @@ impl Client { set_if_some, builder, config.tls_verify_hostname, - verify_hostname + tls_verify_hostname ); - apply_option!(set_if_some_inner, builder, config.tls_identity, identity); - apply_option!(set_if_some_inner, builder, config.tls_keylog, keylog); + apply_option!( + set_if_some_inner, + builder, + config.tls_identity, + tls_identity + ); + apply_option!(set_if_some_inner, builder, config.tls_keylog, tls_keylog); apply_option!(set_if_some_inner, builder, config.tls_options, tls_options); if let Some(verify) = config.tls_verify.take() { builder = match verify { - TlsVerify::Verification(verify) => builder.cert_verification(verify), + TlsVerify::Verification(verify) => builder.tls_cert_verification(verify), TlsVerify::CertificatePath(path_buf) => { let pem_data = std::fs::read(path_buf)?; let store = CertStore::from_pem_stack(pem_data).map_err(Error::Library)?; - builder.cert_store(store) + builder.tls_cert_store(store) + } + TlsVerify::CertificateStore(cert_store) => { + builder.tls_cert_store(cert_store.0) } - TlsVerify::CertificateStore(cert_store) => builder.cert_store(cert_store.0), } } diff --git a/src/client/req.rs b/src/client/req.rs index 460a700f..cee104f7 100644 --- a/src/client/req.rs +++ b/src/client/req.rs @@ -7,7 +7,6 @@ use futures_util::TryFutureExt; use http::header::COOKIE; use pyo3::{PyResult, prelude::*, pybacked::PyBackedStr}; use wreq::Client; -use wreq_util::EmulationOption; use crate::{ client::{ @@ -16,6 +15,7 @@ use crate::{ resp::{Response, WebSocket}, }, cookie::{Cookies, Jar}, + emulate::EmulationLike, error::Error, extractor::Extractor, header::{HeaderMap, OrigHeaderMap}, @@ -29,7 +29,7 @@ use crate::{ #[non_exhaustive] pub struct Request { /// The Emulation settings for the request. - emulation: Option>, + emulation: Option, /// The proxy to use for the request. proxy: Option, @@ -112,7 +112,7 @@ pub struct Request { #[non_exhaustive] pub struct WebSocketRequest { /// The Emulation settings for the request. - emulation: Option>, + emulation: Option, /// The proxy to use for the request. proxy: Option, @@ -141,8 +141,8 @@ pub struct WebSocketRequest { /// The protocols to use for the request. protocols: Option>, - /// Whether to use HTTP/2 for the websocket. - force_http2: Option, + /// The HTTP version to use for the request. + version: Option, /// The authentication to use for the request. auth: Option, @@ -263,7 +263,7 @@ impl FromPyObject<'_, '_> for WebSocketRequest { extract_option!(ob, params, local_addresses); extract_option!(ob, params, interface); - extract_option!(ob, params, force_http2); + extract_option!(ob, params, version); extract_option!(ob, params, headers); extract_option!(ob, params, orig_headers); extract_option!(ob, params, default_headers); @@ -298,7 +298,7 @@ where if let Some(mut request) = request { // Emulation options. - apply_option!(set_if_some_inner, builder, request.emulation, emulation); + apply_option!(set_if_some, builder, request.emulation, emulation); // Version options. apply_option!( @@ -427,11 +427,24 @@ where { // Create the WebSocket builder. let mut builder = client.websocket(url.as_ref()); + if let Some(mut request) = request { - // The protocols to use for the request. + // Emulation options. + apply_option!(set_if_some, builder, request.emulation, emulation); + + // Version options. + apply_option!( + set_if_some_map, + builder, + request.version, + version, + Version::into_ffi + ); + + // Subprotocols options. apply_option!(set_if_some, builder, request.protocols, protocols); - // The WebSocket config + // WebSocket config apply_option!( set_if_some, builder, @@ -464,15 +477,6 @@ where accept_unmasked_frames ); - // Use http2 options. - apply_option!( - set_if_true, - builder, - request.force_http2, - force_http2, - false - ); - // Network options. apply_option!(set_if_some_inner, builder, request.proxy, proxy); apply_option!(set_if_some, builder, request.local_address, local_address); diff --git a/src/client/resp/http.rs b/src/client/resp/http.rs index c5dbcda5..21e51ddb 100644 --- a/src/client/resp/http.rs +++ b/src/client/resp/http.rs @@ -62,12 +62,14 @@ impl Response { } /// Builds a [`wreq::Response`] from the current response metadata and the given body. + #[inline] fn build_response>(&self, body: T) -> wreq::Response { let response = HttpResponse::from_parts(self.parts.clone(), body); wreq::Response::from(response) } /// Creates an empty [`wreq::Response`] with the same metadata but no body content. + #[inline] fn empty_response(&self) -> wreq::Response { self.build_response(Bytes::new()) } @@ -249,22 +251,15 @@ impl Response { /// Close the response. /// - /// **Current behavior:** - /// - When connection pooling is **disabled**: This method closes the network connection. - /// - When connection pooling is **enabled**: This method closes the response, prevents further - /// body reads, and returns the connection to the pool for reuse. - /// - /// **Future changes:** - /// In future versions, this method will be changed to always close the network connection - /// regardless of whether connection pooling is enabled or not. - /// - /// **Recommendation:** - /// It is **not recommended** to manually call this method at present. Instead, use context - /// managers (async with statement) to properly manage response lifecycle. Wait for the - /// improved implementation in future versions. + /// This method closes the network connection regardless of whether connection pooling is + /// enabled or not. It is recommended to use async context managers (`async with` statement) + /// to properly manage response lifecycle instead of calling this method manually. pub async fn close(&self) { Python::attach(|py| { - py.detach(|| self.destroy()); + py.detach(|| { + self.empty_response().forbid_recycle(); + self.destroy() + }); }); } } @@ -416,22 +411,15 @@ impl BlockingResponse { /// Close the response. /// - /// **Current behavior:** - /// - When connection pooling is **disabled**: This method closes the network connection. - /// - When connection pooling is **enabled**: This method closes the response, prevents further - /// body reads, and returns the connection to the pool for reuse. - /// - /// **Future changes:** - /// In future versions, this method will be changed to always close the network connection - /// regardless of whether connection pooling is enabled or not. - /// - /// **Recommendation:** - /// It is **not recommended** to manually call this method at present. Instead, use context - /// managers (with statement) to properly manage response lifecycle. Wait for the improved - /// implementation in future versions. + /// This method closes the network connection regardless of whether connection pooling is + /// enabled or not. It is recommended to use context managers (`with` statement) to properly + /// manage response lifecycle instead of calling this method manually. #[inline] pub fn close(&self, py: Python) { - py.detach(|| self.0.destroy()); + py.detach(|| { + self.0.empty_response().forbid_recycle(); + self.0.destroy(); + }); } } diff --git a/src/cookie.rs b/src/cookie.rs index 78f109bf..53580eed 100644 --- a/src/cookie.rs +++ b/src/cookie.rs @@ -222,22 +222,8 @@ impl FromPyObject<'_, '_> for Cookies { impl Jar { /// Create a new [`Jar`] with an empty cookie store. #[new] - #[pyo3(signature = (compression = None))] - pub fn new(compression: Option) -> Self { - Self(Arc::new(compression.map_or_else( - wreq::cookie::Jar::default, - wreq::cookie::Jar::new, - ))) - } - - /// Clone this [`Jar`], sharing storage but enabling compression. - pub fn compreessed(&self) -> Self { - Self(self.0.compressed()) - } - - /// Clone this [`Jar`], sharing storage but disabling compression. - pub fn uncompressed(&self) -> Self { - Self(self.0.uncompressed()) + pub fn new() -> Self { + Self(Arc::new(wreq::cookie::Jar::default())) } /// Get a cookie by name and URL. diff --git a/src/emulate.rs b/src/emulate.rs new file mode 100644 index 00000000..e8670b3d --- /dev/null +++ b/src/emulate.rs @@ -0,0 +1,209 @@ +use pyo3::prelude::*; + +define_enum!( + /// Selects which client profile the request should look like. + /// + /// This controls the built-in TLS, HTTP/2, and header presets used for the + /// request. Variants cover browser-style profiles as well as other clients, + /// such as OkHttp. + const, struct Emulation, + Profile, + wreq_util::Profile, + Chrome100, + Chrome101, + Chrome104, + Chrome105, + Chrome106, + Chrome107, + Chrome108, + Chrome109, + Chrome110, + Chrome114, + Chrome116, + Chrome117, + Chrome118, + Chrome119, + Chrome120, + Chrome123, + Chrome124, + Chrome126, + Chrome127, + Chrome128, + Chrome129, + Chrome130, + Chrome131, + Chrome132, + Chrome133, + Chrome134, + Chrome135, + Chrome136, + Chrome137, + Chrome138, + Chrome139, + Chrome140, + Chrome141, + Chrome142, + Chrome143, + Chrome144, + Chrome145, + Chrome146, + Chrome147, + + Edge101, + Edge122, + Edge127, + Edge131, + Edge134, + Edge135, + Edge136, + Edge137, + Edge138, + Edge139, + Edge140, + Edge141, + Edge142, + Edge143, + Edge144, + Edge145, + Edge146, + Edge147, + + Firefox109, + Firefox117, + Firefox128, + Firefox133, + Firefox135, + FirefoxPrivate135, + FirefoxAndroid135, + Firefox136, + FirefoxPrivate136, + Firefox139, + Firefox142, + Firefox143, + Firefox144, + Firefox145, + Firefox146, + Firefox147, + Firefox148, + Firefox149, + + SafariIos17_2, + SafariIos17_4_1, + SafariIos16_5, + Safari15_3, + Safari15_5, + Safari15_6_1, + Safari16, + Safari16_5, + Safari17_0, + Safari17_2_1, + Safari17_4_1, + Safari17_5, + Safari18, + SafariIPad18, + Safari18_2, + Safari18_3, + Safari18_3_1, + SafariIos18_1_1, + Safari18_5, + Safari26, + Safari26_1, + Safari26_2, + SafariIos26, + SafariIos26_2, + SafariIPad26, + SafariIpad26_2, + + OkHttp3_9, + OkHttp3_11, + OkHttp3_13, + OkHttp3_14, + OkHttp4_9, + OkHttp4_10, + OkHttp4_12, + OkHttp5, + + Opera116, + Opera117, + Opera118, + Opera119, + Opera120, + Opera121, + Opera122, + Opera123, + Opera124, + Opera125, + Opera126, + Opera127, + Opera128, + Opera129, + Opera130, +); + +define_enum!( + /// Selects which platform the client should look like. + /// + /// This mainly affects platform-specific headers and user-agent details. + /// In most cases you can keep the default unless you need to match a + /// specific Windows, macOS, Linux, Android, or iOS profile. + const, + Platform, + wreq_util::Platform, + Windows, + MacOS, + Linux, + Android, + IOS, +); + +/// Represents the configuration options for emulating a client profile and platform. +/// +/// The `Emulation` struct allows you to configure various aspects of profile and platform +/// emulation, including the profile, platform, and whether to enable certain features +/// like HTTP/2 or headers. +#[derive(Clone)] +#[pyclass(subclass, from_py_object)] +pub struct Emulation(pub wreq_util::Emulation); + +#[pymethods] +impl Emulation { + /// Create a new Emulation option instance. + #[new] + #[pyo3(signature = ( + profile = Profile::Chrome100, + platform = Platform::MacOS, + http2 = true, + headers = true + ))] + fn new(profile: Profile, platform: Platform, http2: bool, headers: bool) -> Self { + let emulation = wreq_util::Emulation::builder() + .profile(profile.into_ffi()) + .platform(platform.into_ffi()) + .http2(http2) + .headers(headers) + .build(); + Self(emulation) + } + + /// Creates a new random Emulation option instance. + #[staticmethod] + fn random() -> Self { + Self(wreq_util::Emulation::random()) + } +} + +/// A helper enum to allow accepting either a Profile or an Emulation in the same parameter. +#[derive(FromPyObject)] +pub enum EmulationLike { + Profile(Profile), + Emulation(Emulation), +} + +impl wreq::IntoEmulation for EmulationLike { + fn into_emulation(self) -> wreq::Emulation { + match self { + EmulationLike::Profile(profile) => profile.into_ffi().into_emulation(), + EmulationLike::Emulation(inner) => inner.0.into_emulation(), + } + } +} diff --git a/src/emulation.rs b/src/emulation.rs deleted file mode 100644 index 4ca7cb65..00000000 --- a/src/emulation.rs +++ /dev/null @@ -1,165 +0,0 @@ -use pyo3::prelude::*; - -define_enum!( - /// An emulation. - const, - Emulation, - wreq_util::Emulation, - Chrome100, - Chrome101, - Chrome104, - Chrome105, - Chrome106, - Chrome107, - Chrome108, - Chrome109, - Chrome110, - Chrome114, - Chrome116, - Chrome117, - Chrome118, - Chrome119, - Chrome120, - Chrome123, - Chrome124, - Chrome126, - Chrome127, - Chrome128, - Chrome129, - Chrome130, - Chrome131, - Chrome132, - Chrome133, - Chrome134, - Chrome135, - Chrome136, - Chrome137, - Chrome138, - Chrome139, - Chrome140, - Chrome141, - Chrome142, - Chrome143, - Chrome144, - Chrome145, - Edge101, - Edge122, - Edge127, - Edge131, - Edge134, - Edge135, - Edge136, - Edge137, - Edge138, - Edge139, - Edge140, - Edge141, - Edge142, - Edge143, - Edge144, - Edge145, - Firefox109, - Firefox117, - Firefox128, - Firefox133, - Firefox135, - FirefoxPrivate135, - FirefoxAndroid135, - Firefox136, - FirefoxPrivate136, - Firefox139, - Firefox142, - Firefox143, - Firefox144, - Firefox145, - Firefox146, - Firefox147, - SafariIos17_2, - SafariIos17_4_1, - SafariIos16_5, - Safari15_3, - Safari15_5, - Safari15_6_1, - Safari16, - Safari16_5, - Safari17_0, - Safari17_2_1, - Safari17_4_1, - Safari17_5, - Safari18, - SafariIPad18, - Safari18_2, - Safari18_3, - Safari18_3_1, - SafariIos18_1_1, - Safari18_5, - Safari26, - Safari26_1, - Safari26_2, - SafariIos26, - SafariIos26_2, - SafariIPad26, - SafariIpad26_2, - OkHttp3_9, - OkHttp3_11, - OkHttp3_13, - OkHttp3_14, - OkHttp4_9, - OkHttp4_10, - OkHttp4_12, - OkHttp5, - Opera116, - Opera117, - Opera118, - Opera119 -); - -define_enum!( - /// An emulation operating system. - const, - EmulationOS, - wreq_util::EmulationOS, - Windows, - MacOS, - Linux, - Android, - IOS, -); - -/// A struct to represent the `EmulationOption` class. -#[derive(Clone)] -#[pyclass(subclass, from_py_object)] -pub struct EmulationOption(pub wreq_util::EmulationOption); - -#[pymethods] -impl EmulationOption { - /// Create a new Emulation option instance. - #[new] - #[pyo3(signature = ( - emulation, - emulation_os = None, - skip_http2 = None, - skip_headers = None - ))] - fn new( - emulation: Emulation, - emulation_os: Option, - skip_http2: Option, - skip_headers: Option, - ) -> Self { - let emulation = wreq_util::EmulationOption::builder() - .emulation(emulation.into_ffi()) - .emulation_os(emulation_os.map(|os| os.into_ffi()).unwrap_or_default()) - .skip_http2(skip_http2.unwrap_or(false)) - .skip_headers(skip_headers.unwrap_or(false)) - .build(); - - Self(emulation) - } - - /// Creates a new random Emulation option instance. - #[staticmethod] - fn random() -> Self { - Self(wreq_util::Emulation::random()) - } -} diff --git a/src/extractor.rs b/src/extractor.rs index ca1e5b4f..003d0ba6 100644 --- a/src/extractor.rs +++ b/src/extractor.rs @@ -2,31 +2,11 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use pyo3::{FromPyObject, prelude::*, types::PyList}; -use crate::{ - emulation::{Emulation, EmulationOption}, - proxy::Proxy, -}; +use crate::proxy::Proxy; /// A generic extractor for various types. pub struct Extractor(pub T); -impl FromPyObject<'_, '_> for Extractor { - type Error = PyErr; - - fn extract(ob: Borrowed) -> PyResult { - if let Ok(impersonate) = ob.cast::() { - let emulation = wreq_util::EmulationOption::builder() - .emulation(impersonate.borrow().into_ffi()) - .build(); - - return Ok(Self(emulation)); - } - - let option = ob.cast::()?.borrow(); - Ok(Self(option.0.clone())) - } -} - impl FromPyObject<'_, '_> for Extractor> { type Error = PyErr; diff --git a/src/lib.rs b/src/lib.rs index 9d947a93..e50384e1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ mod buffer; mod client; mod cookie; mod dns; -mod emulation; +mod emulate; mod error; mod extractor; mod header; @@ -31,7 +31,7 @@ use client::{ }; use cookie::{Cookie, Jar, SameSite}; use dns::{LookupIpStrategy, ResolverOptions}; -use emulation::{Emulation, EmulationOS, EmulationOption}; +use emulate::{Emulation, Platform, Profile}; use error::*; use header::{HeaderMap, OrigHeaderMap}; use http::{Method, StatusCode, Version}; @@ -470,8 +470,8 @@ fn cookie_module(m: &Bound<'_, PyModule>) -> PyResult<()> { #[pymodule(gil_used = false, name = "emulation")] fn emulation_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; - m.add_class::()?; - m.add_class::()?; + m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/src/macros.rs b/src/macros.rs index f7a36410..73f9c544 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -68,7 +68,16 @@ macro_rules! apply_option { }; } +#[allow(unused_macro_rules)] macro_rules! define_enum { + ($(#[$meta:meta])* struct $struct_type:ident, $enum_type:ident, $ffi_type:ty, $($variant:ident),* $(,)?) => { + define_enum!($(#[$meta])* struct $struct_type, $enum_type, $ffi_type, $( ($variant, $variant) ),*); + }; + + ($(#[$meta:meta])* const, struct $struct_type:ident, $enum_type:ident, $ffi_type:ty, $($variant:ident),* $(,)?) => { + define_enum!($(#[$meta])* const, struct $struct_type, $enum_type, $ffi_type, $( ($variant, $variant) ),*); + }; + ($(#[$meta:meta])* $enum_type:ident, $ffi_type:ty, $($variant:ident),* $(,)?) => { define_enum!($(#[$meta])* $enum_type, $ffi_type, $( ($variant, $variant) ),*); }; @@ -77,6 +86,32 @@ macro_rules! define_enum { define_enum!($(#[$meta])* const, $enum_type, $ffi_type, $( ($variant, $variant) ),*); }; + ($(#[$meta:meta])* struct $struct_type:ident, $enum_type:ident, $ffi_type:ty, $(($rust_variant:ident, $ffi_variant:ident)),* $(,)?) => { + define_enum!($(#[$meta])* $enum_type, $ffi_type, $(($rust_variant, $ffi_variant)),*); + + #[pymethods] + #[allow(non_upper_case_globals)] + impl $struct_type { + $( + #[classattr] + const $rust_variant: $enum_type = $enum_type::$rust_variant; + )* + } + }; + + ($(#[$meta:meta])* const, struct $struct_type:ident, $enum_type:ident, $ffi_type:ty, $(($rust_variant:ident, $ffi_variant:ident)),* $(,)?) => { + define_enum!($(#[$meta])* const, $enum_type, $ffi_type, $(($rust_variant, $ffi_variant)),*); + + #[pymethods] + #[allow(non_upper_case_globals)] + impl $struct_type { + $( + #[classattr] + const $rust_variant: $enum_type = $enum_type::$rust_variant; + )* + } + }; + ($(#[$meta:meta])* $enum_type:ident, $ffi_type:ty, $(($rust_variant:ident, $ffi_variant:ident)),* $(,)?) => { $(#[$meta])* #[pyclass(eq, eq_int, frozen, from_py_object)] @@ -107,6 +142,7 @@ macro_rules! define_enum { } impl $enum_type { + #[allow(dead_code)] pub const fn into_ffi(self) -> $ffi_type { match self { $(<$enum_type>::$rust_variant => <$ffi_type>::$ffi_variant,)* diff --git a/src/tls.rs b/src/tls.rs index 7d7cfcc2..e81a323b 100644 --- a/src/tls.rs +++ b/src/tls.rs @@ -3,6 +3,8 @@ mod keylog; mod store; use pyo3::prelude::*; +use wreq::tls::compress::CertificateCompressor; +use wreq_util::emulate::compress; pub use self::{identity::Identity, keylog::KeyLog, store::CertStore}; use crate::buffer::PyBuffer; @@ -45,11 +47,28 @@ define_enum!( HTTP3, ); +define_enum!( + /// Key exchange groups (elliptic curves) for TLS 1.3. + const, + KeyShare, + wreq::tls::KeyShare, + P256, + P384, + P521, + X25519, + X25519_MLKEM768, + X25519_KYBER768_DRAFT00, + P256_KYBER768_DRAFT00, + MLKEM1024, + FFDHE2048, + FFDHE3072, +); + define_enum!( // IANA assigned identifier of compression algorithm. See https://www.rfc-editor.org/rfc/rfc8879.html#name-compression-algorithms const, CertificateCompressionAlgorithm, - wreq::tls::CertificateCompressionAlgorithm, + wreq::tls::compress::CertificateCompressionAlgorithm, ZLIB, BROTLI, ZSTD, @@ -83,8 +102,8 @@ define_enum!( KEY_SHARE, RENEGOTIATE, DELEGATED_CREDENTIAL, + APPLICATION_SETTINGS_OLD, APPLICATION_SETTINGS, - APPLICATION_SETTINGS_NEW, ENCRYPTED_CLIENT_HELLO, CERTIFICATE_TIMESTAMP, NEXT_PROTO_NEG, @@ -153,8 +172,8 @@ struct Builder { /// Whether to skip session tickets when using PSK. psk_skip_session_ticket: Option, - /// Maximum number of key shares to include in ClientHello. - key_shares_limit: Option, + /// Whether to set specific key shares for TLS 1.3 handshakes. + key_shares: Option>, /// Enables PSK with (EC)DHE key establishment (`psk_dhe_ke`). psk_dhe_ke: Option, @@ -171,13 +190,16 @@ struct Builder { /// List of supported elliptic curves. curves_list: Option, + /// List of supported signature algorithms. + sigalgs_list: Option, + /// Cipher suite configuration string. /// /// Uses BoringSSL's mini-language to select, enable, and prioritize ciphers. cipher_list: Option, - /// List of supported signature algorithms. - sigalgs_list: Option, + /// Sets whether to preserve the TLS 1.3 cipher list as configured by [`Self::cipher_list`]. + preserve_tls13_cipher_list: Option, /// Supported certificate compression algorithms ([RFC 8879](https://datatracker.ietf.org/doc/html/rfc8879)). certificate_compression_algorithms: Option>, @@ -188,9 +210,6 @@ struct Builder { /// Overrides AES hardware acceleration. aes_hw_override: Option, - /// Sets whether to preserve the TLS 1.3 cipher list as configured by [`Self::cipher_list`]. - preserve_tls13_cipher_list: Option, - /// Overrides the random AES hardware acceleration. random_aes_hw_override: Option, } @@ -214,7 +233,7 @@ impl FromPyObject<'_, '_> for Builder { extract_option!(ob, params, enable_signed_cert_timestamps); extract_option!(ob, params, record_size_limit); extract_option!(ob, params, psk_skip_session_ticket); - extract_option!(ob, params, key_shares_limit); + extract_option!(ob, params, key_shares); extract_option!(ob, params, psk_dhe_ke); extract_option!(ob, params, renegotiation); extract_option!(ob, params, delegated_credentials); @@ -327,10 +346,11 @@ impl TlsOptions { psk_skip_session_ticket ); apply_option!( - set_if_some, + set_if_some_map, builder, - params.key_shares_limit, - key_shares_limit + params.key_shares, + key_shares, + |v: Vec<_>| v.into_iter().map(KeyShare::into_ffi).collect::>() ); apply_option!(set_if_some, builder, params.psk_dhe_ke, psk_dhe_ke); apply_option!(set_if_some, builder, params.renegotiation, renegotiation); @@ -347,11 +367,23 @@ impl TlsOptions { set_if_some_map, builder, params.certificate_compression_algorithms, - certificate_compression_algorithms, + certificate_compressors, |v: Vec<_>| v .into_iter() - .map(CertificateCompressionAlgorithm::into_ffi) - .collect::>() + .map(|algs| { + match algs { + CertificateCompressionAlgorithm::ZLIB => { + &compress::ZlibCompressor as _ + } + CertificateCompressionAlgorithm::BROTLI => { + &compress::BrotliCompressor as _ + } + CertificateCompressionAlgorithm::ZSTD => { + &compress::ZstdCompressor as _ + } + } + }) + .collect::>() ); apply_option!( set_if_some_map, diff --git a/src/tls/identity.rs b/src/tls/identity.rs index 970274df..c5b9ca69 100644 --- a/src/tls/identity.rs +++ b/src/tls/identity.rs @@ -9,7 +9,7 @@ use crate::error::Error; /// Represents a private key and X509 cert as a client certificate. #[derive(Clone)] #[pyclass(from_py_object)] -pub struct Identity(pub wreq::tls::Identity); +pub struct Identity(pub wreq::tls::trust::Identity); #[pymethods] impl Identity { @@ -28,7 +28,7 @@ impl Identity { #[staticmethod] #[pyo3(signature = (buf, pass))] pub fn from_pkcs12_der(buf: PyBackedBytes, pass: PyBackedStr) -> PyResult { - wreq::tls::Identity::from_pkcs12_der(buf.as_ref(), pass.as_ref()) + wreq::tls::trust::Identity::from_pkcs12_der(buf.as_ref(), pass.as_ref()) .map(Identity) .map_err(Error::Library) .map_err(Into::into) @@ -44,7 +44,7 @@ impl Identity { #[staticmethod] #[pyo3(signature = (buf, key))] pub fn from_pkcs8_pem(buf: PyBackedBytes, key: PyBackedBytes) -> PyResult { - wreq::tls::Identity::from_pkcs8_pem(buf.as_ref(), key.as_ref()) + wreq::tls::trust::Identity::from_pkcs8_pem(buf.as_ref(), key.as_ref()) .map(Identity) .map_err(Error::Library) .map_err(Into::into) diff --git a/src/tls/keylog.rs b/src/tls/keylog.rs index 991255ec..2e87c86a 100644 --- a/src/tls/keylog.rs +++ b/src/tls/keylog.rs @@ -10,19 +10,19 @@ use pyo3::{pyclass, pymethods}; /// with the correct session keys. #[derive(Clone)] #[pyclass(from_py_object)] -pub struct KeyLog(pub wreq::tls::KeyLog); +pub struct KeyLog(pub wreq::tls::keylog::KeyLog); #[pymethods] impl KeyLog { /// Use the environment variable SSLKEYLOGFILE. #[staticmethod] pub fn environment() -> Self { - KeyLog(wreq::tls::KeyLog::from_env()) + KeyLog(wreq::tls::keylog::KeyLog::from_env()) } /// Log keys to the specified file path. #[staticmethod] pub fn file(path: PathBuf) -> Self { - KeyLog(wreq::tls::KeyLog::from_file(path)) + KeyLog(wreq::tls::keylog::KeyLog::from_file(path)) } } diff --git a/src/tls/store.rs b/src/tls/store.rs index ff5425a8..52cf0692 100644 --- a/src/tls/store.rs +++ b/src/tls/store.rs @@ -8,7 +8,7 @@ use crate::error::Error; #[derive(Clone)] #[pyclass(from_py_object)] -pub struct CertStore(pub wreq::tls::CertStore); +pub struct CertStore(pub wreq::tls::trust::CertStore); #[pymethods] impl CertStore { @@ -20,7 +20,7 @@ impl CertStore { pem_certs: Option>, default_paths: bool, ) -> PyResult { - let mut store = wreq::tls::CertStore::builder(); + let mut store = wreq::tls::trust::CertStore::builder(); // Add DER certificates if provided if let Some(der_certs) = der_certs { @@ -48,7 +48,7 @@ impl CertStore { #[staticmethod] #[pyo3(signature = (certs))] pub fn from_der_certs(certs: Vec) -> PyResult { - wreq::tls::CertStore::from_der_certs(&certs) + wreq::tls::trust::CertStore::from_der_certs(&certs) .map(CertStore) .map_err(Error::Library) .map_err(Into::into) @@ -58,7 +58,7 @@ impl CertStore { #[staticmethod] #[pyo3(signature = (certs))] pub fn from_pem_certs(certs: Vec) -> PyResult { - wreq::tls::CertStore::from_pem_certs(&certs) + wreq::tls::trust::CertStore::from_pem_certs(&certs) .map(CertStore) .map_err(Error::Library) .map_err(Into::into) @@ -68,7 +68,7 @@ impl CertStore { #[staticmethod] #[pyo3(signature = (certs))] pub fn from_pem_stack(certs: PyBackedBytes) -> PyResult { - wreq::tls::CertStore::from_pem_stack(certs.as_ref()) + wreq::tls::trust::CertStore::from_pem_stack(certs.as_ref()) .map(CertStore) .map_err(Error::Library) .map_err(Into::into)