From 914e85ae33e224f523471889491cd704c2320278 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 09:18:10 +0000 Subject: [PATCH 1/9] fix: make `JsonSerializable` type alias read-only ty 0.0.83 rejects e.g. `list[str]` where `list[JsonSerializable]` is expected, because `list`/`dict` are invariant. Using `Sequence`/`Mapping` makes plain lists and dicts of JSON values acceptable again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018m46SvjMFWEVxC4RDuegPN --- .../code_examples/cookie_management/retry_restore_cookies.py | 3 ++- src/crawlee/_types.py | 2 +- src/crawlee/storage_clients/_redis/_dataset_client.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/guides/code_examples/cookie_management/retry_restore_cookies.py b/docs/guides/code_examples/cookie_management/retry_restore_cookies.py index 6f785ee662..1ca1c5c0ed 100644 --- a/docs/guides/code_examples/cookie_management/retry_restore_cookies.py +++ b/docs/guides/code_examples/cookie_management/retry_restore_cookies.py @@ -20,7 +20,8 @@ async def handler(context: HttpCrawlingContext) -> None: # First pass: establish cookies once and store them for every session, # then raise to trigger a retry. await context.send_request('https://httpbingo.org/cookies/set?logged_in=1') - state['cookies'] = context.session.cookies.get_cookies_as_dicts() + cookies = context.session.cookies.get_cookies_as_dicts() + state['cookies'] = cast('list[dict]', cookies) raise RuntimeError('retry with cookies') context.log.info(f'Cookies: {context.session.cookies.get_cookies_as_dicts()}') diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 11bd9fb344..2c8518ae48 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -27,7 +27,7 @@ from crawlee.storage_clients import StorageClient from crawlee.storages import KeyValueStore - JsonSerializable = dict[str, 'JsonSerializable'] | list['JsonSerializable'] | str | int | float | bool | None + JsonSerializable = Mapping[str, 'JsonSerializable'] | Sequence['JsonSerializable'] | str | int | float | bool | None else: from pydantic import JsonValue as JsonSerializable diff --git a/src/crawlee/storage_clients/_redis/_dataset_client.py b/src/crawlee/storage_clients/_redis/_dataset_client.py index d05cabf10e..8a2d899d45 100644 --- a/src/crawlee/storage_clients/_redis/_dataset_client.py +++ b/src/crawlee/storage_clients/_redis/_dataset_client.py @@ -133,7 +133,7 @@ async def push_data(self, data: Sequence[Mapping[str, JsonSerializable]] | Mappi items = data if isinstance(data, Sequence) else [data] async with self._get_pipeline() as pipe: - pipe.json().arrappend(self._items_key, '$', *items) + pipe.json().arrappend(self._items_key, '$', *cast('list[Any]', items)) await self._update_metadata( pipe, **_DatasetMetadataUpdateParams( From 6c7e10b7b295510680079dcd3947314bde8939cb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 09:35:17 +0000 Subject: [PATCH 2/9] fix: replace casts with real type fixes - `CookieParam` is now a closed TypedDict, so it is accepted as JSON-serializable. `_from_playwright` builds it from declared keys only (the dropped keys were already ignored by `SessionCookies.set`). - Redis `push_data` encodes items itself instead of going through `arrappend`, whose `JsonType` stub rejects read-only `Sequence`/`Mapping` values. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018m46SvjMFWEVxC4RDuegPN --- .../retry_restore_cookies.py | 3 +- src/crawlee/sessions/_cookies.py | 30 +++++++++++-------- .../storage_clients/_redis/_dataset_client.py | 5 +++- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/docs/guides/code_examples/cookie_management/retry_restore_cookies.py b/docs/guides/code_examples/cookie_management/retry_restore_cookies.py index 1ca1c5c0ed..6f785ee662 100644 --- a/docs/guides/code_examples/cookie_management/retry_restore_cookies.py +++ b/docs/guides/code_examples/cookie_management/retry_restore_cookies.py @@ -20,8 +20,7 @@ async def handler(context: HttpCrawlingContext) -> None: # First pass: establish cookies once and store them for every session, # then raise to trigger a retry. await context.send_request('https://httpbingo.org/cookies/set?logged_in=1') - cookies = context.session.cookies.get_cookies_as_dicts() - state['cookies'] = cast('list[dict]', cookies) + state['cookies'] = context.session.cookies.get_cookies_as_dicts() raise RuntimeError('retry with cookies') context.log.info(f'Cookies: {context.session.cookies.get_cookies_as_dicts()}') diff --git a/src/crawlee/sessions/_cookies.py b/src/crawlee/sessions/_cookies.py index 46dceac0a9..f5e3158622 100644 --- a/src/crawlee/sessions/_cookies.py +++ b/src/crawlee/sessions/_cookies.py @@ -28,7 +28,7 @@ def info(self) -> Message: @docs_group('Session management') -class CookieParam(TypedDict, total=False): +class CookieParam(TypedDict, total=False, closed=True): """Dictionary representation of cookies for `SessionCookies.set` method.""" name: Required[str] @@ -187,17 +187,23 @@ def _to_playwright(self, cookie_dict: CookieParam) -> PlaywrightCookieParam: def _from_playwright(self, cookie_dict: PlaywrightCookieParam) -> CookieParam: """Convert Playwright cookie to internal format.""" - result: dict = dict(cookie_dict) - - if 'httpOnly' in result: - result['http_only'] = result.pop('httpOnly') - if 'sameSite' in result: - result['same_site'] = result.pop('sameSite') - if 'expires' in result: - expires = int(result['expires']) - result['expires'] = None if expires == -1 else expires - - return CookieParam(name=result.pop('name', ''), value=result.pop('value', ''), **result) + result = CookieParam(name=cookie_dict.get('name', ''), value=cookie_dict.get('value', '')) + + if 'domain' in cookie_dict: + result['domain'] = cookie_dict['domain'] + if 'path' in cookie_dict: + result['path'] = cookie_dict['path'] + if 'secure' in cookie_dict: + result['secure'] = cookie_dict['secure'] + if 'httpOnly' in cookie_dict: + result['http_only'] = cookie_dict['httpOnly'] + if 'sameSite' in cookie_dict: + result['same_site'] = cookie_dict['sameSite'] + # Playwright uses -1 for session cookies, which have no expiration. + if 'expires' in cookie_dict and (expires := int(cookie_dict['expires'])) != -1: + result['expires'] = expires + + return result def get_cookies_as_dicts(self) -> list[CookieParam]: """Convert cookies to a list with `CookieParam` dicts.""" diff --git a/src/crawlee/storage_clients/_redis/_dataset_client.py b/src/crawlee/storage_clients/_redis/_dataset_client.py index 8a2d899d45..71e04b8d87 100644 --- a/src/crawlee/storage_clients/_redis/_dataset_client.py +++ b/src/crawlee/storage_clients/_redis/_dataset_client.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from collections.abc import Sequence from logging import getLogger from typing import TYPE_CHECKING, Any, cast @@ -133,7 +134,9 @@ async def push_data(self, data: Sequence[Mapping[str, JsonSerializable]] | Mappi items = data if isinstance(data, Sequence) else [data] async with self._get_pipeline() as pipe: - pipe.json().arrappend(self._items_key, '$', *cast('list[Any]', items)) + # Equivalent of `pipe.json().arrappend(...)`, whose `JsonType` stub only accepts `list`/`dict`, not + # read-only `Sequence`/`Mapping` values. + pipe.execute_command('JSON.ARRAPPEND', self._items_key, '$', *[json.dumps(item) for item in items]) await self._update_metadata( pipe, **_DatasetMetadataUpdateParams( From 6e8f29436a535f575d0f7b0ab60bad8af3ded53b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 09:51:03 +0000 Subject: [PATCH 3/9] chore(deps): bump ty to 0.0.83 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018m46SvjMFWEVxC4RDuegPN --- uv.lock | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/uv.lock b/uv.lock index a02a8fe411..cf7e298d72 100644 --- a/uv.lock +++ b/uv.lock @@ -4916,27 +4916,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.80" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/54/b0/6d1b10e0d422736a3c439e487c950ae785f401d71ff879d5be68bccb6d90/ty-0.0.80.tar.gz", hash = "sha256:fe86bc91327e45ff5e3593b7e306e7f57a44bc0608f38d647b8d97bd99c96013", size = 7183998, upload-time = "2026-09-09T21:18:47.547Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/29/c8/3c93195eca282936ebb574d0ba98843e4b147ee324006f88464777012a92/ty-0.0.80-py3-none-linux_armv6l.whl", hash = "sha256:738d1cfca466c577aea24547c348629c00c63548cf7da2c46b497b3840f85dee", size = 13606509, upload-time = "2026-09-09T21:18:10.476Z" }, - { url = "https://files.pythonhosted.org/packages/e7/48/bbb47f7001c97262a5109bcd92a45bcc8a2fbb21e994c214a9897630bfb7/ty-0.0.80-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:56060164bb8ee43770fa367524fbdba2611fc90f8923a02cc91f80eb37d5a1b0", size = 13203812, upload-time = "2026-09-09T21:18:12.796Z" }, - { url = "https://files.pythonhosted.org/packages/db/8d/fd141160567047b742e466be8ed09a4c4ed80fa045d37f946eb4f4532fee/ty-0.0.80-py3-none-macosx_11_0_arm64.whl", hash = "sha256:da4062e0fbf3923d9b71688157c5753394f243348f00732e9edbb27498397169", size = 13021896, upload-time = "2026-09-09T21:18:15.186Z" }, - { url = "https://files.pythonhosted.org/packages/1e/10/b4177faf9e71bc37bc4d08f512042269660a1ce0ec457c48f96e51fc91d7/ty-0.0.80-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9eb94b2659f506a3a07a9ea1415d5c18aaa1e554adc1612614d617a0ed320e1", size = 13083881, upload-time = "2026-09-09T21:18:17.616Z" }, - { url = "https://files.pythonhosted.org/packages/7f/c2/192d48b9a9acbbe030fc9a4bd73af75671a5566e731949f77e912a68ce68/ty-0.0.80-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e2f5e39a87da48c1af1a3b3135f02be7a83a93da30b0d6a5f5d27806e7c747ce", size = 13354874, upload-time = "2026-09-09T21:18:19.744Z" }, - { url = "https://files.pythonhosted.org/packages/0a/5b/736dfd31efd98bed9dbe018ff91676e5ea83485b1e24afb635fd0247728f/ty-0.0.80-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e00149e7779c6b98f3ba12e10a631a861bd7ca1a42bbf064e2677ada7d2c0c2", size = 14204805, upload-time = "2026-09-09T21:18:21.999Z" }, - { url = "https://files.pythonhosted.org/packages/fd/6f/557c726cb53401998e3589bb632c963a6887a70240e0a4386024f3731c81/ty-0.0.80-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a377268f359fb6e7a2cc9026a09223c764f58d020dec8e6baeaba9caba6ff829", size = 14630014, upload-time = "2026-09-09T21:18:24Z" }, - { url = "https://files.pythonhosted.org/packages/e9/19/7a4b18fe27f6b6bd4ffa56b67c8b09ecb76bb4312d1c64b73f65417b4dbd/ty-0.0.80-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4511dbf1b266b9ce5b73e9b28d94468096c2b6da00e2fe205168c32d3b352ca7", size = 14326946, upload-time = "2026-09-09T21:18:26.679Z" }, - { url = "https://files.pythonhosted.org/packages/f8/95/16dd90805fc7e53ec54ae9fdb1a8b74753fb159a55a170f60f49b9417824/ty-0.0.80-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe95feffa7156800c6f804195acb9fb5846a39671c7192c30fff23351aaf7c31", size = 13705988, upload-time = "2026-09-09T21:18:28.681Z" }, - { url = "https://files.pythonhosted.org/packages/b1/09/4e87992c23ab8a742c7d4914ac5fe66fb9301c8464a69fd2ecbc0be3fbcd/ty-0.0.80-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ba26b39f06bc8c3c2c5acd3a147239482b36620cdbca1b9d02e915294854f2cc", size = 14233729, upload-time = "2026-09-09T21:18:30.961Z" }, - { url = "https://files.pythonhosted.org/packages/36/e9/b8c11fda8e66a1d1cc1a7160ed719a33f4dc0107b1cef6a14e2673965763/ty-0.0.80-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:2e6167320888c115a6fbe69893b63fd1c848f9121a454e75e5f612674528e3f3", size = 13173523, upload-time = "2026-09-09T21:18:33.266Z" }, - { url = "https://files.pythonhosted.org/packages/73/a7/512083fa540c5be1ac8642658f03bfb5bfa6fd168ea4f55e3c0aabe04166/ty-0.0.80-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7aecb62b4de70b479eab07d1779ea66da26dd8b068f50a9330867157312f103c", size = 13373014, upload-time = "2026-09-09T21:18:35.339Z" }, - { url = "https://files.pythonhosted.org/packages/5e/63/cd2f0ca81fd9b8cafef93bbcf022bf636bc4de8ee7465c6aafeec1d4d52b/ty-0.0.80-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4fed06adacd16b7e2d722f37449021b1419598d0440769701c0f4827faddde63", size = 13667827, upload-time = "2026-09-09T21:18:37.241Z" }, - { url = "https://files.pythonhosted.org/packages/0d/5d/ebdfdcb2dc099ae74dc4b75511eef0dd398b2fa27006c543543974b4567f/ty-0.0.80-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:09329af6303ce611ec2cfffc995dc1ddf76c9f47194fc1a4d64a984759c25f5b", size = 13963866, upload-time = "2026-09-09T21:18:39.386Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f6/c286d5e1b560cfd16d6e91fdab683718aacf09fc88938ca06dcc9d1c8502/ty-0.0.80-py3-none-win32.whl", hash = "sha256:a81b3b512f7b4c68e42fbd1835633de658809666e9aafd5defa52bbc5197698d", size = 12881806, upload-time = "2026-09-09T21:18:41.507Z" }, - { url = "https://files.pythonhosted.org/packages/fd/c8/3b3a8ac16d47a23ff34bf57c0d99a491860748aea96e0e886b4bfed54f9b/ty-0.0.80-py3-none-win_amd64.whl", hash = "sha256:8043f99878a2ae434781cb881c8a3840dff70c4a5b60bdc5ae84251f35ee063f", size = 13528883, upload-time = "2026-09-09T21:18:43.687Z" }, - { url = "https://files.pythonhosted.org/packages/2e/71/a6c697930fca76596d7d17f8853120f41a298fd9ea632c901cfca8ca869b/ty-0.0.80-py3-none-win_arm64.whl", hash = "sha256:e277ef034331da5319efc839c064968d24f35715b71c70369cf97d36fe4dcbdb", size = 13370135, upload-time = "2026-09-09T21:18:45.758Z" }, +version = "0.0.83" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/7d/2fd9575bce2d14e2281bec82d0713dee68329b0dc944e9c06c2e36b761fe/ty-0.0.83.tar.gz", hash = "sha256:db118de73c05ac476faceb4d42d59782feaeddfc4d721fd3ae8b807a0a2ae4e4", size = 7373076, upload-time = "2026-09-21T22:47:29.207Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/85/7259431bade2317d8b82fa27220ae4618c21396acf0ca28a7bd75e92afd7/ty-0.0.83-py3-none-linux_armv6l.whl", hash = "sha256:836bc08bad96ecc7e38baf1a6ddaec15aa8c8b46ae99a96365397b5f6481b7a5", size = 13905669, upload-time = "2026-09-21T22:46:36.677Z" }, + { url = "https://files.pythonhosted.org/packages/d2/30/9527386ade8dd71fc52f65cde3c094411af7ad0d505f9063ef0841998702/ty-0.0.83-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0f1fb029fe076c7dca404d98e38fdb5fdbc2038a09407f89d43be1fdd9b2c459", size = 13499505, upload-time = "2026-09-21T22:46:40.114Z" }, + { url = "https://files.pythonhosted.org/packages/a0/54/63080d8cfd59a53810499a69a2ebc995724be27618b70d7f099e73ace317/ty-0.0.83-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3515776d01ee3b1be974dafa73d449d0bc711afbc12ba341f82f70bc641ff4ee", size = 13498980, upload-time = "2026-09-21T22:46:43.197Z" }, + { url = "https://files.pythonhosted.org/packages/49/78/e647f190bd9513d09db30d45309e9b5df070b5caa44af2de3290fab59fbe/ty-0.0.83-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5b80d66c7f9c8d60f25ccfd40c6eaebe8fd7c9005074904e0e164dd89c13d03", size = 13494705, upload-time = "2026-09-21T22:46:46.106Z" }, + { url = "https://files.pythonhosted.org/packages/2e/12/aef645b13c5bd15e3e5993537dbc23f7d65475f3cf1755d07b4d3e56850d/ty-0.0.83-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:57eae4aa55dab1af5d1423227c6517ccab1e658fe265c8e5a58caa584ca2876b", size = 13661466, upload-time = "2026-09-21T22:46:49.392Z" }, + { url = "https://files.pythonhosted.org/packages/d4/c4/11872e823653c568468ef7f0c3b23aed7ad2d33652e20144d28d5bfad58a/ty-0.0.83-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dea3680911340ecf93dff5786dbeefd69c9399df685a22751e1fbd27e6007b50", size = 14479677, upload-time = "2026-09-21T22:46:52.277Z" }, + { url = "https://files.pythonhosted.org/packages/8e/db/5b0d0d24d16d24d781f29ee842ee718903e453dc4068acb02bef860d5540/ty-0.0.83-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8453786c274f48e2a89b78e8a505a4f449323f357f2ce26a9b90a4bd68176f67", size = 14972208, upload-time = "2026-09-21T22:46:55.242Z" }, + { url = "https://files.pythonhosted.org/packages/54/04/0077a53bcf6ceb41365a45d8091b52094c91a5f4b071c2a6d705ed8bb65b/ty-0.0.83-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7701bdae3b52271cf0835abe4202aa3b23b735425d8c5a3e6a8ca1c68d39ead8", size = 14660550, upload-time = "2026-09-21T22:46:58.256Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ff/b6c879f1c3121387edd19ebdb1fb02af30691f88f632278a5d467fb5f348/ty-0.0.83-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa95b4376447e5309fe3ccc98ee75916a9082714e6a029b99427ca62021dc94a", size = 14147376, upload-time = "2026-09-21T22:47:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/0d/bc/82e730247d53cb02acd5a91237c0c62f215b2e0379689a550176d307ed89/ty-0.0.83-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2a4550a08fc1899a33e4a08787a73ec9daa99d292268df4f0a5f1c24c63854c3", size = 14567502, upload-time = "2026-09-21T22:47:04.344Z" }, + { url = "https://files.pythonhosted.org/packages/67/13/bb6f57923231f51b626931625ada71e219d64541f8fadf96065bc5034358/ty-0.0.83-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:feb72080f0835fa2b27485e3b11fa50dc57c30e0ba73808a3f7d64b56db83f6f", size = 13443051, upload-time = "2026-09-21T22:47:07.316Z" }, + { url = "https://files.pythonhosted.org/packages/50/20/9862431662d4453527cd52cf0f30d5975cd1801aa39e570bf45fb299be77/ty-0.0.83-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:3e5206def2d6e1df93c75ed8dc763255233ccc3153647509d125bada10ad770a", size = 13678908, upload-time = "2026-09-21T22:47:10.714Z" }, + { url = "https://files.pythonhosted.org/packages/d7/e3/94ebbe1573b76660fa09f2dbce96da6d0bb69a2d29c7b6ad538b452979d4/ty-0.0.83-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bdf5dfa157f5c127ba096fe0561b47877e6e79562c7fc2b35906ba5e9f6bb723", size = 13937248, upload-time = "2026-09-21T22:47:14.133Z" }, + { url = "https://files.pythonhosted.org/packages/2f/c0/3ea45c2ec3a82fddb7137e457ccb2a452446905430d225adaa4990cdc338/ty-0.0.83-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:626d6ad00f05a5d9b7ea80de44b12931d0c0fb50e64eab20b7fcc13e9aff2a68", size = 14290826, upload-time = "2026-09-21T22:47:16.874Z" }, + { url = "https://files.pythonhosted.org/packages/d8/b2/c941ce83a59e46ef31dd04bdf6067530e7ec77d5f0cdd1e223fbc6fc351b/ty-0.0.83-py3-none-win32.whl", hash = "sha256:0391017dac6ba3e452e0322301f50bba12fbccadac07d7fd888c46951ad37638", size = 13105291, upload-time = "2026-09-21T22:47:19.807Z" }, + { url = "https://files.pythonhosted.org/packages/b2/8a/5eef9458505698b144643cbd2062f465c5d860d8cf86a8710d74505ad9a0/ty-0.0.83-py3-none-win_amd64.whl", hash = "sha256:1b67d86f5bbad4076ae1a2ff23a6a4493219885d72d4b148cb0193e090390060", size = 13901557, upload-time = "2026-09-21T22:47:23.429Z" }, + { url = "https://files.pythonhosted.org/packages/9f/27/a06061513f1de25569c2e8898018bd552ab7b31afa7cfe818703269b3350/ty-0.0.83-py3-none-win_arm64.whl", hash = "sha256:a916a3d7d68fe30b7aacdb9fe16d461fa944d722e59e96ea0fa9d206a1b6625f", size = 13627960, upload-time = "2026-09-21T22:47:26.305Z" }, ] [[package]] From 4cdf838365824b2144e2a6abbc9db249ee0c7b93 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 09:53:47 +0000 Subject: [PATCH 4/9] fix: convert redis dataset items with a typed helper Replace the untyped `execute_command` call with `arrappend` fed by `_to_redis_json`, which ty checks against redis' `list`-based JSON type. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018m46SvjMFWEVxC4RDuegPN --- .../storage_clients/_redis/_dataset_client.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/crawlee/storage_clients/_redis/_dataset_client.py b/src/crawlee/storage_clients/_redis/_dataset_client.py index 71e04b8d87..2afd563dc6 100644 --- a/src/crawlee/storage_clients/_redis/_dataset_client.py +++ b/src/crawlee/storage_clients/_redis/_dataset_client.py @@ -1,7 +1,6 @@ from __future__ import annotations -import json -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from logging import getLogger from typing import TYPE_CHECKING, Any, cast @@ -16,16 +15,31 @@ from ._utils import await_redis_response if TYPE_CHECKING: - from collections.abc import AsyncIterator, Mapping + from collections.abc import AsyncIterator from redis.asyncio import Redis from redis.asyncio.client import Pipeline from crawlee._types import JsonSerializable + # Mirrors redis' own JSON type, which requires arrays to be a `list`. + JsonType = Mapping[str, 'JsonType'] | list['JsonType'] | str | int | float | bool | None + logger = getLogger(__name__) +def _to_redis_json(value: JsonSerializable) -> JsonType: + """Convert a JSON-serializable value to the `JsonType` accepted by redis JSON commands. + + Redis expects arrays as `list`, while `JsonSerializable` allows any read-only `Sequence`. + """ + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Mapping): + return {key: _to_redis_json(item) for key, item in value.items()} + return [_to_redis_json(item) for item in value] + + class _DatasetMetadataUpdateParams(MetadataUpdateParams): """Parameters for updating dataset metadata.""" @@ -134,9 +148,7 @@ async def push_data(self, data: Sequence[Mapping[str, JsonSerializable]] | Mappi items = data if isinstance(data, Sequence) else [data] async with self._get_pipeline() as pipe: - # Equivalent of `pipe.json().arrappend(...)`, whose `JsonType` stub only accepts `list`/`dict`, not - # read-only `Sequence`/`Mapping` values. - pipe.execute_command('JSON.ARRAPPEND', self._items_key, '$', *[json.dumps(item) for item in items]) + pipe.json().arrappend(self._items_key, '$', *[_to_redis_json(item) for item in items]) await self._update_metadata( pipe, **_DatasetMetadataUpdateParams( From 23c9ec5caad1f6baba9cf223e65ce59f3297c5ad Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 11:08:35 +0000 Subject: [PATCH 5/9] refactor: log ignored cookie parameters in `SessionCookies.set` Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018m46SvjMFWEVxC4RDuegPN --- src/crawlee/sessions/_cookies.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/crawlee/sessions/_cookies.py b/src/crawlee/sessions/_cookies.py index f5e3158622..57ea7ce673 100644 --- a/src/crawlee/sessions/_cookies.py +++ b/src/crawlee/sessions/_cookies.py @@ -3,6 +3,7 @@ from copy import deepcopy from email.message import Message from http.cookiejar import Cookie, CookieJar +from logging import getLogger from typing import TYPE_CHECKING, Any, Literal from urllib.request import Request as UrlRequest @@ -14,6 +15,8 @@ from collections.abc import Iterator from typing import TypeGuard +logger = getLogger(__name__) + class _SetCookieResponse: """Minimal response adapter exposing `Set-Cookie` headers to `CookieJar.extract_cookies`.""" @@ -110,7 +113,7 @@ def set( http_only: bool = False, secure: bool = False, same_site: Literal['Lax', 'None', 'Strict'] | None = None, - **_kwargs: Any, # Unknown parameters will be ignored. + **ignored_kwargs: Any, ) -> None: """Create and store a cookie with modern browser attributes. @@ -123,7 +126,11 @@ def set( http_only: Whether cookie is HTTP-only. secure: Whether cookie requires secure context. same_site: SameSite cookie attribute value. + ignored_kwargs: Unknown cookie parameters, which are ignored. """ + if ignored_kwargs: + logger.debug(f'Ignoring unknown parameters of cookie {name!r}: {sorted(ignored_kwargs)}') + cookie = Cookie( version=0, name=name, From 59e078c1947a8e426f7ad997e7fe90860af29d25 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 11:12:50 +0000 Subject: [PATCH 6/9] refactor: drop logging of ignored cookie parameters Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018m46SvjMFWEVxC4RDuegPN --- src/crawlee/sessions/_cookies.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/crawlee/sessions/_cookies.py b/src/crawlee/sessions/_cookies.py index 57ea7ce673..ddb239d82f 100644 --- a/src/crawlee/sessions/_cookies.py +++ b/src/crawlee/sessions/_cookies.py @@ -3,7 +3,6 @@ from copy import deepcopy from email.message import Message from http.cookiejar import Cookie, CookieJar -from logging import getLogger from typing import TYPE_CHECKING, Any, Literal from urllib.request import Request as UrlRequest @@ -15,8 +14,6 @@ from collections.abc import Iterator from typing import TypeGuard -logger = getLogger(__name__) - class _SetCookieResponse: """Minimal response adapter exposing `Set-Cookie` headers to `CookieJar.extract_cookies`.""" @@ -113,7 +110,7 @@ def set( http_only: bool = False, secure: bool = False, same_site: Literal['Lax', 'None', 'Strict'] | None = None, - **ignored_kwargs: Any, + **_ignored_kwargs: Any, ) -> None: """Create and store a cookie with modern browser attributes. @@ -126,11 +123,7 @@ def set( http_only: Whether cookie is HTTP-only. secure: Whether cookie requires secure context. same_site: SameSite cookie attribute value. - ignored_kwargs: Unknown cookie parameters, which are ignored. """ - if ignored_kwargs: - logger.debug(f'Ignoring unknown parameters of cookie {name!r}: {sorted(ignored_kwargs)}') - cookie = Cookie( version=0, name=name, From 3d8694105b55e339061e7f3fa99f68ee2fc7fe59 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 12:27:13 +0000 Subject: [PATCH 7/9] refactor: cast redis dataset items instead of converting them redis' `JsonType` types arrays as `list` although `arrappend` only encodes them, so a documented cast is simpler than copying every item. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018m46SvjMFWEVxC4RDuegPN --- .../storage_clients/_redis/_dataset_client.py | 23 ++++--------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/src/crawlee/storage_clients/_redis/_dataset_client.py b/src/crawlee/storage_clients/_redis/_dataset_client.py index 2afd563dc6..d8a8b6ab45 100644 --- a/src/crawlee/storage_clients/_redis/_dataset_client.py +++ b/src/crawlee/storage_clients/_redis/_dataset_client.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from logging import getLogger from typing import TYPE_CHECKING, Any, cast @@ -15,31 +15,17 @@ from ._utils import await_redis_response if TYPE_CHECKING: - from collections.abc import AsyncIterator + from collections.abc import AsyncIterator, Mapping from redis.asyncio import Redis from redis.asyncio.client import Pipeline + from redis.commands.json._util import JsonType from crawlee._types import JsonSerializable - # Mirrors redis' own JSON type, which requires arrays to be a `list`. - JsonType = Mapping[str, 'JsonType'] | list['JsonType'] | str | int | float | bool | None - logger = getLogger(__name__) -def _to_redis_json(value: JsonSerializable) -> JsonType: - """Convert a JSON-serializable value to the `JsonType` accepted by redis JSON commands. - - Redis expects arrays as `list`, while `JsonSerializable` allows any read-only `Sequence`. - """ - if value is None or isinstance(value, (str, int, float, bool)): - return value - if isinstance(value, Mapping): - return {key: _to_redis_json(item) for key, item in value.items()} - return [_to_redis_json(item) for item in value] - - class _DatasetMetadataUpdateParams(MetadataUpdateParams): """Parameters for updating dataset metadata.""" @@ -148,7 +134,8 @@ async def push_data(self, data: Sequence[Mapping[str, JsonSerializable]] | Mappi items = data if isinstance(data, Sequence) else [data] async with self._get_pipeline() as pipe: - pipe.json().arrappend(self._items_key, '$', *[_to_redis_json(item) for item in items]) + # redis' `JsonType` types arrays as `list`, although `arrappend` only encodes them. + pipe.json().arrappend(self._items_key, '$', *cast('list[JsonType]', items)) await self._update_metadata( pipe, **_DatasetMetadataUpdateParams( From 992bcf9640bc3464cae823c9939353d860ae9897 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 12:31:36 +0000 Subject: [PATCH 8/9] refactor: restore original `SessionCookies.set` kwargs name Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018m46SvjMFWEVxC4RDuegPN --- src/crawlee/sessions/_cookies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crawlee/sessions/_cookies.py b/src/crawlee/sessions/_cookies.py index ddb239d82f..f5e3158622 100644 --- a/src/crawlee/sessions/_cookies.py +++ b/src/crawlee/sessions/_cookies.py @@ -110,7 +110,7 @@ def set( http_only: bool = False, secure: bool = False, same_site: Literal['Lax', 'None', 'Strict'] | None = None, - **_ignored_kwargs: Any, + **_kwargs: Any, # Unknown parameters will be ignored. ) -> None: """Create and store a cookie with modern browser attributes. From 3da0d688c4bc673e768a2770f3fb5f1aa2726294 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 06:59:31 +0000 Subject: [PATCH 9/9] style: move redis items cast to its own line Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018m46SvjMFWEVxC4RDuegPN --- src/crawlee/storage_clients/_redis/_dataset_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/crawlee/storage_clients/_redis/_dataset_client.py b/src/crawlee/storage_clients/_redis/_dataset_client.py index d8a8b6ab45..6c229178a0 100644 --- a/src/crawlee/storage_clients/_redis/_dataset_client.py +++ b/src/crawlee/storage_clients/_redis/_dataset_client.py @@ -135,7 +135,8 @@ async def push_data(self, data: Sequence[Mapping[str, JsonSerializable]] | Mappi async with self._get_pipeline() as pipe: # redis' `JsonType` types arrays as `list`, although `arrappend` only encodes them. - pipe.json().arrappend(self._items_key, '$', *cast('list[JsonType]', items)) + redis_items = cast('list[JsonType]', items) + pipe.json().arrappend(self._items_key, '$', *redis_items) await self._update_metadata( pipe, **_DatasetMetadataUpdateParams(