Skip to content
Merged
2 changes: 1 addition & 1 deletion src/crawlee/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
from crawlee.storage_clients import StorageClient
from crawlee.storages import KeyValueStore

JsonSerializable = dict[str, 'JsonSerializable'] | list['JsonSerializable'] | str | int | float | bool | None
JsonSerializable = Mapping[str, 'JsonSerializable'] | Sequence['JsonSerializable'] | str | int | float | bool | None
else:
from pydantic import JsonValue as JsonSerializable

Expand Down
30 changes: 18 additions & 12 deletions src/crawlee/sessions/_cookies.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def info(self) -> Message:


@docs_group('Session management')
class CookieParam(TypedDict, total=False):
class CookieParam(TypedDict, total=False, closed=True):
"""Dictionary representation of cookies for `SessionCookies.set` method."""

name: Required[str]
Expand Down Expand Up @@ -187,17 +187,23 @@ def _to_playwright(self, cookie_dict: CookieParam) -> PlaywrightCookieParam:

def _from_playwright(self, cookie_dict: PlaywrightCookieParam) -> CookieParam:
"""Convert Playwright cookie to internal format."""
result: dict = dict(cookie_dict)

if 'httpOnly' in result:
result['http_only'] = result.pop('httpOnly')
if 'sameSite' in result:
result['same_site'] = result.pop('sameSite')
if 'expires' in result:
expires = int(result['expires'])
result['expires'] = None if expires == -1 else expires

return CookieParam(name=result.pop('name', ''), value=result.pop('value', ''), **result)
result = CookieParam(name=cookie_dict.get('name', ''), value=cookie_dict.get('value', ''))

if 'domain' in cookie_dict:
result['domain'] = cookie_dict['domain']
if 'path' in cookie_dict:
result['path'] = cookie_dict['path']
if 'secure' in cookie_dict:
result['secure'] = cookie_dict['secure']
if 'httpOnly' in cookie_dict:
result['http_only'] = cookie_dict['httpOnly']
if 'sameSite' in cookie_dict:
result['same_site'] = cookie_dict['sameSite']
# Playwright uses -1 for session cookies, which have no expiration.
if 'expires' in cookie_dict and (expires := int(cookie_dict['expires'])) != -1:
result['expires'] = expires

return result

def get_cookies_as_dicts(self) -> list[CookieParam]:
"""Convert cookies to a list with `CookieParam` dicts."""
Expand Down
5 changes: 4 additions & 1 deletion src/crawlee/storage_clients/_redis/_dataset_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

from redis.asyncio import Redis
from redis.asyncio.client import Pipeline
from redis.commands.json._util import JsonType

from crawlee._types import JsonSerializable

Expand Down Expand Up @@ -133,7 +134,9 @@ async def push_data(self, data: Sequence[Mapping[str, JsonSerializable]] | Mappi
items = data if isinstance(data, Sequence) else [data]

async with self._get_pipeline() as pipe:
pipe.json().arrappend(self._items_key, '$', *items)
# redis' `JsonType` types arrays as `list`, although `arrappend` only encodes them.
redis_items = cast('list[JsonType]', items)
pipe.json().arrappend(self._items_key, '$', *redis_items)
await self._update_metadata(
pipe,
**_DatasetMetadataUpdateParams(
Expand Down
42 changes: 21 additions & 21 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading