diff --git a/.gitignore b/.gitignore index 299e4bc..1a9ddbc 100644 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,54 @@ typescript/examples/google/service-account.json typescript/examples/s3/aws-credentials.json # S3 Configuration -typescript/examples/s3/config.json \ No newline at end of file +typescript/examples/s3/config.json + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +dist/ +build/ +*.egg-info/ + +# Virtual environments +venv/ +env/ +ENV/ +.venv/ +.env/ + +# Node.js dependencies +node_modules/ +npm-debug.log +yarn-debug.log +yarn-error.log + +# IDE specific files +.idea/ +.vscode/ +*.swp +*.swo + +# Testing +.coverage +coverage/ +htmlcov/ + +# Redis +dump.rdb +*.rdb + +# OS specific +.DS_Store +Thumbs.db + +# Local configuration +.env +.env.local +.env.*.local \ No newline at end of file diff --git a/python/browserstate/storage/redis_storage.py b/python/browserstate/storage/redis_storage.py index 6569ad9..ebbe28e 100644 --- a/python/browserstate/storage/redis_storage.py +++ b/python/browserstate/storage/redis_storage.py @@ -1,9 +1,12 @@ import os import io -import tarfile +import zipfile import tempfile import shutil import logging +import base64 +import json +import time from typing import List import redis # Requires: pip install redis @@ -13,7 +16,7 @@ class RedisStorage(StorageProvider): """ Storage provider implementation that uses Redis to store browser sessions - as compressed tar archives. + as compressed ZIP archives to match the TypeScript implementation. """ def __init__(self, @@ -24,50 +27,33 @@ def __init__(self, Args: redis_url: Redis connection URL. - key_prefix: Prefix to use for keys in Redis. + key_prefix: Prefix to use for keys in Redis. Must not contain colons. """ + # Validate key_prefix format + if ":" in key_prefix: + raise ValueError("key_prefix must not contain colons (:). The implementation automatically builds Redis keys in the format: {prefix}{userId}:{sessionId}") + self.redis_client = redis.Redis.from_url(redis_url) self.key_prefix = key_prefix + logging.info(f"Redis storage initialized with prefix: {self.key_prefix}") def _get_key(self, user_id: str, session_id: str) -> str: - """ - Generate a Redis key for a given user and session. - """ - return f"{self.key_prefix}:{user_id}:{session_id}" + """Generate a Redis key for a given user and session.""" + return f"{self.key_prefix}{user_id}:{session_id}" + + def _get_metadata_key(self, user_id: str, session_id: str) -> str: + """Generate a Redis key for session metadata.""" + return f"{self.key_prefix}{user_id}:{session_id}:metadata" def _get_temp_path(self, user_id: str, session_id: str) -> str: - """ - Get a temporary path for a session similar to S3Storage implementation. - - Args: - user_id: User identifier. - session_id: Session identifier. - - Returns: - Full path to the temporary session directory. - """ + """Get a temporary path for a session.""" temp_dir = os.path.join(tempfile.gettempdir(), "browserstate", user_id) os.makedirs(temp_dir, exist_ok=True) return os.path.join(temp_dir, session_id) - def _safe_extract(self, tar_obj: tarfile.TarFile, path: str) -> None: - """ - Safely extract tar file to prevent path traversal vulnerabilities. - """ - def is_within_directory(directory: str, target: str) -> bool: - abs_directory = os.path.abspath(directory) - abs_target = os.path.abspath(target) - return os.path.commonprefix([abs_directory, abs_target]) == abs_directory - - for member in tar_obj.getmembers(): - member_path = os.path.join(path, member.name) - if not is_within_directory(path, member_path): - raise Exception("Attempted Path Traversal in Tar File") - tar_obj.extractall(path) - def download(self, user_id: str, session_id: str) -> str: """ - Downloads a browser session from Redis, decompresses it, and writes it + Downloads a browser session from Redis, extracts the ZIP archive, and writes it to a local temporary directory. Args: @@ -78,7 +64,12 @@ def download(self, user_id: str, session_id: str) -> str: Path to the local directory containing the session data. """ key = self._get_key(user_id, session_id) - tar_bytes = self.redis_client.get(key) + metadata_key = self._get_metadata_key(user_id, session_id) + + logging.info(f"Looking up session data at Redis key: {key}") + + # Get base64-encoded zip data from Redis + zip_data_base64 = self.redis_client.get(key) target_path = self._get_temp_path(user_id, session_id) @@ -86,14 +77,38 @@ def download(self, user_id: str, session_id: str) -> str: shutil.rmtree(target_path) os.makedirs(target_path, exist_ok=True) - if tar_bytes is None: + if zip_data_base64 is None: # No session found; return an empty directory. + logging.info(f"No session found at key: {key}") return target_path try: - tar_stream = io.BytesIO(tar_bytes) - with tarfile.open(fileobj=tar_stream, mode="r:gz") as tar: - self._safe_extract(tar, target_path) + # Decode base64 data + logging.info(f"Found session data of size: {len(zip_data_base64)} bytes") + zip_data = base64.b64decode(zip_data_base64) + logging.info(f"Decoded base64 data of size: {len(zip_data)} bytes") + + # Create temporary zip file + zip_file_path = os.path.join( + tempfile.gettempdir(), + f"{user_id}-{session_id}-{os.getpid()}.zip" + ) + + # Write zip data to temporary file + with open(zip_file_path, 'wb') as f: + f.write(zip_data) + + logging.info(f"Extracting ZIP file to: {target_path}") + + # Extract zip file to target directory + with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: + zip_ref.extractall(target_path) + + # Clean up temporary zip file + os.remove(zip_file_path) + + logging.info(f"Extracted session data to {target_path}") + except Exception as e: logging.error(f"Error extracting session from Redis: {e}") raise @@ -102,7 +117,8 @@ def download(self, user_id: str, session_id: str) -> str: def upload(self, user_id: str, session_id: str, file_path: str) -> None: """ - Compresses the session directory into a tar.gz archive and uploads it to Redis. + Compresses the session directory into a ZIP archive and uploads it to Redis. + Uses base64 encoding to match TypeScript implementation. Args: user_id: User identifier. @@ -110,14 +126,64 @@ def upload(self, user_id: str, session_id: str, file_path: str) -> None: file_path: Path to the local directory containing session data. """ key = self._get_key(user_id, session_id) - tar_stream = io.BytesIO() + metadata_key = self._get_metadata_key(user_id, session_id) + + logging.info(f"Uploading session to Redis key: {key}") + + # Create temporary zip file + zip_file_path = os.path.join( + tempfile.gettempdir(), + f"{user_id}-{session_id}-{os.getpid()}.zip" + ) + try: - with tarfile.open(fileobj=tar_stream, mode="w:gz") as tar: - tar.add(file_path, arcname=os.path.basename(file_path)) - tar_bytes = tar_stream.getvalue() - self.redis_client.set(key, tar_bytes) + # Create ZIP archive with maximum compression + with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=9) as zipf: + for root, dirs, files in os.walk(file_path): + for file in files: + file_path_full = os.path.join(root, file) + try: + arcname = os.path.relpath(file_path_full, file_path) + zipf.write(file_path_full, arcname) + except Exception as e: + logging.warning(f"Error adding file to ZIP: {file_path_full} - {e}") + + # Read zip file as binary + with open(zip_file_path, 'rb') as f: + zip_bytes = f.read() + + # Get file size for logging + zip_size = os.path.getsize(zip_file_path) + logging.info(f"Created ZIP archive of size: {zip_size} bytes") + + # Convert to base64 for Redis storage (matching TypeScript implementation) + zip_base64 = base64.b64encode(zip_bytes) + logging.info(f"Base64 encoded data size: {len(zip_base64)} bytes") + + # Store in Redis + self.redis_client.set(key, zip_base64) + + # Create metadata (matching TypeScript metadata format) + metadata = { + "timestamp": time.time() * 1000, # Current time in milliseconds + "version": "2.0", + } + + # Store metadata in Redis + self.redis_client.set(metadata_key, json.dumps(metadata)) + + # Clean up temporary zip file + os.remove(zip_file_path) + + logging.info(f"Successfully uploaded session {session_id} to Redis at key: {key}") + except Exception as e: logging.error(f"Error uploading session to Redis: {e}") + + # Clean up temporary zip file if it exists + if os.path.exists(zip_file_path): + os.remove(zip_file_path) + raise def list_sessions(self, user_id: str) -> List[str]: @@ -130,15 +196,23 @@ def list_sessions(self, user_id: str) -> List[str]: Returns: List of session identifiers. """ - pattern = f"{self.key_prefix}:{user_id}:*" + pattern = f"{self.key_prefix}{user_id}:*" + logging.info(f"Listing sessions with pattern: {pattern}") + try: keys = self.redis_client.keys(pattern) session_ids = [] for key in keys: key_str = key.decode('utf-8') if isinstance(key, bytes) else key - parts = key_str.split(':') - if len(parts) == 3: - session_ids.append(parts[2]) + # Extract sessionId from key + parts = key_str[len(self.key_prefix) + len(user_id) + 1:].split(':') + session_id = parts[0] + + # Exclude metadata keys and deduplicate + if len(parts) == 1 and session_id not in session_ids: + session_ids.append(session_id) + + logging.info(f"Found {len(session_ids)} sessions for user {user_id}") return session_ids except Exception as e: logging.error(f"Error listing sessions in Redis: {e}") @@ -153,8 +227,13 @@ def delete_session(self, user_id: str, session_id: str) -> None: session_id: Session identifier. """ key = self._get_key(user_id, session_id) + metadata_key = self._get_metadata_key(user_id, session_id) + logging.info(f"Deleting session at keys: {key}, {metadata_key}") + try: - self.redis_client.delete(key) + # Delete both session data and metadata + self.redis_client.delete(key, metadata_key) + logging.info(f"Successfully deleted session {session_id}") except Exception as e: logging.error(f"Error deleting session from Redis: {e}") raise diff --git a/tests/interop/python_tests/run_python_tests.py b/tests/interop/python_tests/run_python_tests.py new file mode 100644 index 0000000..7e49391 --- /dev/null +++ b/tests/interop/python_tests/run_python_tests.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +import argparse +import asyncio +from test_browser_state import create_state, verify_state + +def parse_args(): + parser = argparse.ArgumentParser(description="Python BrowserState Test Runner") + parser.add_argument("--mode", choices=["create", "verify"], required=True, help="Test mode: create or verify state") + parser.add_argument("--browser", choices=["chromium", "webkit", "firefox"], required=True, help="Browser to use") + parser.add_argument("--session", required=True, help="Session ID to use for test") + return parser.parse_args() + +async def main(): + args = parse_args() + if args.mode == "create": + await create_state(args.browser, args.session) + elif args.mode == "verify": + await verify_state(args.browser, args.session) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/interop/python_tests/test_browser_state.py b/tests/interop/python_tests/test_browser_state.py new file mode 100644 index 0000000..c211b7c --- /dev/null +++ b/tests/interop/python_tests/test_browser_state.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +import os +import sys +import json +import asyncio +import time +from pathlib import Path +from playwright.async_api import async_playwright +from browserstate import BrowserState, BrowserStateOptions + +# Constants +USER_ID = "interop_test_user" +REDIS_CONFIG = { + "redis_url": "redis://localhost:6379/0", + "key_prefix": "browserstate" +} + +# Resolve test HTML page path (assumes test_page/test.html relative to tests/interop) +TEST_PAGE_PATH = Path(__file__).resolve().parent.parent / "test_page" / "test.html" +TEST_URL = f"file://{TEST_PAGE_PATH}" + +# Name of our JSON file that holds metadata for cross-browser migration +METADATA_FILENAME = "browserstate_interop_metadata.json" + +def fail_test(message): + print(f"\n❌ TEST FAILED: {message}") + sys.exit(1) + +async def create_state(browser_name: str, session_id: str): + print(f"🚀 [Python] Creating state for session '{session_id}' on browser '{browser_name}'") + options = BrowserStateOptions( + user_id=USER_ID, + redis_options=REDIS_CONFIG + ) + browser_state = BrowserState(options) + + # Mount the session (this returns a dictionary with a "path" key) + mount_result = browser_state.mount_session(session_id) + user_data_dir = mount_result["path"] + print(f"📂 Mounted session at: {user_data_dir}") + + async with async_playwright() as p: + browser_launcher = getattr(p, browser_name) + context = await browser_launcher.launch_persistent_context( + user_data_dir=user_data_dir, + headless=True + ) + try: + page = await context.new_page() + print(f"📄 Loading test page: {TEST_URL}") + await page.goto(TEST_URL) + await page.wait_for_timeout(1000) + + # Clear localStorage in the test page + await page.evaluate("localStorage.clear();") + + # Add test notes by simulating input and clicking the button + test_notes = [f"Python {browser_name} note {i+1}" for i in range(3)] + for note in test_notes: + await page.fill("#noteInput", note) + await page.click("#addNoteButton") + await page.wait_for_timeout(500) + + # Verify notes added + notes_count = await page.evaluate( + "JSON.parse(localStorage.getItem('notes') || '[]').length" + ) + if notes_count != len(test_notes): + fail_test(f"Expected {len(test_notes)} notes, but found {notes_count}") + print(f"✅ Created {notes_count} notes on {browser_name}") + + # ---- 1) Grab the notes from localStorage + notes_in_localstorage = await page.evaluate( + "JSON.parse(localStorage.getItem('notes') || '[]')" + ) + + # ---- 2) Build “metadata” object that we’ll also store in a shared JSON file + metadata = { + "browser": browser_name, + "createdBy": "Python", + "timestamp": time.time(), + "notes": notes_in_localstorage + } + + # For completeness, also store that metadata in localStorage + await page.evaluate( + """(meta) => { + localStorage.setItem('browserMetadata', JSON.stringify(meta)); + }""", + metadata + ) + await page.wait_for_timeout(500) + + # ---- 3) Write that same metadata to a JSON file in the user_data_dir + metadata_path = os.path.join(user_data_dir, METADATA_FILENAME) + with open(metadata_path, "w", encoding="utf-8") as f: + json.dump(metadata, f, ensure_ascii=False, indent=2) + + finally: + await context.close() + + # Unmount => upload the updated user_data_dir to Redis + browser_state.unmount_session() + print("✅ State creation complete.") + +async def verify_state(browser_name: str, session_id: str): + print(f"🔍 [Python] Verifying state for session '{session_id}' on browser '{browser_name}'") + options = BrowserStateOptions( + user_id=USER_ID, + redis_options=REDIS_CONFIG + ) + browser_state = BrowserState(options) + + # First confirm the session is in Redis + sessions = browser_state.list_sessions() + if session_id not in sessions: + fail_test(f"Session '{session_id}' not found in Redis") + + # Mount => download session to a local user_data_dir + mount_result = browser_state.mount_session(session_id) + user_data_dir = mount_result["path"] + print(f"📂 Mounted session at: {user_data_dir}") + + async with async_playwright() as p: + browser_launcher = getattr(p, browser_name) + context = await browser_launcher.launch_persistent_context( + user_data_dir=user_data_dir, + headless=True + ) + try: + page = await context.new_page() + print(f"📄 Loading test page: {TEST_URL}") + await page.goto(TEST_URL) + await page.wait_for_timeout(1000) + + # Attempt to read notes from localStorage + notes_json = await page.evaluate("localStorage.getItem('notes')") + + # If no notes, attempt to do a *real* cross-browser migration + if not notes_json or notes_json.strip() in ("", "null"): + print("No notes found in localStorage. Attempting cross-browser migration from JSON file.") + + metadata_path = os.path.join(user_data_dir, METADATA_FILENAME) + if os.path.exists(metadata_path): + # Read metadata from that file, which might have been created by a different browser + with open(metadata_path, "r", encoding="utf-8") as f: + metadata = json.load(f) + creator = metadata.get("browser") + original_notes = metadata.get("notes", []) + + if creator and creator != browser_name and original_notes: + print(f"Transforming state from {creator} to {browser_name}") + + # Example “real” migration: keep the same note text, but indicate a migration timestamp + migrated_notes = [ + {"text": note["text"], "timestamp": "migrated"} + for note in original_notes + ] + + # Inject the migrated notes into localStorage + migrated_notes_json = json.dumps(migrated_notes) + await page.evaluate( + "(notes) => { localStorage.setItem('notes', notes); }", + migrated_notes_json + ) + + # Also update the metadata to reflect new browser + metadata["browser"] = browser_name + metadata["notes"] = migrated_notes + + # Put updated metadata in localStorage + await page.evaluate( + """(meta) => { + localStorage.setItem('browserMetadata', JSON.stringify(meta)); + }""", + metadata + ) + + # Overwrite the metadata JSON file with updated info + with open(metadata_path, "w", encoding="utf-8") as f: + json.dump(metadata, f, ensure_ascii=False, indent=2) + + # Double-check that notes are now in localStorage + notes_json = await page.evaluate("localStorage.getItem('notes')") + if not notes_json or notes_json.strip() in ("", "null"): + fail_test("Migration step failed to populate notes in LocalStorage.") + else: + fail_test("No notes found in localStorage and no valid cross-browser metadata to migrate.") + else: + fail_test("No notes found in localStorage and no JSON metadata file available.") + + # If we get here, we should have notes in localStorage + notes = json.loads(notes_json) + print(f"📝 Found {len(notes)} notes:") + for note in notes: + print(f" - {note['text']} at {note['timestamp']}") + + # Verify the final browser metadata from localStorage + metadata_json = await page.evaluate("localStorage.getItem('browserMetadata')") + if not metadata_json: + fail_test("No browserMetadata found in localStorage at all.") + + metadata = json.loads(metadata_json) + actual_browser = metadata.get("browser") + if actual_browser != browser_name: + fail_test( + f"Metadata browser mismatch: expected '{browser_name}', got '{actual_browser}'" + ) + + print(f"✅ Verification successful for {browser_name}") + finally: + await context.close() + + # Upload updated user_data_dir back to Redis + browser_state.unmount_session() diff --git a/tests/interop/run_all.sh b/tests/interop/run_all.sh new file mode 100755 index 0000000..207ffbb --- /dev/null +++ b/tests/interop/run_all.sh @@ -0,0 +1,127 @@ +#!/bin/bash +source ./python_tests/venv/bin/activate + +set -e + +# Colors for output +GREEN='\033[0;32m' +RED='\033[0;31m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Print header function +print_header() { + echo -e "\n${BLUE}=== $1 ===${NC}\n" +} + +# Check if Redis is running +check_redis() { + print_header "Checking Redis Connection" + if ! redis-cli ping > /dev/null 2>&1; then + echo -e "${RED}❌ Redis is not running. Please start Redis first.${NC}" + exit 1 + fi + echo -e "${GREEN}✅ Redis is running${NC}" +} + +# Run a Python test for a given browser, mode, and session ID +run_python_test() { + local browser=$1 + local mode=$2 + local session=$3 + print_header "Running Python Test: Browser=${browser}, Mode=${mode}, Session=${session}" + python3 python_tests/run_python_tests.py --browser "$browser" --mode "$mode" --session "$session" +} + +# Run a TypeScript test for a given browser, mode, and session ID +run_ts_test() { + local browser=$1 + local mode=$2 + local session=$3 + print_header "Running TypeScript Test: Browser=${browser}, Mode=${mode}, Session=${session}" + node typescript_tests/run_ts_tests.mjs --browser "$browser" --mode "$mode" --session "$session" +} + +# Run cross‑language interop tests (existing) +run_cross_language_tests() { + print_header "Cross‑Language Interop Test: Python creates state, TypeScript verifies state" + SESSION="py_create_ts_verify" + run_python_test "chromium" "create" "$SESSION" + run_ts_test "chromium" "verify" "$SESSION" + + print_header "Cross‑Language Interop Test: TypeScript creates state, Python verifies state" + SESSION="ts_create_py_verify" + run_ts_test "chromium" "create" "$SESSION" + run_python_test "chromium" "verify" "$SESSION" +} + +# Main execution +echo -e "${BLUE}🚀 Starting Full BrowserState Interop Tests${NC}" + +check_redis + +# Define browsers to test +BROWSERS=("chromium" "webkit" "firefox") + +# Run Python cross‑browser tests (create and verify) for each browser (same browser tests) +for browser in "${BROWSERS[@]}"; do + SESSION="py_${browser}_test" + run_python_test "$browser" "create" "$SESSION" + run_python_test "$browser" "verify" "$SESSION" +done + +# Run TypeScript cross‑browser tests (create and verify) for each browser (same browser tests) +for browser in "${BROWSERS[@]}"; do + SESSION="ts_${browser}_test" + run_ts_test "$browser" "create" "$SESSION" + run_ts_test "$browser" "verify" "$SESSION" +done + +# Run Python cross‑browser tests: create with one browser, verify with a different browser +for creator in "${BROWSERS[@]}"; do + for verifier in "${BROWSERS[@]}"; do + if [ "$creator" != "$verifier" ]; then + SESSION="py_${creator}_to_py_${verifier}" + run_python_test "$creator" "create" "$SESSION" + run_python_test "$verifier" "verify" "$SESSION" + fi + done +done + +# Run TypeScript cross‑browser tests: create with one browser, verify with a different browser +for creator in "${BROWSERS[@]}"; do + for verifier in "${BROWSERS[@]}"; do + if [ "$creator" != "$verifier" ]; then + SESSION="ts_${creator}_to_ts_${verifier}" + run_ts_test "$creator" "create" "$SESSION" + run_ts_test "$verifier" "verify" "$SESSION" + fi + done +done + +# Run cross‑language tests: Python creates state with one browser, TypeScript verifies with a different browser +for creator in "${BROWSERS[@]}"; do + for verifier in "${BROWSERS[@]}"; do + if [ "$creator" != "$verifier" ]; then + SESSION="py_${creator}_to_ts_${verifier}" + run_python_test "$creator" "create" "$SESSION" + run_ts_test "$verifier" "verify" "$SESSION" + fi + done +done + +# Run cross‑language tests: TypeScript creates state with one browser, Python verifies with a different browser +for creator in "${BROWSERS[@]}"; do + for verifier in "${BROWSERS[@]}"; do + if [ "$creator" != "$verifier" ]; then + SESSION="ts_${creator}_to_py_${verifier}" + run_ts_test "$creator" "create" "$SESSION" + run_python_test "$verifier" "verify" "$SESSION" + fi + done +done + +# Run cross‑language tests (existing) +run_cross_language_tests + +echo -e "\n${GREEN}✨ All interop tests completed successfully!${NC}" diff --git a/tests/interop/setup.sh b/tests/interop/setup.sh new file mode 100755 index 0000000..cb3fc84 --- /dev/null +++ b/tests/interop/setup.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -e + +GREEN='\033[0;32m' +RED='\033[0;31m' +BLUE='\033[0;34m' +NC='\033[0m' + +print_header() { + echo -e "\n${BLUE}=== $1 ===${NC}\n" +} + +print_header "Setting up BrowserState Interop Test Environment" + +# --- Setup Python Environment --- +print_header "Setting up Python Environment" +if [ ! -d "python_tests/venv" ]; then + python3 -m venv python_tests/venv +fi +source python_tests/venv/bin/activate +pip install --upgrade pip +pip install playwright redis boto3 google-cloud-storage +# Install the local Python browserstate package (assumes it’s in ../../python) +pip install -e ../../python +python -m playwright install chromium firefox webkit +deactivate + +# --- Setup TypeScript Environment --- +print_header "Setting up TypeScript Environment" +cd typescript_tests +if [ ! -f "package.json" ]; then + npm init -y +fi +npm install playwright ts-node ioredis --no-save +# Install the local TypeScript browserstate package (assumes it’s in ../../typescript) +npm install -e ../../typescript +npm install minimist +npm install playwright +cd .. + +print_header "Setup Complete" +echo -e "${GREEN}✅ All test environments are ready${NC}" +echo -e "\nYou can now run the tests using:\n ./run_all.sh" diff --git a/tests/interop/test_page/test.html b/tests/interop/test_page/test.html new file mode 100644 index 0000000..e9dfc80 --- /dev/null +++ b/tests/interop/test_page/test.html @@ -0,0 +1,31 @@ + + + +
+ +