From bb62a7466efdebf0701bf1c3296551a00094c2b9 Mon Sep 17 00:00:00 2001 From: Sagar Jaiswal Date: Sun, 30 Mar 2025 22:17:49 -0400 Subject: [PATCH] Added interoperability tests, also making changes based on the interoperability --- pytest.ini | 4 + python/browserstate/storage/redis_storage.py | 212 ++++++++-- python/tests/test_redis_storage.py | 11 +- setup.py | 18 + tests/__init__.py | 3 + tests/integration/__init__.py | 3 + tests/integration/compatible_redis_storage.py | 308 ++++++++++++++ .../test_compatible_redis_storage.py | 199 +++++++++ .../integration/test_local_storage_interop.py | 202 +++++++++ tests/integration/test_py_ts_interop.py | 218 ++++++++++ tests/integration/test_s3_storage_interop.py | 238 +++++++++++ tests/integration/ts_local_helper.js | 191 +++++++++ tests/integration/ts_redis_helper.js | 270 ++++++++++++ tests/integration/ts_s3_helper.js | 201 +++++++++ tests/integration/ts_storage_helper.py | 73 ++++ typescript/package-lock.json | 151 ++++--- typescript/package.json | 7 +- typescript/src/storage/LocalStorage.ts | 2 +- typescript/src/storage/RedisStorage.ts | 383 ++++++++++++++---- typescript/src/types/external.d.ts | 47 +++ typescript/src/utils/DynamicImport.ts | 40 +- 21 files changed, 2621 insertions(+), 160 deletions(-) create mode 100644 pytest.ini create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/compatible_redis_storage.py create mode 100644 tests/integration/test_compatible_redis_storage.py create mode 100644 tests/integration/test_local_storage_interop.py create mode 100644 tests/integration/test_py_ts_interop.py create mode 100644 tests/integration/test_s3_storage_interop.py create mode 100644 tests/integration/ts_local_helper.js create mode 100644 tests/integration/ts_redis_helper.js create mode 100644 tests/integration/ts_s3_helper.js create mode 100644 tests/integration/ts_storage_helper.py create mode 100644 typescript/src/types/external.d.ts diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..075173d --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +python_paths = . +testpaths = tests +python_files = test_*.py \ No newline at end of file diff --git a/python/browserstate/storage/redis_storage.py b/python/browserstate/storage/redis_storage.py index 6569ad9..8c3918d 100644 --- a/python/browserstate/storage/redis_storage.py +++ b/python/browserstate/storage/redis_storage.py @@ -1,10 +1,13 @@ import os import io +import json import tarfile +import zipfile import tempfile import shutil import logging -from typing import List +import base64 +from typing import List, Dict, Optional, Union import redis # Requires: pip install redis @@ -12,22 +15,33 @@ class RedisStorage(StorageProvider): """ - Storage provider implementation that uses Redis to store browser sessions - as compressed tar archives. + Storage provider implementation that uses Redis to store browser sessions. + + Supports both TAR.GZ and ZIP formats for cross-implementation compatibility. + Can handle sessions created by both Python and TypeScript implementations. """ def __init__(self, redis_url: str = "redis://localhost:6379/0", - key_prefix: str = "browserstate"): + key_prefix: str = "browserstate", + format: str = "zip"): """ Initialize RedisStorage provider. Args: redis_url: Redis connection URL. key_prefix: Prefix to use for keys in Redis. + format: Format to use for storing sessions ("tar.gz" or "zip"). + "tar.gz" is the Python-native format + "zip" is the TypeScript-compatible format """ + if format not in ["tar.gz", "zip"]: + raise ValueError('format must be "tar.gz" or "zip"') + self.redis_client = redis.Redis.from_url(redis_url) self.key_prefix = key_prefix + self.format = format + self.logger = logging.getLogger("RedisStorage") def _get_key(self, user_id: str, session_id: str) -> str: """ @@ -35,6 +49,13 @@ def _get_key(self, user_id: str, session_id: str) -> str: """ return f"{self.key_prefix}:{user_id}:{session_id}" + def _get_metadata_key(self, user_id: str, session_id: str) -> str: + """ + Generate a Redis key for session metadata. + Used for TypeScript compatibility. + """ + return f"{self.key_prefix}:{user_id}:{session_id}:metadata" + def _get_temp_path(self, user_id: str, session_id: str) -> str: """ Get a temporary path for a session similar to S3Storage implementation. @@ -50,6 +71,34 @@ def _get_temp_path(self, user_id: str, session_id: str) -> str: os.makedirs(temp_dir, exist_ok=True) return os.path.join(temp_dir, session_id) + def _detect_format(self, data: bytes) -> str: + """ + Detect the format of the stored data. + + Args: + data: Raw bytes from Redis + + Returns: + "tar.gz", "zip", or "unknown" + """ + # Check for gzip magic bytes (first 2 bytes are 0x1F8B) + if data[:2] == b'\x1f\x8b': + return "tar.gz" + + # Check if it might be base64-encoded ZIP + try: + decoded = base64.b64decode(data) + if decoded[:4] == b'PK\x03\x04': # ZIP magic bytes + return "zip" + except: + pass + + # Try directly checking for ZIP magic bytes + if data[:4] == b'PK\x03\x04': + return "zip" + + return "unknown" + def _safe_extract(self, tar_obj: tarfile.TarFile, path: str) -> None: """ Safely extract tar file to prevent path traversal vulnerabilities. @@ -70,6 +119,9 @@ def download(self, user_id: str, session_id: str) -> str: Downloads a browser session from Redis, decompresses it, and writes it to a local temporary directory. + Automatically detects and handles both TAR.GZ and ZIP formats for + compatibility with TypeScript implementation. + Args: user_id: User identifier. session_id: Session identifier. @@ -77,8 +129,12 @@ def download(self, user_id: str, session_id: str) -> str: Returns: Path to the local directory containing the session data. """ - key = self._get_key(user_id, session_id) - tar_bytes = self.redis_client.get(key) + session_key = self._get_key(user_id, session_id) + metadata_key = self._get_metadata_key(user_id, session_id) + + # Get session data + session_data = self.redis_client.get(session_key) + metadata_data = self.redis_client.get(metadata_key) target_path = self._get_temp_path(user_id, session_id) @@ -86,44 +142,133 @@ def download(self, user_id: str, session_id: str) -> str: shutil.rmtree(target_path) os.makedirs(target_path, exist_ok=True) - if tar_bytes is None: + if session_data is None: # No session found; return an empty directory. return target_path + # Parse metadata if available + metadata = None + if metadata_data: + try: + metadata = json.loads(metadata_data) + self.logger.debug(f"Found metadata for session {session_id}: {metadata}") + except json.JSONDecodeError: + self.logger.warning(f"Failed to parse metadata for session {session_id}") + + # Detect format + format_type = self._detect_format(session_data) + self.logger.debug(f"Detected format for session {session_id}: {format_type}") + try: - tar_stream = io.BytesIO(tar_bytes) - with tarfile.open(fileobj=tar_stream, mode="r:gz") as tar: - self._safe_extract(tar, target_path) + if format_type == "tar.gz": + # Handle TAR.GZ format (Python style) + tar_stream = io.BytesIO(session_data) + with tarfile.open(fileobj=tar_stream, mode="r:gz") as tar: + self._safe_extract(tar, target_path) + + elif format_type == "zip": + # Handle ZIP format (TypeScript style) + # First try to handle as base64-encoded ZIP + try: + zip_data = base64.b64decode(session_data) + except: + # If not base64, use raw data + zip_data = session_data + + zip_path = os.path.join(tempfile.gettempdir(), f"{user_id}_{session_id}_{os.urandom(4).hex()}.zip") + + try: + # Write the zip data to a file + with open(zip_path, "wb") as f: + f.write(zip_data) + + # Extract the zip + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(target_path) + finally: + # Clean up temporary zip file + if os.path.exists(zip_path): + os.remove(zip_path) + + else: + raise Exception(f"Unknown format: {format_type}") + except Exception as e: - logging.error(f"Error extracting session from Redis: {e}") + self.logger.error(f"Error extracting session from Redis: {e}") raise return target_path def upload(self, user_id: str, session_id: str, file_path: str) -> None: """ - Compresses the session directory into a tar.gz archive and uploads it to Redis. + Compresses the session directory and uploads it to Redis. + + Can use either TAR.GZ (Python-native) or ZIP (TypeScript-compatible) format + based on the 'format' parameter passed to the constructor. Args: user_id: User identifier. session_id: Session identifier. file_path: Path to the local directory containing session data. """ - key = self._get_key(user_id, session_id) - tar_stream = io.BytesIO() + session_key = self._get_key(user_id, session_id) + metadata_key = self._get_metadata_key(user_id, session_id) + try: - with tarfile.open(fileobj=tar_stream, mode="w:gz") as tar: - tar.add(file_path, arcname=os.path.basename(file_path)) - tar_bytes = tar_stream.getvalue() - self.redis_client.set(key, tar_bytes) + if self.format == "tar.gz": + # Python-style TAR.GZ format + tar_stream = io.BytesIO() + with tarfile.open(fileobj=tar_stream, mode="w:gz") as tar: + tar.add(file_path, arcname=os.path.basename(file_path)) + + session_data = tar_stream.getvalue() + + # Store the data (no separate metadata) + self.redis_client.set(session_key, session_data) + + else: # ZIP format + # TypeScript-style ZIP format with metadata + zip_path = os.path.join(tempfile.gettempdir(), f"{user_id}_{session_id}_{os.urandom(4).hex()}.zip") + + try: + # Create ZIP archive + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zip_ref: + for root, dirs, files in os.walk(file_path): + for file in files: + file_path_full = os.path.join(root, file) + arc_name = os.path.relpath(file_path_full, file_path) + zip_ref.write(file_path_full, arcname=arc_name) + + # Read as base64 + with open(zip_path, "rb") as f: + zip_data = base64.b64encode(f.read()).decode('utf-8') + + # Create metadata like TypeScript version + metadata = { + "timestamp": int(os.path.getmtime(file_path) * 1000), # Convert to JS timestamp + "fileCount": sum(len(files) for _, _, files in os.walk(file_path)), + "version": "2.0" # Version from TypeScript implementation + } + + # Store both data and metadata + self.redis_client.set(session_key, zip_data) + self.redis_client.set(metadata_key, json.dumps(metadata)) + + finally: + # Clean up + if os.path.exists(zip_path): + os.remove(zip_path) + except Exception as e: - logging.error(f"Error uploading session to Redis: {e}") + self.logger.error(f"Error uploading session to Redis: {e}") raise def list_sessions(self, user_id: str) -> List[str]: """ Lists all available sessions for a user from Redis. + Handles sessions created by both Python and TypeScript implementations. + Args: user_id: User identifier. @@ -131,30 +276,45 @@ def list_sessions(self, user_id: str) -> List[str]: List of session identifiers. """ pattern = f"{self.key_prefix}:{user_id}:*" + try: keys = self.redis_client.keys(pattern) - session_ids = [] + session_ids = set() + for key in keys: key_str = key.decode('utf-8') if isinstance(key, bytes) else key parts = key_str.split(':') + + # Skip metadata keys + if len(parts) > 3 and parts[3] == "metadata": + continue + if len(parts) == 3: - session_ids.append(parts[2]) - return session_ids + session_ids.add(parts[2]) + + return list(session_ids) + except Exception as e: - logging.error(f"Error listing sessions in Redis: {e}") + self.logger.error(f"Error listing sessions in Redis: {e}") return [] def delete_session(self, user_id: str, session_id: str) -> None: """ Deletes a browser session from Redis. + Deletes both the session data and metadata for TypeScript compatibility. + Args: user_id: User identifier. session_id: Session identifier. """ - key = self._get_key(user_id, session_id) + session_key = self._get_key(user_id, session_id) + metadata_key = self._get_metadata_key(user_id, session_id) + try: - self.redis_client.delete(key) + # Delete both the session data and metadata + self.redis_client.delete(session_key) + self.redis_client.delete(metadata_key) except Exception as e: - logging.error(f"Error deleting session from Redis: {e}") + self.logger.error(f"Error deleting session from Redis: {e}") raise diff --git a/python/tests/test_redis_storage.py b/python/tests/test_redis_storage.py index 78f057a..ab6e465 100644 --- a/python/tests/test_redis_storage.py +++ b/python/tests/test_redis_storage.py @@ -13,8 +13,15 @@ def test_redis_storage_upload_download(fake_redis, dummy_session_dir): original_file = os.path.join(dummy_session_dir, "test.txt") downloaded_file = os.path.join(downloaded_path, os.path.basename(dummy_session_dir), "test.txt") - assert os.path.exists(downloaded_file) - assert filecmp.cmp(original_file, downloaded_file, shallow=False) + + # Based on the format, the file will be either a tar.gz or a zip + if storage.format == "tar.gz": + expected_file = os.path.join(downloaded_path, os.path.basename(dummy_session_dir), "test.txt") + else: # zip format (TypeScript-compatible) + expected_file = os.path.join(downloaded_path, "test.txt") + + assert os.path.exists(expected_file) + assert filecmp.cmp(original_file, expected_file, shallow=False) sessions = storage.list_sessions(user_id) assert session_id in sessions diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4696297 --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +""" +BrowserState Package Setup +""" + +from setuptools import setup, find_packages + +setup( + name="browserstate", + version="0.1.0", + packages=find_packages(include=["python", "python.*", "tests", "tests.*"]), + install_requires=[ + "redis", + "boto3", + "moto", + "google-cloud-storage", + ], + python_requires=">=3.7", +) \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..384e819 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +""" +BrowserState test package +""" \ No newline at end of file diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..9614403 --- /dev/null +++ b/tests/integration/__init__.py @@ -0,0 +1,3 @@ +""" +Integration tests package for BrowserState cross-implementation compatibility testing. +""" \ No newline at end of file diff --git a/tests/integration/compatible_redis_storage.py b/tests/integration/compatible_redis_storage.py new file mode 100644 index 0000000..1412118 --- /dev/null +++ b/tests/integration/compatible_redis_storage.py @@ -0,0 +1,308 @@ +""" +Enhanced Redis Storage with cross-format compatibility. + +This module provides an enhanced version of the Redis storage provider +that can handle both Python's TAR.GZ and TypeScript's ZIP formats, +allowing full interoperability between the implementations. +""" + +import os +import io +import json +import tarfile +import zipfile +import tempfile +import shutil +import logging +import base64 +from typing import List, Dict, Optional, Union, Tuple + +import redis + +from python.browserstate.storage.storage_provider import StorageProvider + +class CompatibleRedisStorage(StorageProvider): + """ + Enhanced Redis storage provider that supports both TAR.GZ and ZIP formats + for maximum interoperability between Python and TypeScript implementations. + + Features: + - Auto-detection of storage format (TAR.GZ or ZIP) + - Supports both Python-style keys and TypeScript-style keys with metadata + - Can read and write sessions created by either implementation + - Preserves metadata when possible + - Improved error handling and diagnostics + """ + + def __init__(self, + redis_url: str = "redis://localhost:6379/0", + key_prefix: str = "browserstate", + preferred_format: str = "tar.gz"): + """ + Initialize the compatible Redis storage provider. + + Args: + redis_url: Redis connection URL + key_prefix: Prefix to use for keys in Redis + preferred_format: Format to use when creating new sessions ("tar.gz" or "zip") + """ + if preferred_format not in ["tar.gz", "zip"]: + raise ValueError('preferred_format must be "tar.gz" or "zip"') + + self.redis_client = redis.Redis.from_url(redis_url) + self.key_prefix = key_prefix + self.preferred_format = preferred_format + self.logger = logging.getLogger("CompatibleRedisStorage") + + def _get_session_key(self, user_id: str, session_id: str) -> str: + """Generate a Redis key for a session.""" + return f"{self.key_prefix}:{user_id}:{session_id}" + + def _get_metadata_key(self, user_id: str, session_id: str) -> str: + """Generate a Redis key for session metadata.""" + return f"{self.key_prefix}:{user_id}:{session_id}:metadata" + + def _get_temp_path(self, user_id: str, session_id: str) -> str: + """Get a temporary path for a session.""" + temp_dir = os.path.join(tempfile.gettempdir(), "browserstate", user_id) + os.makedirs(temp_dir, exist_ok=True) + return os.path.join(temp_dir, session_id) + + def _detect_format(self, data: bytes) -> str: + """ + Detect the format of the stored data. + + Args: + data: Raw bytes from Redis + + Returns: + "tar.gz", "zip", or "unknown" + """ + # Check for gzip magic bytes (first 2 bytes are 0x1F8B) + if data[:2] == b'\x1f\x8b': + return "tar.gz" + + # Check if it might be base64-encoded ZIP + try: + decoded = base64.b64decode(data) + if decoded[:4] == b'PK\x03\x04': # ZIP magic bytes + return "zip" + except: + pass + + # Try directly checking for ZIP magic bytes + if data[:4] == b'PK\x03\x04': + return "zip" + + return "unknown" + + def _safe_extract_tar(self, tar_obj: tarfile.TarFile, path: str) -> None: + """Safely extract tar file to prevent path traversal vulnerabilities.""" + def is_within_directory(directory: str, target: str) -> bool: + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + return os.path.commonprefix([abs_directory, abs_target]) == abs_directory + + for member in tar_obj.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + tar_obj.extractall(path) + + def download(self, user_id: str, session_id: str) -> str: + """ + Downloads a browser session from Redis, auto-detecting and handling + both TAR.GZ and ZIP formats. + + Args: + user_id: User identifier + session_id: Session identifier + + Returns: + Path to the local directory containing the session data + """ + session_key = self._get_session_key(user_id, session_id) + metadata_key = self._get_metadata_key(user_id, session_id) + + # Get both the session data and metadata (if available) + session_data = self.redis_client.get(session_key) + metadata_data = self.redis_client.get(metadata_key) + + target_path = self._get_temp_path(user_id, session_id) + + if os.path.exists(target_path): + shutil.rmtree(target_path) + os.makedirs(target_path, exist_ok=True) + + if session_data is None: + # No session found; return an empty directory + return target_path + + metadata = None + if metadata_data: + try: + metadata = json.loads(metadata_data) + self.logger.info(f"Found metadata for session {session_id}: {metadata}") + except json.JSONDecodeError: + self.logger.warning(f"Failed to parse metadata for session {session_id}") + + # Detect format + format_type = self._detect_format(session_data) + self.logger.info(f"Detected format for session {session_id}: {format_type}") + + try: + if format_type == "tar.gz": + # Handle TAR.GZ format (Python style) + tar_stream = io.BytesIO(session_data) + with tarfile.open(fileobj=tar_stream, mode="r:gz") as tar: + self._safe_extract_tar(tar, target_path) + + elif format_type == "zip": + # Handle ZIP format (TypeScript style) + # First try to handle as base64-encoded ZIP + try: + zip_data = base64.b64decode(session_data) + except: + # If not base64, use raw data + zip_data = session_data + + zip_path = os.path.join(tempfile.gettempdir(), f"{user_id}_{session_id}_{os.urandom(4).hex()}.zip") + + try: + # Write the zip data to a file + with open(zip_path, "wb") as f: + f.write(zip_data) + + # Extract the zip + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(target_path) + finally: + # Clean up temporary zip file + if os.path.exists(zip_path): + os.remove(zip_path) + + else: + raise Exception(f"Unknown format: {format_type}") + + except Exception as e: + self.logger.error(f"Error extracting session from Redis: {e}") + # Don't delete the target directory - leave it as an empty dir + raise + + return target_path + + def upload(self, user_id: str, session_id: str, file_path: str) -> None: + """ + Uploads a browser session to Redis using the preferred format. + + Args: + user_id: User identifier + session_id: Session identifier + file_path: Path to the local directory containing session data + """ + session_key = self._get_session_key(user_id, session_id) + metadata_key = self._get_metadata_key(user_id, session_id) + + try: + if self.preferred_format == "tar.gz": + # Python-style TAR.GZ format + tar_stream = io.BytesIO() + with tarfile.open(fileobj=tar_stream, mode="w:gz") as tar: + tar.add(file_path, arcname=os.path.basename(file_path)) + + session_data = tar_stream.getvalue() + + # Store the data (no separate metadata) + metadata = {"format": "tar.gz"} + self.redis_client.set(metadata_key, json.dumps(metadata)) + self.redis_client.set(session_key, session_data) + + else: # ZIP format + # TypeScript-style ZIP format with metadata + zip_path = os.path.join(tempfile.gettempdir(), f"{user_id}_{session_id}_{os.urandom(4).hex()}.zip") + + try: + # Create ZIP archive + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zip_ref: + for root, dirs, files in os.walk(file_path): + for file in files: + file_path_full = os.path.join(root, file) + arc_name = os.path.relpath(file_path_full, file_path) + zip_ref.write(file_path_full, arcname=arc_name) + + # Read as base64 + with open(zip_path, "rb") as f: + zip_data = base64.b64encode(f.read()).decode('utf-8') + + # Create metadata like TypeScript version + metadata = { + "timestamp": int(os.path.getmtime(file_path) * 1000), # Convert to JS timestamp + "fileCount": sum(len(files) for _, _, files in os.walk(file_path)), + "version": "2.0", # Version from TypeScript implementation + "format": "zip" # Explicitly record the format + } + + # Store both data and metadata + self.redis_client.set(session_key, zip_data) + self.redis_client.set(metadata_key, json.dumps(metadata)) + + finally: + # Clean up + if os.path.exists(zip_path): + os.remove(zip_path) + + except Exception as e: + self.logger.error(f"Error uploading session to Redis: {e}") + raise + + def list_sessions(self, user_id: str) -> List[str]: + """ + Lists all available sessions for a user from Redis. + + Args: + user_id: User identifier + + Returns: + List of session identifiers + """ + pattern = f"{self.key_prefix}:{user_id}:*" + + try: + keys = self.redis_client.keys(pattern) + session_ids = [] + + for key in keys: + key_str = key.decode('utf-8') if isinstance(key, bytes) else key + parts = key_str.split(':') + + # Skip metadata keys + if len(parts) > 3 and parts[3] == "metadata": + continue + + if len(parts) == 3: + session_ids.append(parts[2]) + + return list(set(session_ids)) # Remove duplicates + + except Exception as e: + self.logger.error(f"Error listing sessions in Redis: {e}") + return [] + + def delete_session(self, user_id: str, session_id: str) -> None: + """ + Deletes a browser session and its metadata from Redis. + + Args: + user_id: User identifier + session_id: Session identifier + """ + session_key = self._get_session_key(user_id, session_id) + metadata_key = self._get_metadata_key(user_id, session_id) + + try: + # Delete both session data and metadata + self.redis_client.delete(session_key) + self.redis_client.delete(metadata_key) + except Exception as e: + self.logger.error(f"Error deleting session from Redis: {e}") + raise \ No newline at end of file diff --git a/tests/integration/test_compatible_redis_storage.py b/tests/integration/test_compatible_redis_storage.py new file mode 100644 index 0000000..19b3c5e --- /dev/null +++ b/tests/integration/test_compatible_redis_storage.py @@ -0,0 +1,199 @@ +import os +import json +import shutil +import tempfile +import unittest +import time + +import redis + +# Use a relative import for the compatible_redis_storage module +from .compatible_redis_storage import CompatibleRedisStorage + +class TestCompatibleRedisStorage(unittest.TestCase): + """Test enhanced Redis storage with cross-format compatibility.""" + + def setUp(self): + """Set up the test environment.""" + # Redis connection + self.redis_url = "redis://localhost:6379/0" + self.redis_client = redis.Redis.from_url(self.redis_url) + self.key_prefix = "browserstate_test_compat" + + # Clean up any leftover keys from previous test runs + for key in self.redis_client.keys(f"{self.key_prefix}:*"): + self.redis_client.delete(key) + + # Test user and session IDs + self.user_id = "test_compat_user" + self.session_id_tar = "test_session_tar" + self.session_id_zip = "test_session_zip" + + # Create temporary directory for test files + self.temp_dir = tempfile.mkdtemp() + self.session_dir = os.path.join(self.temp_dir, "session") + os.makedirs(self.session_dir, exist_ok=True) + + # Create test files with unique content + with open(os.path.join(self.session_dir, "test.txt"), "w") as f: + f.write("Test data for compatibility testing") + + # Create a subdirectory with files to test nested structure handling + subdir = os.path.join(self.session_dir, "subdir") + os.makedirs(subdir, exist_ok=True) + with open(os.path.join(subdir, "nested.txt"), "w") as f: + f.write("Nested file for testing directory structure") + + # Create different storage providers for testing + self.storage_tar = CompatibleRedisStorage( + redis_url=self.redis_url, + key_prefix=self.key_prefix, + preferred_format="tar.gz" + ) + + self.storage_zip = CompatibleRedisStorage( + redis_url=self.redis_url, + key_prefix=self.key_prefix, + preferred_format="zip" + ) + + def tearDown(self): + """Clean up resources after tests.""" + # Clean Redis + for key in self.redis_client.keys(f"{self.key_prefix}:*"): + self.redis_client.delete(key) + + # Remove temporary directory + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def verify_files_exist(self, directory, expected_files): + """ + Verify that expected files exist in the specified directory. + If there's a single top-level folder, descend into it before checking. + Returns the final directory path after any descent. + """ + entries = os.listdir(directory) + if len(entries) == 1: + candidate = os.path.join(directory, entries[0]) + if os.path.isdir(candidate): + directory = candidate + + for file_path in expected_files: + full_path = os.path.join(directory, file_path) + self.assertTrue(os.path.exists(full_path), f"File {file_path} does not exist") + + return directory + + def test_tar_to_zip_format_compatibility(self): + """Test that TAR.GZ created sessions can be read by ZIP-compatible storage.""" + # Create a session using TAR.GZ format + self.storage_tar.upload(self.user_id, self.session_id_tar, self.session_dir) + + # Verify key exists in Redis + session_key = f"{self.key_prefix}:{self.user_id}:{self.session_id_tar}" + self.assertIsNotNone(self.redis_client.get(session_key)) + + # Try to download with ZIP-compatible storage + download_path = self.storage_zip.download(self.user_id, self.session_id_tar) + + # Verify files are correctly extracted and get the final directory path + expected_files = ["test.txt", os.path.join("subdir", "nested.txt")] + download_path = self.verify_files_exist(download_path, expected_files) + + # Verify content is preserved + with open(os.path.join(download_path, "test.txt"), "r") as f: + content = f.read() + self.assertEqual(content, "Test data for compatibility testing") + + def test_zip_to_tar_format_compatibility(self): + """Test that ZIP created sessions can be read by TAR.GZ-compatible storage.""" + # Create a session using ZIP format + self.storage_zip.upload(self.user_id, self.session_id_zip, self.session_dir) + + # Verify key exists in Redis and metadata is created + session_key = f"{self.key_prefix}:{self.user_id}:{self.session_id_zip}" + metadata_key = f"{self.key_prefix}:{self.user_id}:{self.session_id_zip}:metadata" + + self.assertIsNotNone(self.redis_client.get(session_key)) + self.assertIsNotNone(self.redis_client.get(metadata_key)) + + # Try to download with TAR.GZ-compatible storage + download_path = self.storage_tar.download(self.user_id, self.session_id_zip) + + # Verify files are correctly extracted and get the final directory path + expected_files = ["test.txt", os.path.join("subdir", "nested.txt")] + download_path = self.verify_files_exist(download_path, expected_files) + + # Verify content is preserved + with open(os.path.join(download_path, "test.txt"), "r") as f: + content = f.read() + self.assertEqual(content, "Test data for compatibility testing") + + def test_metadata_preservation(self): + """Test that metadata is preserved across formats.""" + # Create a session with metadata (ZIP format) + self.storage_zip.upload(self.user_id, self.session_id_zip, self.session_dir) + + # Verify metadata is created + metadata_key = f"{self.key_prefix}:{self.user_id}:{self.session_id_zip}:metadata" + metadata_raw = self.redis_client.get(metadata_key) + self.assertIsNotNone(metadata_raw) + + # Parse metadata + metadata = json.loads(metadata_raw) + self.assertIn("timestamp", metadata) + self.assertIn("version", metadata) + + # List sessions should include this session + sessions = self.storage_tar.list_sessions(self.user_id) + self.assertIn(self.session_id_zip, sessions) + + # Delete session + self.storage_tar.delete_session(self.user_id, self.session_id_zip) + + # Verify both session and metadata are deleted + session_key = f"{self.key_prefix}:{self.user_id}:{self.session_id_zip}" + self.assertIsNone(self.redis_client.get(session_key)) + self.assertIsNone(self.redis_client.get(metadata_key)) + + def test_list_sessions_across_formats(self): + """Test that list_sessions works across different formats.""" + # Create sessions in both formats + self.storage_tar.upload(self.user_id, self.session_id_tar, self.session_dir) + self.storage_zip.upload(self.user_id, self.session_id_zip, self.session_dir) + + # List sessions using TAR.GZ storage + tar_sessions = self.storage_tar.list_sessions(self.user_id) + self.assertIn(self.session_id_tar, tar_sessions) + self.assertIn(self.session_id_zip, tar_sessions) + + # List sessions using ZIP storage + zip_sessions = self.storage_zip.list_sessions(self.user_id) + self.assertIn(self.session_id_tar, zip_sessions) + self.assertIn(self.session_id_zip, zip_sessions) + + def test_overwrite_session_different_format(self): + """Test overwriting a session with a different format.""" + # Create session with TAR.GZ format + self.storage_tar.upload(self.user_id, self.session_id_tar, self.session_dir) + + # Modify the session dir with new content + with open(os.path.join(self.session_dir, "updated.txt"), "w") as f: + f.write("Updated content") + + # Overwrite with ZIP format + self.storage_zip.upload(self.user_id, self.session_id_tar, self.session_dir) + + # Verify we can still read it back + download_path = self.storage_tar.download(self.user_id, self.session_id_tar) + + # Verify original and new files, and get the final directory path + expected_files = ["test.txt", "updated.txt", os.path.join("subdir", "nested.txt")] + download_path = self.verify_files_exist(download_path, expected_files) + + # Now both files (test.txt and updated.txt) and the subdirectory file exist + with open(os.path.join(download_path, "updated.txt"), "r") as f: + self.assertEqual(f.read(), "Updated content") + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/test_local_storage_interop.py b/tests/integration/test_local_storage_interop.py new file mode 100644 index 0000000..904142f --- /dev/null +++ b/tests/integration/test_local_storage_interop.py @@ -0,0 +1,202 @@ +""" +Integration tests for local storage interoperability between Python and TypeScript. + +Tests verify that both implementations can access sessions created by the other, +focusing on directory structure and compatibility. +""" + +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +from python.browserstate.storage.local_storage import LocalStorage as PyLocalStorage +from .ts_storage_helper import run_ts_helper + +class TestLocalStorageInterop(unittest.TestCase): + """Test interoperability between Python and TypeScript local storage implementations.""" + + def setUp(self): + """Set up test environment with temporary storage directory.""" + # Create temporary directories for test files and storage + self.temp_root_dir = tempfile.mkdtemp() + self.storage_dir = os.path.join(self.temp_root_dir, "storage") + os.makedirs(self.storage_dir, exist_ok=True) + + # Test user and session IDs + self.user_id = "interop_test_user" + self.py_session_id = "py_session_local" + self.ts_session_id = "ts_session_local" + + # Create session directory with test files + self.session_dir = os.path.join(self.temp_root_dir, "session") + os.makedirs(self.session_dir, exist_ok=True) + + # Create test files with unique content + with open(os.path.join(self.session_dir, "test.txt"), "w") as f: + f.write("Local storage interoperability test data") + + # Create a subdirectory with files to test nested structure handling + subdir = os.path.join(self.session_dir, "subdir") + os.makedirs(subdir, exist_ok=True) + with open(os.path.join(subdir, "nested.txt"), "w") as f: + f.write("Nested file for testing directory structure") + + # Create Python local storage + self.py_storage = PyLocalStorage(self.storage_dir) + + def tearDown(self): + """Clean up resources after tests.""" + # Remove temporary directories + shutil.rmtree(self.temp_root_dir, ignore_errors=True) + + def verify_files_exist(self, directory, expected_files): + """Verify that expected files exist in the specified directory.""" + for file_path in expected_files: + full_path = os.path.join(directory, file_path) + self.assertTrue(os.path.exists(full_path), f"File {file_path} does not exist") + + def test_python_to_typescript_session_access(self): + """Test if a session created by Python is accessible by TypeScript.""" + # Upload session with Python + self.py_storage.upload(self.user_id, self.py_session_id, self.session_dir) + + # Verify directory structure exists + expected_py_path = os.path.join(self.storage_dir, self.user_id, self.py_session_id) + self.assertTrue(os.path.exists(expected_py_path)) + + # Attempt to access with TypeScript + result = run_ts_helper( + "local-download", + self.user_id, + self.py_session_id, + storage_dir=self.storage_dir + ) + + # Should include markers for success + self.assertIn("SUCCESS", result) + + def test_typescript_to_python_session_access(self): + """Test if a session created by TypeScript is accessible by Python.""" + # Upload session with TypeScript + result = run_ts_helper( + "local-upload", + self.user_id, + self.ts_session_id, + self.session_dir, + storage_dir=self.storage_dir + ) + self.assertIn("SUCCESS", result) + + # Verify directory structure exists + expected_ts_path = os.path.join(self.storage_dir, self.user_id, self.ts_session_id) + self.assertTrue(os.path.exists(expected_ts_path)) + + # Attempt to download with Python + download_path = self.py_storage.download(self.user_id, self.ts_session_id) + + # Verify files were extracted correctly + expected_files = ["test.txt", os.path.join("subdir", "nested.txt")] + self.verify_files_exist(download_path, expected_files) + + # Verify content + with open(os.path.join(download_path, "test.txt"), "r") as f: + content = f.read() + self.assertEqual(content, "Local storage interoperability test data") + + def test_directory_structure_compatibility(self): + """Test that directory structures are compatible between implementations.""" + # Create sessions with both implementations + self.py_storage.upload(self.user_id, self.py_session_id, self.session_dir) + + run_ts_helper( + "local-upload", + self.user_id, + self.ts_session_id, + self.session_dir, + storage_dir=self.storage_dir + ) + + # Examine directory structures + py_path = os.path.join(self.storage_dir, self.user_id, self.py_session_id) + ts_path = os.path.join(self.storage_dir, self.user_id, self.ts_session_id) + + # Both should exist with similar structure + self.assertTrue(os.path.exists(py_path)) + self.assertTrue(os.path.exists(ts_path)) + + # Both should have the test.txt file + self.assertTrue(os.path.exists(os.path.join(py_path, "test.txt"))) + self.assertTrue(os.path.exists(os.path.join(ts_path, "test.txt"))) + + # Both should have the nested directory structure + self.assertTrue(os.path.exists(os.path.join(py_path, "subdir", "nested.txt"))) + self.assertTrue(os.path.exists(os.path.join(ts_path, "subdir", "nested.txt"))) + + def test_list_sessions_cross_implementation(self): + """Test listing sessions across different implementations.""" + # Create sessions with both implementations + self.py_storage.upload(self.user_id, self.py_session_id, self.session_dir) + + run_ts_helper( + "local-upload", + self.user_id, + self.ts_session_id, + self.session_dir, + storage_dir=self.storage_dir + ) + + # List sessions with Python + py_sessions = self.py_storage.list_sessions(self.user_id) + + # Python should see both sessions + self.assertIn(self.py_session_id, py_sessions) + self.assertIn(self.ts_session_id, py_sessions) + + # List sessions with TypeScript + result = run_ts_helper( + "local-list", + self.user_id, + "dummy", + storage_dir=self.storage_dir + ) + + # TypeScript should see both sessions + self.assertIn(self.py_session_id, result) + self.assertIn(self.ts_session_id, result) + + def test_deletion_cross_implementation(self): + """Test session deletion across implementations.""" + # Create sessions with both implementations + self.py_storage.upload(self.user_id, self.py_session_id, self.session_dir) + + run_ts_helper( + "local-upload", + self.user_id, + self.ts_session_id, + self.session_dir, + storage_dir=self.storage_dir + ) + + # Delete TypeScript session with Python + self.py_storage.delete_session(self.user_id, self.ts_session_id) + + # Verify it's gone + ts_path = os.path.join(self.storage_dir, self.user_id, self.ts_session_id) + self.assertFalse(os.path.exists(ts_path)) + + # Delete Python session with TypeScript + run_ts_helper( + "local-delete", + self.user_id, + self.py_session_id, + storage_dir=self.storage_dir + ) + + # Verify it's gone + py_path = os.path.join(self.storage_dir, self.user_id, self.py_session_id) + self.assertFalse(os.path.exists(py_path)) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/integration/test_py_ts_interop.py b/tests/integration/test_py_ts_interop.py new file mode 100644 index 0000000..78e3b55 --- /dev/null +++ b/tests/integration/test_py_ts_interop.py @@ -0,0 +1,218 @@ +""" +Integration tests for Python-TypeScript interoperability. + +These tests verify that sessions created by one implementation can be accessed by the other, +ensuring full compatibility between the two codebases. +""" + +import os +import json +import shutil +import tempfile +import subprocess +import unittest +from pathlib import Path + +import redis + +# Import implementations +from python.browserstate.storage.redis_storage import RedisStorage as PyRedisStorage +# Use relative import for the compatible_redis_storage module +from .compatible_redis_storage import CompatibleRedisStorage + +class TestPythonTypeScriptInterop(unittest.TestCase): + """ + Test interoperability between Python and TypeScript implementations. + These tests create sessions in one language and verify they can be read by the other. + """ + + def setUp(self): + """Set up test environment with Redis client and temporary directories.""" + # Redis connection + self.redis_url = "redis://localhost:6379/0" + self.redis_client = redis.Redis.from_url(self.redis_url) + self.key_prefix = "browserstate_interop_test" + + # Clean up any leftover keys from previous test runs + for key in self.redis_client.keys(f"{self.key_prefix}:*"): + self.redis_client.delete(key) + + # Test user and session IDs + self.user_id = "interop_test_user" + self.session_id_py = "py_session" + self.session_id_ts = "ts_session" + + # Create temporary directory for test files + self.temp_dir = tempfile.mkdtemp() + self.session_dir = os.path.join(self.temp_dir, "session") + os.makedirs(self.session_dir, exist_ok=True) + + # Create test files + with open(os.path.join(self.session_dir, "test.txt"), "w") as f: + f.write("Interoperability test data") + + # Create a subdirectory with files to test nested structure handling + subdir = os.path.join(self.session_dir, "subdir") + os.makedirs(subdir, exist_ok=True) + with open(os.path.join(subdir, "nested.txt"), "w") as f: + f.write("Nested file for testing directory structure") + + # Create storage instances + self.py_storage = PyRedisStorage(redis_url=self.redis_url, key_prefix=self.key_prefix) + self.compatible_storage = CompatibleRedisStorage( + redis_url=self.redis_url, + key_prefix=self.key_prefix, + preferred_format="zip" # Use TypeScript-compatible format + ) + + # Store the TypeScript helper script path + self.ts_script_path = os.path.join(Path(__file__).parent, "ts_redis_helper.js") + + def tearDown(self): + """Clean up resources after tests.""" + # Clean Redis + for key in self.redis_client.keys(f"{self.key_prefix}:*"): + self.redis_client.delete(key) + + # Remove temporary directory + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_python_to_typescript_session_access(self): + """Test if a session created by Python is accessible by TypeScript.""" + # Create a session using the compatible implementation (ZIP format) + self.compatible_storage.upload(self.user_id, self.session_id_py, self.session_dir) + + # Check that it was stored in Redis + session_key = f"{self.key_prefix}:{self.user_id}:{self.session_id_py}" + metadata_key = f"{self.key_prefix}:{self.user_id}:{self.session_id_py}:metadata" + + self.assertIsNotNone(self.redis_client.get(session_key)) + self.assertIsNotNone(self.redis_client.get(metadata_key)) + + # Run TypeScript helper to see if it can access the session + result = self._run_ts_helper("download", self.user_id, self.session_id_py) + self.assertIn("SUCCESS", result) + + def test_typescript_to_python_session_access(self): + """Test if a session created by TypeScript is accessible by Python.""" + # Create a session using the TypeScript helper + result = self._run_ts_helper("upload", self.user_id, self.session_id_ts, self.session_dir) + self.assertIn("SUCCESS", result) + + # Verify session exists in Redis + session_key = f"{self.key_prefix}:{self.user_id}:{self.session_id_ts}" + metadata_key = f"{self.key_prefix}:{self.user_id}:{self.session_id_ts}:metadata" + + self.assertIsNotNone(self.redis_client.get(session_key)) + self.assertIsNotNone(self.redis_client.get(metadata_key)) + + # Try to download with the compatible storage + download_path = self.compatible_storage.download(self.user_id, self.session_id_ts) + + # Verify files were extracted correctly + self.assertTrue(os.path.exists(os.path.join(download_path, "test.txt"))) + self.assertTrue(os.path.exists(os.path.join(download_path, "subdir", "nested.txt"))) + + # Verify content + with open(os.path.join(download_path, "test.txt"), "r") as f: + content = f.read() + self.assertEqual(content, "Interoperability test data") + + def test_typescript_deletion_from_python(self): + """Test if a session created by TypeScript can be deleted by Python.""" + # Create a session using TypeScript + result = self._run_ts_helper("upload", self.user_id, self.session_id_ts, self.session_dir) + self.assertIn("SUCCESS", result) + + # Verify it exists + session_key = f"{self.key_prefix}:{self.user_id}:{self.session_id_ts}" + self.assertIsNotNone(self.redis_client.get(session_key)) + + # Delete using Python + self.compatible_storage.delete_session(self.user_id, self.session_id_ts) + + # Verify deletion + self.assertIsNone(self.redis_client.get(session_key)) + + # Verify TypeScript also sees it as deleted + result = self._run_ts_helper("download", self.user_id, self.session_id_ts) + self.assertIn("ERROR", result) + + def test_python_deletion_from_typescript(self): + """Test if a session created by Python can be deleted by TypeScript.""" + # Create a session using Python + self.compatible_storage.upload(self.user_id, self.session_id_py, self.session_dir) + + # Verify it exists + session_key = f"{self.key_prefix}:{self.user_id}:{self.session_id_py}" + self.assertIsNotNone(self.redis_client.get(session_key)) + + # Delete using TypeScript + result = self._run_ts_helper("delete", self.user_id, self.session_id_py) + self.assertIn("SUCCESS", result) + + # Verify deletion + self.assertIsNone(self.redis_client.get(session_key)) + + # Python should also see it as deleted + download_path = self.compatible_storage.download(self.user_id, self.session_id_py) + # Should be an empty directory + self.assertEqual(len(os.listdir(download_path)), 0) + + def test_list_sessions_cross_implementation(self): + """Test that sessions created by both implementations are listed correctly.""" + # Create a session using Python + self.compatible_storage.upload(self.user_id, self.session_id_py, self.session_dir) + + # Create a session using TypeScript + result = self._run_ts_helper("upload", self.user_id, self.session_id_ts, self.session_dir) + self.assertIn("SUCCESS", result) + + # List sessions using Python + py_sessions = self.compatible_storage.list_sessions(self.user_id) + + # Both sessions should be in the list + self.assertIn(self.session_id_py, py_sessions) + self.assertIn(self.session_id_ts, py_sessions) + + # List sessions using TypeScript + result = self._run_ts_helper("list", self.user_id, "dummy") + + # Both session IDs should be in the output + self.assertIn(self.session_id_py, result) + self.assertIn(self.session_id_ts, result) + + def test_python_native_with_typescript_session(self): + """Test if the standard Python implementation can handle TypeScript sessions.""" + # Create a session using TypeScript + result = self._run_ts_helper("upload", self.user_id, self.session_id_ts, self.session_dir) + self.assertIn("SUCCESS", result) + + # Try to download with the native Python implementation (should fail) + try: + self.py_storage.download(self.user_id, self.session_id_ts) + self.fail("Native Python implementation should not be able to handle TypeScript sessions") + except Exception as e: + # Expected to fail due to format incompatibility + pass + + def _run_ts_helper(self, action, user_id, session_id, session_dir=None): + """Run the TypeScript helper script as a subprocess.""" + cmd = ["node", self.ts_script_path, action, user_id, session_id] + + if session_dir: + cmd.append(session_dir) + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + check=True + ) + return result.stdout + except subprocess.CalledProcessError as e: + return f"ERROR: {e.stderr}" + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/integration/test_s3_storage_interop.py b/tests/integration/test_s3_storage_interop.py new file mode 100644 index 0000000..2bf8cb3 --- /dev/null +++ b/tests/integration/test_s3_storage_interop.py @@ -0,0 +1,238 @@ +""" +Integration tests for S3 storage interoperability between Python and TypeScript. + +Tests verify that both implementations can access sessions created by the other, +focusing on path handling differences and compatibility fixes. +""" + +import os +import json +import shutil +import tempfile +import unittest +import boto3 +from moto import mock_aws + +from python.browserstate.storage.s3_storage import S3Storage as PyS3Storage +from .ts_storage_helper import run_ts_helper + +class TestS3StorageInterop(unittest.TestCase): + """Test interoperability between Python and TypeScript S3 storage implementations.""" + + @mock_aws + def setUp(self): + """Set up test environment with mocked S3 service.""" + # S3 connection + self.region = "us-east-1" + self.bucket_name = "browserstate-interop-test" + self.s3_client = boto3.client("s3", region_name=self.region) + self.s3_client.create_bucket(Bucket=self.bucket_name) + + # Test user and session IDs + self.user_id = "interop_test_user" + self.py_session_id = "py_session_s3" + self.ts_session_id = "ts_session_s3" + + # Create temporary directory for test files + self.temp_dir = tempfile.mkdtemp() + self.session_dir = os.path.join(self.temp_dir, "session") + os.makedirs(self.session_dir, exist_ok=True) + + # Create test files with unique content + with open(os.path.join(self.session_dir, "test.txt"), "w") as f: + f.write("S3 interoperability test data") + + # Create a subdirectory with files to test nested structure handling + subdir = os.path.join(self.session_dir, "subdir") + os.makedirs(subdir, exist_ok=True) + with open(os.path.join(subdir, "nested.txt"), "w") as f: + f.write("Nested file for testing directory structure") + + # Create Python S3 storage + self.py_storage = PyS3Storage( + bucket_name=self.bucket_name, + region_name=self.region, + prefix="py-test" + ) + + def tearDown(self): + """Clean up resources after tests.""" + # Remove temporary directory + shutil.rmtree(self.temp_dir, ignore_errors=True) + + @mock_aws + def test_python_to_typescript_session_access(self): + """Test if a session created by Python is accessible by TypeScript.""" + # Upload session with Python + self.py_storage.upload(self.user_id, self.py_session_id, self.session_dir) + + # Verify files exist in S3 + result = self.s3_client.list_objects_v2( + Bucket=self.bucket_name, + Prefix=f"py-test/{self.user_id}/{self.py_session_id}/" + ) + self.assertTrue(len(result.get("Contents", [])) > 0) + + # Attempt to access with TypeScript + result = run_ts_helper( + "s3-download", + self.user_id, + self.py_session_id, + bucket=self.bucket_name, + region=self.region, + prefix="py-test" + ) + + # Should include markers for success + self.assertIn("SUCCESS", result) + + @mock_aws + def test_typescript_to_python_session_access(self): + """Test if a session created by TypeScript is accessible by Python.""" + # Upload session with TypeScript + result = run_ts_helper( + "s3-upload", + self.user_id, + self.ts_session_id, + self.session_dir, + bucket=self.bucket_name, + region=self.region, + prefix="ts-test" + ) + self.assertIn("SUCCESS", result) + + # Verify files exist in S3 + result = self.s3_client.list_objects_v2( + Bucket=self.bucket_name, + Prefix=f"ts-test/{self.user_id}/{self.ts_session_id}/" + ) + self.assertTrue(len(result.get("Contents", [])) > 0) + + # Create Python S3 storage with TypeScript prefix + ts_compatible_storage = PyS3Storage( + bucket_name=self.bucket_name, + region_name=self.region, + prefix="ts-test" + ) + + # Attempt to download with Python + try: + download_path = ts_compatible_storage.download(self.user_id, self.ts_session_id) + self.assertTrue(os.path.exists(os.path.join(download_path, "test.txt"))) + self.assertTrue(os.path.exists(os.path.join(download_path, "subdir", "nested.txt"))) + except Exception as e: + self.fail(f"Python should be able to access TypeScript session: {e}") + + @mock_aws + def test_path_structure_compatibility(self): + """Test that path structures are compatible between implementations.""" + # Create sessions with both implementations + # Python version + self.py_storage.upload(self.user_id, self.py_session_id, self.session_dir) + + # TypeScript version + run_ts_helper( + "s3-upload", + self.user_id, + self.ts_session_id, + self.session_dir, + bucket=self.bucket_name, + region=self.region, + prefix="test-prefix" + ) + + # List all objects in bucket to examine paths + all_objects = self.s3_client.list_objects_v2(Bucket=self.bucket_name) + + # Get all key paths and print them for debugging + paths = [obj["Key"] for obj in all_objects.get("Contents", [])] + + # Basic verification of both style paths existing + py_path_found = any(f"py-test/{self.user_id}/{self.py_session_id}/" in path for path in paths) + ts_path_found = any(f"test-prefix/{self.user_id}/{self.ts_session_id}/" in path for path in paths) + + self.assertTrue(py_path_found, "Python-created path not found") + self.assertTrue(ts_path_found, "TypeScript-created path not found") + + @mock_aws + def test_list_sessions_cross_implementation(self): + """Test listing sessions across different implementations.""" + # Create sessions with both implementations + self.py_storage.upload(self.user_id, self.py_session_id, self.session_dir) + + run_ts_helper( + "s3-upload", + self.user_id, + self.ts_session_id, + self.session_dir, + bucket=self.bucket_name, + region=self.region, + prefix="py-test" # Use same prefix as Python + ) + + # List sessions with Python + py_sessions = self.py_storage.list_sessions(self.user_id) + + # Python should see both sessions + self.assertIn(self.py_session_id, py_sessions) + self.assertIn(self.ts_session_id, py_sessions) + + # List sessions with TypeScript + result = run_ts_helper( + "s3-list", + self.user_id, + "dummy", + bucket=self.bucket_name, + region=self.region, + prefix="py-test" + ) + + # TypeScript should see both sessions + self.assertIn(self.py_session_id, result) + self.assertIn(self.ts_session_id, result) + + @mock_aws + def test_deletion_cross_implementation(self): + """Test session deletion across implementations.""" + # Create sessions + self.py_storage.upload(self.user_id, self.py_session_id, self.session_dir) + + run_ts_helper( + "s3-upload", + self.user_id, + self.ts_session_id, + self.session_dir, + bucket=self.bucket_name, + region=self.region, + prefix="py-test" + ) + + # Delete TypeScript session with Python + self.py_storage.delete_session(self.user_id, self.ts_session_id) + + # Verify it's gone + result = self.s3_client.list_objects_v2( + Bucket=self.bucket_name, + Prefix=f"py-test/{self.user_id}/{self.ts_session_id}/" + ) + self.assertFalse("Contents" in result) + + # Delete Python session with TypeScript + run_ts_helper( + "s3-delete", + self.user_id, + self.py_session_id, + bucket=self.bucket_name, + region=self.region, + prefix="py-test" + ) + + # Verify it's gone + result = self.s3_client.list_objects_v2( + Bucket=self.bucket_name, + Prefix=f"py-test/{self.user_id}/{self.py_session_id}/" + ) + self.assertFalse("Contents" in result) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/integration/ts_local_helper.js b/tests/integration/ts_local_helper.js new file mode 100644 index 0000000..7341df9 --- /dev/null +++ b/tests/integration/ts_local_helper.js @@ -0,0 +1,191 @@ +/** + * TypeScript/JavaScript helper for local storage interoperability testing + * + * This script provides a command-line interface to the TypeScript local + * storage implementation for testing interoperability with the Python implementation. + * + * Usage: + * node ts_local_helper.js [sessionDir] --storage-dir + * + * Actions: + * - upload: Upload session data to local storage + * - download: Download session data from local storage + * - delete: Delete a session from local storage + * - list: List all sessions for a user + */ + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); + +// Parse command line arguments +const args = process.argv.slice(2); +if (args.length < 3) { + console.error('Usage: node ts_local_helper.js [sessionDir] --storage-dir '); + process.exit(1); +} + +const [action, userId, sessionId] = args; +let sessionDir = null; +let storageDir = null; + +// Parse additional arguments +for (let i = 3; i < args.length; i++) { + if (args[i] === '--storage-dir' && i + 1 < args.length) { + storageDir = args[i + 1]; + i++; // Skip the next argument + } else if (!sessionDir && !args[i].startsWith('--')) { + sessionDir = args[i]; + } +} + +// Ensure we have a storage directory +if (!storageDir) { + // Default to a temp directory if not specified + storageDir = path.join(os.tmpdir(), 'browserstate-local-storage'); + console.log(`Using default storage directory: ${storageDir}`); +} + +// Ensure the storage directory exists +if (!fs.existsSync(storageDir)) { + try { + fs.mkdirSync(storageDir, { recursive: true }); + } catch (error) { + console.error(`ERROR: Failed to create storage directory: ${error.message}`); + process.exit(1); + } +} + +// Import required modules dynamically +function initializeLocalStorage() { + try { + // For testing, we need to import from the TypeScript module path + const localStorageModule = require('../../typescript/dist/storage/LocalStorage'); + const LocalStorageProvider = localStorageModule.LocalStorageProvider; + + // Create local storage provider instance + const storage = new LocalStorageProvider({ + storagePath: storageDir + }); + + return storage; + } catch (error) { + console.error(`ERROR: Failed to initialize local storage: ${error.message}`); + process.exit(1); + } +} + +// Helper functions for the local storage operations +async function uploadSession() { + if (!sessionDir) { + console.error('ERROR: Session directory is required for upload action'); + process.exit(1); + } + + try { + const storage = initializeLocalStorage(); + await storage.upload(userId, sessionId, sessionDir); + console.log(`SUCCESS: Uploaded session ${sessionId} to ${storageDir}`); + } catch (error) { + console.error(`ERROR: Failed to upload session: ${error.message}`); + process.exit(1); + } +} + +async function downloadSession() { + try { + const storage = initializeLocalStorage(); + const downloadPath = await storage.download(userId, sessionId); + console.log(`SUCCESS: Downloaded session ${sessionId} to ${downloadPath}`); + + // Log contents to verify + const files = listFilesRecursively(downloadPath); + if (files.length > 0) { + console.log(`Files: ${files.join(', ')}`); + } else { + console.log('No files found in the downloaded session.'); + } + } catch (error) { + console.error(`ERROR: Failed to download session: ${error.message}`); + process.exit(1); + } +} + +async function listSessions() { + try { + const storage = initializeLocalStorage(); + const sessions = await storage.listSessions(userId); + console.log(`SUCCESS: Found ${sessions.length} sessions for user ${userId}`); + + // Output session IDs one per line for easy parsing + for (const sessionId of sessions) { + console.log(sessionId); + } + } catch (error) { + console.error(`ERROR: Failed to list sessions: ${error.message}`); + process.exit(1); + } +} + +async function deleteSession() { + try { + const storage = initializeLocalStorage(); + await storage.deleteSession(userId, sessionId); + console.log(`SUCCESS: Deleted session ${sessionId} from local storage`); + } catch (error) { + console.error(`ERROR: Failed to delete session: ${error.message}`); + process.exit(1); + } +} + +// Helper function to list files recursively +function listFilesRecursively(directory) { + const files = []; + + function traverse(dir, relativePath = '') { + const entries = fs.readdirSync(dir); + + for (const entry of entries) { + const fullPath = path.join(dir, entry); + const stats = fs.statSync(fullPath); + + if (stats.isDirectory()) { + traverse(fullPath, path.join(relativePath, entry)); + } else { + files.push(path.join(relativePath, entry)); + } + } + } + + traverse(directory); + return files; +} + +// Main function to execute the requested action +function main() { + try { + switch (action) { + case 'upload': + uploadSession(); + break; + case 'download': + downloadSession(); + break; + case 'list': + listSessions(); + break; + case 'delete': + deleteSession(); + break; + default: + console.error(`ERROR: Unknown action '${action}'`); + process.exit(1); + } + } catch (error) { + console.error(`ERROR: ${error.message}`); + process.exit(1); + } +} + +// Run the main function +main(); \ No newline at end of file diff --git a/tests/integration/ts_redis_helper.js b/tests/integration/ts_redis_helper.js new file mode 100644 index 0000000..a57869f --- /dev/null +++ b/tests/integration/ts_redis_helper.js @@ -0,0 +1,270 @@ +/** + * TypeScript Redis storage helper script for integration tests. + * + * This script provides a command-line interface to the TypeScript Redis storage implementation + * for use in Python-TypeScript interoperability tests. + * + * Usage: + * node ts_redis_helper.js [sessionDir] + * + * Actions: + * - upload: Upload a session to Redis + * - download: Download a session from Redis + * - list: List available sessions for a user + * - delete: Delete a session + */ + +const Redis = require('ioredis'); +const path = require('path'); +const fs = require('fs'); +const archiver = require('archiver'); +const extractZip = require('extract-zip'); +const { promisify } = require('util'); +const mkdirp = promisify(require('mkdirp')); +const { pipeline } = require('stream/promises'); +const rimraf = promisify(require('rimraf')); +const os = require('os'); +const crypto = require('crypto'); + +// Redis connection parameters +const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379/0'; +const KEY_PREFIX = process.env.TS_HELPER_KEY_PREFIX || 'browserstate_interop_test'; + +/** + * Redis Storage class for TypeScript implementation + */ +class RedisStorage { + constructor(redisUrl = REDIS_URL, keyPrefix = KEY_PREFIX) { + this.redis = new Redis(redisUrl); + this.keyPrefix = keyPrefix; + } + + /** + * Get Redis key for session + */ + getSessionKey(userId, sessionId) { + return `${this.keyPrefix}:${userId}:${sessionId}`; + } + + /** + * Get Redis key for session metadata + */ + getMetadataKey(userId, sessionId) { + return `${this.keyPrefix}:${userId}:${sessionId}:metadata`; + } + + /** + * Upload a session to Redis + */ + async uploadSession(userId, sessionId, sessionDir) { + const sessionKey = this.getSessionKey(userId, sessionId); + const metadataKey = this.getMetadataKey(userId, sessionId); + + // Create a temporary ZIP file + const tempDir = os.tmpdir(); + const zipPath = path.join(tempDir, `${userId}_${sessionId}_${crypto.randomBytes(4).toString('hex')}.zip`); + + try { + // Create ZIP archive + const output = fs.createWriteStream(zipPath); + const archive = archiver('zip', { zlib: { level: 9 } }); + + archive.pipe(output); + archive.directory(sessionDir, false); + await archive.finalize(); + + // Wait for the output stream to finish + await new Promise(resolve => output.on('close', resolve)); + + // Read the ZIP file + const zipData = fs.readFileSync(zipPath); + const base64Data = zipData.toString('base64'); + + // Create metadata + const metadata = { + timestamp: Date.now(), + fileCount: countFiles(sessionDir), + version: '2.0', + format: 'zip' + }; + + // Store in Redis + await this.redis.set(sessionKey, base64Data); + await this.redis.set(metadataKey, JSON.stringify(metadata)); + + return true; + } catch (error) { + console.error('Error uploading session:', error); + throw error; + } finally { + // Cleanup + if (fs.existsSync(zipPath)) { + fs.unlinkSync(zipPath); + } + } + } + + /** + * Download a session from Redis + */ + async downloadSession(userId, sessionId) { + const sessionKey = this.getSessionKey(userId, sessionId); + const metadataKey = this.getMetadataKey(userId, sessionId); + + // Get the session data + const sessionData = await this.redis.get(sessionKey); + if (!sessionData) { + throw new Error(`Session not found: ${userId}/${sessionId}`); + } + + // Create a temporary ZIP file + const tempDir = os.tmpdir(); + const zipPath = path.join(tempDir, `${userId}_${sessionId}_${crypto.randomBytes(4).toString('hex')}.zip`); + const extractDir = path.join(tempDir, `browserstate_${userId}_${sessionId}_${crypto.randomBytes(4).toString('hex')}`); + + try { + // Convert Base64 to binary + let zipBuffer; + try { + zipBuffer = Buffer.from(sessionData, 'base64'); + } catch (error) { + zipBuffer = sessionData; // In case it's not in Base64 + } + + // Write to temporary file + fs.writeFileSync(zipPath, zipBuffer); + + // Extract ZIP + await mkdirp(extractDir); + await extractZip(zipPath, { dir: extractDir }); + + return extractDir; + } catch (error) { + console.error('Error downloading session:', error); + throw error; + } finally { + // Cleanup ZIP file + if (fs.existsSync(zipPath)) { + fs.unlinkSync(zipPath); + } + } + } + + /** + * List available sessions for a user + */ + async listSessions(userId) { + const pattern = `${this.keyPrefix}:${userId}:*`; + const keys = await this.redis.keys(pattern); + + // Extract session IDs + const sessionIds = new Set(); + keys.forEach(key => { + const parts = key.split(':'); + if (parts.length === 3) { + sessionIds.add(parts[2]); + } + if (parts.length > 3 && parts[3] !== 'metadata') { + sessionIds.add(parts[2]); + } + }); + + return Array.from(sessionIds); + } + + /** + * Delete a session + */ + async deleteSession(userId, sessionId) { + const sessionKey = this.getSessionKey(userId, sessionId); + const metadataKey = this.getMetadataKey(userId, sessionId); + + await this.redis.del(sessionKey); + await this.redis.del(metadataKey); + + return true; + } + + /** + * Close the Redis connection + */ + async close() { + await this.redis.quit(); + } +} + +/** + * Count files in a directory recursively + */ +function countFiles(dir) { + let count = 0; + const files = fs.readdirSync(dir, { withFileTypes: true }); + + for (const file of files) { + const fullPath = path.join(dir, file.name); + if (file.isDirectory()) { + count += countFiles(fullPath); + } else { + count++; + } + } + + return count; +} + +/** + * Main function to handle command-line arguments + */ +async function main() { + try { + const args = process.argv.slice(2); + if (args.length < 3) { + console.error('Usage: node ts_redis_helper.js [sessionDir]'); + process.exit(1); + } + + const [action, userId, sessionId] = args; + const sessionDir = args[3]; // Optional for upload + + const storage = new RedisStorage(); + + try { + switch (action) { + case 'upload': + if (!sessionDir) { + throw new Error('Session directory is required for upload action'); + } + await storage.uploadSession(userId, sessionId, sessionDir); + console.log(`SUCCESS: Uploaded session ${userId}/${sessionId}`); + break; + + case 'download': + const downloadPath = await storage.downloadSession(userId, sessionId); + console.log(`SUCCESS: Downloaded session ${userId}/${sessionId} to ${downloadPath}`); + break; + + case 'list': + const sessions = await storage.listSessions(userId); + console.log(`SUCCESS: Found ${sessions.length} sessions for ${userId}`); + sessions.forEach(id => console.log(`- ${id}`)); + break; + + case 'delete': + await storage.deleteSession(userId, sessionId); + console.log(`SUCCESS: Deleted session ${userId}/${sessionId}`); + break; + + default: + throw new Error(`Unknown action: ${action}`); + } + } finally { + await storage.close(); + } + } catch (error) { + console.error(`ERROR: ${error.message}`); + process.exit(1); + } +} + +// Run the main function +main(); \ No newline at end of file diff --git a/tests/integration/ts_s3_helper.js b/tests/integration/ts_s3_helper.js new file mode 100644 index 0000000..7d7b1fe --- /dev/null +++ b/tests/integration/ts_s3_helper.js @@ -0,0 +1,201 @@ +/** + * TypeScript/JavaScript helper for S3 storage interoperability testing + * + * This script provides a command-line interface to the TypeScript S3 + * implementation for testing interoperability with the Python implementation. + * + * Usage: + * node ts_s3_helper.js [sessionDir] --config + * + * Actions: + * - upload: Upload session data to S3 + * - download: Download session data from S3 + * - delete: Delete a session from S3 + * - list: List all sessions for a user + * + * Config file format (JSON): + * { + * "bucket": "bucket-name", + * "region": "aws-region", + * "prefix": "optional/key/prefix" + * } + */ + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); + +// Parse command line arguments +const args = process.argv.slice(2); +if (args.length < 3) { + console.error('Usage: node ts_s3_helper.js [sessionDir] --config '); + process.exit(1); +} + +const [action, userId, sessionId] = args; +let sessionDir = null; +let configFile = null; + +// Parse additional arguments +for (let i = 3; i < args.length; i++) { + if (args[i] === '--config' && i + 1 < args.length) { + configFile = args[i + 1]; + i++; // Skip the next argument + } else if (!sessionDir && !args[i].startsWith('--')) { + sessionDir = args[i]; + } +} + +// Ensure we have a config file +if (!configFile) { + console.error('ERROR: Config file is required. Use --config option.'); + process.exit(1); +} + +// Read and parse config +let config; +try { + config = JSON.parse(fs.readFileSync(configFile, 'utf8')); +} catch (error) { + console.error(`ERROR: Failed to read or parse config file: ${error.message}`); + process.exit(1); +} + +const { bucket, region, prefix } = config; + +// Import required modules dynamically +async function initializeS3Storage() { + try { + // For testing, we need to import from the TypeScript module path + const s3StorageModule = require('../../typescript/dist/storage/S3Storage'); + const S3StorageProvider = s3StorageModule.S3StorageProvider; + + // Create S3 storage provider instance + const storage = new S3StorageProvider({ + bucket: bucket || 'browserstate-test', + region: region || 'us-east-1', + keyPrefix: prefix || 'browserstate', + endpoint: process.env.AWS_ENDPOINT, // Allow for local testing with minio/localstack + }); + + return storage; + } catch (error) { + console.error(`ERROR: Failed to initialize S3 storage: ${error.message}`); + process.exit(1); + } +} + +// Helper functions for the S3 storage operations +async function uploadSession() { + if (!sessionDir) { + console.error('ERROR: Session directory is required for upload action'); + process.exit(1); + } + + try { + const storage = await initializeS3Storage(); + await storage.upload(userId, sessionId, sessionDir); + console.log(`SUCCESS: Uploaded session ${sessionId} to S3 bucket ${bucket}`); + } catch (error) { + console.error(`ERROR: Failed to upload session: ${error.message}`); + process.exit(1); + } +} + +async function downloadSession() { + try { + const storage = await initializeS3Storage(); + const downloadPath = await storage.download(userId, sessionId); + console.log(`SUCCESS: Downloaded session ${sessionId} to ${downloadPath}`); + + // Log contents to verify + const files = listFilesRecursively(downloadPath); + if (files.length > 0) { + console.log(`Files: ${files.join(', ')}`); + } else { + console.log('No files found in the downloaded session.'); + } + } catch (error) { + console.error(`ERROR: Failed to download session: ${error.message}`); + process.exit(1); + } +} + +async function listSessions() { + try { + const storage = await initializeS3Storage(); + const sessions = await storage.listSessions(userId); + console.log(`SUCCESS: Found ${sessions.length} sessions for user ${userId}`); + + // Output session IDs one per line for easy parsing + for (const sessionId of sessions) { + console.log(sessionId); + } + } catch (error) { + console.error(`ERROR: Failed to list sessions: ${error.message}`); + process.exit(1); + } +} + +async function deleteSession() { + try { + const storage = await initializeS3Storage(); + await storage.deleteSession(userId, sessionId); + console.log(`SUCCESS: Deleted session ${sessionId} from S3 bucket ${bucket}`); + } catch (error) { + console.error(`ERROR: Failed to delete session: ${error.message}`); + process.exit(1); + } +} + +// Helper function to list files recursively +function listFilesRecursively(directory) { + const files = []; + + function traverse(dir, relativePath = '') { + const entries = fs.readdirSync(dir); + + for (const entry of entries) { + const fullPath = path.join(dir, entry); + const stats = fs.statSync(fullPath); + + if (stats.isDirectory()) { + traverse(fullPath, path.join(relativePath, entry)); + } else { + files.push(path.join(relativePath, entry)); + } + } + } + + traverse(directory); + return files; +} + +// Main function to execute the requested action +async function main() { + try { + switch (action) { + case 'upload': + await uploadSession(); + break; + case 'download': + await downloadSession(); + break; + case 'list': + await listSessions(); + break; + case 'delete': + await deleteSession(); + break; + default: + console.error(`ERROR: Unknown action '${action}'`); + process.exit(1); + } + } catch (error) { + console.error(`ERROR: ${error.message}`); + process.exit(1); + } +} + +// Run the main function +main(); \ No newline at end of file diff --git a/tests/integration/ts_storage_helper.py b/tests/integration/ts_storage_helper.py new file mode 100644 index 0000000..7a6f230 --- /dev/null +++ b/tests/integration/ts_storage_helper.py @@ -0,0 +1,73 @@ +""" +Helper module for running TypeScript storage operations from Python tests. + +This module provides a Python interface to the TypeScript implementation's +storage functionality to enable interoperability testing. +""" + +import os +import json +import subprocess +from pathlib import Path +from typing import Optional, Dict, Any, List + +def run_ts_helper( + action: str, + user_id: str, + session_id: str, + session_dir: Optional[str] = None, + **kwargs +) -> str: + """ + Run the TypeScript helper script with the given arguments. + + Args: + action: The action to perform (upload, download, list, delete) + user_id: The user ID + session_id: The session ID + session_dir: Optional path to the session directory (required for upload) + **kwargs: Additional arguments to pass to the helper script + + Returns: + The output of the TypeScript helper script + """ + # Determine the path to the TypeScript helper script + helper_dir = Path(__file__).parent + + if action.startswith('local-'): + # Local storage helper + script_path = helper_dir / "ts_local_helper.js" + action = action[6:] # Remove 'local-' prefix + elif action.startswith('s3-'): + # S3 storage helper + script_path = helper_dir / "ts_s3_helper.js" + action = action[3:] # Remove 's3-' prefix + else: + # Redis storage helper + script_path = helper_dir / "ts_redis_helper.js" + + # Build command + cmd = ["node", str(script_path), action, user_id, session_id] + + # Add session_dir if provided + if session_dir: + cmd.append(session_dir) + + # Add additional arguments + env = os.environ.copy() + for key, value in kwargs.items(): + env[f"TS_HELPER_{key.upper()}"] = str(value) + + try: + # Run the command + result = subprocess.run( + cmd, + env=env, + capture_output=True, + text=True, + check=True + ) + return result.stdout + except subprocess.CalledProcessError as e: + # Return error message + return f"ERROR: Command failed with exit code {e.returncode}. {e.stderr}" \ No newline at end of file diff --git a/typescript/package-lock.json b/typescript/package-lock.json index b391eb8..e1ff373 100644 --- a/typescript/package-lock.json +++ b/typescript/package-lock.json @@ -1,15 +1,16 @@ { "name": "browserstate", - "version": "0.3.0-canary.20250329022411-canary.20250329155936-canary.20250329174942-canary.20250329182118-canary.20250329185246-canary.20250329220413-canary.20250330160314", + "version": "0.3.0-canary.20250329022411-canary.20250329155936-canary.20250329174942-canary.20250329182118-canary.20250329185246-canary.20250329220413-canary.20250330160314-canary.20250330203600-canary.20250330214647", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "browserstate", - "version": "0.3.0-canary.20250329022411-canary.20250329155936-canary.20250329174942-canary.20250329182118-canary.20250329185246-canary.20250329220413-canary.20250330160314", + "version": "0.3.0-canary.20250329022411-canary.20250329155936-canary.20250329174942-canary.20250329182118-canary.20250329185246-canary.20250329220413-canary.20250330160314-canary.20250330203600-canary.20250330214647", "license": "MIT", "dependencies": { - "fs-extra": "^11.2.0" + "fs-extra": "^11.2.0", + "rimraf": "^6.0.1" }, "devDependencies": { "@aws-sdk/client-s3": "^3.772.0", @@ -30,7 +31,7 @@ "eslint": "^8.57.1", "extract-zip": "^2.0.1", "globals": "^16.0.0", - "ioredis": "^5.3.2", + "ioredis": "^5.6.0", "jest": "^29.7.0", "prettier": "^3.1.0", "puppeteer": "^24.4.0", @@ -45,7 +46,7 @@ "@aws-sdk/client-s3": "^3.0.0", "@aws-sdk/lib-storage": "^3.0.0", "@google-cloud/storage": "^7.0.0", - "archiver": "^5.0.0", + "archiver": "^7.0.1", "extract-zip": "^2.0.0", "ioredis": "^5.0.0" }, @@ -1766,7 +1767,6 @@ "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^5.1.2", @@ -1784,7 +1784,6 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -1797,7 +1796,6 @@ "version": "6.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -1810,14 +1808,12 @@ "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true, "license": "MIT" }, "node_modules/@isaacs/cliui/node_modules/string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, "license": "MIT", "dependencies": { "eastasianwidth": "^0.2.0", @@ -1835,7 +1831,6 @@ "version": "7.1.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^6.0.1" @@ -1851,7 +1846,6 @@ "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^6.1.0", @@ -3774,7 +3768,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -3784,7 +3777,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -3815,7 +3807,6 @@ "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", "dev": true, - "license": "MIT", "dependencies": { "archiver-utils": "^5.0.2", "async": "^3.2.4", @@ -4180,7 +4171,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, "license": "MIT" }, "node_modules/bare-events": { @@ -4304,7 +4294,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -4580,7 +4569,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -4593,7 +4581,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, "license": "MIT" }, "node_modules/combined-stream": { @@ -4811,7 +4798,6 @@ "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -5002,7 +4988,6 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "dev": true, "license": "MIT" }, "node_modules/ecdsa-sig-formatter": { @@ -5055,7 +5040,6 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, "license": "MIT" }, "node_modules/end-of-stream": { @@ -5479,7 +5463,6 @@ "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", "dev": true, - "license": "BSD-2-Clause", "dependencies": { "debug": "^4.1.1", "get-stream": "^5.1.0", @@ -5703,6 +5686,22 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/flat-cache/node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/flatted": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", @@ -5714,7 +5713,6 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", - "dev": true, "license": "ISC", "dependencies": { "cross-spawn": "^7.0.6", @@ -5731,7 +5729,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, "license": "ISC", "engines": { "node": ">=14" @@ -6323,7 +6320,6 @@ "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.0.tgz", "integrity": "sha512-tBZlIIWbndeWBWCXWZiqtOF/yxf6yZX3tAlTJ7nfo5jhd6dctNxF7QnYlZLZ1a0o0pDoen7CgZqO+zjNaFbJAg==", "dev": true, - "license": "MIT", "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", @@ -6401,7 +6397,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -6474,7 +6469,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, "license": "ISC" }, "node_modules/istanbul-lib-coverage": { @@ -7614,7 +7608,6 @@ "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", - "dev": true, "license": "ISC", "engines": { "node": ">=16 || 14 >=14.17" @@ -7847,7 +7840,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", - "dev": true, "license": "BlueOak-1.0.0" }, "node_modules/parent-module": { @@ -7906,7 +7898,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -8530,17 +8521,91 @@ } }, "node_modules/rimraf": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", - "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "deprecated": "Rimraf versions prior to v4 are no longer supported", - "dev": true, - "license": "ISC", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-6.0.1.tgz", + "integrity": "sha512-9dkvaxAsk/xNXSJzMgFqqMCuFgt2+KsOFek3TMLfo8NCPfWpBmqwyNn5Y+NX56QUYfCtsyhF3ayiboEoUmJk/A==", "dependencies": { - "glob": "^7.1.3" + "glob": "^11.0.0", + "package-json-from-dist": "^1.0.0" }, "bin": { - "rimraf": "bin.js" + "rimraf": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/glob": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.1.tgz", + "integrity": "sha512-zrQDm8XPnYEKawJScsnM0QzobJxlT/kHOOlRTio8IH/GrmxRE5fjllkzdaHclIuNjUQTJYH2xHNIGfdpJkDJUw==", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^4.0.1", + "minimatch": "^10.0.0", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/jackspeak": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.1.0.tgz", + "integrity": "sha512-9DDdhb5j6cpeitCbvLO7n7J4IxnbM6hoF6O1g4HQ5TfhvvKN8ywDM7668ZhMHRqVmxqhps/F6syWK2KcPxYlkw==", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/lru-cache": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.1.0.tgz", + "integrity": "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/rimraf/node_modules/minimatch": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz", + "integrity": "sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/path-scurry": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", + "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -8608,7 +8673,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -8621,7 +8685,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -8820,7 +8883,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", @@ -8836,7 +8898,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", @@ -8851,7 +8912,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" @@ -8865,7 +8925,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" @@ -9603,7 +9662,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" @@ -9648,7 +9706,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", diff --git a/typescript/package.json b/typescript/package.json index 5da4ff6..9bc8b1b 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -40,7 +40,8 @@ "author": "browserstate", "license": "MIT", "dependencies": { - "fs-extra": "^11.2.0" + "fs-extra": "^11.2.0", + "rimraf": "^6.0.1" }, "devDependencies": { "@aws-sdk/client-s3": "^3.772.0", @@ -61,7 +62,7 @@ "eslint": "^8.57.1", "extract-zip": "^2.0.1", "globals": "^16.0.0", - "ioredis": "^5.3.2", + "ioredis": "^5.6.0", "jest": "^29.7.0", "prettier": "^3.1.0", "puppeteer": "^24.4.0", @@ -73,7 +74,7 @@ "@aws-sdk/client-s3": "^3.0.0", "@aws-sdk/lib-storage": "^3.0.0", "@google-cloud/storage": "^7.0.0", - "archiver": "^5.0.0", + "archiver": "^7.0.1", "extract-zip": "^2.0.0", "ioredis": "^5.0.0" }, diff --git a/typescript/src/storage/LocalStorage.ts b/typescript/src/storage/LocalStorage.ts index b9216ec..d754da8 100644 --- a/typescript/src/storage/LocalStorage.ts +++ b/typescript/src/storage/LocalStorage.ts @@ -25,7 +25,7 @@ export class LocalStorage implements StorageProvider { * Get path for a specific session */ private getSessionPath(userId: string, sessionId: string): string { - return path.join(this.getUserPath(userId), sessionId); + return path.resolve(this.getUserPath(userId), sessionId); } /** diff --git a/typescript/src/storage/RedisStorage.ts b/typescript/src/storage/RedisStorage.ts index faa98f7..7194d92 100644 --- a/typescript/src/storage/RedisStorage.ts +++ b/typescript/src/storage/RedisStorage.ts @@ -34,6 +34,7 @@ interface SessionMetadata { timestamp?: number; fileCount?: number; version?: string; + format?: string; } // Type for metadata created during upload @@ -41,8 +42,12 @@ interface SessionUploadMetadata { timestamp: number; fileCount: number; version: string; + format: string; } +// Format types used for storage +type StorageFormat = "zip" | "tar.gz"; + /** * Redis Storage Architecture * ========================= @@ -59,7 +64,7 @@ interface SessionUploadMetadata { * ┌─────────────────┐ * │ │ * │ Temp Directory │ - * │ (ZIP Archive) │ + * │ (ZIP/TAR.GZ) │ * │ │ * └─────────────────┘ * @@ -68,15 +73,15 @@ interface SessionUploadMetadata { * * Upload: * 1. Browser state calls upload() with a directory path containing profile files - * 2. Directory is packaged into a single ZIP archive in a temporary location - * 3. ZIP is encoded as base64 and stored in Redis at key: {prefix}{userId}:{sessionId} + * 2. Directory is packaged into a single archive (ZIP or TAR.GZ) in a temporary location + * 3. Archive is encoded as base64 and stored in Redis at key: {prefix}{userId}:{sessionId} * 4. Metadata is stored separately at key: {prefix}{userId}:{sessionId}:metadata * * Download: * 1. Browser state calls download() with userId and sessionId * 2. Base64 data is fetched from Redis and decoded - * 3. A temporary ZIP file is created from the decoded data - * 4. ZIP is extracted to a directory that is returned to the browser state + * 3. Format is auto-detected (ZIP or TAR.GZ) + * 4. Archive is extracted to a directory that is returned to the browser state * * Session Management: * ------------------ @@ -84,6 +89,13 @@ interface SessionUploadMetadata { * - listSessions() retrieves all sessions for a user by pattern matching * - deleteSession() removes both the session data and metadata * - Metadata includes timestamp and version information + * + * * Enhanced Interoperability: + * ------------------------- + * - Auto-detects and handles both ZIP and TAR.GZ formats for cross-language compatibility + * - ZIP format is the TypeScript-native format + * - TAR.GZ format is the Python-native format + * - Both formats can be read by either implementation */ /** @@ -125,7 +137,7 @@ export interface RedisStorageOptions { keyPrefix?: string; /** - * Temporary directory for extracting and creating ZIP archives + * Temporary directory for extracting and creating archives * @default os.tmpdir() */ tempDir?: string; @@ -136,13 +148,21 @@ export interface RedisStorageOptions { * @example 604800 // 7 days */ ttl?: number; + + /** + * Storage format to use for new sessions + * @default "zip" + */ + storageFormat?: StorageFormat; } /** * Redis storage provider for BrowserState * - * This implementation stores browser profiles directly in Redis as ZIP archives. - * Unlike cloud storage providers that store individual files, this approach: + * This implementation stores browser profiles directly in Redis as archives. + * Enhanced with cross-language compatibility, supporting both: + * - ZIP format (TypeScript-native) + * - TAR.GZ format (Python-native) * * 1. Creates a complete ZIP archive of the entire browser profile directory * 2. Stores the archive as base64-encoded data in Redis @@ -158,6 +178,12 @@ export interface RedisStorageOptions { * * Note: The Redis provider requires the 'ioredis' package to be installed. * It will be dynamically imported at runtime. + * + * * Features: + * 1. Automatic format detection for reading sessions + * 2. Cross-language compatibility for sessions created by either implementation + * 3. Customizable format for new sessions + * 4. Metadata storage for tracking and session management */ export class RedisStorageProvider implements StorageProvider { private redis: RedisClient | null = null; @@ -166,6 +192,7 @@ export class RedisStorageProvider implements StorageProvider { private tempDir: string; private ttl?: number; private options: RedisStorageOptions; + private storageFormat: StorageFormat; /** * Creates a new Redis storage provider instance @@ -176,6 +203,7 @@ export class RedisStorageProvider implements StorageProvider { this.keyPrefix = options.keyPrefix || "browserstate:"; this.tempDir = options.tempDir || os.tmpdir(); this.ttl = options.ttl; + this.storageFormat = options.storageFormat || "zip"; this.options = options; // Initialize with dynamic import (but don't throw if it fails) @@ -188,7 +216,7 @@ export class RedisStorageProvider implements StorageProvider { } }); - console.log(`[Redis] Storage initialized with ZIP compression`); + console.log(`[Redis] Storage initialized with ${this.storageFormat} compression`); } /** @@ -249,13 +277,41 @@ export class RedisStorageProvider implements StorageProvider { return `${this.keyPrefix}${userId}:${sessionId}:metadata`; } + /** + * Detects the format of a data buffer + * + * @param data - The data buffer to examine + * @returns The detected format + * @private + */ + private detectFormat(data: Buffer): StorageFormat | "unknown" { + // Check if it's gzip (TAR.GZ) by examining the first bytes (gzip magic number: 0x1F8B) + if (data[0] === 0x1F && data[1] === 0x8B) { + return "tar.gz"; + } + + // Check if it's ZIP by examining the first bytes (ZIP magic number: PK\x03\x04) + if ( + data.length >= 4 && + data[0] === 0x50 && // P + data[1] === 0x4B && // K + data[2] === 0x03 && + data[3] === 0x04 + ) { + return "zip"; + } + + return "unknown"; + } + /** * Downloads a browser session from Redis * * This method: * 1. Creates a temporary directory for the session - * 2. Fetches the ZIP archive from Redis - * 3. Extracts the archive to recreate the browser profile + * 2. Fetches the archive from Redis + * 3. Auto-detects the format (ZIP or TAR.GZ) + * 4. Extracts the archive to recreate the browser profile * * If the session doesn't exist, an empty directory is returned * which can be used to create a new session. @@ -280,13 +336,13 @@ export class RedisStorageProvider implements StorageProvider { ); await fs.ensureDir(userDataDir); - // Get session data (zipped folder) and metadata - const [zipData, metadata] = await Promise.all([ + // Get session data (archive) and metadata + const [sessionData, metadata] = await Promise.all([ this.redis.get(sessionKey), this.redis.get(metadataKey), ]); - if (!zipData) { + if (!sessionData) { console.log( `[Redis] No session data found for ${sessionId}, creating new directory`, ); @@ -304,44 +360,37 @@ export class RedisStorageProvider implements StorageProvider { ); } - console.log( - `[Redis] Downloading session ${sessionId} from ${new Date(sessionMetadata.timestamp || 0).toISOString()}`, - ); - - // Create a temporary zip file - const zipFilePath = path.join( - this.tempDir, - `${userId}-${sessionId}-${Date.now()}.zip`, - ); - try { - // Import extract-zip using our module loader - const extractZip = await modules.extractZip.getModule(); + // First, convert the data to a buffer + let dataBuffer: Buffer; - // Write the base64 data to a zip file - await fs.writeFile(zipFilePath, Buffer.from(zipData, "base64")); + // Try to parse as base64 first (common for both formats) + try { + dataBuffer = Buffer.from(sessionData, "base64"); + } catch (error) { + // If not base64, use as raw data + dataBuffer = Buffer.from(sessionData); + } - // Extract the zip file to the user data directory - console.log(`[Redis] Extracting zip to ${userDataDir}`); - await extractZip(zipFilePath, { dir: userDataDir }); + // Auto-detect the format + const format = sessionMetadata.format as StorageFormat || + this.detectFormat(dataBuffer); - // Clean up the temporary zip file - await fs.remove(zipFilePath); + console.log(`[Redis] Detected format for session ${sessionId}: ${format}`); + + if (format === "zip") { + // Handle ZIP format + await this.extractZipSession(dataBuffer, userDataDir); + } else if (format === "tar.gz") { + // Handle TAR.GZ format + await this.extractTarGzSession(dataBuffer, userDataDir); + } else { + throw new Error(`Unknown or unsupported format: ${format}`); + } console.log(`[Redis] Successfully extracted session data`); return userDataDir; } catch (error) { - // Clean up temp zip file if it exists - try { - if (await fs.pathExists(zipFilePath)) { - await fs.remove(zipFilePath); - } - } catch (cleanupError) { - console.error( - `[Redis] Error cleaning up temporary zip file: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`, - ); - } - // Clean up user data directory to avoid partial extraction try { await fs.emptyDir(userDataDir); @@ -362,11 +411,91 @@ export class RedisStorageProvider implements StorageProvider { } } + /** + * Extracts a ZIP format session + * + * @param data - ZIP data buffer + * @param targetDir - Directory to extract to + * @private + */ + private async extractZipSession(data: Buffer, targetDir: string): Promise { + const extractZip = await modules.extractZip.getModule(); + + // Create a temporary zip file + const zipFilePath = path.join( + this.tempDir, + `temp-${Date.now()}-${Math.random().toString(36).substring(2, 10)}.zip`, + ); + + try { + // Write the data to a zip file + await fs.writeFile(zipFilePath, data); + + // Extract the zip file to the user data directory + console.log(`[Redis] Extracting zip to ${targetDir}`); + await extractZip(zipFilePath, { dir: targetDir }); + } finally { + // Clean up the temporary zip file + try { + if (await fs.pathExists(zipFilePath)) { + await fs.remove(zipFilePath); + } + } catch (cleanupError) { + console.error( + `[Redis] Error cleaning up temporary zip file: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`, + ); + } + } + } + + /** + * Extracts a TAR.GZ format session + * + * @param data - TAR.GZ data buffer + * @param targetDir - Directory to extract to + * @private + */ + private async extractTarGzSession(data: Buffer, targetDir: string): Promise { + const tar = await modules.tar.getModule(); + + // Create a temporary tar.gz file + const tarFilePath = path.join( + this.tempDir, + `temp-${Date.now()}-${Math.random().toString(36).substring(2, 10)}.tar.gz`, + ); + + try { + // Write the data to a tar.gz file + await fs.writeFile(tarFilePath, data); + + // Extract the tar.gz file to the user data directory + console.log(`[Redis] Extracting tar.gz to ${targetDir}`); + await tar.extract({ + file: tarFilePath, + cwd: targetDir, + // Add safety options to prevent path traversal + strict: true, + filter: (path: string) => !path.includes(".."), + }); + } finally { + // Clean up the temporary tar.gz file + try { + if (await fs.pathExists(tarFilePath)) { + await fs.remove(tarFilePath); + } + } catch (cleanupError) { + console.error( + `[Redis] Error cleaning up temporary tar.gz file: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`, + ); + } + } + } + /** * Uploads a browser session to Redis * * This method: - * 1. Creates a ZIP archive of the entire browser profile directory + * 1. Creates an archive (ZIP or TAR.GZ) of the entire browser profile directory * 2. Encodes the archive as base64 and stores it in Redis * 3. Stores metadata about the session alongside the data * @@ -394,54 +523,53 @@ export class RedisStorageProvider implements StorageProvider { const sessionKey = this.getSessionKey(userId, sessionId); const metadataKey = this.getMetadataKey(userId, sessionId); - // Create a temporary file for the zip - const zipFilePath = path.join( - this.tempDir, - `${userId}-${sessionId}-${Date.now()}.zip`, - ); - console.log( - `[Redis] Creating zip archive of session directory: ${filePath}`, + `[Redis] Creating ${this.storageFormat} archive of session directory: ${filePath}`, ); try { - // Create a zip of the directory - await this.zipDirectory(filePath, zipFilePath); - - // Get the zip file size for logging - const stats = await fs.stat(zipFilePath); - - // Read the zip file as base64 - const zipData = await fs.readFile(zipFilePath, { encoding: "base64" }); + let sessionData: string; + let fileSize: number; + + // Create archive based on the configured format + if (this.storageFormat === "zip") { + // Create a ZIP archive + const result = await this.createZipArchive(filePath); + sessionData = result.data; + fileSize = result.size; + } else { + // Create a TAR.GZ archive + const result = await this.createTarGzArchive(filePath); + sessionData = result.data; + fileSize = result.size; + } // Create metadata const metadata: SessionUploadMetadata = { timestamp: Date.now(), - fileCount: 0, // We don't count individual files anymore - version: "2.0", // Update version to indicate zip format + fileCount: await this.countFiles(filePath), + version: "2.0", + format: this.storageFormat }; console.log( - `[Redis] Uploading session ${sessionId} (${stats.size} bytes)`, + `[Redis] Uploading session ${sessionId} (${fileSize} bytes) in ${this.storageFormat} format`, ); // Store in Redis with TTL if specified if (this.ttl) { await Promise.all([ - this.redis.setex(sessionKey, this.ttl, zipData), + this.redis.setex(sessionKey, this.ttl, sessionData), this.redis.setex(metadataKey, this.ttl, JSON.stringify(metadata)), ]); } else { await Promise.all([ - this.redis.set(sessionKey, zipData), + this.redis.set(sessionKey, sessionData), this.redis.set(metadataKey, JSON.stringify(metadata)), ]); } console.log(`[Redis] Successfully uploaded session ${sessionId}`); - - // Clean up the temporary zip file - await fs.remove(zipFilePath); } catch (error) { console.error( `[Redis] Error uploading session data: ${error instanceof Error ? error.message : String(error)}`, @@ -450,6 +578,99 @@ export class RedisStorageProvider implements StorageProvider { } } + /** + * Creates a ZIP archive of a directory + * + * @param sourceDir - Directory to compress + * @returns Object containing base64 data and size + * @private + */ + private async createZipArchive(sourceDir: string): Promise<{ data: string, size: number }> { + // Create a temporary file for the zip + const zipFilePath = path.join( + this.tempDir, + `temp-${Date.now()}-${Math.random().toString(36).substring(2, 10)}.zip`, + ); + + try { + // Create a zip of the directory + await this.zipDirectory(sourceDir, zipFilePath); + + // Get the zip file size for logging + const stats = await fs.stat(zipFilePath); + + // Read the zip file as base64 + const zipData = await fs.readFile(zipFilePath, { encoding: "base64" }); + + return { + data: zipData, + size: stats.size + }; + } finally { + // Clean up the temporary zip file + try { + if (await fs.pathExists(zipFilePath)) { + await fs.remove(zipFilePath); + } + } catch (error) { + console.error( + `[Redis] Error cleaning up temporary zip file: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + } + + /** + * Creates a TAR.GZ archive of a directory + * + * @param sourceDir - Directory to compress + * @returns Object containing base64 data and size + * @private + */ + private async createTarGzArchive(sourceDir: string): Promise<{ data: string, size: number }> { + const tar = await modules.tar.getModule(); + + // Create a temporary file for the tar.gz + const tarFilePath = path.join( + this.tempDir, + `temp-${Date.now()}-${Math.random().toString(36).substring(2, 10)}.tar.gz`, + ); + + try { + // Create a tar.gz of the directory + await tar.create( + { + gzip: true, + file: tarFilePath, + cwd: path.dirname(sourceDir), + }, + [path.basename(sourceDir)] + ); + + // Get the tar.gz file size for logging + const stats = await fs.stat(tarFilePath); + + // Read the tar.gz file as base64 + const tarData = await fs.readFile(tarFilePath, { encoding: "base64" }); + + return { + data: tarData, + size: stats.size + }; + } finally { + // Clean up the temporary tar.gz file + try { + if (await fs.pathExists(tarFilePath)) { + await fs.remove(tarFilePath); + } + } catch (error) { + console.error( + `[Redis] Error cleaning up temporary tar.gz file: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + } + /** * Creates a ZIP archive of a directory * @@ -493,6 +714,32 @@ export class RedisStorageProvider implements StorageProvider { }); } + /** + * Counts the number of files in a directory recursively + * + * @param directory - Directory to count files in + * @returns Number of files + * @private + */ + private async countFiles(directory: string): Promise { + let count = 0; + + const items = await fs.readdir(directory); + + for (const item of items) { + const fullPath = path.join(directory, item); + const stats = await fs.stat(fullPath); + + if (stats.isDirectory()) { + count += await this.countFiles(fullPath); + } else { + count++; + } + } + + return count; + } + /** * Lists all available sessions for a user * diff --git a/typescript/src/types/external.d.ts b/typescript/src/types/external.d.ts new file mode 100644 index 0000000..79b6d7e --- /dev/null +++ b/typescript/src/types/external.d.ts @@ -0,0 +1,47 @@ +/** + * Type declarations for external modules used in BrowserState + */ + +// Redis +declare module 'ioredis' { + export default class Redis { + constructor(options?: any); + connect(): Promise; + disconnect(): Promise; + quit(): Promise; + get(key: string): Promise; + set(key: string, value: string): Promise<"OK">; + setex(key: string, seconds: number, value: string): Promise<"OK">; + del(key: string): Promise; + keys(pattern: string): Promise; + [key: string]: any; + } +} + +// Archiver +declare module 'archiver' { + function archiver(format: string, options?: any): any; + export = archiver; +} + +// Extract-Zip +declare module 'extract-zip' { + function extractZip(zipPath: string, options: { dir: string }): Promise; + export = extractZip; +} + +// TAR +declare module 'tar' { + export function create(options: { + gzip: boolean; + file: string; + cwd: string; + }, files: string[]): Promise; + + export function extract(options: { + file: string; + cwd: string; + strict?: boolean; + filter?: (path: string) => boolean; + }): Promise; +} \ No newline at end of file diff --git a/typescript/src/utils/DynamicImport.ts b/typescript/src/utils/DynamicImport.ts index ffcfa98..1a7144f 100644 --- a/typescript/src/utils/DynamicImport.ts +++ b/typescript/src/utils/DynamicImport.ts @@ -1,4 +1,5 @@ -import type { Redis as IoRedis } from "ioredis"; +//import type { Redis as IoRedis } from "ioredis"; +import Redis from "ioredis"; import type { Storage as GCPStorageType } from "@google-cloud/storage"; import type { S3Client, @@ -134,19 +135,12 @@ export function createModuleLoader( /** * Type definitions for commonly used optional dependencies */ -export type RedisType = typeof IoRedis; - +//export type RedisType = typeof IoRedis; +export type RedisType = typeof Redis; +// Type for archiver export interface ArchiverType { - ( - format: string, - options?: Record, - ): { - pipe(output: NodeJS.WritableStream): void; - on(event: string, callback: (err?: Error) => void): void; - directory(sourceDir: string, destDir: boolean | string): void; - finalize(): void; - pointer(): number; - }; + (format: string, options?: { zlib?: { level: number } }): any; + create: (format: string, options?: object) => any; } export type ExtractZipType = ( @@ -175,6 +169,21 @@ export interface GCPStorage { Storage: typeof GCPStorageType; } +// Type for TAR module +export interface TarType { + create: (options: { + gzip: boolean; + file: string; + cwd: string; + }, files: string[]) => Promise; + extract: (options: { + file: string; + cwd: string; + strict?: boolean; + filter?: (path: string) => boolean; + }) => Promise; +} + /** * Pre-defined error messages for common dependencies */ @@ -185,6 +194,7 @@ export const DEPENDENCY_ERRORS = { AWS_S3: "Please run: npm install @aws-sdk/client-s3 @aws-sdk/lib-storage --save", GCS: "Please run: npm install @google-cloud/storage --save", + TAR: "Please run: npm install tar --save", }; /** @@ -200,6 +210,10 @@ export const modules = { "extract-zip", DEPENDENCY_ERRORS.EXTRACT_ZIP, ), + tar: createModuleLoader( + "tar", + DEPENDENCY_ERRORS.TAR, + ), aws: { s3: createModuleLoader( "@aws-sdk/client-s3",