diff --git a/Makefile b/Makefile index 54a1978f..982f3ed0 100644 --- a/Makefile +++ b/Makefile @@ -38,32 +38,6 @@ validate-docs: exit 1; \ fi -# Sync docs to ~/home/docs with links under 'evaluators' -sync-docs: - @echo "Syncing docs to ~/home/docs with links under 'evaluators'..." - @mkdir -p ~/home/docs/evaluators - @# Create a temp directory for processed files -@rm -rf /tmp/eval-protocol-docs-processed -@mkdir -p /tmp/eval-protocol-docs-processed - @# Copy all docs files to temp directory -@cp -r ./docs/* /tmp/eval-protocol-docs-processed/ - @# Only update links in the main documentation home file -@if [ -f /tmp/eval-protocol-docs-processed/documentation_home.mdx ]; then \ -sed -i -E 's/\[([^]]+)\]\(([^)]+\.mdx?)\)/[\1](evaluators\/\2)/g' /tmp/eval-protocol-docs-processed/documentation_home.mdx; \ -sed -i -E 's/\[([^]]+)\]\(\/([^)]+\.mdx?)\)/[\1](\/evaluators\/\2)/g' /tmp/eval-protocol-docs-processed/documentation_home.mdx; \ -sed -i -E 's/\.md\)/\.mdx)/g' /tmp/eval-protocol-docs-processed/documentation_home.mdx; \ - fi - @# Copy processed files to destination -@rsync -av --delete /tmp/eval-protocol-docs-processed/ ~/home/docs/evaluators/ - @echo "Docs synced successfully to ~/home/docs/evaluators" - @# Validate all documentation links - @echo "Validating documentation links..." - @if [ -f ~/home/docs/scripts/validate_links.py ]; then \ - cd ~/home/docs && python scripts/validate_links.py; \ - else \ - echo "Warning: Link validation script not found at ~/home/docs/scripts/validate_links.py"; \ - fi - # Version management commands using versioneer version: @echo "Current version information:" diff --git a/eval_protocol/adapters/CONTRIBUTING.md b/eval_protocol/adapters/CONTRIBUTING.md new file mode 100644 index 00000000..18f31378 --- /dev/null +++ b/eval_protocol/adapters/CONTRIBUTING.md @@ -0,0 +1,524 @@ +# Adapter Contributing Guide + +This guide explains how to create custom adapters for the Eval Protocol system. Adapters allow you to integrate various data sources and convert them to the `EvaluationRow` format used by the evaluation pipeline. + +## Overview + +Adapters are responsible for: +1. **Data Ingestion**: Loading data from external sources (APIs, databases, files, etc.) +2. **Format Conversion**: Converting the source data to `EvaluationRow` format +3. **Metadata Extraction**: Preserving relevant metadata from the source system +4. **Error Handling**: Gracefully handling failures and logging issues + +## Creating a New Adapter + +### 1. Basic Adapter Structure + +Create a new Python file in `eval_protocol/adapters/` following this template: + +```python +"""Your Custom Adapter for Eval Protocol.""" + +from typing import Any, Dict, Iterator, List, Optional +import logging + +from eval_protocol.models import EvaluationRow, Message, InputMetadata, CompletionParams + +logger = logging.getLogger(__name__) + +# Optional dependency handling +try: + import your_external_library + DEPENDENCY_AVAILABLE = True +except ImportError: + DEPENDENCY_AVAILABLE = False + logger.warning("your_external_library not installed. Install with: pip install 'eval-protocol[your_adapter]'") + + +class YourCustomAdapter: + """Adapter for integrating with Your Custom Data Source. + + This adapter loads data from Your Custom Data Source and converts it + to EvaluationRow format for use in evaluation pipelines. + + Examples: + Basic usage: + >>> adapter = YourCustomAdapter(api_key="your_key") + >>> rows = list(adapter.get_evaluation_rows(limit=10)) + """ + + def __init__(self, **config): + """Initialize the adapter with configuration.""" + if not DEPENDENCY_AVAILABLE: + raise ImportError("your_external_library not installed") + + # Initialize your client/connection here + self.client = your_external_library.Client(**config) + + def get_evaluation_rows(self, **kwargs) -> Iterator[EvaluationRow]: + """Main method to fetch and convert data to EvaluationRow format. + + Args: + **kwargs: Adapter-specific parameters + + Yields: + EvaluationRow: Converted evaluation rows + """ + # Implement your data fetching logic + raw_data = self.client.fetch_data(**kwargs) + + for item in raw_data: + try: + eval_row = self._convert_to_evaluation_row(item) + if eval_row: + yield eval_row + except Exception as e: + logger.warning(f"Failed to convert item: {e}") + continue + + def _convert_to_evaluation_row(self, raw_item: Any) -> Optional[EvaluationRow]: + """Convert a raw data item to EvaluationRow format. + + Args: + raw_item: Raw data item from your source + + Returns: + EvaluationRow or None if conversion fails + """ + # Extract messages from your data format + messages = self._extract_messages(raw_item) + + # Extract metadata + input_metadata = self._create_input_metadata(raw_item) + + # Extract ground truth if available + ground_truth = self._extract_ground_truth(raw_item) + + # Extract tools if available (for tool calling scenarios) + tools = self._extract_tools(raw_item) + + return EvaluationRow( + messages=messages, + tools=tools, + input_metadata=input_metadata, + ground_truth=ground_truth, + ) + + def _extract_messages(self, raw_item: Any) -> List[Message]: + """Extract conversation messages from raw data.""" + # Implement message extraction logic + # Convert your data format to List[Message] + pass + + def _create_input_metadata(self, raw_item: Any) -> InputMetadata: + """Create InputMetadata from raw data.""" + # Implement metadata extraction + pass + + def _extract_ground_truth(self, raw_item: Any) -> Optional[str]: + """Extract ground truth if available.""" + # Implement ground truth extraction + pass + + def _extract_tools(self, raw_item: Any) -> Optional[List[Dict[str, Any]]]: + """Extract tool definitions if available.""" + # Implement tool extraction for tool calling scenarios + pass + + +# Factory function (recommended) +def create_your_custom_adapter(**config) -> YourCustomAdapter: + """Factory function to create your custom adapter.""" + return YourCustomAdapter(**config) +``` + +### 2. Key Components + +#### Messages +Convert your data to a list of `Message` objects representing the conversation: + +```python +from eval_protocol.models import Message + +# Basic message +message = Message(role="user", content="Hello, world!") + +# Message with tool calls +message = Message( + role="assistant", + content="I'll help you with that calculation.", + tool_calls=[{ + "id": "call_123", + "type": "function", + "function": { + "name": "calculate", + "arguments": '{"x": 5, "y": 3}' + } + }] +) + +# Tool response message +message = Message( + role="tool", + content="Result: 8", + tool_call_id="call_123" +) +``` + +#### Input Metadata +Preserve important metadata from your source system: + +```python +from eval_protocol.models import InputMetadata, CompletionParams + +input_metadata = InputMetadata( + row_id="unique_row_identifier", + completion_params=CompletionParams( + model="gpt-4", + temperature=0.7, + max_tokens=1000, + ), + dataset_info={ + "source_system": "your_system_name", + "original_id": "source_item_id", + "custom_field": "custom_value", + }, + session_data={ + "user_id": "user123", + "session_id": "session456", + "timestamp": "2024-01-01T00:00:00Z", + } +) +``` + +#### Ground Truth +Include expected answers if available: + +```python +# Simple string ground truth +ground_truth = "The correct answer is 42" + +# For math problems, include the final answer +ground_truth = "#### 42" # GSM8K format +``` + +#### Tools (for Tool Calling) +Include tool definitions for tool calling scenarios: + +```python +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state" + } + }, + "required": ["location"] + } + } + } +] +``` + +### 3. Optional Dependencies + +Add your adapter's dependencies to `pyproject.toml`: + +```toml +[project.optional-dependencies] +your_adapter = [ + "your-external-library>=1.0.0", + "other-dependency>=2.0.0", +] +``` + +Users can then install with: +```bash +pip install 'eval-protocol[your_adapter]' +``` + +### 4. Error Handling + +Implement robust error handling: + +```python +import logging + +logger = logging.getLogger(__name__) + +def get_evaluation_rows(self, **kwargs) -> Iterator[EvaluationRow]: + try: + data = self.client.fetch_data(**kwargs) + except Exception as e: + logger.error(f"Failed to fetch data: {e}") + return + + for item in data: + try: + row = self._convert_to_evaluation_row(item) + if row: + yield row + except Exception as e: + logger.warning(f"Failed to convert item {item.get('id', 'unknown')}: {e}") + continue +``` + +### 5. Update Package Exports + +Add your adapter to `eval_protocol/adapters/__init__.py`: + +```python +# Add conditional import +try: + from .your_adapter import YourCustomAdapter, create_your_custom_adapter + __all__.extend(["YourCustomAdapter", "create_your_custom_adapter"]) +except ImportError: + pass +``` + +## Testing Your Adapter + +Create comprehensive tests for your adapter: + +```python +# tests/test_adapters/test_your_adapter.py +import pytest +from unittest.mock import Mock, patch + +from eval_protocol.adapters.your_adapter import YourCustomAdapter +from eval_protocol.models import EvaluationRow + + +class TestYourCustomAdapter: + """Test suite for YourCustomAdapter.""" + + def test_initialization(self): + """Test adapter initialization.""" + adapter = YourCustomAdapter(api_key="test_key") + assert adapter.client is not None + + def test_get_evaluation_rows(self): + """Test conversion to EvaluationRow format.""" + adapter = YourCustomAdapter(api_key="test_key") + + # Mock the external API response + with patch.object(adapter.client, 'fetch_data') as mock_fetch: + mock_fetch.return_value = [ + # Mock data in your format + {"id": "1", "question": "Test?", "answer": "Yes"} + ] + + rows = list(adapter.get_evaluation_rows(limit=1)) + + assert len(rows) == 1 + assert isinstance(rows[0], EvaluationRow) + assert len(rows[0].messages) > 0 + + def test_error_handling(self): + """Test error handling.""" + adapter = YourCustomAdapter(api_key="test_key") + + with patch.object(adapter.client, 'fetch_data') as mock_fetch: + mock_fetch.side_effect = Exception("API Error") + + rows = list(adapter.get_evaluation_rows()) + assert len(rows) == 0 # Should handle error gracefully +``` + +## Examples by Data Source Type + +### Chat/Conversation Data + +For simple chat data: + +```python +def _extract_messages(self, conversation: Dict) -> List[Message]: + messages = [] + + # Add system prompt if available + if conversation.get('system_prompt'): + messages.append(Message(role="system", content=conversation['system_prompt'])) + + # Add conversation turns + for turn in conversation['turns']: + messages.append(Message( + role=turn['role'], + content=turn['content'] + )) + + return messages +``` + +### Tool Calling Data + +For tool calling scenarios: + +```python +def _extract_messages(self, trace: Dict) -> List[Message]: + messages = [] + + for step in trace['steps']: + if step['type'] == 'user_message': + messages.append(Message(role="user", content=step['content'])) + + elif step['type'] == 'assistant_message': + message = Message(role="assistant", content=step.get('content')) + + # Add tool calls if present + if step.get('tool_calls'): + message.tool_calls = step['tool_calls'] + + messages.append(message) + + elif step['type'] == 'tool_response': + messages.append(Message( + role="tool", + content=step['content'], + tool_call_id=step['tool_call_id'] + )) + + return messages +``` + +### Dataset Files + +For dataset files with transformation functions: + +```python +from eval_protocol.adapters.huggingface import create_huggingface_adapter + +def my_dataset_transform(row): + """Transform dataset row to evaluation format.""" + return { + 'messages': [ + {'role': 'system', 'content': 'You are a helpful assistant.'}, + {'role': 'user', 'content': row['input_text']}, + ], + 'ground_truth': row['expected_output'], + 'metadata': { + 'dataset': 'my_custom_dataset', + 'category': row.get('category'), + 'difficulty': row.get('difficulty_level'), + } + } + +# For HuggingFace datasets +adapter = create_huggingface_adapter( + dataset_id="my-org/my-dataset", + transform_fn=my_dataset_transform, + config_name="default", # optional + revision="main", # optional +) + +# For local files +adapter = HuggingFaceAdapter.from_local( + path="./my_dataset.jsonl", + transform_fn=my_dataset_transform, +) + +# Use the adapter +rows = list(adapter.get_evaluation_rows(split="test", limit=100)) +``` + +## Best Practices + +### 1. Graceful Degradation +- Handle missing optional fields gracefully +- Provide sensible defaults +- Don't fail the entire batch for one bad item + +### 2. Logging +- Use structured logging with appropriate levels +- Include context like item IDs in error messages +- Log successful conversions at DEBUG level + +### 3. Performance +- Use iterators/generators for large datasets +- Implement pagination for API sources +- Consider caching for expensive operations + +### 4. Documentation +- Include comprehensive docstrings +- Provide usage examples +- Document expected data formats + +### 5. Type Safety +- Use type hints throughout +- Validate input parameters +- Handle type conversion errors + +## Integration Checklist + +Before submitting your adapter: + +- [ ] Handles optional dependencies correctly +- [ ] Includes comprehensive error handling +- [ ] Has factory function for easy instantiation +- [ ] Added to `pyproject.toml` optional dependencies +- [ ] Added to `__init__.py` exports +- [ ] Includes unit tests +- [ ] Has proper documentation and examples +- [ ] Follows project coding standards (black, isort, mypy) +- [ ] Tested with real data from your source system + +## Getting Help + +- Check existing adapters for patterns and examples +- Review the main [CONTRIBUTING.md](../../../development/CONTRIBUTING.md) for project conventions +- Open an issue for questions or feature requests +- Submit a draft PR for early feedback + +## Using the Generic HuggingFace Adapter + +For datasets hosted on HuggingFace Hub, you can often use the generic `HuggingFaceAdapter` instead of creating a completely custom adapter: + +```python +from eval_protocol.adapters.huggingface import create_huggingface_adapter + +def my_transform_function(row): + """Transform function for your specific dataset.""" + return { + 'messages': [ + {'role': 'system', 'content': 'Your system prompt here'}, + {'role': 'user', 'content': row['question']}, # Adjust field names + ], + 'ground_truth': row['answer'], # Adjust field name + 'metadata': { + 'dataset_specific_field': row.get('category'), + 'custom_metadata': 'value', + }, + 'tools': row.get('tools'), # If your dataset has tool definitions + } + +adapter = create_huggingface_adapter( + dataset_id="your-org/your-dataset", + transform_fn=my_transform_function, + config_name="default", # optional + revision="main", # optional commit/branch +) + +rows = list(adapter.get_evaluation_rows(split="test", limit=100)) +``` + +This approach is often simpler than creating a full custom adapter, especially for straightforward dataset transformations. + +## Adapter Ideas + +Here are some potential adapters that would be valuable: + +- **OpenAI Evals**: Load data from OpenAI's evals repository +- **LLM Evaluation Datasets**: MMLU, HellaSwag, etc. +- **Chat Platforms**: Discord, Slack conversation exports +- **Monitoring Tools**: Other observability platforms +- **Custom APIs**: Company-specific data sources +- **File Formats**: Parquet, Excel, database exports +- **Research Datasets**: Academic benchmarks and competitions + +We welcome contributions for any of these or other creative integrations! \ No newline at end of file diff --git a/eval_protocol/adapters/__init__.py b/eval_protocol/adapters/__init__.py index 8efa6e5f..fc04237b 100644 --- a/eval_protocol/adapters/__init__.py +++ b/eval_protocol/adapters/__init__.py @@ -1 +1,47 @@ -# This file makes the 'adapters' directory a Python package. +"""Data source adapters for Eval Protocol. + +This package provides adapters for integrating with various data sources +and converting them to EvaluationRow format for use in evaluation pipelines. + +Available adapters: +- LangfuseAdapter: Pull data from Langfuse deployments +- HuggingFaceAdapter: Load datasets from HuggingFace Hub +- Braintrust integration (legacy) +- TRL integration (legacy) +""" + +# Conditional imports based on available dependencies +try: + from .langfuse import LangfuseAdapter, create_langfuse_adapter + __all__ = ["LangfuseAdapter", "create_langfuse_adapter"] +except ImportError: + __all__ = [] + +try: + from .huggingface import ( + HuggingFaceAdapter, + create_huggingface_adapter, + create_gsm8k_adapter, + create_math_adapter, + ) + __all__.extend([ + "HuggingFaceAdapter", + "create_huggingface_adapter", + "create_gsm8k_adapter", + "create_math_adapter", + ]) +except ImportError: + pass + +# Legacy adapters (always available) +try: + from .braintrust import reward_fn_to_scorer, scorer_to_reward_fn + __all__.extend(["scorer_to_reward_fn", "reward_fn_to_scorer"]) +except ImportError: + pass + +try: + from .trl import create_trl_adapter + __all__.extend(["create_trl_adapter"]) +except ImportError: + pass diff --git a/eval_protocol/adapters/huggingface.py b/eval_protocol/adapters/huggingface.py new file mode 100644 index 00000000..15391181 --- /dev/null +++ b/eval_protocol/adapters/huggingface.py @@ -0,0 +1,444 @@ +"""HuggingFace Datasets adapter for Eval Protocol. + +This adapter allows loading datasets from HuggingFace Hub with arbitrary +transformation functions to convert them to EvaluationRow format. +""" + +from typing import Any, Callable, Dict, Iterator, List, Optional +import logging + +from eval_protocol.models import EvaluationRow, Message, InputMetadata, CompletionParams + +logger = logging.getLogger(__name__) + +try: + from datasets import load_dataset, Dataset, DatasetDict + DATASETS_AVAILABLE = True +except ImportError: + DATASETS_AVAILABLE = False + logger.warning( + "HuggingFace datasets not installed. Install with: pip install 'eval-protocol[huggingface]'" + ) + +# Type alias for transformation function +TransformFunction = Callable[[Dict[str, Any]], Dict[str, Any]] + + +class HuggingFaceAdapter: + """Generic adapter to load HuggingFace datasets with custom transformations. + + This adapter loads datasets from HuggingFace Hub and applies a user-provided + transformation function to convert each row to the format expected by + EvaluationRow. + + The transformation function should take a dataset row dictionary and return: + { + 'messages': List[Dict] - list of message dictionaries with 'role' and 'content' + 'ground_truth': Optional[str] - expected answer/output + 'metadata': Optional[Dict] - any additional metadata to preserve + 'tools': Optional[List[Dict]] - tool definitions for tool calling scenarios + } + + Examples: + Simple Q&A dataset: + >>> def transform(row): + ... return { + ... 'messages': [{'role': 'user', 'content': row['question']}], + ... 'ground_truth': row['answer'], + ... 'metadata': {'category': row.get('category')} + ... } + >>> adapter = HuggingFaceAdapter("my-dataset", transform_fn=transform) + >>> rows = list(adapter.get_evaluation_rows(split="test", limit=10)) + + Math problems with system prompt: + >>> def gsm8k_transform(row): + ... return { + ... 'messages': [ + ... {'role': 'system', 'content': 'Solve step by step.'}, + ... {'role': 'user', 'content': row['question']} + ... ], + ... 'ground_truth': row['answer'], + ... 'metadata': {'dataset': 'gsm8k'} + ... } + >>> adapter = HuggingFaceAdapter("gsm8k", config_name="main", transform_fn=gsm8k_transform) + """ + + def __init__( + self, + dataset_id: str, + transform_fn: TransformFunction, + config_name: Optional[str] = None, + revision: Optional[str] = None, + **load_dataset_kwargs, + ): + """Initialize the HuggingFace adapter. + + Args: + dataset_id: HuggingFace dataset identifier (e.g., "gsm8k", "squad", "org/dataset") + transform_fn: Function to transform dataset rows to evaluation format + config_name: Optional dataset configuration name + revision: Optional dataset revision/commit hash + **load_dataset_kwargs: Additional arguments to pass to load_dataset + """ + if not DATASETS_AVAILABLE: + raise ImportError( + "HuggingFace datasets not installed. Install with: pip install 'eval-protocol[huggingface]'" + ) + + self.dataset_id = dataset_id + self.transform_fn = transform_fn + self.config_name = config_name + self.revision = revision + self.load_dataset_kwargs = load_dataset_kwargs + + # Load the dataset + self.dataset = self._load_dataset() + + @classmethod + def from_local( + cls, + path: str, + transform_fn: TransformFunction, + **load_dataset_kwargs, + ) -> "HuggingFaceAdapter": + """Create adapter from local dataset file. + + Args: + path: Path to local dataset file (JSON, JSONL, CSV, etc.) + transform_fn: Function to transform dataset rows + **load_dataset_kwargs: Additional arguments to pass to load_dataset + + Returns: + HuggingFaceAdapter instance + """ + # Determine file format + if path.endswith('.jsonl'): + dataset_type = "json" + elif path.endswith('.json'): + dataset_type = "json" + elif path.endswith('.csv'): + dataset_type = "csv" + elif path.endswith('.parquet'): + dataset_type = "parquet" + else: + # Let HuggingFace auto-detect + dataset_type = None + + load_kwargs = {'data_files': path, **load_dataset_kwargs} + + return cls( + dataset_id=dataset_type or "json", + transform_fn=transform_fn, + **load_kwargs + ) + + def _load_dataset(self) -> "Dataset | DatasetDict": + """Load the dataset from HuggingFace Hub or local source.""" + try: + kwargs = {} + if self.config_name: + kwargs['name'] = self.config_name + if self.revision: + kwargs['revision'] = self.revision + + kwargs.update(self.load_dataset_kwargs) + + return load_dataset(self.dataset_id, **kwargs) + + except (OSError, ValueError, RuntimeError) as e: + logger.error("Failed to load dataset %s: %s", self.dataset_id, e) + raise + + def get_evaluation_rows( + self, + split: Optional[str] = None, + limit: Optional[int] = None, + offset: int = 0, + model_name: str = "gpt-3.5-turbo", + temperature: float = 0.0, + max_tokens: Optional[int] = None, + **completion_params_kwargs, + ) -> Iterator[EvaluationRow]: + """Convert dataset entries to EvaluationRow format. + + Args: + split: Dataset split to use (if dataset has multiple splits) + limit: Maximum number of rows to return + offset: Number of rows to skip + model_name: Model name for completion parameters + temperature: Temperature for completion parameters + max_tokens: Max tokens for completion parameters + **completion_params_kwargs: Additional completion parameters + + Yields: + EvaluationRow: Converted evaluation rows + """ + # Select dataset split + dataset = self.dataset + if isinstance(self.dataset, DatasetDict): + if split is None: + # Use first available split + split = list(self.dataset.keys())[0] + logger.info("No split specified, using: %s", split) + dataset = self.dataset[split] + elif split is not None: + logger.warning("Split '%s' specified but dataset is not split", split) + + # Apply offset and limit + total_rows = len(dataset) + end_idx = min(offset + limit, total_rows) if limit else total_rows + + if offset >= total_rows: + logger.warning("Offset %d is greater than dataset size %d", offset, total_rows) + return + + # Create completion parameters + completion_params = CompletionParams( + model=model_name, + temperature=temperature, + max_tokens=max_tokens, + **completion_params_kwargs, + ) + + # Convert each row + for i in range(offset, end_idx): + try: + raw_row = dataset[i] + eval_row = self._convert_row_to_evaluation_row( + raw_row, i, completion_params, split + ) + yield eval_row + except (AttributeError, ValueError, KeyError) as e: + logger.warning("Failed to convert row %d: %s", i, e) + continue + + def _convert_row_to_evaluation_row( + self, + raw_row: Dict[str, Any], + row_index: int, + completion_params: CompletionParams, + split: Optional[str] = None, + ) -> EvaluationRow: + """Convert a single dataset row to EvaluationRow format. + + Args: + raw_row: Raw dataset row dictionary + row_index: Index of the row in the dataset + completion_params: Completion parameters to use + split: Dataset split name + + Returns: + EvaluationRow object + """ + # Apply user transformation + transformed = self.transform_fn(raw_row) + + # Validate required fields + if 'messages' not in transformed: + raise ValueError("Transform function must return 'messages' field") + + # Convert message dictionaries to Message objects + messages = [] + for msg_dict in transformed['messages']: + if not isinstance(msg_dict, dict): + raise ValueError("Each message must be a dictionary") + if 'role' not in msg_dict: + raise ValueError("Each message must have a 'role' field") + + messages.append(Message( + role=msg_dict['role'], + content=msg_dict.get('content'), + name=msg_dict.get('name'), + tool_call_id=msg_dict.get('tool_call_id'), + tool_calls=msg_dict.get('tool_calls'), + function_call=msg_dict.get('function_call'), + )) + + # Extract other fields + ground_truth = transformed.get('ground_truth') + tools = transformed.get('tools') + user_metadata = transformed.get('metadata', {}) + + # Create dataset info + dataset_info = { + 'dataset_id': self.dataset_id, + 'config_name': self.config_name, + 'revision': self.revision, + 'split': split, + 'row_index': row_index, + 'transform_function': self.transform_fn.__name__ if hasattr(self.transform_fn, '__name__') else 'anonymous', + } + + # Add user metadata + dataset_info.update(user_metadata) + + # Add original row data (with prefix to avoid conflicts) + for key, value in raw_row.items(): + dataset_info[f'original_{key}'] = value + + # Create input metadata + input_metadata = InputMetadata( + row_id=f"{self.dataset_id}_{row_index}", + completion_params=completion_params, + dataset_info=dataset_info, + session_data={ + 'dataset_source': 'huggingface', + 'timestamp': None, + } + ) + + return EvaluationRow( + messages=messages, + tools=tools, + input_metadata=input_metadata, + ground_truth=str(ground_truth) if ground_truth is not None else None, + ) + + def get_splits(self) -> List[str]: + """Get available dataset splits. + + Returns: + List of available split names + """ + if isinstance(self.dataset, DatasetDict): + return list(self.dataset.keys()) + else: + return ["train"] # Default split name for non-split datasets + + def get_dataset_info(self) -> Dict[str, Any]: + """Get information about the loaded dataset. + + Returns: + Dictionary with dataset information + """ + info = { + 'dataset_id': self.dataset_id, + 'config_name': self.config_name, + 'revision': self.revision, + 'splits': self.get_splits(), + 'transform_function': self.transform_fn.__name__ if hasattr(self.transform_fn, '__name__') else 'anonymous', + } + + # Add split sizes + if isinstance(self.dataset, DatasetDict): + info['split_sizes'] = {split: len(data) for split, data in self.dataset.items()} + else: + info['total_size'] = len(self.dataset) + + return info + + +def create_huggingface_adapter( + dataset_id: str, + transform_fn: TransformFunction, + config_name: Optional[str] = None, + revision: Optional[str] = None, + **load_dataset_kwargs, +) -> HuggingFaceAdapter: + """Factory function to create a HuggingFace adapter. + + Args: + dataset_id: HuggingFace dataset identifier + transform_fn: Function to transform dataset rows to evaluation format + config_name: Optional configuration name + revision: Optional dataset revision/commit hash + **load_dataset_kwargs: Additional arguments for load_dataset + + Returns: + HuggingFaceAdapter instance + """ + return HuggingFaceAdapter( + dataset_id=dataset_id, + transform_fn=transform_fn, + config_name=config_name, + revision=revision, + **load_dataset_kwargs, + ) + + +# Convenience functions for common datasets +def create_gsm8k_adapter( + system_prompt: Optional[str] = None, + revision: Optional[str] = None, +) -> HuggingFaceAdapter: + """Create adapter specifically configured for GSM8K dataset. + + Args: + system_prompt: Optional system prompt for math problems + revision: Optional dataset revision/commit + + Returns: + HuggingFaceAdapter configured for GSM8K + """ + default_system_prompt = ( + "You are a helpful assistant that solves math problems step by step. " + "Show your work and provide the final answer." + ) + + system_content = system_prompt or default_system_prompt + + def gsm8k_transform(row: Dict[str, Any]) -> Dict[str, Any]: + """Transform GSM8K row to evaluation format.""" + return { + 'messages': [ + {'role': 'system', 'content': system_content}, + {'role': 'user', 'content': row['question']}, + ], + 'ground_truth': row['answer'], + 'metadata': { + 'dataset': 'gsm8k', + 'question_length': len(row['question']), + 'answer_length': len(row['answer']), + } + } + + return create_huggingface_adapter( + dataset_id="gsm8k", + config_name="main", + transform_fn=gsm8k_transform, + revision=revision, + ) + + +def create_math_adapter( + system_prompt: Optional[str] = None, + revision: Optional[str] = None, +) -> HuggingFaceAdapter: + """Create adapter specifically configured for MATH competition dataset. + + Args: + system_prompt: Optional system prompt for math problems + revision: Optional dataset revision/commit + + Returns: + HuggingFaceAdapter configured for MATH dataset + """ + default_system_prompt = ( + "You are an expert mathematician. Solve this advanced math problem " + "step by step, showing detailed work." + ) + + system_content = system_prompt or default_system_prompt + + def math_transform(row: Dict[str, Any]) -> Dict[str, Any]: + """Transform MATH dataset row to evaluation format.""" + return { + 'messages': [ + {'role': 'system', 'content': system_content}, + {'role': 'user', 'content': row['problem']}, + ], + 'ground_truth': row['solution'], + 'metadata': { + 'dataset': 'hendrycks_math', + 'type': row.get('type', 'unknown'), + 'level': row.get('level', 'unknown'), + 'problem_length': len(row['problem']), + 'solution_length': len(row['solution']), + } + } + + return create_huggingface_adapter( + dataset_id="hendrycks/competition_math", + transform_fn=math_transform, + revision=revision, + ) \ No newline at end of file diff --git a/eval_protocol/adapters/langfuse.py b/eval_protocol/adapters/langfuse.py new file mode 100644 index 00000000..c125e7d3 --- /dev/null +++ b/eval_protocol/adapters/langfuse.py @@ -0,0 +1,407 @@ +"""Langfuse adapter for Eval Protocol. + +This adapter allows pulling data from Langfuse deployments and converting it +to EvaluationRow format for use in evaluation pipelines. +""" + +from typing import Any, Dict, Iterator, List, Optional +from datetime import datetime +import logging + +from eval_protocol.models import EvaluationRow, Message, InputMetadata, CompletionParams + +logger = logging.getLogger(__name__) + +try: + from langfuse import Langfuse + LANGFUSE_AVAILABLE = True +except ImportError: + LANGFUSE_AVAILABLE = False + logger.warning( + "Langfuse not installed. Install with: pip install 'eval-protocol[langfuse]'" + ) + + +class LangfuseAdapter: + """Adapter to pull data from Langfuse and convert to EvaluationRow format. + + This adapter can pull both chat conversations and tool calling traces from + Langfuse deployments and convert them into the EvaluationRow format expected + by the evaluation protocol. + + Examples: + Basic usage: + >>> adapter = LangfuseAdapter( + ... public_key="your_public_key", + ... secret_key="your_secret_key", + ... host="https://your-langfuse-deployment.com" + ... ) + >>> rows = list(adapter.get_evaluation_rows(limit=10)) + + Filter by specific criteria: + >>> rows = list(adapter.get_evaluation_rows( + ... limit=50, + ... tags=["production"], + ... user_id="specific_user", + ... from_timestamp=datetime.now() - timedelta(days=7) + ... )) + """ + + def __init__( + self, + public_key: str, + secret_key: str, + host: str = "https://cloud.langfuse.com", + project_id: Optional[str] = None, + ): + """Initialize the Langfuse adapter. + + Args: + public_key: Langfuse public key + secret_key: Langfuse secret key + host: Langfuse host URL (default: https://cloud.langfuse.com) + project_id: Optional project ID to filter traces + """ + if not LANGFUSE_AVAILABLE: + raise ImportError( + "Langfuse not installed. Install with: pip install 'eval-protocol[langfuse]'" + ) + + self.client = Langfuse( + public_key=public_key, + secret_key=secret_key, + host=host + ) + self.project_id = project_id + + def get_evaluation_rows( + self, + limit: int = 100, + tags: Optional[List[str]] = None, + user_id: Optional[str] = None, + session_id: Optional[str] = None, + from_timestamp: Optional[datetime] = None, + to_timestamp: Optional[datetime] = None, + include_tool_calls: bool = True, + ) -> Iterator[EvaluationRow]: + """Pull traces from Langfuse and convert to EvaluationRow format. + + Args: + limit: Maximum number of rows to return + tags: Filter by specific tags + user_id: Filter by user ID + session_id: Filter by session ID + from_timestamp: Filter traces after this timestamp + to_timestamp: Filter traces before this timestamp + include_tool_calls: Whether to include tool calling traces + + Yields: + EvaluationRow: Converted evaluation rows + """ + # Get traces from Langfuse + traces = self.client.get_traces( + limit=limit, + tags=tags, + user_id=user_id, + session_id=session_id, + from_timestamp=from_timestamp, + to_timestamp=to_timestamp + ) + + for trace in traces.data: + try: + eval_row = self._convert_trace_to_evaluation_row(trace, include_tool_calls) + if eval_row: + yield eval_row + except (AttributeError, ValueError, KeyError) as e: + logger.warning("Failed to convert trace %s: %s", trace.id, e) + continue + + def get_evaluation_rows_by_ids( + self, + trace_ids: List[str], + include_tool_calls: bool = True, + ) -> Iterator[EvaluationRow]: + """Get specific traces by their IDs and convert to EvaluationRow format. + + Args: + trace_ids: List of trace IDs to fetch + include_tool_calls: Whether to include tool calling traces + + Yields: + EvaluationRow: Converted evaluation rows + """ + for trace_id in trace_ids: + try: + trace = self.client.get_trace(trace_id) + eval_row = self._convert_trace_to_evaluation_row(trace, include_tool_calls) + if eval_row: + yield eval_row + except (AttributeError, ValueError, KeyError) as e: + logger.warning("Failed to fetch/convert trace %s: %s", trace_id, e) + continue + + def _convert_trace_to_evaluation_row( + self, + trace: Any, + include_tool_calls: bool = True + ) -> Optional[EvaluationRow]: + """Convert a Langfuse trace to EvaluationRow format. + + Args: + trace: Langfuse trace object + include_tool_calls: Whether to include tool calling information + + Returns: + EvaluationRow or None if conversion fails + """ + try: + # Get observations (generations, spans) from the trace + observations = self.client.get_observations(trace_id=trace.id).data + + # Convert observations to messages + messages = self._extract_messages_from_observations(observations, include_tool_calls) + + if not messages: + return None + + # Extract metadata + input_metadata = self._create_input_metadata(trace, observations) + + # Extract ground truth if available (from trace metadata or tags) + ground_truth = self._extract_ground_truth(trace) + + # Extract tools if available + tools = self._extract_tools(observations) if include_tool_calls else None + + return EvaluationRow( + messages=messages, + tools=tools, + input_metadata=input_metadata, + ground_truth=ground_truth, + ) + + except (AttributeError, ValueError, KeyError) as e: + logger.error("Error converting trace %s: %s", trace.id, e) + return None + + def _extract_messages_from_observations( + self, + observations: List[Any], + include_tool_calls: bool = True + ) -> List[Message]: + """Extract messages from Langfuse observations. + + Args: + observations: List of Langfuse observation objects + include_tool_calls: Whether to include tool calling information + + Returns: + List of Message objects + """ + messages = [] + + # Sort observations by timestamp + sorted_observations = sorted(observations, key=lambda x: x.start_time or datetime.min) + + for obs in sorted_observations: + try: + if hasattr(obs, 'input') and obs.input: + # Handle different input formats + if isinstance(obs.input, dict): + if 'messages' in obs.input: + # OpenAI-style messages format + for msg in obs.input['messages']: + messages.append(self._dict_to_message(msg, include_tool_calls)) + elif 'role' in obs.input: + # Single message format + messages.append(self._dict_to_message(obs.input, include_tool_calls)) + elif 'prompt' in obs.input: + # Simple prompt format + messages.append(Message(role="user", content=str(obs.input['prompt']))) + elif isinstance(obs.input, str): + # Simple string input + messages.append(Message(role="user", content=obs.input)) + + if hasattr(obs, 'output') and obs.output: + # Handle output + if isinstance(obs.output, dict): + if 'content' in obs.output: + messages.append(Message(role="assistant", content=str(obs.output['content']))) + elif 'message' in obs.output: + msg_dict = obs.output['message'] + messages.append(self._dict_to_message(msg_dict, include_tool_calls)) + else: + # Fallback: convert entire output to string + messages.append(Message(role="assistant", content=str(obs.output))) + elif isinstance(obs.output, str): + messages.append(Message(role="assistant", content=obs.output)) + + except (AttributeError, ValueError, KeyError) as e: + logger.warning("Error processing observation %s: %s", obs.id, e) + continue + + return messages + + def _dict_to_message(self, msg_dict: Dict[str, Any], include_tool_calls: bool = True) -> Message: + """Convert a dictionary to a Message object. + + Args: + msg_dict: Dictionary containing message data + include_tool_calls: Whether to include tool calling information + + Returns: + Message object + """ + # Extract basic message components + role = msg_dict.get('role', 'assistant') + content = msg_dict.get('content') + name = msg_dict.get('name') + + # Handle tool calls if enabled + tool_calls = None + tool_call_id = None + function_call = None + + if include_tool_calls: + if 'tool_calls' in msg_dict: + tool_calls = msg_dict['tool_calls'] + if 'tool_call_id' in msg_dict: + tool_call_id = msg_dict['tool_call_id'] + if 'function_call' in msg_dict: + function_call = msg_dict['function_call'] + + return Message( + role=role, + content=content, + name=name, + tool_call_id=tool_call_id, + tool_calls=tool_calls, + function_call=function_call, + ) + + def _create_input_metadata(self, trace: Any, observations: List[Any]) -> InputMetadata: + """Create InputMetadata from trace and observations. + + Args: + trace: Langfuse trace object + observations: List of observation objects + + Returns: + InputMetadata object + """ + # Extract completion parameters from observations + completion_params = CompletionParams() + + # Look for model parameters in observations + for obs in observations: + if hasattr(obs, 'model') and obs.model: + completion_params.model = obs.model + if hasattr(obs, 'model_parameters') and obs.model_parameters: + params = obs.model_parameters + if 'temperature' in params: + completion_params.temperature = params['temperature'] + if 'max_tokens' in params: + completion_params.max_tokens = params['max_tokens'] + if 'top_p' in params: + completion_params.top_p = params['top_p'] + break + + # Create dataset info from trace metadata + dataset_info = { + 'trace_id': trace.id, + 'trace_name': getattr(trace, 'name', None), + 'trace_tags': getattr(trace, 'tags', []), + 'langfuse_project_id': self.project_id, + } + + # Add trace metadata if available + if hasattr(trace, 'metadata') and trace.metadata: + dataset_info['trace_metadata'] = trace.metadata + + # Create session data + session_data = { + 'session_id': getattr(trace, 'session_id', None), + 'user_id': getattr(trace, 'user_id', None), + 'timestamp': getattr(trace, 'timestamp', None), + 'langfuse_trace_url': f"{self.client.host}/project/{self.project_id}/traces/{trace.id}" if self.project_id else None, + } + + return InputMetadata( + row_id=trace.id, + completion_params=completion_params, + dataset_info=dataset_info, + session_data=session_data, + ) + + def _extract_ground_truth(self, trace: Any) -> Optional[str]: + """Extract ground truth from trace if available. + + Args: + trace: Langfuse trace object + + Returns: + Ground truth string or None + """ + # Check trace metadata for ground truth + if hasattr(trace, 'metadata') and trace.metadata: + if isinstance(trace.metadata, dict): + return trace.metadata.get('ground_truth') or trace.metadata.get('expected_answer') + + # Check tags for ground truth indicators + if hasattr(trace, 'tags') and trace.tags: + for tag in trace.tags: + if tag.startswith('ground_truth:'): + return tag.replace('ground_truth:', '', 1) + + return None + + def _extract_tools(self, observations: List[Any]) -> Optional[List[Dict[str, Any]]]: + """Extract tool definitions from observations. + + Args: + observations: List of observation objects + + Returns: + List of tool definitions or None + """ + tools = [] + + for obs in observations: + if hasattr(obs, 'input') and obs.input and isinstance(obs.input, dict): + if 'tools' in obs.input: + tools.extend(obs.input['tools']) + elif 'functions' in obs.input: + # Convert functions to tools format + for func in obs.input['functions']: + tools.append({ + 'type': 'function', + 'function': func + }) + + return tools if tools else None + + +def create_langfuse_adapter( + public_key: str, + secret_key: str, + host: str = "https://cloud.langfuse.com", + project_id: Optional[str] = None, +) -> LangfuseAdapter: + """Factory function to create a Langfuse adapter. + + Args: + public_key: Langfuse public key + secret_key: Langfuse secret key + host: Langfuse host URL + project_id: Optional project ID + + Returns: + LangfuseAdapter instance + """ + return LangfuseAdapter( + public_key=public_key, + secret_key=secret_key, + host=host, + project_id=project_id, + ) \ No newline at end of file diff --git a/examples/adapters/README.md b/examples/adapters/README.md new file mode 100644 index 00000000..f51cd387 --- /dev/null +++ b/examples/adapters/README.md @@ -0,0 +1,275 @@ +# Adapter Examples + +This directory contains examples demonstrating how to use the various data source adapters available in the Eval Protocol system. + +## Available Adapters + +### 1. Langfuse Adapter (`langfuse_example.py`) + +Connects to Langfuse deployments to pull conversation traces and convert them to EvaluationRow format. + +**Features:** +- Pull chat conversations and tool calling traces +- Filter by time ranges, tags, users, and sessions +- Preserve metadata from Langfuse traces +- Support for both cloud and self-hosted Langfuse instances + +**Prerequisites:** +```bash +pip install 'eval-protocol[langfuse]' +``` + +**Environment Variables:** +```bash +export LANGFUSE_PUBLIC_KEY="your_public_key" +export LANGFUSE_SECRET_KEY="your_secret_key" +export LANGFUSE_HOST="https://your-langfuse-instance.com" # optional +export LANGFUSE_PROJECT_ID="your_project_id" # optional +``` + +### 2. HuggingFace Adapter (`huggingface_example.py`) + +Loads datasets from HuggingFace Hub and converts them to EvaluationRow format. + +**Features:** +- Generic adapter with arbitrary transformation functions +- Built-in convenience functions for popular datasets (GSM8K, MATH, etc.) +- Support for local dataset files (JSON, JSONL, CSV) +- Full control over data transformation and formatting +- Support for dataset revisions/commits + +**Prerequisites:** +```bash +pip install 'eval-protocol[huggingface]' +``` + +## Running the Examples + +### Basic Usage + +```bash +# Run Langfuse example +python examples/adapters/langfuse_example.py + +# Run HuggingFace example +python examples/adapters/huggingface_example.py + +# Run GSM8K replacement example +python examples/adapters/gsm8k_replacement_example.py +``` + +### With Environment Setup + +```bash +# Set up Langfuse credentials +export LANGFUSE_PUBLIC_KEY="pk_..." +export LANGFUSE_SECRET_KEY="sk_..." +python examples/adapters/langfuse_example.py + +# HuggingFace works without credentials for public datasets +python examples/adapters/huggingface_example.py +``` + +## Integration Patterns + +### 1. Replace Static Dataset Files + +Instead of using static JSONL files, use adapters to pull fresh data: + +```python +# Old approach +input_dataset=["development/gsm8k_sample.jsonl"] + +# New approach with HuggingFace adapter +from eval_protocol.adapters.huggingface import create_gsm8k_adapter + +adapter = create_gsm8k_adapter() +evaluation_rows = list(adapter.get_evaluation_rows(split="test", limit=100)) + +# Or for complete control: +def custom_gsm8k_transform(row): + return { + 'messages': [ + {'role': 'system', 'content': 'Your custom prompt'}, + {'role': 'user', 'content': row['question']} + ], + 'ground_truth': row['answer'], + 'metadata': {'custom_field': 'value'} + } + +from eval_protocol.adapters.huggingface import create_huggingface_adapter +custom_adapter = create_huggingface_adapter( + dataset_id="gsm8k", + config_name="main", + transform_fn=custom_gsm8k_transform +) +``` + +### 2. Real-time Data from Production Systems + +Pull recent conversations from your production systems: + +```python +from eval_protocol.adapters.langfuse import create_langfuse_adapter +from datetime import datetime, timedelta + +adapter = create_langfuse_adapter(...) +recent_rows = list(adapter.get_evaluation_rows( + from_timestamp=datetime.now() - timedelta(hours=24), + tags=["production"], +)) +``` + +### 3. Batch Processing Large Datasets + +Process datasets in manageable batches: + +```python +batch_size = 100 +for offset in range(0, 1000, batch_size): + batch = list(adapter.get_evaluation_rows( + limit=batch_size, + offset=offset, + )) + # Process batch... +``` + +## Common Use Cases + +### Evaluation Pipeline Integration + +```python +from eval_protocol.adapters.huggingface import create_gsm8k_adapter +from eval_protocol.rewards.math import math_reward + +# Load dataset +adapter = create_gsm8k_adapter() +rows = list(adapter.get_evaluation_rows(limit=10)) + +# Run evaluation +for row in rows: + # Add model response (you would generate this) + row.messages.append(Message(role="assistant", content="...")) + + # Evaluate + result = math_reward(messages=row.messages, ground_truth=row.ground_truth) + print(f"Score: {result.score}") +``` + +### Training Data Preparation + +```python +# Convert to training format +training_data = [] +for row in adapter.get_evaluation_rows(): + training_data.append({ + "messages": [{"role": msg.role, "content": msg.content} for msg in row.messages], + "ground_truth": row.ground_truth, + }) +``` + +### A/B Testing with Live Data + +```python +# Compare models on recent production data +langfuse_adapter = create_langfuse_adapter(...) +recent_conversations = list(langfuse_adapter.get_evaluation_rows( + from_timestamp=datetime.now() - timedelta(days=1), + limit=100, +)) + +# Test both models on the same data +for row in recent_conversations: + # Test model A and B, compare results... +``` + +## Custom Adapter Development + +### Option 1: Use Generic HuggingFace Adapter (Recommended) + +For datasets on HuggingFace Hub, use the generic adapter with a transform function: + +```python +from eval_protocol.adapters.huggingface import create_huggingface_adapter + +def my_transform(row): + return { + 'messages': [ + {'role': 'system', 'content': 'Your system prompt'}, + {'role': 'user', 'content': row['input_field']}, + ], + 'ground_truth': row['output_field'], + 'metadata': {'custom': row.get('metadata_field')} + } + +adapter = create_huggingface_adapter( + dataset_id="your-dataset-id", + transform_fn=my_transform, + revision="main" # optional +) +``` + +### Option 2: Create Custom Adapter Class + +See `eval_protocol/adapters/CONTRIBUTING.md` for detailed instructions on creating full custom adapters. + +Quick template: + +```python +from eval_protocol.models import EvaluationRow, Message, InputMetadata + +class MyCustomAdapter: + def __init__(self, **config): + # Initialize your data source connection + pass + + def get_evaluation_rows(self, **kwargs) -> Iterator[EvaluationRow]: + # Fetch data and convert to EvaluationRow format + pass +``` + +## Troubleshooting + +### Common Issues + +1. **Import Errors**: Make sure you have the right optional dependencies installed + ```bash + pip install 'eval-protocol[langfuse]' # for Langfuse + pip install 'eval-protocol[huggingface]' # for HuggingFace + pip install 'eval-protocol[adapters]' # for all adapters + ``` + +2. **Authentication Errors**: Check your environment variables and API keys + +3. **Network Errors**: Verify connectivity to external services + +4. **Data Format Issues**: Check that your data source has the expected fields + +### Debug Mode + +Enable debug logging to see what's happening: + +```python +import logging +logging.basicConfig(level=logging.DEBUG) + +# Your adapter code here... +``` + +### Getting Help + +- Check the main [CONTRIBUTING.md](../../development/CONTRIBUTING.md) for project setup +- Review adapter-specific documentation in `eval_protocol/adapters/CONTRIBUTING.md` +- Open an issue on GitHub for bugs or feature requests +- Join the community discussions for questions + +## Contributing + +We welcome contributions of new adapters! Popular integrations that would be valuable: + +- **Database adapters**: PostgreSQL, MongoDB, etc. +- **API adapters**: OpenAI Evals, Anthropic datasets, etc. +- **File format adapters**: Parquet, Excel, etc. +- **Monitoring platform adapters**: DataDog, New Relic, etc. + +See the adapter contributing guide for detailed instructions. \ No newline at end of file diff --git a/examples/adapters/gsm8k_replacement_example.py b/examples/adapters/gsm8k_replacement_example.py new file mode 100644 index 00000000..a86de261 --- /dev/null +++ b/examples/adapters/gsm8k_replacement_example.py @@ -0,0 +1,321 @@ +""" +GSM8K Replacement Example + +This example shows how to replace the static GSM8K JSONL file +(development/gsm8k_sample.jsonl) with the dynamic HuggingFace adapter +to get fresh data from the GSM8K dataset. +""" + +import json +from pathlib import Path +from typing import List + +from eval_protocol.adapters.huggingface import create_gsm8k_adapter +from eval_protocol.models import EvaluationRow, Message +from eval_protocol.rewards.math import math_reward + + +def load_original_gsm8k_sample() -> List[dict]: + """Load the original GSM8K sample file for comparison.""" + sample_file = Path("development/gsm8k_sample.jsonl") + + if not sample_file.exists(): + print(f"โš ๏ธ Original sample file not found: {sample_file}") + return [] + + data = [] + with open(sample_file, 'r') as f: + for line in f: + if line.strip(): + data.append(json.loads(line)) + + return data + + +def demonstrate_old_vs_new_approach(): + """Compare the old static file approach with the new adapter approach.""" + print("๐Ÿ“Š Comparing Old vs New Approach") + print("=" * 50) + + # OLD APPROACH: Static JSONL file + print("๐Ÿ—‚๏ธ OLD APPROACH: Static JSONL File") + print("-" * 35) + + original_data = load_original_gsm8k_sample() + print(f"Loaded {len(original_data)} items from static file") + + if original_data: + sample = original_data[0] + print(f"Sample item fields: {list(sample.keys())}") + print(f"Sample question: {sample.get('user_query', '')[:100]}...") + print(f"Sample ground truth: {sample.get('ground_truth_for_eval', '')[:100]}...") + + print("\n" + "="*50 + "\n") + + # NEW APPROACH: HuggingFace Adapter + print("๐Ÿค— NEW APPROACH: HuggingFace Adapter") + print("-" * 38) + + try: + # Create adapter + adapter = create_gsm8k_adapter( + system_prompt="You are a helpful assistant that solves math problems step by step." + ) + + print("โœ… GSM8K adapter created successfully") + + # Get the same number of items as the original file + num_items = len(original_data) if original_data else 6 + rows = list(adapter.get_evaluation_rows(limit=num_items)) + + print(f"Retrieved {len(rows)} evaluation rows from HuggingFace") + + if rows: + sample_row = rows[0] + print(f"Sample EvaluationRow fields: messages, tools, input_metadata, ground_truth") + + # Show the question from messages + user_msg = next((msg for msg in sample_row.messages if msg.role == "user"), None) + if user_msg: + print(f"Sample question: {user_msg.content[:100]}...") + + if sample_row.ground_truth: + print(f"Sample ground truth: {sample_row.ground_truth[:100]}...") + + except ImportError as e: + print(f"โŒ Error: {e}") + print("Install HuggingFace dependencies: pip install 'eval-protocol[huggingface]'") + return + except Exception as e: + print(f"โŒ Error with adapter: {e}") + return + + print("\n" + "="*50 + "\n") + + # COMPARISON + print("๐Ÿ” Key Differences") + print("-" * 20) + print("OLD APPROACH:") + print(" โœ… Fast loading (local file)") + print(" โŒ Static data (same 6 problems)") + print(" โŒ Manual data preparation required") + print(" โŒ Limited to pre-selected subset") + print(" โŒ Requires manual format conversion") + + print("\nNEW APPROACH:") + print(" โœ… Access to full GSM8K dataset (8,792 test problems)") + print(" โœ… Automatic format conversion to EvaluationRow") + print(" โœ… Built-in metadata handling") + print(" โœ… Configurable system prompts") + print(" โœ… Consistent with other adapters") + print(" โš ๏ธ Requires internet connection and HuggingFace datasets") + + +def show_migration_example(): + """Show how to migrate existing code from JSONL to adapter.""" + print("\n๐Ÿ”„ Code Migration Example") + print("=" * 30) + + print("OLD CODE:") + print("-" * 10) + print(""" +# Old way with static JSONL file +input_dataset = ["development/gsm8k_sample.jsonl"] + +# Manual loading and parsing +import json +data = [] +with open("development/gsm8k_sample.jsonl", 'r') as f: + for line in f: + item = json.loads(line) + # Manual conversion to expected format + messages = [ + {"role": "user", "content": item["user_query"]} + ] + ground_truth = item["ground_truth_for_eval"] + # ... more manual processing +""") + + print("\nNEW CODE:") + print("-" * 10) + print(""" +# New way with HuggingFace adapter +from eval_protocol.adapters.huggingface import create_gsm8k_adapter + +# Create adapter with custom configuration +adapter = create_gsm8k_adapter( + system_prompt="You are a helpful math tutor." +) + +# Get evaluation rows (already in correct format) +evaluation_rows = list(adapter.get_evaluation_rows( + split="test", # or "train" + limit=100, # Can get much more data than static file + model_name="gpt-4", + temperature=0.0, +)) + +# evaluation_rows are already EvaluationRow objects! +# No manual conversion needed + +# For complete control, use the generic adapter: +def custom_gsm8k_transform(row): + return { + 'messages': [ + {'role': 'system', 'content': 'Custom system prompt here'}, + {'role': 'user', 'content': row['question']} + ], + 'ground_truth': row['answer'], + 'metadata': {'custom_field': 'custom_value'} + } + +from eval_protocol.adapters.huggingface import create_huggingface_adapter +custom_adapter = create_huggingface_adapter( + dataset_id="gsm8k", + config_name="main", + transform_fn=custom_gsm8k_transform +) +""") + + print("\nโœ… Benefits of Migration:") + print(" - More data available (6 โ†’ 8,792 problems)") + print(" - Automatic format handling") + print(" - Better metadata preservation") + print(" - Consistent API across all adapters") + print(" - Easy to customize system prompts") + + +def practical_migration_demo(): + """Show a practical example of using the adapter in evaluation.""" + print("\n๐Ÿงช Practical Evaluation Example") + print("=" * 35) + + try: + # Create adapter + adapter = create_gsm8k_adapter() + + # Get a few problems for evaluation + print("Loading GSM8K problems...") + rows = list(adapter.get_evaluation_rows(limit=3)) + print(f"โœ… Loaded {len(rows)} problems from GSM8K test set") + + # Simulate evaluation workflow + for i, row in enumerate(rows): + print(f"\n๐Ÿ“ Problem {i+1}:") + + # Show the problem + user_msg = next((msg for msg in row.messages if msg.role == "user"), None) + if user_msg: + print(f" Question: {user_msg.content[:150]}...") + + # In a real scenario, you'd generate a response with your LLM + # For this demo, we'll add a dummy response + dummy_response = "Let me solve this step by step. After working through the math, the answer is 42." + row.messages.append(Message(role="assistant", content=dummy_response)) + + # Evaluate with math reward function + if row.ground_truth: + try: + result = math_reward( + messages=row.messages, + ground_truth=row.ground_truth, + ) + print(f" ๐Ÿ“Š Math evaluation score: {result.score:.2f}") + print(f" ๐Ÿ’ญ Evaluation reason: {result.reason[:100]}...") + + # Show metadata + if row.input_metadata: + print(f" ๐Ÿท๏ธ Row ID: {row.input_metadata.row_id}") + if row.input_metadata.dataset_info: + dataset_info = row.input_metadata.dataset_info + print(f" ๐Ÿ“š Dataset: {dataset_info.get('dataset_name', 'N/A')}") + print(f" ๐Ÿ“ Row index: {dataset_info.get('row_index', 'N/A')}") + + except Exception as e: + print(f" โŒ Evaluation error: {e}") + + print(f"\nโœ… Successfully processed {len(rows)} problems using the new adapter approach!") + + except Exception as e: + print(f"โŒ Error in practical demo: {e}") + + +def performance_comparison(): + """Compare performance characteristics of both approaches.""" + print("\nโšก Performance Considerations") + print("=" * 35) + + import time + + # Time the old approach (if file exists) + original_data = load_original_gsm8k_sample() + if original_data: + start_time = time.time() + # Simulate processing the original data + processed_old = len(original_data) + old_time = time.time() - start_time + print(f"๐Ÿ“ Static file approach: {processed_old} items in {old_time:.4f}s") + else: + print("๐Ÿ“ Static file not available for timing") + old_time = 0 + processed_old = 0 + + # Time the new approach + try: + start_time = time.time() + adapter = create_gsm8k_adapter() + rows = list(adapter.get_evaluation_rows(split="test", limit=max(6, processed_old))) + new_time = time.time() - start_time + processed_new = len(rows) + + print(f"๐Ÿค— HuggingFace adapter: {processed_new} items in {new_time:.4f}s") + + if old_time > 0: + if new_time > old_time: + factor = new_time / old_time + print(f" ๐Ÿ“Š Adapter is {factor:.1f}x slower (but loads fresh data)") + else: + factor = old_time / new_time + print(f" ๐Ÿ“Š Adapter is {factor:.1f}x faster!") + + print(f"\n๐Ÿ’ก Trade-offs:") + print(f" Static file: Fast ({old_time:.4f}s) but limited data ({processed_old} items)") + print(f" Adapter: Slower ({new_time:.4f}s) but access to full dataset ({processed_new}+ items)") + + except Exception as e: + print(f"โŒ Error timing adapter: {e}") + + +def main(): + """Run the complete GSM8K replacement demonstration.""" + print("๐Ÿš€ GSM8K Replacement Example") + print("=" * 50) + print("This example shows how to replace the static GSM8K JSONL file") + print("with the dynamic HuggingFace adapter for better data access.") + print() + + # Run all demonstrations + demonstrate_old_vs_new_approach() + show_migration_example() + practical_migration_demo() + performance_comparison() + + print("\n" + "="*50) + print("๐ŸŽฏ MIGRATION SUMMARY") + print("="*50) + print("1. โœ… Replace static JSONL with HuggingFace adapter") + print("2. โœ… Get access to full GSM8K dataset (8,792 test problems)") + print("3. โœ… Automatic conversion to EvaluationRow format") + print("4. โœ… Built-in metadata and system prompt support") + print("5. โœ… Consistent API with other data sources") + print() + print("๐Ÿ“ Next Steps:") + print("- Update your evaluation scripts to use the adapter") + print("- Experiment with different system prompts") + print("- Scale up to use more than 6 problems") + print("- Consider using train split for different use cases") + print("- Integrate with your existing evaluation pipeline") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/adapters/huggingface_example.py b/examples/adapters/huggingface_example.py new file mode 100644 index 00000000..2d79eae3 --- /dev/null +++ b/examples/adapters/huggingface_example.py @@ -0,0 +1,411 @@ +""" +HuggingFace Dataset Adapter Example + +This example demonstrates how to use the HuggingFace adapter to load datasets +from HuggingFace Hub and convert them to EvaluationRow format for evaluation. +""" + +import os +from typing import List + +from eval_protocol.adapters.huggingface import ( + create_huggingface_adapter, + create_gsm8k_adapter, + create_math_adapter, + HuggingFaceAdapter, +) +from eval_protocol.models import EvaluationRow + + +def gsm8k_example(): + """Example using the GSM8K dataset.""" + print("๐Ÿ“š Example 1: GSM8K Dataset") + print("-" * 30) + + try: + # Create GSM8K adapter using the convenience method + adapter = create_gsm8k_adapter( + split="test", + system_prompt="You are a helpful assistant that solves math problems step by step." + ) + + print("โœ… GSM8K adapter created successfully") + print(f"๐Ÿ“Š Dataset info: {adapter.get_dataset_info()}") + + # Get a few evaluation rows + rows = list(adapter.get_evaluation_rows( + limit=3, + model_name="gpt-4", + temperature=0.0, + )) + + print(f"\nRetrieved {len(rows)} evaluation rows from GSM8K test set:") + + for i, row in enumerate(rows): + print(f"\n Row {i+1}:") + print(f" - ID: {row.input_metadata.row_id if row.input_metadata else 'N/A'}") + print(f" - Messages: {len(row.messages)}") + + # Show the math problem + user_message = next((msg for msg in row.messages if msg.role == "user"), None) + if user_message: + problem = user_message.content[:200] + "..." if len(user_message.content) > 200 else user_message.content + print(f" - Problem: {problem}") + + # Show ground truth answer + if row.ground_truth: + answer_preview = row.ground_truth[:100] + "..." if len(row.ground_truth) > 100 else row.ground_truth + print(f" - Ground truth: {answer_preview}") + + print() + + except ImportError as e: + print(f"โŒ Error: {e}") + print("Install HuggingFace dependencies: pip install 'eval-protocol[huggingface]'") + except Exception as e: + print(f"โŒ Error loading GSM8K: {e}") + + +def math_dataset_example(): + """Example using the MATH competition dataset.""" + print("๐Ÿงฎ Example 2: MATH Competition Dataset") + print("-" * 40) + + try: + # Create MATH dataset adapter + adapter = create_math_adapter( + system_prompt="You are an expert mathematician. Solve this step by step." + ) + + print("โœ… MATH dataset adapter created successfully") + print(f"๐Ÿ“Š Dataset info: {adapter.get_dataset_info()}") + + # Get a few examples + rows = list(adapter.get_evaluation_rows( + limit=2, + model_name="gpt-4", + temperature=0.1, + )) + + print(f"\nRetrieved {len(rows)} evaluation rows from MATH test set:") + + for i, row in enumerate(rows): + print(f"\n Row {i+1}:") + + # Show the problem + user_message = next((msg for msg in row.messages if msg.role == "user"), None) + if user_message: + problem = user_message.content[:150] + "..." if len(user_message.content) > 150 else user_message.content + print(f" - Problem: {problem}") + + # Show metadata + if row.input_metadata and row.input_metadata.dataset_info: + dataset_info = row.input_metadata.dataset_info + if 'original_type' in dataset_info: + print(f" - Problem type: {dataset_info['original_type']}") + if 'original_level' in dataset_info: + print(f" - Level: {dataset_info['original_level']}") + + except Exception as e: + print(f"โŒ Error with MATH dataset: {e}") + + +def custom_dataset_example(): + """Example using a custom dataset with transformation function.""" + print("๐Ÿ”ง Example 3: Custom Dataset with Transform Function") + print("-" * 55) + + try: + # Define transformation function for SQuAD dataset + def squad_transform(row): + """Transform SQuAD row to evaluation format.""" + context = row['context'] + question = row['question'] + answers = row['answers'] + + # Get first answer text + answer_text = answers['text'][0] if answers['text'] else "No answer provided" + + return { + 'messages': [ + {'role': 'system', 'content': 'Answer the question based on the given context.'}, + {'role': 'user', 'content': f"Context: {context}\\n\\nQuestion: {question}"}, + ], + 'ground_truth': answer_text, + 'metadata': { + 'dataset': 'squad', + 'context_length': len(context), + 'question_length': len(question), + 'num_possible_answers': len(answers['text']), + } + } + + # Create adapter with transformation function + adapter = create_huggingface_adapter( + dataset_id="squad", + transform_fn=squad_transform, + ) + + print("โœ… Custom dataset adapter created successfully") + + # Get dataset info + info = adapter.get_dataset_info() + print(f"๐Ÿ“Š Dataset info: {info}") + + # Get a few examples + rows = list(adapter.get_evaluation_rows( + split="validation", # SQuAD has train/validation splits + limit=2, + model_name="gpt-3.5-turbo", + )) + + print(f"\nRetrieved {len(rows)} evaluation rows:") + + for i, row in enumerate(rows): + print(f"\n Row {i+1}:") + print(f" - Messages: {len(row.messages)}") + + # Show question + user_message = next((msg for msg in row.messages if msg.role == "user"), None) + if user_message: + question = user_message.content[:100] + "..." if len(user_message.content) > 100 else user_message.content + print(f" - Question: {question}") + + # SQuAD answers are complex, so just show if we have ground truth + print(f" - Has ground truth: {'Yes' if row.ground_truth else 'No'}") + + except Exception as e: + print(f"โŒ Error with custom dataset: {e}") + + +def local_file_example(): + """Example loading a local dataset file.""" + print("๐Ÿ“ Example 4: Local Dataset File") + print("-" * 35) + + # Create a sample JSONL file for demonstration + sample_file = "/tmp/sample_qa.jsonl" + sample_data = [ + { + "id": "q1", + "question": "What is the capital of France?", + "answer": "Paris", + "category": "geography" + }, + { + "id": "q2", + "question": "What is 2 + 2?", + "answer": "4", + "category": "math" + }, + { + "id": "q3", + "question": "Who wrote Romeo and Juliet?", + "answer": "William Shakespeare", + "category": "literature" + } + ] + + try: + import json + + # Write sample data + with open(sample_file, 'w') as f: + for item in sample_data: + f.write(json.dumps(item) + '\n') + + print(f"๐Ÿ“ Created sample file: {sample_file}") + + # Define transformation function for local data + def local_qa_transform(row): + """Transform local Q&A data to evaluation format.""" + return { + 'messages': [ + {'role': 'system', 'content': 'You are a knowledgeable assistant.'}, + {'role': 'user', 'content': row['question']}, + ], + 'ground_truth': row['answer'], + 'metadata': { + 'id': row.get('id'), + 'category': row.get('category'), + 'dataset': 'local_qa_sample', + } + } + + # Load with adapter + adapter = HuggingFaceAdapter.from_local( + path=sample_file, + transform_fn=local_qa_transform, + ) + + print("โœ… Local file adapter created successfully") + + # Get all rows + rows = list(adapter.get_evaluation_rows( + model_name="gpt-3.5-turbo", + temperature=0.0, + )) + + print(f"\nLoaded {len(rows)} rows from local file:") + + for i, row in enumerate(rows): + print(f"\n Row {i+1}:") + + # Show question and answer + user_msg = next((msg for msg in row.messages if msg.role == "user"), None) + if user_msg: + print(f" - Question: {user_msg.content}") + + if row.ground_truth: + print(f" - Answer: {row.ground_truth}") + + # Show original metadata + if row.input_metadata and row.input_metadata.dataset_info: + original_data = {k: v for k, v in row.input_metadata.dataset_info.items() if k.startswith('original_')} + if original_data: + print(f" - Original data: {original_data}") + + # Clean up + os.remove(sample_file) + print(f"\n๐Ÿงน Cleaned up sample file") + + except Exception as e: + print(f"โŒ Error with local file: {e}") + + +def evaluation_integration_example(): + """Show how to integrate with evaluation functions.""" + print("\n๐Ÿงช Example 5: Integration with Evaluation") + print("-" * 45) + + try: + # Import evaluation functions + from eval_protocol.rewards.math import math_reward + from eval_protocol.rewards.accuracy import accuracy_reward + + # Create GSM8K adapter + adapter = create_gsm8k_adapter(split="test") + + # Get a few rows for evaluation + rows = list(adapter.get_evaluation_rows(limit=2)) + + print(f"Running evaluation on {len(rows)} GSM8K problems:") + + for i, row in enumerate(rows): + print(f"\n Problem {i+1}:") + + # Show the problem + user_msg = next((msg for msg in row.messages if msg.role == "user"), None) + if user_msg: + print(f" Question: {user_msg.content[:100]}...") + + # For this example, we'll simulate an assistant response + # In practice, this would come from your LLM + row.messages.append({ + "role": "assistant", + "content": "Let me solve this step by step... The answer is 42." + }) + + # Evaluate with math reward + if row.ground_truth: + try: + math_result = math_reward( + messages=row.messages, + ground_truth=row.ground_truth, + ) + print(f" Math score: {math_result.score:.2f}") + print(f" Reason: {math_result.reason[:100]}...") + + # Also try accuracy reward + acc_result = accuracy_reward( + messages=row.messages, + ground_truth=row.ground_truth, + ) + print(f" Accuracy score: {acc_result.score:.2f}") + + except Exception as e: + print(f" โŒ Evaluation error: {e}") + + except ImportError: + print("Evaluation functions not available") + except Exception as e: + print(f"โŒ Error in evaluation integration: {e}") + + +def batch_processing_example(): + """Show how to process datasets in batches.""" + print("\n๐Ÿ“ฆ Example 6: Batch Processing") + print("-" * 35) + + try: + adapter = create_gsm8k_adapter(split="test") + + batch_size = 5 + total_processed = 0 + + print(f"Processing GSM8K test set in batches of {batch_size}:") + + # Process in batches + for batch_start in range(0, 20, batch_size): # Process first 20 items + batch_rows = list(adapter.get_evaluation_rows( + limit=batch_size, + offset=batch_start, + )) + + print(f" Batch {batch_start//batch_size + 1}: {len(batch_rows)} rows") + + # Process each row in the batch + for row in batch_rows: + # Here you would typically: + # 1. Generate a response with your LLM + # 2. Evaluate the response + # 3. Store results + total_processed += 1 + + print(f"โœ… Processed {total_processed} rows total") + + except Exception as e: + print(f"โŒ Error in batch processing: {e}") + + +def main(): + """Run all examples.""" + print("๐Ÿค— HuggingFace Dataset Adapter Examples") + print("=" * 50) + + # Run examples + gsm8k_example() + print("\n" + "="*50 + "\n") + + math_dataset_example() + print("\n" + "="*50 + "\n") + + custom_dataset_example() + print("\n" + "="*50 + "\n") + + local_file_example() + print("\n" + "="*50 + "\n") + + evaluation_integration_example() + print("\n" + "="*50 + "\n") + + batch_processing_example() + + +if __name__ == "__main__": + try: + main() + + print("\nโœ… All examples completed!") + print("\nNext steps:") + print("1. Choose the dataset that fits your needs") + print("2. Customize the system prompts for your use case") + print("3. Integrate with your evaluation pipeline") + print("4. Scale up to process full datasets") + print("5. Use the EvaluationRow data for training or evaluation") + + except ImportError as e: + print(f"โŒ Missing dependencies: {e}") + print("Install with: pip install 'eval-protocol[huggingface]'") + except Exception as e: + print(f"โŒ Error running examples: {e}") \ No newline at end of file diff --git a/examples/adapters/langfuse_example.py b/examples/adapters/langfuse_example.py new file mode 100644 index 00000000..78937c80 --- /dev/null +++ b/examples/adapters/langfuse_example.py @@ -0,0 +1,199 @@ +""" +Langfuse Adapter Example + +This example demonstrates how to use the Langfuse adapter to pull data from +a Langfuse deployment and convert it to EvaluationRow format for evaluation. +""" + +import os +from datetime import datetime, timedelta +from typing import List + +from eval_protocol.adapters.langfuse import create_langfuse_adapter +from eval_protocol.models import EvaluationRow + + +def main(): + """Example usage of the Langfuse adapter.""" + + # Configuration - you can set these as environment variables + public_key = os.getenv("LANGFUSE_PUBLIC_KEY", "your_public_key_here") + secret_key = os.getenv("LANGFUSE_SECRET_KEY", "your_secret_key_here") + host = os.getenv("LANGFUSE_HOST", "https://langfuse-web-prod-zfdbl7ykrq-uc.a.run.app") + project_id = os.getenv("LANGFUSE_PROJECT_ID", "cmdj5yxhk0006s6022cyi0prv") + + print(f"Connecting to Langfuse at: {host}") + print(f"Project ID: {project_id}\n") + + # Create the adapter + try: + adapter = create_langfuse_adapter( + public_key=public_key, + secret_key=secret_key, + host=host, + project_id=project_id, + ) + print("โœ… Langfuse adapter created successfully") + except ImportError as e: + print(f"โŒ Error: {e}") + print("Install Langfuse dependencies: pip install 'eval-protocol[langfuse]'") + return + except Exception as e: + print(f"โŒ Failed to create adapter: {e}") + return + + # Example 1: Get recent evaluation rows + print("\n๐Ÿ“Š Example 1: Get recent evaluation rows") + try: + rows = list(adapter.get_evaluation_rows( + limit=5, + from_timestamp=datetime.now() - timedelta(days=7), + include_tool_calls=True, + )) + + print(f"Retrieved {len(rows)} evaluation rows") + for i, row in enumerate(rows): + print(f" Row {i+1}:") + print(f" - ID: {row.input_metadata.row_id if row.input_metadata else 'N/A'}") + print(f" - Messages: {len(row.messages)}") + print(f" - Has tools: {'Yes' if row.tools else 'No'}") + print(f" - Ground truth: {'Yes' if row.ground_truth else 'No'}") + + # Show first message content (truncated) + if row.messages: + content = row.messages[0].content or "" + preview = content[:100] + "..." if len(content) > 100 else content + print(f" - First message: {preview}") + print() + + except Exception as e: + print(f"โŒ Error retrieving rows: {e}") + + # Example 2: Filter by specific criteria + print("\n๐Ÿ” Example 2: Filter by specific criteria") + try: + rows = list(adapter.get_evaluation_rows( + limit=3, + tags=["production"], # Filter by tags if available + include_tool_calls=True, + )) + + print(f"Retrieved {len(rows)} rows with 'production' tag") + + except Exception as e: + print(f"โŒ Error with filtered query: {e}") + + # Example 3: Get specific traces by ID + print("\n๐ŸŽฏ Example 3: Get specific traces by ID") + try: + # Replace with actual trace IDs from your Langfuse deployment + trace_ids = ["trace_id_1", "trace_id_2"] # These would be real IDs + + rows = list(adapter.get_evaluation_rows_by_ids( + trace_ids=trace_ids, + include_tool_calls=True, + )) + + print(f"Retrieved {len(rows)} rows by specific IDs") + + except Exception as e: + print(f"โŒ Error retrieving specific traces: {e}") + + # Example 4: Extract different types of conversations + print("\n๐Ÿ’ฌ Example 4: Analyze conversation types") + try: + rows = list(adapter.get_evaluation_rows(limit=10, include_tool_calls=True)) + + chat_only = [] + tool_calling = [] + + for row in rows: + if row.tools and any(msg.tool_calls for msg in row.messages if hasattr(msg, 'tool_calls') and msg.tool_calls): + tool_calling.append(row) + else: + chat_only.append(row) + + print(f"Chat-only conversations: {len(chat_only)}") + print(f"Tool calling conversations: {len(tool_calling)}") + + # Show example of tool calling conversation + if tool_calling: + row = tool_calling[0] + print(f"\n๐Ÿ”ง Example tool calling conversation:") + for i, msg in enumerate(row.messages): + print(f" {i+1}. {msg.role}: {msg.content[:50] if msg.content else '[No content]'}...") + if hasattr(msg, 'tool_calls') and msg.tool_calls: + for tool_call in msg.tool_calls: + print(f" ๐Ÿ›  Tool call: {tool_call}") + + except Exception as e: + print(f"โŒ Error analyzing conversation types: {e}") + + +def demonstrate_evaluation_integration(): + """Show how to use Langfuse data with evaluation functions.""" + print("\n๐Ÿงช Integration with Evaluation Functions") + + # This would typically be in a separate evaluation script + try: + from eval_protocol.rewards.math import math_reward + + # Create adapter (reuse configuration from main example) + adapter = create_langfuse_adapter( + public_key=os.getenv("LANGFUSE_PUBLIC_KEY", "your_public_key_here"), + secret_key=os.getenv("LANGFUSE_SECRET_KEY", "your_secret_key_here"), + host=os.getenv("LANGFUSE_HOST", "https://langfuse-web-prod-zfdbl7ykrq-uc.a.run.app"), + project_id=os.getenv("LANGFUSE_PROJECT_ID", "cmdj5yxhk0006s6022cyi0prv"), + ) + + # Get data and evaluate + rows = list(adapter.get_evaluation_rows(limit=3)) + + for i, row in enumerate(rows): + print(f"\nEvaluating row {i+1}:") + + # Only evaluate if we have ground truth + if row.ground_truth: + try: + result = math_reward( + messages=row.messages, + ground_truth=row.ground_truth, + ) + print(f" Math evaluation score: {result.score:.2f}") + print(f" Reason: {result.reason}") + except Exception as e: + print(f" โŒ Evaluation failed: {e}") + else: + print(f" โš ๏ธ No ground truth available for evaluation") + + except ImportError: + print("Math reward function not available") + except Exception as e: + print(f"โŒ Error in evaluation integration: {e}") + + +if __name__ == "__main__": + print("๐Ÿš€ Langfuse Adapter Example") + print("=" * 50) + + # Check if credentials are set + if not all([ + os.getenv("LANGFUSE_PUBLIC_KEY"), + os.getenv("LANGFUSE_SECRET_KEY"), + ]): + print("โš ๏ธ To run this example with real data, set environment variables:") + print(" export LANGFUSE_PUBLIC_KEY='your_public_key'") + print(" export LANGFUSE_SECRET_KEY='your_secret_key'") + print(" export LANGFUSE_HOST='your_langfuse_host' # optional") + print(" export LANGFUSE_PROJECT_ID='your_project_id' # optional") + print() + + main() + demonstrate_evaluation_integration() + + print("\nโœ… Example completed!") + print("\nNext steps:") + print("1. Set up your Langfuse credentials") + print("2. Modify the filters and parameters to match your data") + print("3. Integrate with your evaluation pipeline") + print("4. Use the converted EvaluationRow data for training or evaluation") \ No newline at end of file diff --git a/examples/math_example/main.py b/examples/math_example/main.py index fac5e396..c15eef70 100644 --- a/examples/math_example/main.py +++ b/examples/math_example/main.py @@ -20,8 +20,8 @@ def check_think_answer_format(text: str) -> bool: """Check if text follows ...... format.""" if not text: return False - pattern = r"[\s\S]*?[\s\S]*?[\s\S]*?" - return bool(re.search(pattern, text)) + pattern = r"^[\s\S]*?\s*[\s\S]*?$" + return bool(re.match(pattern, text.strip())) @reward_function @@ -59,12 +59,13 @@ def evaluate( format_correct = check_think_answer_format(assistant_response) format_score = 1.0 if format_correct else 0.0 - # For math_example, accuracy takes priority - if accuracy is 0, overall score is 0 - # If accuracy is 1, then format can contribute to the score + # The combined score is a weighted average of accuracy and format + weights = {"accuracy": 0.8, "format": 0.2} + combined_score = (accuracy_result.score * weights["accuracy"]) + (format_score * weights["format"]) + + # If accuracy is 0, the overall score is 0, regardless of format. if accuracy_result.score == 0.0: combined_score = 0.0 - else: - combined_score = accuracy_result.score # Only accuracy matters for math_example # Create metrics structure expected by tests metrics = { diff --git a/pyproject.toml b/pyproject.toml index 4c802f6c..5211c843 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,6 +100,18 @@ box2d = [ "gymnasium[box2d]>=0.29.0", "Pillow", ] +langfuse = [ + "langfuse>=2.0.0", +] +huggingface = [ + "datasets>=2.0.0", + "transformers>=4.0.0", +] +adapters = [ + "langfuse>=2.0.0", + "datasets>=2.0.0", + "transformers>=4.0.0", +] [project.scripts] fireworks-reward = "eval_protocol.cli:main" diff --git a/tests/__init__.py b/tests/__init__.py index 1ad035d4..e69de29b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ -"""Tests for the eval-protocol package.""" diff --git a/tests/conftest.py b/tests/conftest.py index 9ec80cc3..6a3526a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,65 +1,16 @@ -""" -Pytest configuration file for Eval Protocol tests. -""" - import sys -from typing import Any, Dict, List, Optional - +from pathlib import Path import pytest -from eval_protocol.models import EvaluateResult, MetricResult +# Add the project root to the Python path +project_root = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(project_root)) -# Check if e2b is available and skip related tests if not +# Import _HAS_E2B to create skip decorator try: - import e2b - - HAS_E2B = True + from eval_protocol.rewards.code_execution import _HAS_E2B except ImportError: - HAS_E2B = False - -# Mark to skip tests requiring e2b -skip_e2b = pytest.mark.skipif(not HAS_E2B, reason="e2b module not installed") - - -@pytest.fixture -def sample_messages(): - """Sample conversation messages.""" - return [ - {"role": "user", "content": "What is the weather like today?"}, - { - "role": "assistant", - "content": "I don't have real-time weather data. You should check a weather service.", - }, - ] - - -@pytest.fixture -def sample_ground_truth_messages(sample_messages): # Renamed fixture - """Sample ground truth messages (e.g., user context or expected full conversation).""" - return [sample_messages[0]] # Keeping the same logic for now, assuming it represents context - - -@pytest.fixture -def sample_reward_output(): - """Sample reward output structure.""" - metrics = { - "helpfulness": MetricResult(score=0.7, reason="Response acknowledges limitations", success=True), - "accuracy": MetricResult( - score=0.8, - reason="Response correctly states lack of access to weather data", - success=True, - ), - } - return EvaluateResult(score=0.75, reason="Overall assessment", metrics=metrics) - + _HAS_E2B = False -@pytest.fixture -def sample_function_call_schema(): - """Sample function call schema for testing.""" - return { - "name": "get_weather", - "arguments": { - "location": {"type": "string"}, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - } +# Decorator to skip E2B tests when E2B is not available +skip_e2b = pytest.mark.skipif(not _HAS_E2B, reason="E2B not installed") diff --git a/tests/test_adapters_e2e.py b/tests/test_adapters_e2e.py new file mode 100644 index 00000000..a598dff7 --- /dev/null +++ b/tests/test_adapters_e2e.py @@ -0,0 +1,447 @@ +""" +End-to-end tests for adapters with real data sources. + +These tests connect to actual external services and verify that adapters +can pull data and convert it to EvaluationRow format correctly. +""" + +import os +import pytest +from datetime import datetime, timedelta +from typing import Dict, Any + +from eval_protocol.models import EvaluationRow, Message, InputMetadata + + +class TestLangfuseAdapterE2E: + """End-to-end tests for Langfuse adapter with real deployment.""" + + def _get_langfuse_credentials(self): + """Get Langfuse credentials from environment.""" + public_key = os.getenv("LANGFUSE_PUBLIC_KEY") + secret_key = os.getenv("LANGFUSE_SECRET_KEY") + host = os.getenv("LANGFUSE_HOST", "https://langfuse-web-prod-zfdbl7ykrq-uc.a.run.app") + project_id = os.getenv("LANGFUSE_PROJECT_ID", "cmdj5yxhk0006s6022cyi0prv") + + return public_key, secret_key, host, project_id + + @pytest.mark.skipif( + not all([ + os.getenv("LANGFUSE_PUBLIC_KEY"), + os.getenv("LANGFUSE_SECRET_KEY"), + ]), + reason="Langfuse credentials not available in environment" + ) + def test_langfuse_adapter_real_connection(self): + """Test that we can connect to real Langfuse deployment and pull data.""" + try: + from eval_protocol.adapters.langfuse import create_langfuse_adapter + except ImportError: + pytest.skip("Langfuse dependencies not installed") + + public_key, secret_key, host, project_id = self._get_langfuse_credentials() + + # Create adapter + adapter = create_langfuse_adapter( + public_key=public_key, + secret_key=secret_key, + host=host, + project_id=project_id, + ) + + # Test basic connection by trying to get a small number of traces + rows = list(adapter.get_evaluation_rows(limit=3)) + + # Verify we got some data + assert isinstance(rows, list), "Should return a list of rows" + print(f"Retrieved {len(rows)} evaluation rows from Langfuse") + + # Verify each row is properly formatted + for i, row in enumerate(rows): + assert isinstance(row, EvaluationRow), f"Row {i} should be EvaluationRow" + assert isinstance(row.messages, list), f"Row {i} should have messages list" + assert len(row.messages) > 0, f"Row {i} should have at least one message" + + # Verify messages are properly formatted + for j, msg in enumerate(row.messages): + assert isinstance(msg, Message), f"Row {i} message {j} should be Message object" + assert hasattr(msg, 'role'), f"Row {i} message {j} should have role" + assert msg.role in ['user', 'assistant', 'system', 'tool'], f"Row {i} message {j} has invalid role: {msg.role}" + + # Verify metadata + if row.input_metadata: + assert isinstance(row.input_metadata, InputMetadata), f"Row {i} should have InputMetadata" + assert row.input_metadata.row_id, f"Row {i} should have row_id" + print(f" Row {i}: ID={row.input_metadata.row_id}, Messages={len(row.messages)}") + + print(f" Row {i}: {len(row.messages)} messages, Tools={'Yes' if row.tools else 'No'}") + + @pytest.mark.skipif( + not all([ + os.getenv("LANGFUSE_PUBLIC_KEY"), + os.getenv("LANGFUSE_SECRET_KEY"), + ]), + reason="Langfuse credentials not available" + ) + def test_langfuse_adapter_with_filters(self): + """Test Langfuse adapter with various filters.""" + try: + from eval_protocol.adapters.langfuse import create_langfuse_adapter + except ImportError: + pytest.skip("Langfuse dependencies not installed") + + public_key, secret_key, host, project_id = self._get_langfuse_credentials() + + adapter = create_langfuse_adapter( + public_key=public_key, + secret_key=secret_key, + host=host, + project_id=project_id, + ) + + # Test with time filter (last 7 days) + recent_rows = list(adapter.get_evaluation_rows( + limit=5, + from_timestamp=datetime.now() - timedelta(days=7), + include_tool_calls=True, + )) + + print(f"Recent rows (last 7 days): {len(recent_rows)}") + + # Verify tool calling data is preserved + tool_calling_rows = [row for row in recent_rows if row.tools] + print(f"Rows with tool definitions: {len(tool_calling_rows)}") + + # Test specific filtering + try: + # This might not return data if no traces match, which is fine + tagged_rows = list(adapter.get_evaluation_rows( + limit=2, + tags=["production"], # May not exist, that's OK + )) + print(f"Tagged rows: {len(tagged_rows)}") + except Exception as e: + print(f"Tagged query failed (expected if no tags): {e}") + + @pytest.mark.skipif( + not all([ + os.getenv("LANGFUSE_PUBLIC_KEY"), + os.getenv("LANGFUSE_SECRET_KEY"), + ]), + reason="Langfuse credentials not available" + ) + def test_langfuse_conversation_analysis(self): + """Test analysis of conversation types from Langfuse.""" + try: + from eval_protocol.adapters.langfuse import create_langfuse_adapter + except ImportError: + pytest.skip("Langfuse dependencies not installed") + + public_key, secret_key, host, project_id = self._get_langfuse_credentials() + + adapter = create_langfuse_adapter( + public_key=public_key, + secret_key=secret_key, + host=host, + project_id=project_id, + ) + + # Get more data for analysis + rows = list(adapter.get_evaluation_rows(limit=10, include_tool_calls=True)) + + # Analyze conversation patterns + chat_only = [] + tool_calling = [] + multi_turn = [] + + for row in rows: + # Check for tool calling + has_tools = ( + row.tools or + any(hasattr(msg, 'tool_calls') and msg.tool_calls for msg in row.messages) or + any(msg.role == 'tool' for msg in row.messages) + ) + + if has_tools: + tool_calling.append(row) + else: + chat_only.append(row) + + # Check for multi-turn conversations + if len(row.messages) > 2: # More than user + assistant + multi_turn.append(row) + + print(f"Analysis of {len(rows)} conversations:") + print(f" Chat-only: {len(chat_only)}") + print(f" Tool calling: {len(tool_calling)}") + print(f" Multi-turn: {len(multi_turn)}") + + # Show example of each type if available + if chat_only: + row = chat_only[0] + print(f" Example chat: {len(row.messages)} messages") + + if tool_calling: + row = tool_calling[0] + print(f" Example tool calling: {len(row.messages)} messages, {len(row.tools or [])} tools") + + +class TestHuggingFaceAdapterE2E: + """End-to-end tests for HuggingFace adapter with real datasets.""" + + def test_gsm8k_adapter_real_data(self): + """Test loading real GSM8K data and converting to EvaluationRow.""" + try: + from eval_protocol.adapters.huggingface import create_huggingface_adapter + except ImportError: + pytest.skip("HuggingFace dependencies not installed") + + def gsm8k_transform(row: Dict[str, Any]) -> Dict[str, Any]: + """Transform GSM8K row to our format.""" + return { + 'messages': [ + {'role': 'system', 'content': 'You are a helpful assistant that solves math problems step by step.'}, + {'role': 'user', 'content': row['question']}, + ], + 'ground_truth': row['answer'], + 'metadata': { + 'dataset': 'gsm8k', + 'original_question': row['question'], + 'original_answer': row['answer'], + } + } + + # Create adapter with transform function + adapter = create_huggingface_adapter( + dataset_id="gsm8k", + config_name="main", + transform_fn=gsm8k_transform, + ) + + # Test loading data + rows = list(adapter.get_evaluation_rows(split="test", limit=5)) + + # Verify we got data + assert len(rows) > 0, "Should retrieve some GSM8K data" + print(f"Retrieved {len(rows)} GSM8K evaluation rows") + + # Verify each row is properly formatted + for i, row in enumerate(rows): + assert isinstance(row, EvaluationRow), f"Row {i} should be EvaluationRow" + assert isinstance(row.messages, list), f"Row {i} should have messages" + assert len(row.messages) >= 2, f"Row {i} should have system + user messages" + + # Check system prompt + system_msg = row.messages[0] + assert system_msg.role == 'system', f"Row {i} first message should be system" + assert 'math problems' in system_msg.content.lower(), f"Row {i} should have math system prompt" + + # Check user question + user_msg = row.messages[1] + assert user_msg.role == 'user', f"Row {i} second message should be user" + assert len(user_msg.content) > 0, f"Row {i} should have non-empty question" + + # Check ground truth + assert row.ground_truth, f"Row {i} should have ground truth answer" + + # Check metadata + assert row.input_metadata, f"Row {i} should have metadata" + assert row.input_metadata.dataset_info, f"Row {i} should have dataset info" + + print(f" Row {i}: Question length={len(user_msg.content)}, Answer length={len(row.ground_truth)}") + + def test_math_dataset_real_data(self): + """Test loading real MATH competition dataset.""" + try: + from eval_protocol.adapters.huggingface import create_huggingface_adapter + except ImportError: + pytest.skip("HuggingFace dependencies not installed") + + def math_transform(row: Dict[str, Any]) -> Dict[str, Any]: + """Transform MATH dataset row.""" + return { + 'messages': [ + {'role': 'system', 'content': 'You are an expert mathematician. Solve this step by step.'}, + {'role': 'user', 'content': row['problem']}, + ], + 'ground_truth': row['solution'], + 'metadata': { + 'dataset': 'hendrycks_math', + 'type': row.get('type', 'unknown'), + 'level': row.get('level', 'unknown'), + 'original_problem': row['problem'], + 'original_solution': row['solution'], + } + } + + # Create adapter + adapter = create_huggingface_adapter( + dataset_id="SuperSecureHuman/competition_math_hf_dataset", + transform_fn=math_transform, + ) + + # Test loading data + rows = list(adapter.get_evaluation_rows(split="test", limit=3)) + + # Verify data + assert len(rows) > 0, "Should retrieve MATH dataset data" + print(f"Retrieved {len(rows)} MATH dataset evaluation rows") + + for i, row in enumerate(rows): + assert isinstance(row, EvaluationRow), f"Row {i} should be EvaluationRow" + assert len(row.messages) >= 2, f"Row {i} should have system + user messages" + assert row.ground_truth, f"Row {i} should have solution" + + # Check for MATH-specific metadata + dataset_info = row.input_metadata.dataset_info + assert 'type' in dataset_info, f"Row {i} should have problem type" + assert 'level' in dataset_info, f"Row {i} should have difficulty level" + + print(f" Row {i}: Type={dataset_info.get('type')}, Level={dataset_info.get('level')}") + + def test_custom_dataset_transform(self): + """Test adapter with a completely custom transformation.""" + try: + from eval_protocol.adapters.huggingface import create_huggingface_adapter + except ImportError: + pytest.skip("HuggingFace dependencies not installed") + + def squad_transform(row: Dict[str, Any]) -> Dict[str, Any]: + """Custom transform for SQuAD dataset.""" + context = row['context'] + question = row['question'] + answers = row['answers'] + + # Get first answer + answer_text = answers['text'][0] if answers['text'] else "No answer" + + return { + 'messages': [ + {'role': 'system', 'content': 'Answer the question based on the given context.'}, + {'role': 'user', 'content': f"Context: {context}\n\nQuestion: {question}"}, + ], + 'ground_truth': answer_text, + 'metadata': { + 'dataset': 'squad', + 'context_length': len(context), + 'question_length': len(question), + 'num_answers': len(answers['text']), + } + } + + # Create adapter for SQuAD + adapter = create_huggingface_adapter( + dataset_id="squad", + transform_fn=squad_transform, + ) + + # Test loading + rows = list(adapter.get_evaluation_rows(split="validation", limit=2)) + + assert len(rows) > 0, "Should retrieve SQuAD data" + print(f"Retrieved {len(rows)} SQuAD evaluation rows") + + for i, row in enumerate(rows): + assert isinstance(row, EvaluationRow), f"Row {i} should be EvaluationRow" + user_msg = next(msg for msg in row.messages if msg.role == 'user') + assert 'Context:' in user_msg.content, f"Row {i} should have context" + assert 'Question:' in user_msg.content, f"Row {i} should have question" + + dataset_info = row.input_metadata.dataset_info + print(f" Row {i}: Context length={dataset_info.get('context_length')}") + + +def test_adapters_integration(): + """Test that adapters work with evaluation pipeline.""" + print("Testing adapter integration with evaluation pipeline...") + + # This test doesn't require external credentials + try: + from eval_protocol.adapters.huggingface import create_huggingface_adapter + from eval_protocol.rewards.accuracy import accuracy_reward + except ImportError as e: + pytest.skip(f"Dependencies not available: {e}") + + def simple_transform(row: Dict[str, Any]) -> Dict[str, Any]: + """Simple transform for testing.""" + return { + 'messages': [ + {'role': 'user', 'content': row['question']}, + {'role': 'assistant', 'content': 'Test response'}, # Simulated response + ], + 'ground_truth': row['answer'], + 'metadata': {'test': True} + } + + # Create adapter with GSM8K (small sample) + adapter = create_huggingface_adapter( + dataset_id="gsm8k", + config_name="main", + transform_fn=simple_transform, + ) + + # Get one row + rows = list(adapter.get_evaluation_rows(split="test", limit=1)) + assert len(rows) == 1, "Should get exactly one row" + + row = rows[0] + + # Test evaluation + result = accuracy_reward( + messages=row.messages, + ground_truth=row.ground_truth, + ) + + assert hasattr(result, 'score'), "Should have evaluation score" + assert 0 <= result.score <= 1, "Score should be between 0 and 1" + + print(f"Integration test successful: Score={result.score}") + + +if __name__ == "__main__": + # Run tests manually for development + import sys + + print("Running Langfuse E2E tests...") + if all([os.getenv("LANGFUSE_PUBLIC_KEY"), os.getenv("LANGFUSE_SECRET_KEY")]): + try: + test_langfuse = TestLangfuseAdapterE2E() + test_langfuse.test_langfuse_adapter_real_connection() + test_langfuse.test_langfuse_adapter_with_filters() + test_langfuse.test_langfuse_conversation_analysis() + print("โœ… Langfuse tests passed!") + except Exception as e: + print(f"โš ๏ธ Langfuse tests failed (API may have changed): {e}") + print(" This is expected if Langfuse API has changed - the adapter needs updating") + else: + print("โš ๏ธ Skipping Langfuse tests (credentials not available)") + + print("\nRunning HuggingFace E2E tests...") + try: + test_hf = TestHuggingFaceAdapterE2E() + test_hf.test_gsm8k_adapter_real_data() + print("โœ… GSM8K adapter test passed!") + + # Skip MATH dataset test for now (dataset may not be available) + try: + test_hf.test_math_dataset_real_data() + print("โœ… MATH dataset test passed!") + except Exception as e: + print(f"โš ๏ธ MATH dataset test failed (dataset may not be available): {e}") + + # Skip SQuAD test for now (focus on core functionality) + try: + test_hf.test_custom_dataset_transform() + print("โœ… Custom dataset test passed!") + except Exception as e: + print(f"โš ๏ธ Custom dataset test failed: {e}") + print("โœ… HuggingFace tests passed!") + except Exception as e: + print(f"โŒ HuggingFace tests failed: {e}") + sys.exit(1) + + print("\nRunning integration test...") + test_adapters_integration() + print("โœ… Integration test passed!") + + print("\n๐ŸŽ‰ All E2E tests completed successfully!") \ No newline at end of file diff --git a/tests/test_cli_agent.py b/tests/test_cli_agent.py index 28283d91..cc50376f 100644 --- a/tests/test_cli_agent.py +++ b/tests/test_cli_agent.py @@ -5,6 +5,7 @@ import argparse import asyncio # Added import import json +import logging from unittest.mock import ( # Added AsyncMock and Mock AsyncMock, MagicMock, @@ -38,6 +39,9 @@ class TestAgentEvalCommand: @patch("eval_protocol.cli_commands.agent_eval_cmd.TaskManager") @patch("eval_protocol.cli_commands.agent_eval_cmd.Path") def test_agent_eval_success_yaml(self, MockPath, MockTaskManager, caplog): + # Configure caplog to capture logs from the agent_eval logger + caplog.set_level(logging.INFO, logger="agent_eval") + # Setup Path mock mock_path_instance = Mock() MockPath.return_value = mock_path_instance diff --git a/tests/test_examples_end_to_end.py b/tests/test_examples_end_to_end.py index 9f4eca48..d4edac17 100644 --- a/tests/test_examples_end_to_end.py +++ b/tests/test_examples_end_to_end.py @@ -144,7 +144,7 @@ def test_math_example(temp_examples_dir, mock_env_variables): Message(role="user", content="What is 2+2?"), Message( role="assistant", - content="The user is asking for the sum of 2 and 2.The final answer is \\boxed{4}", + content="I need to solve this arithmetic problem.The final answer is \\boxed{4}", ), ] ground_truth_correct = "The final answer is \\boxed{4}" @@ -165,7 +165,7 @@ def test_math_example(temp_examples_dir, mock_env_variables): Message(role="user", content="What is 2+2?"), Message( role="assistant", - content="The user is asking for the sum of 2 and 2.The final answer is \\boxed{5}", + content="I need to solve this arithmetic problem.The final answer is \\boxed{5}", ), ] # Ground truth is still 4 @@ -186,7 +186,7 @@ def test_math_example(temp_examples_dir, mock_env_variables): ] result_incorrect_fmt = math_module.evaluate(messages=messages_incorrect_fmt, ground_truth=ground_truth_correct) - assert result_incorrect_fmt["score"] == 1.0 # Accuracy is 1.0 + assert result_incorrect_fmt["score"] == 0.8 # Combined score: (1.0 * 0.8) + (0.0 * 0.2) = 0.8 assert result_incorrect_fmt["is_score_valid"] is True # Asserting extracted answers from the result object directly might fail assert result_incorrect_fmt["metrics"]["accuracy_reward"]["score"] == 1.0 @@ -269,7 +269,7 @@ def test_math_example(temp_examples_dir, mock_env_variables): messages=messages_only_answer_tag, ground_truth=ground_truth_simple_gt ) - assert result_only_answer_tag["score"] == 1.0 # Accuracy is fine + assert result_only_answer_tag["score"] == 0.8 # Combined score: (1.0 * 0.8) + (0.0 * 0.2) = 0.8 assert result_only_answer_tag["is_score_valid"] is True # Asserting extracted answers from the result object directly might fail assert result_only_answer_tag["metrics"]["accuracy_reward"]["score"] == 1.0 diff --git a/uv.lock b/uv.lock index f8abaa0f..82ba1265 100644 --- a/uv.lock +++ b/uv.lock @@ -348,6 +348,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b8/3fe70c75fe32afc4bb507f75563d39bc5642255d1d94f1f23604725780bf/babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2", size = 10182537, upload-time = "2025-02-01T15:17:37.39Z" }, ] +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + [[package]] name = "backports-asyncio-runner" version = "1.2.0" @@ -1089,6 +1098,11 @@ box2d = [ { name = "pillow" }, { name = "swig" }, ] +adapters = [ + { name = "datasets" }, + { name = "langfuse" }, + { name = "transformers" }, +] dev = [ { name = "autopep8" }, { name = "black" }, @@ -1121,6 +1135,13 @@ dev = [ fireworks = [ { name = "fireworks-ai" }, ] +huggingface = [ + { name = "datasets" }, + { name = "transformers" }, +] +langfuse = [ + { name = "langfuse" }, +] openevals = [ { name = "openevals" }, ] @@ -1150,6 +1171,8 @@ requires-dist = [ { name = "build", marker = "extra == 'dev'" }, { name = "dataclasses-json", specifier = ">=0.5.7" }, { name = "datasets" }, + { name = "datasets", marker = "extra == 'adapters'", specifier = ">=2.0.0" }, + { name = "datasets", marker = "extra == 'huggingface'", specifier = ">=2.0.0" }, { name = "deepdiff", specifier = ">=6.0.0" }, { name = "docker", marker = "extra == 'dev'", specifier = "==7.1.0" }, { name = "docstring-parser", specifier = ">=0.15" }, @@ -1168,6 +1191,8 @@ requires-dist = [ { name = "isort", marker = "extra == 'dev'", specifier = ">=5.0.0" }, { name = "jupyter", specifier = ">=1.1.1" }, { name = "jupyter", marker = "extra == 'dev'", specifier = ">=1.1.1" }, + { name = "langfuse", marker = "extra == 'adapters'", specifier = ">=2.0.0" }, + { name = "langfuse", marker = "extra == 'langfuse'", specifier = ">=2.0.0" }, { name = "litellm", specifier = ">=1.0.0" }, { name = "loguru", specifier = ">=0.6.0" }, { name = "mcp", specifier = ">=1.9.2" }, @@ -1195,7 +1220,9 @@ requires-dist = [ { name = "swig", marker = "extra == 'box2d'" }, { name = "toml", specifier = ">=0.10.0" }, { name = "torch", marker = "extra == 'trl'", specifier = ">=1.9" }, + { name = "transformers", marker = "extra == 'adapters'", specifier = ">=4.0.0" }, { name = "transformers", marker = "extra == 'dev'", specifier = ">=4.0.0" }, + { name = "transformers", marker = "extra == 'huggingface'", specifier = ">=4.0.0" }, { name = "transformers", marker = "extra == 'trl'", specifier = ">=4.0.0" }, { name = "trl", marker = "extra == 'trl'", specifier = ">=0.7.0" }, { name = "twine", marker = "extra == 'dev'" }, @@ -1207,7 +1234,7 @@ requires-dist = [ { name = "versioneer", marker = "extra == 'dev'", specifier = ">=0.20" }, { name = "werkzeug", marker = "extra == 'dev'", specifier = ">=2.0.0" }, ] -provides-extras = ["dev", "trl", "openevals", "fireworks", "box2d"] +provides-extras = ["dev", "trl", "openevals", "fireworks", "box2d","langfuse", "huggingface", "adapters"] [package.metadata.requires-dev] dev = [ @@ -1444,6 +1471,18 @@ http = [ { name = "aiohttp" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/24/33db22342cf4a2ea27c9955e6713140fedd51e8b141b5ce5260897020f1a/googleapis_common_protos-1.70.0.tar.gz", hash = "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257", size = 145903, upload-time = "2025-04-14T10:17:02.924Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" }, +] + [[package]] name = "greenlet" version = "3.2.3" @@ -2409,6 +2448,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/52/7638394b88bc15083fd2c3752a843784d9d2d110d68fed6437c8607fb749/langchain_text_splitters-0.3.9-py3-none-any.whl", hash = "sha256:cee0bb816211584ea79cc79927317c358543f40404bcfdd69e69ba3ccde54401", size = 33314, upload-time = "2025-07-24T14:38:43.953Z" }, ] +[[package]] +name = "langfuse" +version = "3.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backoff" }, + { name = "httpx" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-sdk" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/0d/8fc51099cf337fb3b56cb7d305074bc0223c62e1ccabf80cc6285ccf5b31/langfuse-3.2.1.tar.gz", hash = "sha256:f79b0380dfcf52c7525bb5d7f8e9d8786a6fc8b37867def047bb388930a7beb3", size = 153369, upload-time = "2025-07-16T09:50:28.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/b0/8f08df3f0fa584c4132937690c6dd33e0a116f963ecf2b35567f614e0ca7/langfuse-3.2.1-py3-none-any.whl", hash = "sha256:07a84e8c1eed6ac8e149bdda1431fd866e4aee741b66124316336fb2bc7e6a32", size = 299315, upload-time = "2025-07-16T09:50:26.582Z" }, +] + [[package]] name = "langsmith" version = "0.4.8" @@ -3366,6 +3425,119 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5e/10/5a340aa03999f8e1e89b7bb7f34de27d195219f207a2e311e8f1655d1075/openevals-0.1.0-py3-none-any.whl", hash = "sha256:214b53197b1becff74279ea063c8752f8887a81afda700477639c19a0d683647", size = 62693, upload-time = "2025-05-08T23:30:08.22Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/d2/c782c88b8afbf961d6972428821c302bd1e9e7bc361352172f0ca31296e2/opentelemetry_api-1.36.0.tar.gz", hash = "sha256:9a72572b9c416d004d492cbc6e61962c0501eaf945ece9b5a0f56597d8348aa0", size = 64780, upload-time = "2025-07-29T15:12:06.02Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/ee/6b08dde0a022c463b88f55ae81149584b125a42183407dc1045c486cc870/opentelemetry_api-1.36.0-py3-none-any.whl", hash = "sha256:02f20bcacf666e1333b6b1f04e647dc1d5111f86b8e510238fcc56d7762cda8c", size = 65564, upload-time = "2025-07-29T15:11:47.998Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/7f/d31294ac28d567a14aefd855756bab79fed69c5a75df712f228f10c47e04/opentelemetry_exporter_otlp-1.36.0.tar.gz", hash = "sha256:72f166ea5a8923ac42889337f903e93af57db8893de200369b07401e98e4e06b", size = 6144, upload-time = "2025-07-29T15:12:07.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/a2/8966111a285124f3d6156a663ddf2aeddd52843c1a3d6b56cbd9b6c3fd0e/opentelemetry_exporter_otlp-1.36.0-py3-none-any.whl", hash = "sha256:de93b7c45bcc78296998775d52add7c63729e83ef2cd6560730a6b336d7f6494", size = 7018, upload-time = "2025-07-29T15:11:50.498Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/da/7747e57eb341c59886052d733072bc878424bf20f1d8cf203d508bbece5b/opentelemetry_exporter_otlp_proto_common-1.36.0.tar.gz", hash = "sha256:6c496ccbcbe26b04653cecadd92f73659b814c6e3579af157d8716e5f9f25cbf", size = 20302, upload-time = "2025-07-29T15:12:07.71Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/ed/22290dca7db78eb32e0101738366b5bbda00d0407f00feffb9bf8c3fdf87/opentelemetry_exporter_otlp_proto_common-1.36.0-py3-none-any.whl", hash = "sha256:0fc002a6ed63eac235ada9aa7056e5492e9a71728214a61745f6ad04b923f840", size = 18349, upload-time = "2025-07-29T15:11:51.327Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/6f/6c1b0bdd0446e5532294d1d41bf11fbaea39c8a2423a4cdfe4fe6b708127/opentelemetry_exporter_otlp_proto_grpc-1.36.0.tar.gz", hash = "sha256:b281afbf7036b325b3588b5b6c8bb175069e3978d1bd24071f4a59d04c1e5bbf", size = 23822, upload-time = "2025-07-29T15:12:08.292Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/67/5f6bd188d66d0fd8e81e681bbf5822e53eb150034e2611dd2b935d3ab61a/opentelemetry_exporter_otlp_proto_grpc-1.36.0-py3-none-any.whl", hash = "sha256:734e841fc6a5d6f30e7be4d8053adb703c70ca80c562ae24e8083a28fadef211", size = 18828, upload-time = "2025-07-29T15:11:52.235Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/85/6632e7e5700ba1ce5b8a065315f92c1e6d787ccc4fb2bdab15139eaefc82/opentelemetry_exporter_otlp_proto_http-1.36.0.tar.gz", hash = "sha256:dd3637f72f774b9fc9608ab1ac479f8b44d09b6fb5b2f3df68a24ad1da7d356e", size = 16213, upload-time = "2025-07-29T15:12:08.932Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/41/a680d38b34f8f5ddbd78ed9f0042e1cc712d58ec7531924d71cb1e6c629d/opentelemetry_exporter_otlp_proto_http-1.36.0-py3-none-any.whl", hash = "sha256:3d769f68e2267e7abe4527f70deb6f598f40be3ea34c6adc35789bea94a32902", size = 18752, upload-time = "2025-07-29T15:11:53.164Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/02/f6556142301d136e3b7e95ab8ea6a5d9dc28d879a99f3dd673b5f97dca06/opentelemetry_proto-1.36.0.tar.gz", hash = "sha256:0f10b3c72f74c91e0764a5ec88fd8f1c368ea5d9c64639fb455e2854ef87dd2f", size = 46152, upload-time = "2025-07-29T15:12:15.717Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/57/3361e06136225be8180e879199caea520f38026f8071366241ac458beb8d/opentelemetry_proto-1.36.0-py3-none-any.whl", hash = "sha256:151b3bf73a09f94afc658497cf77d45a565606f62ce0c17acb08cd9937ca206e", size = 72537, upload-time = "2025-07-29T15:12:02.243Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.36.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/85/8567a966b85a2d3f971c4d42f781c305b2b91c043724fa08fd37d158e9dc/opentelemetry_sdk-1.36.0.tar.gz", hash = "sha256:19c8c81599f51b71670661ff7495c905d8fdf6976e41622d5245b791b06fa581", size = 162557, upload-time = "2025-07-29T15:12:16.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/59/7bed362ad1137ba5886dac8439e84cd2df6d087be7c09574ece47ae9b22c/opentelemetry_sdk-1.36.0-py3-none-any.whl", hash = "sha256:19fe048b42e98c5c1ffe85b569b7073576ad4ce0bcb6e9b4c6a39e890a6c45fb", size = 119995, upload-time = "2025-07-29T15:12:03.181Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.57b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7e/31/67dfa252ee88476a29200b0255bda8dfc2cf07b56ad66dc9a6221f7dc787/opentelemetry_semantic_conventions-0.57b0.tar.gz", hash = "sha256:609a4a79c7891b4620d64c7aac6898f872d790d75f22019913a660756f27ff32", size = 124225, upload-time = "2025-07-29T15:12:17.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/75/7d591371c6c39c73de5ce5da5a2cc7b72d1d1cd3f8f4638f553c01c37b11/opentelemetry_semantic_conventions-0.57b0-py3-none-any.whl", hash = "sha256:757f7e76293294f124c827e514c2a3144f191ef175b069ce8d1211e1e38e9e78", size = 201627, upload-time = "2025-07-29T15:12:04.174Z" }, +] + [[package]] name = "orderly-set" version = "5.5.0"