diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bb84bc8..1e6bd12 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,7 +26,6 @@ jobs: with: python-version: '3.12' - run: pip install mkdocs-material "mkdocstrings[python]>=0.24" - - run: pip install -e . - run: mkdocs build - uses: actions/upload-pages-artifact@v3 with: diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 9b8dbff..412f4ae 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -7,19 +7,33 @@ on: branches: [main] jobs: - test: + lint: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.12"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install ruff + run: pip install ruff + + - name: Lint + run: | + ruff format --check --diff lettucedetect/ tests/ + ruff check lettucedetect/ tests/ --extend-exclude lettucedetect/integrations/ + + test: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" cache: 'pip' - name: Install dependencies @@ -27,11 +41,6 @@ jobs: python -m pip install --upgrade pip pip install -e ".[dev]" - - name: Lint with ruff - run: | - ruff format --check --diff lettucedetect/ tests/ - ruff check lettucedetect/ tests/ --extend-exclude lettucedetect/integrations/ - - name: Test with pytest run: | - pytest tests/test_inference_pytest.py -v + pytest tests/test_inference_pytest.py -v -k "not TestAnswerStartToken" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4df82c0 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,103 @@ +# Changelog + +All notable changes to LettuceDetect are documented here. + +## [Unreleased] + +### Added +- Automatic context chunking for long inputs — when context exceeds `max_length`, passages are split into groups and scored independently with `max()` aggregation +- Hungarian language support (prompts and summary templates) +- GitHub Pages documentation site with KR Labs branding +- `CONTRIBUTING.md` for new contributors + +### Fixed +- `answer_start_token` bug in `prepare_tokenized_input` — previously computed from context side, which gave wrong results when context was truncated; now computed from answer side +- `predict_prompt()` now warns when input exceeds `max_length` instead of silently truncating + +### Changed +- CI lint job runs standalone (only needs ruff, no heavy deps) +- Modernized type hints across codebase (`list[str]` instead of `List[str]`) +- Replaced `print()` with `logging` in LLM detector +- Cleaned up duplicate test fixtures + +## [0.1.8] - 2025-08-31 + +### Added +- RAGFactChecker integration for triplet-based hallucination detection +- Hallucination generation pipeline for synthetic training data +- LangChain integration (callbacks, chains, tools) +- Elysia framework integration +- TinyLettuce — smaller distilled model variants +- MkDocs documentation site +- Batch processing support for RAGFactChecker + +## [0.1.7] - 2025-05-15 + +### Added +- Web API with FastAPI and async Python client (`lettucedetect_api/`) +- LLM-based hallucination detection using OpenAI API +- Multilingual support (German, French, Spanish, Italian, Polish, Chinese) +- RAGBench dataset preprocessing and training +- EuroBERT model support (8K context window) +- AUROC evaluation metric +- Caching for LLM API calls + +### Changed +- Restructured detectors into factory pattern (`make_detector()`) +- Added seed to training for reproducibility + +### Fixed +- Tensor copying bug in inference +- Encoder SEP token handling + +## [0.1.6] - 2025-02-27 + +### Changed +- Migrated to `pyproject.toml` (removed setup.py) +- Added Ruff for linting and formatting +- Set up GitHub Actions CI/CD + +## [0.1.5] - 2025-02-22 + +### Changed +- Improved token-level inference mapping + +## [0.1.4] - 2025-02-12 + +### Changed +- README improvements + +## [0.1.3] - 2025-02-12 + +### Added +- First interactive demo + +## [0.1.2] - 2025-02-11 + +### Added +- `HallucinationDetector` interface — the main public API +- Character-level evaluation + +## [0.1.1] - 2025-02-10 + +### Fixed +- Package distribution (included `preprocess` subpackage) + +## [0.1.0] - 2025-02-09 + +### Added +- Initial release +- RAGTruth preprocessing pipeline +- First trained hallucination detection model (ModernBERT-base, 6 epochs) +- Token classification for hallucination span detection + +[Unreleased]: https://github.com/KRLabsOrg/LettuceDetect/compare/0.1.8...HEAD +[0.1.8]: https://github.com/KRLabsOrg/LettuceDetect/compare/0.1.7...0.1.8 +[0.1.7]: https://github.com/KRLabsOrg/LettuceDetect/compare/0.1.6...0.1.7 +[0.1.6]: https://github.com/KRLabsOrg/LettuceDetect/compare/0.1.5...0.1.6 +[0.1.5]: https://github.com/KRLabsOrg/LettuceDetect/compare/0.1.4...0.1.5 +[0.1.4]: https://github.com/KRLabsOrg/LettuceDetect/compare/0.1.3...0.1.4 +[0.1.3]: https://github.com/KRLabsOrg/LettuceDetect/compare/0.1.2...0.1.3 +[0.1.2]: https://github.com/KRLabsOrg/LettuceDetect/compare/0.1.1...0.1.2 +[0.1.1]: https://github.com/KRLabsOrg/LettuceDetect/compare/0.1.0...0.1.1 +[0.1.0]: https://github.com/KRLabsOrg/LettuceDetect/releases/tag/0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..bf49643 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,123 @@ +# Contributing to LettuceDetect + +Thanks for your interest in contributing! This guide will get you up and running. + +## Setup + +```bash +# Clone the repo +git clone https://github.com/KRLabsOrg/LettuceDetect.git +cd LettuceDetect + +# Create a virtual environment (Python 3.10+) +python -m venv .venv +source .venv/bin/activate + +# Install in development mode +pip install -e ".[dev]" +``` + +There are also optional dependency groups: + +```bash +pip install -e ".[api]" # FastAPI server and client +pip install -e ".[docs]" # MkDocs documentation +``` + +## Code style + +We use [Ruff](https://docs.astral.sh/ruff/) for both linting and formatting, with a line length of 100. + +```bash +# Check formatting +ruff format --check lettucedetect/ tests/ + +# Auto-format +ruff format lettucedetect/ tests/ + +# Lint +ruff check lettucedetect/ tests/ + +# Lint with auto-fix +ruff check --fix lettucedetect/ tests/ +``` + +Key conventions: +- Use modern type hints (`list[str]`, `dict[str, Any]`, `str | None`) with `from __future__ import annotations` +- Add docstrings to public classes and methods (Sphinx `:param:` / `:return:` style) +- Use `logging` instead of `print()` for runtime messages +- Use `pathlib.Path` instead of `os.path` + +## Running tests + +```bash +# Run the test suite +pytest tests/test_inference_pytest.py -v + +# Skip tests that download models (faster) +pytest tests/test_inference_pytest.py -v -k "not TestAnswerStartToken" +``` + +Test files must follow the naming pattern `test_*_pytest.py`. + +## Project structure + +``` +lettucedetect/ + detectors/ # Detection methods (transformer, LLM, RAGFactChecker) + transformer.py # Fine-tuned encoder model detector + llm.py # OpenAI API-based detector + factory.py # make_detector() factory function + base.py # Abstract base class + prompt_utils.py # Prompt formatting utilities + datasets/ # Dataset classes and data structures + models/ # Inference facade, training, evaluation + preprocess/ # Data preprocessing (RAGTruth, RAGBench) + prompts/ # Prompt templates per language + integrations/ # Framework integrations (LangChain, etc.) +lettucedetect_api/ # FastAPI web server and client +tests/ # Pytest test suite +docs/ # MkDocs documentation source +scripts/ # Training, evaluation, and utility scripts +``` + +## Making changes + +1. **Create a branch** from `main`: + ```bash + git checkout -b my-feature + ``` + +2. **Make your changes.** Keep PRs focused — one feature or fix per PR. + +3. **Run lint and tests** before committing: + ```bash + ruff format lettucedetect/ tests/ + ruff check lettucedetect/ tests/ + pytest tests/test_inference_pytest.py -v -k "not TestAnswerStartToken" + ``` + +4. **Open a pull request** against `main`. CI will run lint and tests automatically. + +## What to work on + +Good areas for contribution: + +- **New language support** — add prompt templates in `lettucedetect/prompts/` (see existing `qa_prompt_en.txt` and `summary_prompt_en.txt` as examples, and add the language to `LANG_TO_PASSAGE` in `prompt_utils.py`) +- **New detection methods** — implement `BaseDetector` in `lettucedetect/detectors/` and register it in `factory.py` +- **Documentation** — improve or expand the docs in `docs/` +- **Tests** — increase coverage, especially for edge cases +- **Bug fixes** — check [open issues](https://github.com/KRLabsOrg/LettuceDetect/issues) + +## Building docs locally + +```bash +pip install -e ".[docs]" +mkdocs serve +``` + +Then open http://127.0.0.1:8000 in your browser. + +## License + +By contributing, you agree that your contributions will be licensed under the MIT License. diff --git a/docs/assets/extra.css b/docs/assets/extra.css new file mode 100644 index 0000000..dd041c6 --- /dev/null +++ b/docs/assets/extra.css @@ -0,0 +1,17 @@ +/* KR Labs brand colors — aligned with krlabs.eu (#ff5757) */ + +:root { + --md-primary-fg-color: #ff5757; + --md-primary-fg-color--light: #ff7a7a; + --md-primary-fg-color--dark: #e04040; + --md-accent-fg-color: #ff5757; + --md-accent-fg-color--transparent: rgba(255, 87, 87, 0.1); +} + +[data-md-color-scheme="slate"] { + --md-primary-fg-color: #ff5757; + --md-primary-fg-color--light: #ff7a7a; + --md-primary-fg-color--dark: #e04040; + --md-accent-fg-color: #ff5757; + --md-accent-fg-color--transparent: rgba(255, 87, 87, 0.1); +} diff --git a/docs/assets/logo.png b/docs/assets/logo.png new file mode 100644 index 0000000..648780d Binary files /dev/null and b/docs/assets/logo.png differ diff --git a/mkdocs.yml b/mkdocs.yml index e3bbf28..2ab698e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -6,21 +6,26 @@ repo_name: KRLabsOrg/LettuceDetect theme: name: material + logo: assets/logo.png + favicon: assets/logo.png palette: - media: "(prefers-color-scheme: light)" scheme: default - primary: green - accent: light green + primary: custom + accent: custom toggle: icon: material/brightness-7 name: Switch to dark mode - media: "(prefers-color-scheme: dark)" scheme: slate - primary: green - accent: light green + primary: custom + accent: custom toggle: icon: material/brightness-4 name: Switch to light mode + font: + text: Inter + code: JetBrains Mono features: - navigation.sections - navigation.expand @@ -31,6 +36,9 @@ theme: icon: repo: fontawesome/brands/github +extra_css: + - assets/extra.css + markdown_extensions: - admonition - pymdownx.details