diff --git a/application/tests/harvester_test/__init__.py b/application/tests/harvester_test/__init__.py new file mode 100644 index 000000000..06b0f7440 --- /dev/null +++ b/application/tests/harvester_test/__init__.py @@ -0,0 +1,3 @@ +""" +tests for Module A configuration layer (empty for now) +""" diff --git a/application/tests/harvester_test/config_loader_test.py b/application/tests/harvester_test/config_loader_test.py new file mode 100644 index 000000000..01c404f18 --- /dev/null +++ b/application/tests/harvester_test/config_loader_test.py @@ -0,0 +1,63 @@ +from pathlib import Path +import unittest + +from application.utils.harvester.config_loader import ( + ConfigLoaderError, + ConfigFileNotFoundError, + load_repo_config, +) + +FIXTURES_DIR = Path(__file__).parent / "fixtures" + + +class ConfigLoaderTests(unittest.TestCase): + def test_load_valid_config(self): + config_path = FIXTURES_DIR / "valid_repos.yaml" + + config = load_repo_config(config_path) + + self.assertEqual(len(config.repositories), 1) + + repo = config.repositories[0] + + self.assertEqual(repo.id, "owasp-asvs") + self.assertEqual(repo.owner, "OWASP") + self.assertEqual(repo.repo, "ASVS") + + def test_missing_repository_id(self): + config_path = FIXTURES_DIR / "invalid_missing_id.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_invalid_chunk_size(self): + config_path = FIXTURES_DIR / "invalid_chunk_size.yaml" + + with self.assertRaisesRegex(ConfigLoaderError, "max_tokens"): + load_repo_config(config_path) + + def test_invalid_yaml_syntax(self): + config_path = FIXTURES_DIR / "invalid_yaml.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_missing_config_file(self): + with self.assertRaises(ConfigFileNotFoundError): + load_repo_config("does_not_exist.yaml") + + def test_invalid_polling_interval(self): + config_path = FIXTURES_DIR / "invalid_polling_interval.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + def test_empty_include_paths(self): + config_path = FIXTURES_DIR / "empty_include_paths.yaml" + + with self.assertRaises(ConfigLoaderError): + load_repo_config(config_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/fixtures/duplicate_include_paths.yaml b/application/tests/harvester_test/fixtures/duplicate_include_paths.yaml new file mode 100644 index 000000000..5db642199 --- /dev/null +++ b/application/tests/harvester_test/fixtures/duplicate_include_paths.yaml @@ -0,0 +1,22 @@ +repositories: + - id: duplicate-includes + type: github + enabled: true + + owner: OWASP + repo: ASVS + branch: master + + paths: + include: + - "docs/**/*.md" + - "docs/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/duplicate_repo_ids.yaml b/application/tests/harvester_test/fixtures/duplicate_repo_ids.yaml new file mode 100644 index 000000000..0cc4c1c1e --- /dev/null +++ b/application/tests/harvester_test/fixtures/duplicate_repo_ids.yaml @@ -0,0 +1,34 @@ +repositories: + - id: asvs + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 + + - id: asvs + type: github + owner: OWASP + repo: CheatSheetSeries + + paths: + include: + - "cheatsheets/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/duplicate_repositories.yaml b/application/tests/harvester_test/fixtures/duplicate_repositories.yaml new file mode 100644 index 000000000..41614d39c --- /dev/null +++ b/application/tests/harvester_test/fixtures/duplicate_repositories.yaml @@ -0,0 +1,34 @@ +repositories: + - id: asvs-1 + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 + + - id: asvs-2 + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/empty_include_paths.yaml b/application/tests/harvester_test/fixtures/empty_include_paths.yaml new file mode 100644 index 000000000..abe0bab41 --- /dev/null +++ b/application/tests/harvester_test/fixtures/empty_include_paths.yaml @@ -0,0 +1,16 @@ +repositories: + - id: asvs + type: github + owner: OWASP + repo: ASVS + + paths: + include: [] + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/empty_owner.yaml b/application/tests/harvester_test/fixtures/empty_owner.yaml new file mode 100644 index 000000000..8063a3f1b --- /dev/null +++ b/application/tests/harvester_test/fixtures/empty_owner.yaml @@ -0,0 +1,21 @@ +repositories: + - id: empty-owner + type: github + enabled: true + + owner: "" + repo: ASVS + branch: master + + paths: + include: + - "docs/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/invalid_chunk_size.yaml b/application/tests/harvester_test/fixtures/invalid_chunk_size.yaml new file mode 100644 index 000000000..79d4cc8f0 --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_chunk_size.yaml @@ -0,0 +1,18 @@ +repositories: + - id: owasp-asvs + + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 0 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/invalid_chunking_strategy.yaml b/application/tests/harvester_test/fixtures/invalid_chunking_strategy.yaml new file mode 100644 index 000000000..c4cb10905 --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_chunking_strategy.yaml @@ -0,0 +1,21 @@ +repositories: + - id: invalid-strategy + type: github + enabled: true + + owner: OWASP + repo: ASVS + branch: master + + paths: + include: + - "docs/**/*.md" + + chunking: + strategy: invalid_strategy + max_tokens: 1200 + overlap_tokens: 100 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/invalid_missing_id.yaml b/application/tests/harvester_test/fixtures/invalid_missing_id.yaml new file mode 100644 index 000000000..102e675b9 --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_missing_id.yaml @@ -0,0 +1,17 @@ +repositories: + - type: github + + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/invalid_polling_interval.yaml b/application/tests/harvester_test/fixtures/invalid_polling_interval.yaml new file mode 100644 index 000000000..3f34f8baa --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_polling_interval.yaml @@ -0,0 +1,17 @@ +repositories: + - id: asvs + type: github + owner: OWASP + repo: ASVS + + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 0 diff --git a/application/tests/harvester_test/fixtures/invalid_polling_mode.yaml b/application/tests/harvester_test/fixtures/invalid_polling_mode.yaml new file mode 100644 index 000000000..0aead47ad --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_polling_mode.yaml @@ -0,0 +1,21 @@ +repositories: + - id: invalid-polling + type: github + enabled: true + + owner: OWASP + repo: ASVS + branch: master + + paths: + include: + - "docs/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + + polling: + mode: realtime + interval_minutes: 60 diff --git a/application/tests/harvester_test/fixtures/invalid_yaml.yaml b/application/tests/harvester_test/fixtures/invalid_yaml.yaml new file mode 100644 index 000000000..c60d3e23b --- /dev/null +++ b/application/tests/harvester_test/fixtures/invalid_yaml.yaml @@ -0,0 +1,4 @@ +repositories: + - id: broken + paths: + include: [unclosed diff --git a/application/tests/harvester_test/fixtures/valid_repos.yaml b/application/tests/harvester_test/fixtures/valid_repos.yaml new file mode 100644 index 000000000..fe52c8430 --- /dev/null +++ b/application/tests/harvester_test/fixtures/valid_repos.yaml @@ -0,0 +1,16 @@ +repositories: + - id: owasp-asvs + type: github + owner: OWASP + repo: ASVS + paths: + include: + - "4.0/en/**/*.md" + + chunking: + strategy: markdown_heading + max_tokens: 1200 + + polling: + mode: incremental + interval_minutes: 60 diff --git a/application/tests/harvester_test/repos_validator_test.py b/application/tests/harvester_test/repos_validator_test.py new file mode 100644 index 000000000..3e0347d41 --- /dev/null +++ b/application/tests/harvester_test/repos_validator_test.py @@ -0,0 +1,58 @@ +from pathlib import Path +import unittest + +from application.utils.harvester.config_loader import ( + load_repo_config, +) +from application.utils.harvester.repos_validator import ( + RepositoryValidationError, + validate_repositories, +) + +FIXTURES_DIR = Path(__file__).parent / "fixtures" + + +class ReposValidatorTests(unittest.TestCase): + def test_duplicate_repository_ids(self): + config_path = FIXTURES_DIR / "duplicate_repo_ids.yaml" + + config = load_repo_config(config_path) + + with self.assertRaisesRegex( + RepositoryValidationError, + "Duplicate repository id", + ): + validate_repositories(config) + + def test_duplicate_repositories(self): + config_path = FIXTURES_DIR / "duplicate_repositories.yaml" + + config = load_repo_config(config_path) + + with self.assertRaisesRegex( + RepositoryValidationError, + "Duplicate repository detected", + ): + validate_repositories(config) + + def test_duplicate_include_paths(self): + config_path = FIXTURES_DIR / "duplicate_include_paths.yaml" + + config = load_repo_config(config_path) + + with self.assertRaisesRegex( + RepositoryValidationError, + "duplicate include paths", + ): + validate_repositories(config) + + def test_validate_valid_repositories(self): + config_path = FIXTURES_DIR / "valid_repos.yaml" + + config = load_repo_config(config_path) + + validate_repositories(config) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/harvester/__init__.py b/application/utils/harvester/__init__.py new file mode 100644 index 000000000..9c74939c7 --- /dev/null +++ b/application/utils/harvester/__init__.py @@ -0,0 +1,27 @@ +from .config_loader import ( + ConfigLoaderError, + load_repo_config, +) +from .repos_validator import ( + RepositoryValidationError, + validate_repositories, +) +from .schemas import ( + ChunkingConfig, + PathRules, + PollingConfig, + RepositoryConfig, + ReposFile, +) + +__all__ = [ + "ChunkingConfig", + "ConfigLoaderError", + "PathRules", + "PollingConfig", + "RepositoryConfig", + "RepositoryValidationError", + "ReposFile", + "load_repo_config", + "validate_repositories", +] diff --git a/application/utils/harvester/config_loader.py b/application/utils/harvester/config_loader.py new file mode 100644 index 000000000..db0431e11 --- /dev/null +++ b/application/utils/harvester/config_loader.py @@ -0,0 +1,32 @@ +from pathlib import Path +import yaml +from pydantic import ValidationError +from .schemas import ReposFile + + +class ConfigLoaderError(Exception): + """Base class for configuration loading errors.""" + + +class ConfigFileNotFoundError(ConfigLoaderError): + """Raised when the configuration file cannot be found.""" + + +def load_repo_config(path: str | Path) -> ReposFile: + config_path = Path(path) + + if not config_path.is_file(): + raise ConfigFileNotFoundError(f"Configuration file not found: {config_path}") + + try: + with config_path.open("r", encoding="utf-8") as file: + raw_config = yaml.safe_load(file) + except yaml.YAMLError as exc: + raise ConfigLoaderError(f"Invalid YAML syntax in {config_path}") from exc + + try: + return ReposFile.model_validate(raw_config) + except ValidationError as exc: + raise ConfigLoaderError( + f"Schema validation failed for {config_path}: {exc}" + ) from exc diff --git a/application/utils/harvester/exclude_patterns.txt b/application/utils/harvester/exclude_patterns.txt new file mode 100644 index 000000000..499850ae8 --- /dev/null +++ b/application/utils/harvester/exclude_patterns.txt @@ -0,0 +1,18 @@ +# Placeholder for repository-level exclude patterns. + +# This file will be consumed by the Week 4 noise-reduction pipeline + +# to filter non-documentation files during harvesting. + +**/.git/* +**/node_modules/** +**/__pycache__/** +**/.claude/** +**/.cursor/** +**/*.png +**/*.jpg +**/*.jpeg +**/*.svg +**/*.gif +**/*.pdf +**/archive/** diff --git a/application/utils/harvester/repos.yaml b/application/utils/harvester/repos.yaml new file mode 100644 index 000000000..f39d07257 --- /dev/null +++ b/application/utils/harvester/repos.yaml @@ -0,0 +1,43 @@ +repositories: + - id: owasp-asvs + type: github + enabled: true + owner: OWASP + repo: ASVS + branch: master + paths: + include: + - "4.0/en/**/*.md" + + exclude: + - "**/archive/**" + + chunking: + strategy: markdown + max_tokens: 1200 + overlap_tokens: 100 + + polling: + mode: incremental + interval_minutes: 60 + + - id: owasp-cheatsheets + type: github + enabled: true + + owner: OWASP + repo: CheatSheetSeries + branch: master + + paths: + include: + - "cheatsheets/**/*.md" + + chunking: + strategy: markdown + max_tokens: 1000 + overlap_tokens: 100 + + polling: + mode: incremental + interval_minutes: 120 diff --git a/application/utils/harvester/repos_validator.py b/application/utils/harvester/repos_validator.py new file mode 100644 index 000000000..81cdc8273 --- /dev/null +++ b/application/utils/harvester/repos_validator.py @@ -0,0 +1,33 @@ +# as the name suggests +from .schemas import ReposFile + + +class RepositoryValidationError(Exception): + """Raised when repository configuration fails semantic validation.""" + + +def validate_repositories(config: ReposFile) -> None: + seen_ids: set[str] = set() + seen_repositories: set[tuple[str, str]] = set() + + for repository in config.repositories: + repo_id_key = repository.id.casefold() + if repo_id_key in seen_ids: + raise RepositoryValidationError( + f"Duplicate repository id found: {repository.id}" + ) + seen_ids.add(repo_id_key) + repository_key = ( + repository.owner.casefold(), + repository.repo.casefold(), + ) + if repository_key in seen_repositories: + raise RepositoryValidationError( + f"Duplicate repository detected: {repository.owner}/{repository.repo}" + ) + seen_repositories.add(repository_key) + include_patterns = set(repository.paths.include) + if len(include_patterns) != len(repository.paths.include): + raise RepositoryValidationError( + f"Repository '{repository.id}' has duplicate include paths" + ) diff --git a/application/utils/harvester/schemas.py b/application/utils/harvester/schemas.py new file mode 100644 index 000000000..c65d62b60 --- /dev/null +++ b/application/utils/harvester/schemas.py @@ -0,0 +1,104 @@ +from typing import Literal +from pydantic import BaseModel, Field, ConfigDict, model_validator + + +# this will control which repo paths are included and excluded during ingestions +class PathRules(BaseModel): + model_config = ConfigDict(extra="forbid") + + include: list[str] = Field( + ..., + min_length=1, + description="Glob patterns to include during ingestions", + ) + exclude: list[str] = Field( + default_factory=list, description="Glob patterns to exclude during ingestions" + ) + + +# this will define how the harvested data should be chunked before downstream +class ChunkingConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + strategy: Literal["markdown_heading", "html_readability", "fixed_size"] = Field( + ..., + description="Chunking strategy used for text segmentation", + ) + + max_tokens: int = Field(..., gt=0, description="max token size per chunk") + + overlap_tokens: int = Field( + ge=0, + default=20, + description="token overlap between adjacent chunks", + ) + + @model_validator(mode="after") + def overlap_must_be_less_than_max(self) -> "ChunkingConfig": + if self.overlap_tokens >= self.max_tokens: + raise ValueError( + f"overlap_tokens ({self.overlap_tokens}) must be less than " + f"max_tokens ({self.max_tokens})" + ) + return self + + +# this one defines repository synchronize behaviour +class PollingConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + mode: Literal["full", "incremental"] = Field( + ..., description="repository sync mode" + ) + + interval_minutes: int = Field(..., gt=0, description="polling interval in minutes") + + +# top level repository ingestion configuration +class RepositoryConfig(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str = Field( + ..., + min_length=1, + description="unique repository identifier.", + ) + + type: Literal["github"] = Field( + ..., + description="repository source type.", + ) + enabled: bool = Field( + default=True, + description="whether ingestion is enabled for this repository.", + ) + owner: str = Field( + ..., + min_length=1, + description="repository organization.", + ) + repo: str = Field( + ..., + min_length=1, + description="repository name.", + ) + branch: str = Field( + default="main", + min_length=1, + description="Repository branch to ingest.", + ) + + paths: PathRules + chunking: ChunkingConfig + polling: PollingConfig + + +# Root configuration object loaded from repos.yaml. +class ReposFile(BaseModel): + model_config = ConfigDict(extra="forbid") + + repositories: list[RepositoryConfig] = Field( + ..., + min_length=1, + description="List of repositories configured for ingestion.", + )