From 1af6cfde1d8a7ccf518e2dbe0f3fd790a2342d05 Mon Sep 17 00:00:00 2001 From: Prerna Gandhi Date: Wed, 22 Jul 2026 23:08:53 +0530 Subject: [PATCH] Ingestion is done, planned AST chunking --- chunk/src/ingest.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/chunk/src/ingest.py b/chunk/src/ingest.py index e69de29..8c76458 100644 --- a/chunk/src/ingest.py +++ b/chunk/src/ingest.py @@ -0,0 +1,21 @@ +from pathlib import Path + +SKIP_DIRS = {"tests", ".git", "docs", "__pycache__"} + + +def load_python_files(root: Path): + + for path in root.rglob("*.py"): + if any(part in SKIP_DIRS or part.endswith(".egg-info") for part in path.parts): + continue + + try: + content = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + print(f"Skipping (bad encoding): {path}") + continue + + yield { + "path": str(path.relative_to(root)), + "content": content, + } \ No newline at end of file