diff --git a/rag/github.py b/rag/github.py index 4ea9638..9c00f7d 100644 --- a/rag/github.py +++ b/rag/github.py @@ -1,6 +1,9 @@ -"""GitHub API helpers used by scripts/sync.py.""" +"""GitHub API and clone helpers used by scripts/ingest.py and scripts/sync.py.""" +import base64 import json +import os +import subprocess import urllib.error import urllib.request @@ -33,16 +36,29 @@ def parse_repo_url(repo_url: str) -> tuple[str, str]: return parts[-2], parts[-1].removesuffix(".git") -def authed_clone_url(repo_url: str, token: str | None) -> str: - """Embed a token into an https GitHub URL so private repos clone non-interactively. +def clone_env(repo_url: str, token: str | None) -> dict[str, str]: + """Environment for `git clone` that authenticates to github.com with the token. - Returns the URL unchanged when no token is set or it isn't an https github.com - URL. Keep the original (token-free) URL for any logging — never print the result. + The token travels as an HTTP header set through GIT_CONFIG_* variables, so it never + appears in the command line, git's own output, or a CalledProcessError message. """ - prefix = "https://github.com/" - if token and repo_url.startswith(prefix): - return f"https://x-access-token:{token}@github.com/{repo_url[len(prefix):]}" - return repo_url + env = os.environ.copy() + if not token or not repo_url.startswith("https://github.com/"): + return env + basic = base64.b64encode(f"x-access-token:{token}".encode()).decode() + n = int(env.get("GIT_CONFIG_COUNT", "0")) + env["GIT_CONFIG_COUNT"] = str(n + 1) + env[f"GIT_CONFIG_KEY_{n}"] = "http.https://github.com/.extraheader" + env[f"GIT_CONFIG_VALUE_{n}"] = f"AUTHORIZATION: basic {basic}" + return env + + +def shallow_clone(repo_url: str, dest: str, token: str | None, ref: str | None = None) -> None: + """Shallow-clone a repo (optionally at a branch or tag), authenticating private repos.""" + cmd = ["git", "clone", "--depth=1"] + if ref: + cmd += ["--branch", ref] + subprocess.run([*cmd, repo_url, dest], check=True, env=clone_env(repo_url, token)) def get_latest_sha(owner: str, repo: str, token: str | None) -> str: diff --git a/scripts/ingest.py b/scripts/ingest.py index 7e64a5d..2b4d862 100644 --- a/scripts/ingest.py +++ b/scripts/ingest.py @@ -37,7 +37,7 @@ load_dotenv(Path(__file__).parent.parent / ".env") from rag.db import get_conn, register_namespace, sanitise_namespace, setup_db, setup_meta_table, upsert_chunks -from rag.github import authed_clone_url +from rag.github import shallow_clone from rag.query import embed_batch # ── Skip rules ──────────────────────────────────────────────────────────────── @@ -241,18 +241,11 @@ def main() -> None: else: if not args.repo: parser.error("--repo URL is required when no local repo_path is given") - clone_cmd = ["git", "clone", "--depth=1"] - if args.ref: - clone_cmd += ["--branch", args.ref] - token = os.environ.get("GITHUB_TOKEN") with tempfile.TemporaryDirectory() as tmpdir: repo_path = os.path.join(tmpdir, "repo") ref_label = args.ref or "default branch" print(f"Cloning {args.repo} @ {ref_label} (shallow)...") - subprocess.run( - clone_cmd + [authed_clone_url(args.repo, token), repo_path], - check=True, - ) + shallow_clone(args.repo, repo_path, os.environ.get("GITHUB_TOKEN"), ref=args.ref) ingest(repo_path, args.namespace, repo_url=args.repo) diff --git a/scripts/sync.py b/scripts/sync.py index a00eb4e..9c08ff2 100644 --- a/scripts/sync.py +++ b/scripts/sync.py @@ -18,7 +18,6 @@ import argparse import os -import subprocess import sys import tempfile from pathlib import Path @@ -36,7 +35,7 @@ update_namespace_sha, upsert_chunks, ) -from rag.github import authed_clone_url, get_changed_files, get_latest_sha, parse_repo_url +from rag.github import get_changed_files, get_latest_sha, parse_repo_url, shallow_clone from rag.query import embed_batch from scripts.ingest import EMBED_BATCH, chunk_file, ingest, should_skip @@ -46,10 +45,7 @@ def _full_reindex(repo_url: str, namespace: str, token: str | None) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_path = os.path.join(tmpdir, "repo") print(f" Cloning {repo_url} (shallow)...") - subprocess.run( - ["git", "clone", "--depth=1", authed_clone_url(repo_url, token), repo_path], - check=True, - ) + shallow_clone(repo_url, repo_path, token) ingest(repo_path, namespace) # repo_url omitted — caller handles SHA registration @@ -106,10 +102,7 @@ def sync_namespace(conn, ns: dict, token: str | None) -> None: with tempfile.TemporaryDirectory() as tmpdir: repo_path = os.path.join(tmpdir, "repo") print(f" Cloning {repo_url} (shallow)...") - subprocess.run( - ["git", "clone", "--depth=1", authed_clone_url(repo_url, token), repo_path], - check=True, - ) + shallow_clone(repo_url, repo_path, token) repo_root = Path(repo_path) for rel_path in changed: