diff --git a/repo-troubleshooting/reclone-failed-repos/README.md b/repo-troubleshooting/reclone-failed-repos/README.md index 43bfb7b..41f61dc 100644 --- a/repo-troubleshooting/reclone-failed-repos/README.md +++ b/repo-troubleshooting/reclone-failed-repos/README.md @@ -1,11 +1,14 @@ # `reclone_failed_repos.py` -List every repository on a Sourcegraph instance whose last clone or fetch -failed, and optionally reclone them all. +Report every repository on a Sourcegraph instance whose last clone or fetch +failed, with its mirror diagnostics, and optionally fetch or reclone them all. -Recloning a repository deletes it from gitserver disk, marks it as not cloned, -and starts a fresh clone. The script lists the failed repositories and exits -unless `--apply` is given. +- Default: read-only. Lists the failed repositories and writes a CSV. +- `--fetch`: also queues a fetch of each repository's existing clone + (`updateMirrorRepository`). Use this when the remote was flaky. +- `--reclone`: instead deletes each repository from gitserver disk, marks it + as not cloned, and starts a fresh clone (`recloneRepository`). Use this when + the on-disk copy is corrupt and a fetch will not fix it. ## Requirements @@ -28,11 +31,14 @@ export SRC_ACCESS_TOKEN="sgp_..." $env:SRC_ENDPOINT = "https://sourcegraph.example.com" $env:SRC_ACCESS_TOKEN = "sgp_..." -# List failed repositories (read-only) +# Report failed repositories (read-only) python3 reclone_failed_repos.py +# Fetch them +python3 reclone_failed_repos.py --fetch + # Reclone them -python3 reclone_failed_repos.py --apply +python3 reclone_failed_repos.py --reclone ``` The script also reads a `.env` file in the current directory when the @@ -50,15 +56,15 @@ but environment variables or `.env` keep the token out of shell history. ```sh # Only the first 5 failed repositories, for a small test run -python3 reclone_failed_repos.py --apply --max-repos 5 +python3 reclone_failed_repos.py --reclone --max-repos 5 -# recloneRepository mutations packed into each GraphQL request (default 10) -python3 reclone_failed_repos.py --apply --reclone-batch-size 20 +# Mutations packed into each GraphQL request (default 10) +python3 reclone_failed_repos.py --fetch --batch-size 20 -# Reclone requests sent at once (default 8); each recloneRepository call +# Mutation requests sent at once (default 8); each recloneRepository call # deletes the repo on every gitserver shard before returning, so higher # values add load on gitserver, not just the frontend -python3 reclone_failed_repos.py --apply --reclone-parallelism 2 +python3 reclone_failed_repos.py --reclone --parallelism 2 # Failed repositories fetched per GraphQL query page when listing (default 100) python3 reclone_failed_repos.py --list-repos-page-size 500 @@ -66,17 +72,27 @@ python3 reclone_failed_repos.py --list-repos-page-size 500 Repositories that already have a reclone in progress are skipped and counted separately. The script exits non-zero if listing fails or if any -reclone fails for another reason. +fetch or reclone fails for another reason. ## Output -Each run writes `yyyy-mm-dd-hh-mm-ss-reclone-failed-repos.csv` (local time) -to the current directory, one row per repository: - -- `repo_name`: e.g. `github.com/torvalds/linux` -- `action`: `listed (dry run)`, `reclone triggered`, `skipped`, or - `reclone failed` -- `result`: last fetch error (dry run), or the reclone error / skip message +Each run writes `yyyy-mm-dd-hh-mm-ss-failed-repos.csv` (local time) to the +current directory, one row per repository: + +- `repo_name`, `sourcegraph_url`, `remote_url`, `gitserver_shard`, `size_mb` +- `cloned`, `clone_in_progress`, `is_corrupted` +- `last_successful_fetch`, `time_since_last_successful_fetch` +- `next_sync`, `time_until_next_sync` +- `update_schedule_due`, `time_until_update_schedule_due`, + `update_schedule_interval_seconds` +- `update_queue_position`, `currently_updating` +- `last_error`, `last_sync_output`: collapsed to one line each +- `action`: `listed (dry run)`, `fetch triggered`, `fetch failed`, + `reclone triggered`, `reclone skipped`, or `reclone failed` +- `result`: the fetch / reclone error or skip message, if any + +Timestamps are RFC 3339 as returned by the API; the `time_*` columns are +relative to when the script ran (e.g. `3h 12m ago`, `in 45s`). ## Development diff --git a/repo-troubleshooting/reclone-failed-repos/reclone_failed_repos.py b/repo-troubleshooting/reclone-failed-repos/reclone_failed_repos.py index 8ed1e18..a7c82e3 100755 --- a/repo-troubleshooting/reclone-failed-repos/reclone_failed_repos.py +++ b/repo-troubleshooting/reclone-failed-repos/reclone_failed_repos.py @@ -1,14 +1,17 @@ #!/usr/bin/env python3 -"""Reclone every repository on a Sourcegraph instance whose last clone or fetch failed. +"""Report, fetch, or reclone every repository on a Sourcegraph instance whose +last clone or fetch failed. -Lists repositories matching `repositories(failedFetch: true)`, then, with -`--apply`, calls the `recloneRepository` mutation on each one. Recloning -deletes the repository from gitserver disk, marks it as not cloned, and -starts a fresh clone. +Lists repositories matching `repositories(failedFetch: true)` and writes their +mirror diagnostics (shard, size, schedule, queue position, last error, ...) to +a CSV. With `--fetch`, also calls `updateMirrorRepository` on each one, which +queues a fetch of the existing clone. With `--reclone`, calls +`recloneRepository` instead, which deletes the repository from gitserver disk, +marks it as not cloned, and starts a fresh clone. -Reclone mutations are packed into one GraphQL request per batch, using field -aliases, and batches are sent in parallel over keep-alive connections. +Mutations are packed into one GraphQL request per batch, using field aliases, +and batches are sent in parallel over keep-alive connections. Requires an access token with site-admin (REPO_MANAGEMENT#WRITE) permission. Uses only the Python standard library. @@ -25,7 +28,7 @@ import sys import threading from concurrent.futures import ThreadPoolExecutor -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Any from urllib.parse import urlparse @@ -35,7 +38,36 @@ REQUEST_TIMEOUT_SECONDS = 60 MAX_ERROR_CHARS = 160 RECLONE_IN_PROGRESS_MESSAGE = "another reclone is in progress" -CSV_COLUMNS = ("repo_name", "action", "result") + +# action name -> (GraphQL mutation field, name of its repository ID argument) +ACTIONS = { + "fetch": ("updateMirrorRepository", "repository"), + "reclone": ("recloneRepository", "repo"), +} + +CSV_COLUMNS: tuple[str, ...] = ( + "repo_name", + "sourcegraph_url", + "remote_url", + "gitserver_shard", + "size_mb", + "cloned", + "clone_in_progress", + "is_corrupted", + "last_successful_fetch", + "time_since_last_successful_fetch", + "next_sync", + "time_until_next_sync", + "update_schedule_due", + "time_until_update_schedule_due", + "update_schedule_interval_seconds", + "update_queue_position", + "currently_updating", + "last_error", + "last_sync_output", + "action", + "result", +) FAILED_REPOSITORIES_QUERY = """ query FailedRepositories($first: Int!, $after: String) { @@ -43,10 +75,20 @@ nodes { id name + url mirrorInfo { + remoteURL + shard + byteSize + cloned cloneInProgress + isCorrupted updatedAt + nextSyncAt + updateSchedule { intervalSeconds due } + updateQueue { index updating } lastError + lastSyncOutput } } pageInfo { @@ -58,24 +100,24 @@ """ Repository = dict[str, Any] -# (repository, error message) — error is None when the reclone was triggered +# (repository, error message) — error is None when the mutation was triggered Outcome = tuple[Repository, str | None] -# (repo_name, action, result), matching CSV_COLUMNS -CsvRow = tuple[str, str, str] +CsvRow = dict[str, Any] class GraphQLError(Exception): """The Sourcegraph API returned an HTTP or GraphQL error.""" -def reclone_mutation(count: int) -> str: - """Build a mutation that reclones `count` repositories via aliased fields.""" +def batched_mutation(action: str, count: int) -> str: + """Build a mutation that applies `action` to `count` repositories via aliased fields.""" + field, argument = ACTIONS[action] declarations = ", ".join(f"$r{index}: ID!" for index in range(count)) fields = "\n".join( - f" r{index}: recloneRepository(repo: $r{index}) {{ alwaysNil }}" + f" r{index}: {field}({argument}: $r{index}) {{ alwaysNil }}" for index in range(count) ) - return f"mutation Reclone({declarations}) {{\n{fields}\n}}" + return f"mutation {action.capitalize()}({declarations}) {{\n{fields}\n}}" class SourcegraphClient: @@ -85,6 +127,7 @@ def __init__(self, endpoint: str, token: str) -> None: parsed = urlparse(endpoint) if parsed.scheme not in ("http", "https") or not parsed.hostname: raise ValueError(f"endpoint must be an http(s) URL: {endpoint!r}") + self.endpoint = endpoint.rstrip("/") self.hostname = parsed.hostname self.port = parsed.port self.secure = parsed.scheme == "https" @@ -165,14 +208,18 @@ def failed_repositories( return repositories after = connection["pageInfo"]["endCursor"] - def reclone_batch(self, repositories: list[Repository]) -> list[Outcome]: - """Reclone a batch in one request; return an outcome per repository.""" + def mutate_batch( + self, action: str, repositories: list[Repository] + ) -> list[Outcome]: + """Apply `action` to a batch in one request; return an outcome per repository.""" variables = { f"r{index}": repository["id"] for index, repository in enumerate(repositories) } try: - payload = self.graphql(reclone_mutation(len(repositories)), variables) + payload = self.graphql( + batched_mutation(action, len(repositories)), variables + ) except (GraphQLError, http.client.HTTPException, OSError) as error: return [(repository, str(error)) for repository in repositories] @@ -182,7 +229,7 @@ def reclone_batch(self, repositories: list[Repository]) -> list[Outcome]: message = error.get("message") or json.dumps(error) errors_by_alias.setdefault(path[0], []).append(message) if None in errors_by_alias: - # Request-level error (e.g. permission denied): nothing was recloned + # Request-level error (e.g. permission denied): nothing was changed message = "; ".join(errors_by_alias[None]) return [(repository, message) for repository in repositories] return [ @@ -205,14 +252,25 @@ def load_dotenv() -> None: def parse_args() -> argparse.Namespace: load_dotenv() parser = argparse.ArgumentParser( - description="Reclone all repositories with failed clones or fetches " - "on a Sourcegraph instance. Default is dry run; use --apply to reclone.", + description="Report all repositories with failed clones or fetches on a " + "Sourcegraph instance, and optionally fetch or reclone them. " + "Default is a read-only report; use --fetch or --reclone to act.", ) basic = parser.add_argument_group("basic options") - basic.add_argument( - "--apply", - action="store_true", - help="Trigger reclones. Without this flag, only list failed repositories", + action = basic.add_mutually_exclusive_group() + action.add_argument( + "--fetch", + action="store_const", + const="fetch", + dest="action", + help="Queue a fetch of each failed repository's existing clone", + ) + action.add_argument( + "--reclone", + action="store_const", + const="reclone", + dest="action", + help="Delete each failed repository from gitserver disk and reclone it", ) basic.add_argument( "--src-endpoint", @@ -230,8 +288,8 @@ def parse_args() -> argparse.Namespace: "--max-repos", type=int, metavar="COUNT", - help="Only list (and with --apply, reclone) the first COUNT failed " - "repos (default: all failed repos)", + help="Only report (and with --fetch / --reclone, act on) the first COUNT " + "failed repos (default: all failed repos)", ) advanced.add_argument( "--list-repos-page-size", @@ -242,18 +300,20 @@ def parse_args() -> argparse.Namespace: "request (default: 100)", ) advanced.add_argument( - "--reclone-batch-size", + "--batch-size", type=int, default=10, metavar="COUNT", - help="With --apply, batch COUNT repos into each reclone request (default: 10)", + help="With --fetch / --reclone, batch COUNT repos into each mutation " + "request (default: 10)", ) advanced.add_argument( - "--reclone-parallelism", + "--parallelism", type=int, default=8, metavar="COUNT", - help="With --apply, send up to COUNT reclone requests in parallel (default: 8)", + help="With --fetch / --reclone, send up to COUNT mutation requests in " + "parallel (default: 8)", ) args = parser.parse_args() if not args.src_endpoint or not args.src_access_token: @@ -261,22 +321,46 @@ def parse_args() -> argparse.Namespace: "set SRC_ENDPOINT and SRC_ACCESS_TOKEN in the environment or a .env file, " "or pass --src-endpoint and --src-access-token" ) - if ( - min( - args.reclone_batch_size, args.reclone_parallelism, args.list_repos_page_size - ) - < 1 - ): + if min(args.batch_size, args.parallelism, args.list_repos_page_size) < 1: parser.error( - "--list-repos-page-size, --reclone-batch-size, and --reclone-parallelism must be >= 1" + "--list-repos-page-size, --batch-size, and --parallelism must be >= 1" ) return args +def one_line(text: str | None) -> str: + """Collapse whitespace so multi-line API text fits one CSV cell / log line. + + Also flattens literal `\\n` / `\\t`, which appear when gitserver quotes + git's output inside the error message. + """ + return " ".join((text or "").replace("\\n", " ").replace("\\t", " ").split()) + + +def relative_time(iso_timestamp: str | None, now: datetime) -> str: + """Render an RFC 3339 timestamp as e.g. '3h 12m ago' or 'in 45s'.""" + if not iso_timestamp: + return "" + timestamp = datetime.fromisoformat(iso_timestamp.replace("Z", "+00:00")) + seconds = int((timestamp - now).total_seconds()) + days, remainder = divmod(abs(seconds), 86400) + hours, remainder = divmod(remainder, 3600) + minutes, remainder = divmod(remainder, 60) + if days: + duration = f"{days}d {hours}h" + elif hours: + duration = f"{hours}h {minutes}m" + elif minutes: + duration = f"{minutes}m" + else: + duration = f"{remainder}s" + return f"in {duration}" if seconds > 0 else f"{duration} ago" + + def describe(repository: Repository) -> str: mirror_info = repository.get("mirrorInfo") or {} status = "cloning" if mirror_info.get("cloneInProgress") else "failed" - error = last_error(repository) + error = one_line(mirror_info.get("lastError")) if len(error) > MAX_ERROR_CHARS: error = error[:MAX_ERROR_CHARS] + "..." error_summary = f", error: {error}" if error else "" @@ -286,47 +370,77 @@ def describe(repository: Repository) -> str: ) -def last_error(repository: Repository) -> str: - """Return the repository's last fetch error on one line, or ''.""" +def csv_row( + repository: Repository, action: str, result: str, endpoint: str, now: datetime +) -> CsvRow: + """Flatten one repository's mirror diagnostics plus the action taken on it.""" mirror_info = repository.get("mirrorInfo") or {} - return " ".join((mirror_info.get("lastError") or "").split()) + schedule = mirror_info.get("updateSchedule") or {} + queue = mirror_info.get("updateQueue") or {} + return { + "repo_name": repository["name"], + "sourcegraph_url": endpoint + repository["url"], + "remote_url": mirror_info.get("remoteURL") or "", + "gitserver_shard": mirror_info.get("shard") or "", + "size_mb": f"{int(mirror_info.get('byteSize') or 0) / (1024 * 1024):.1f}", + "cloned": mirror_info.get("cloned"), + "clone_in_progress": mirror_info.get("cloneInProgress"), + "is_corrupted": mirror_info.get("isCorrupted"), + "last_successful_fetch": mirror_info.get("updatedAt") or "", + "time_since_last_successful_fetch": relative_time( + mirror_info.get("updatedAt"), now + ), + "next_sync": mirror_info.get("nextSyncAt") or "", + "time_until_next_sync": relative_time(mirror_info.get("nextSyncAt"), now), + "update_schedule_due": schedule.get("due") or "", + "time_until_update_schedule_due": relative_time(schedule.get("due"), now), + "update_schedule_interval_seconds": schedule.get("intervalSeconds", ""), + "update_queue_position": queue.get("index", ""), + "currently_updating": queue.get("updating", ""), + "last_error": one_line(mirror_info.get("lastError")), + "last_sync_output": one_line(mirror_info.get("lastSyncOutput")), + "action": action, + "result": result, + } -def reclone_all( +def mutate_all( client: SourcegraphClient, + action: str, repositories: list[Repository], - reclone_batch_size: int, - reclone_parallelism: int, -) -> list[CsvRow]: - """Reclone in parallel batches; return one CSV row per repository.""" + batch_size: int, + parallelism: int, +) -> list[Outcome]: + """Apply `action` in parallel batches; return one outcome per repository.""" batches = [ - repositories[start : start + reclone_batch_size] - for start in range(0, len(repositories), reclone_batch_size) + repositories[start : start + batch_size] + for start in range(0, len(repositories), batch_size) ] - rows: list[CsvRow] = [] - with ThreadPoolExecutor(max_workers=reclone_parallelism) as executor: - for outcomes in executor.map(client.reclone_batch, batches): - for repository, error in outcomes: - name = repository["name"] - if error is None: - rows.append((name, "reclone triggered", "")) - logger.info("Triggered reclone: %s", name) - elif RECLONE_IN_PROGRESS_MESSAGE in error: - rows.append((name, "skipped", error)) - logger.info("Reclone already in progress: %s", name) - else: - rows.append((name, "reclone failed", error)) - logger.error("Failed to reclone %s: %s", name, error) - return rows + outcomes: list[Outcome] = [] + with ThreadPoolExecutor(max_workers=parallelism) as executor: + for batch_outcomes in executor.map( + lambda batch: client.mutate_batch(action, batch), batches + ): + outcomes.extend(batch_outcomes) + return outcomes + + +def outcome_action(action: str, error: str | None) -> str: + """Map a mutation outcome to the CSV `action` value.""" + if error is None: + return f"{action} triggered" + if RECLONE_IN_PROGRESS_MESSAGE in error: + return f"{action} skipped" + return f"{action} failed" def write_csv(rows: list[CsvRow]) -> Path: """Write rows to a timestamped CSV in the current directory; return its path.""" timestamp = datetime.now().astimezone().strftime("%Y-%m-%d-%H-%M-%S") - path = Path(f"{timestamp}-reclone-failed-repos.csv") + path = Path(f"{timestamp}-failed-repos.csv") with path.open("w", newline="", encoding="utf-8") as csv_file: - writer = csv.writer(csv_file) - writer.writerow(CSV_COLUMNS) + writer = csv.DictWriter(csv_file, fieldnames=CSV_COLUMNS) + writer.writeheader() writer.writerows(rows) return path @@ -351,26 +465,38 @@ def main() -> int: for index, repository in enumerate(repositories, 1): logger.info(" %3d. %s", index, describe(repository)) - if args.apply: - rows = reclone_all( - client, repositories, args.reclone_batch_size, args.reclone_parallelism + now = datetime.now(timezone.utc) + if args.action: + outcomes = mutate_all( + client, args.action, repositories, args.batch_size, args.parallelism ) + rows = [] + for repository, error in outcomes: + action = outcome_action(args.action, error) + rows.append(csv_row(repository, action, error or "", client.endpoint, now)) + if action.endswith("failed"): + logger.error("%s: %s: %s", action, repository["name"], error) + else: + logger.info("%s: %s", action, repository["name"]) else: rows = [ - (repository["name"], "listed (dry run)", last_error(repository)) + csv_row(repository, "listed (dry run)", "", client.endpoint, now) for repository in repositories ] - logger.info("Dry run; rerun with --apply to reclone these repositories") + logger.info( + "Dry run; rerun with --fetch or --reclone to act on these repositories" + ) csv_path = write_csv(rows) logger.info("Wrote %s", csv_path) - failed = sum(action == "reclone failed" for _, action, _ in rows) - if args.apply: + failed = sum(row["action"].endswith("failed") for row in rows) + if args.action: logger.info( - "Triggered %d reclones, skipped %d already in progress, %d failed", - sum(action == "reclone triggered" for _, action, _ in rows), - sum(action == "skipped" for _, action, _ in rows), + "%s: triggered %d, skipped %d already in progress, %d failed", + args.action, + sum(row["action"].endswith("triggered") for row in rows), + sum(row["action"].endswith("skipped") for row in rows), failed, ) return 1 if failed else 0