diff --git a/.github/scripts/git_ref_lock_cleaner.py b/.github/scripts/git_ref_lock_cleaner.py index 2cce75d..ebdc6bf 100644 --- a/.github/scripts/git_ref_lock_cleaner.py +++ b/.github/scripts/git_ref_lock_cleaner.py @@ -5,6 +5,22 @@ from abc import ABC, abstractmethod +def remove_empty_parents(path: str, stop_dir: str): + current = os.path.dirname(path) + stop_dir = os.path.normpath(stop_dir) + while current: + norm_current = os.path.normpath(current) + # Stop at stop_dir itself or if path is outside stop_dir + if norm_current == stop_dir or not norm_current.startswith(stop_dir + os.sep): + break + try: + os.rmdir(current) + print(f"Removed empty directory: {current}") + except OSError: + break + current = os.path.dirname(current) + + class RefLockErrorHandler(ABC): """抽象基类，定义处理接口""" @@ -16,17 +32,20 @@ def match(self, error_output: str) -> bool: def parse_branch(self, error_output: str) -> Optional[str]: pass - def handle(self, error_output: str): + def handle(self, error_output: str) -> bool: branch = self.parse_branch(error_output) if branch: print(f"Detected error, attempting to delete ref for branch: {branch}") - self.delete_ref(branch) + return self.delete_ref(branch) else: print("Error parsing branch name.") + return False - def delete_ref(self, branch_name: str): + def delete_ref(self, branch_name: str) -> bool: try: subprocess.run(["git", "update-ref", "-d", branch_name], check=True) + self.cleanup_ref_dirs(branch_name) + return True except subprocess.CalledProcessError as e: print(f"git update-ref failed: {e}") lock_files = [f".git/{branch_name}.lock", f".git/logs/{branch_name}.lock"] @@ -40,32 +59,79 @@ def delete_ref(self, branch_name: str): # retry try: subprocess.run(["git", "update-ref", "-d", branch_name], check=True) + self.cleanup_ref_dirs(branch_name) + return True except subprocess.CalledProcessError as e2: print(f"Still failed to delete ref after removing lock: {e2}") + return False + + def cleanup_ref_dirs(self, branch_name: str): + # `branch_name` looks like refs/remotes/origin/dev/foo. + ref_file = os.path.join(".git", branch_name) + reflog_file = os.path.join(".git", "logs", branch_name) + + if not os.path.isfile(ref_file): + if os.path.isdir(ref_file): + # ref path is a directory (conflict case), remove it + try: + os.rmdir(ref_file) + print(f"Removed conflicting ref directory: {ref_file}") + except OSError: + pass + remove_empty_parents(ref_file, os.path.join(".git", "refs")) + if not os.path.isfile(reflog_file): + if os.path.isdir(reflog_file): + try: + os.rmdir(reflog_file) + print(f"Removed conflicting reflog directory: {reflog_file}") + except OSError: + pass + remove_empty_parents(reflog_file, os.path.join(".git", "logs", "refs")) class Type1Handler(RefLockErrorHandler): # error: cannot lock ref 'refs/remotes/origin/fix/3.0/TD-32817': is at 7af5 but expected eaba - # match the branch name before ‘is at’ with a regular expression + # match the branch name before 'is at' with a regular expression def match(self, error_output: str) -> bool: return "is at" in error_output and "but expected" in error_output - def parse_branch(self, error_output: str) -> str: + def parse_branch(self, error_output: str) -> Optional[str]: + # 匹配 cannot lock ref 部分，兼容中英文 match = re.search( - r"error: cannot lock ref '(refs/remotes/origin/[^']+)': is at", error_output + r"cannot lock ref '(refs/remotes/origin/[^']+)': is at", error_output ) return match.group(1) if match else None class Type2Handler(RefLockErrorHandler): - # match the branch name before ‘exists; cannot create’ with a regular expression + # match the branch name before 'exists; cannot create' with a regular expression def match(self, error_output: str) -> bool: return "exists; cannot create" in error_output - def parse_branch(self, error_output: str) -> str: + def parse_branch(self, error_output: str) -> Optional[str]: match = re.search(r"'(refs/remotes/origin/[^']+)' exists;", error_output) return match.group(1) if match else None + def handle(self, error_output: str) -> bool: + # Example: + # cannot lock ref 'refs/remotes/origin/dev': + # 'refs/remotes/origin/dev/trigger-ci-for-3.0' exists; cannot create 'refs/remotes/origin/dev' + match = re.search( + r"cannot lock ref '(refs/remotes/origin/[^']+)':\s*'(refs/remotes/origin/[^']+)' exists; cannot create '(refs/remotes/origin/[^']+)'", + error_output, + ) + if match: + target_ref = match.group(1) + blocking_ref = match.group(2) + print( + f"Detected conflict: blocking ref {blocking_ref} prevents creating {target_ref}" + ) + fixed = self.delete_ref(blocking_ref) + # Ensure parent directories of target ref are not left as empty dirs. + self.cleanup_ref_dirs(target_ref) + return fixed + return super().handle(error_output) + class Type3Handler(RefLockErrorHandler): # match the branch name before the first single quote before 'Unable to' with a regular expression @@ -73,7 +139,7 @@ class Type3Handler(RefLockErrorHandler): def match(self, error_output: str) -> bool: return "Unable to create" in error_output and "File exists" in error_output - def parse_branch(self, error_output: str) -> str: + def parse_branch(self, error_output: str) -> Optional[str]: match = re.search( r"(?:error|references): cannot lock ref '(refs/remotes/origin/[^']+)': Unable to", error_output, @@ -97,14 +163,42 @@ def get_handler(cls, error_output: str): def handle_error(error_output): handler = RefLockErrorHandlerFactory.get_handler(error_output) if handler: - handler.handle(error_output) + return handler.handle(error_output) else: print("No handler found for this error.") + return False + + +def clean_gc_log(): + """Remove stale .git/gc.log to unblock automatic gc and prevent fetch from hanging.""" + gc_log = os.path.join(".git", "gc.log") + if os.path.exists(gc_log): + print(f"Found stale {gc_log}, removing to unblock gc.") + try: + os.remove(gc_log) + except OSError as e: + print(f"Failed to remove {gc_log}: {e}") + + +def run_gc(): + """Run git gc to repack loose objects and prevent slow fetches.""" + print("Running: git gc --auto") + result = subprocess.run( + ["git", "gc", "--auto"], + capture_output=True, + text=True, + timeout=300, + ) + if result.returncode != 0: + print(f"git gc --auto failed: {result.stderr}") + else: + print("git gc --auto successful.") + return result def git_fetch(): print("Running: git fetch") - result = subprocess.run(["git", "fetch"], capture_output=True, text=True) + result = subprocess.run(["git", "fetch"], capture_output=True, text=True, timeout=600) if result.returncode != 0: print("git fetch failed:") print(result.stderr) @@ -116,7 +210,7 @@ def git_fetch(): def git_prune(): print("Running: git remote prune origin") result = subprocess.run( - ["git", "remote", "prune", "origin"], capture_output=True, text=True + ["git", "remote", "prune", "origin"], capture_output=True, text=True, timeout=300 ) if result.returncode != 0: print("git remote prune origin failed:") @@ -127,10 +221,22 @@ def git_prune(): def main(): - fetch_result = git_fetch() - if fetch_result.returncode != 0: - handle_error(fetch_result.stderr) - return + clean_gc_log() + run_gc() + + max_retries = 2 + for attempt in range(max_retries + 1): + fetch_result = git_fetch() + if fetch_result.returncode == 0: + break + + error_output = "\n".join( + part for part in [fetch_result.stderr, fetch_result.stdout] if part + ) + fixed = handle_error(error_output) + if not fixed or attempt == max_retries: + return + print(f"Retrying git fetch... ({attempt + 1}/{max_retries})") prune_result = git_prune() if prune_result.returncode != 0: diff --git a/.github/scripts/prepare_test_env.py b/.github/scripts/prepare_test_env.py index d70638b..f8faf28 100644 --- a/.github/scripts/prepare_test_env.py +++ b/.github/scripts/prepare_test_env.py @@ -336,8 +336,9 @@ def get_testing_params(self): "extra_param", extra_param, env_file=os.getenv("GITHUB_ENV", "") ) - def _execute_remote_command(self, host_config, command): + def _execute_remote_command(self, host_config, command, timeout=600): """Execute a command on remote host via SSH""" + host = host_config["host"] try: import paramiko @@ -352,7 +353,8 @@ def _execute_remote_command(self, host_config, command): ) # Execute command - stdin, stdout, stderr = ssh.exec_command(command) + logger.info(f"[{host}] Executing: {command[:120]}...") + stdin, stdout, stderr = ssh.exec_command(command, timeout=timeout) stdout_text = stdout.read().decode("utf-8") stderr_text = stderr.read().decode("utf-8") exit_code = stdout.channel.recv_exit_status() @@ -361,9 +363,12 @@ def _execute_remote_command(self, host_config, command): success = exit_code == 0 output = stdout_text if success else stderr_text + if not success: + logger.warning(f"[{host}] Command failed (exit {exit_code}): {stderr_text[:200]}") return success, output except Exception as e: + logger.error(f"[{host}] Remote command error: {e}") return False, str(e) def _prepare_repositories_remote(self, host_config):