Add directory cleanup after ref deletion

.github/scripts/git_ref_lock_cleaner.py

@@ -5,6 +5,22 @@
 from abc import ABC, abstractmethod
 
 
+def remove_empty_parents(path: str, stop_dir: str):
+    current = os.path.dirname(path)
+    stop_dir = os.path.normpath(stop_dir)
+    while current:
+        norm_current = os.path.normpath(current)
+        # Stop at stop_dir itself or if path is outside stop_dir
+        if norm_current == stop_dir or not norm_current.startswith(stop_dir + os.sep):
+            break
+        try:
+            os.rmdir(current)
+            print(f"Removed empty directory: {current}")
+        except OSError:
+            break
+        current = os.path.dirname(current)
+
+
 class RefLockErrorHandler(ABC):
     """抽象基类，定义处理接口"""
 
@@ -16,17 +32,20 @@ def match(self, error_output: str) -> bool:
     def parse_branch(self, error_output: str) -> Optional[str]:
         pass
 
-    def handle(self, error_output: str):
+    def handle(self, error_output: str) -> bool:
         branch = self.parse_branch(error_output)
         if branch:
             print(f"Detected error, attempting to delete ref for branch: {branch}")
-            self.delete_ref(branch)
+            return self.delete_ref(branch)
         else:
             print("Error parsing branch name.")
+            return False
 
-    def delete_ref(self, branch_name: str):
+    def delete_ref(self, branch_name: str) -> bool:
         try:
             subprocess.run(["git", "update-ref", "-d", branch_name], check=True)
+            self.cleanup_ref_dirs(branch_name)
+            return True
         except subprocess.CalledProcessError as e:
             print(f"git update-ref failed: {e}")
             lock_files = [f".git/{branch_name}.lock", f".git/logs/{branch_name}.lock"]
@@ -40,40 +59,87 @@ def delete_ref(self, branch_name: str):
             # retry
             try:
                 subprocess.run(["git", "update-ref", "-d", branch_name], check=True)
+                self.cleanup_ref_dirs(branch_name)
+                return True
             except subprocess.CalledProcessError as e2:
                 print(f"Still failed to delete ref after removing lock: {e2}")
+                return False
+
+    def cleanup_ref_dirs(self, branch_name: str):
+        # `branch_name` looks like refs/remotes/origin/dev/foo.
+        ref_file = os.path.join(".git", branch_name)
+        reflog_file = os.path.join(".git", "logs", branch_name)
+
+        if not os.path.isfile(ref_file):
+            if os.path.isdir(ref_file):
+                # ref path is a directory (conflict case), remove it
+                try:
+                    os.rmdir(ref_file)
+                    print(f"Removed conflicting ref directory: {ref_file}")
+                except OSError:
+                    pass
+            remove_empty_parents(ref_file, os.path.join(".git", "refs"))
+        if not os.path.isfile(reflog_file):
+            if os.path.isdir(reflog_file):
+                try:
+                    os.rmdir(reflog_file)
+                    print(f"Removed conflicting reflog directory: {reflog_file}")
+                except OSError:
+                    pass
+            remove_empty_parents(reflog_file, os.path.join(".git", "logs", "refs"))
 
 
 class Type1Handler(RefLockErrorHandler):
     # error: cannot lock ref 'refs/remotes/origin/fix/3.0/TD-32817': is at 7af5 but expected eaba
-    # match the branch name before ‘is at’ with a regular expression
+    # match the branch name before 'is at' with a regular expression
     def match(self, error_output: str) -> bool:
         return "is at" in error_output and "but expected" in error_output
 
-    def parse_branch(self, error_output: str) -> str:
+    def parse_branch(self, error_output: str) -> Optional[str]:
+        # 匹配 cannot lock ref 部分，兼容中英文
         match = re.search(
-            r"error: cannot lock ref '(refs/remotes/origin/[^']+)': is at", error_output
+            r"cannot lock ref '(refs/remotes/origin/[^']+)': is at", error_output
         )
         return match.group(1) if match else None
 
 
 class Type2Handler(RefLockErrorHandler):
-    # match the branch name before ‘exists; cannot create’ with a regular expression
+    # match the branch name before 'exists; cannot create' with a regular expression
     def match(self, error_output: str) -> bool:
         return "exists; cannot create" in error_output
 
-    def parse_branch(self, error_output: str) -> str:
+    def parse_branch(self, error_output: str) -> Optional[str]:
         match = re.search(r"'(refs/remotes/origin/[^']+)' exists;", error_output)
         return match.group(1) if match else None
 
+    def handle(self, error_output: str) -> bool:
+        # Example:
+        # cannot lock ref 'refs/remotes/origin/dev':
+        # 'refs/remotes/origin/dev/trigger-ci-for-3.0' exists; cannot create 'refs/remotes/origin/dev'
+        match = re.search(
+            r"cannot lock ref '(refs/remotes/origin/[^']+)':\s*'(refs/remotes/origin/[^']+)' exists; cannot create '(refs/remotes/origin/[^']+)'",
+            error_output,
+        )
+        if match:
+            target_ref = match.group(1)
+            blocking_ref = match.group(2)
+            print(
+                f"Detected conflict: blocking ref {blocking_ref} prevents creating {target_ref}"
+            )
+            fixed = self.delete_ref(blocking_ref)
+            # Ensure parent directories of target ref are not left as empty dirs.
+            self.cleanup_ref_dirs(target_ref)
+            return fixed
+        return super().handle(error_output)
+
 
 class Type3Handler(RefLockErrorHandler):
     # match the branch name before the first single quote before 'Unable to' with a regular expression
     # git error: could not delete references: cannot lock ref 'refs/remotes/origin/test/3.0/TS-4893': Unable to create 'D:/workspace/main/TDinternal/community/.git/refs/remotes/origin/test/3.0/TS-4893.lock': File exists
     def match(self, error_output: str) -> bool:
         return "Unable to create" in error_output and "File exists" in error_output
 
-    def parse_branch(self, error_output: str) -> str:
+    def parse_branch(self, error_output: str) -> Optional[str]:
         match = re.search(
             r"(?:error|references): cannot lock ref '(refs/remotes/origin/[^']+)': Unable to",
             error_output,
@@ -97,14 +163,42 @@ def get_handler(cls, error_output: str):
 def handle_error(error_output):
     handler = RefLockErrorHandlerFactory.get_handler(error_output)
     if handler:
-        handler.handle(error_output)
+        return handler.handle(error_output)
     else:
         print("No handler found for this error.")
+        return False
+
+
+def clean_gc_log():
+    """Remove stale .git/gc.log to unblock automatic gc and prevent fetch from hanging."""
+    gc_log = os.path.join(".git", "gc.log")
+    if os.path.exists(gc_log):
+        print(f"Found stale {gc_log}, removing to unblock gc.")
+        try:
+            os.remove(gc_log)
+        except OSError as e:
+            print(f"Failed to remove {gc_log}: {e}")
+
+
+def run_gc():
+    """Run git gc to repack loose objects and prevent slow fetches."""
+    print("Running: git gc --auto")
+    result = subprocess.run(
+        ["git", "gc", "--auto"],
+        capture_output=True,
+        text=True,
+        timeout=300,
+    )
+    if result.returncode != 0:
+        print(f"git gc --auto failed: {result.stderr}")
+    else:
+        print("git gc --auto successful.")
+    return result
 
 
 def git_fetch():
     print("Running: git fetch")
-    result = subprocess.run(["git", "fetch"], capture_output=True, text=True)
+    result = subprocess.run(["git", "fetch"], capture_output=True, text=True, timeout=600)
     if result.returncode != 0:
         print("git fetch failed:")
         print(result.stderr)
@@ -116,7 +210,7 @@ def git_fetch():
 def git_prune():
     print("Running: git remote prune origin")
     result = subprocess.run(
-        ["git", "remote", "prune", "origin"], capture_output=True, text=True
+        ["git", "remote", "prune", "origin"], capture_output=True, text=True, timeout=300
     )
     if result.returncode != 0:
         print("git remote prune origin failed:")
@@ -127,10 +221,22 @@ def git_prune():
 
 
 def main():
-    fetch_result = git_fetch()
-    if fetch_result.returncode != 0:
-        handle_error(fetch_result.stderr)
-        return
+    clean_gc_log()
+    run_gc()
+
+    max_retries = 2
+    for attempt in range(max_retries + 1):
+        fetch_result = git_fetch()
+        if fetch_result.returncode == 0:
+            break
+
+        error_output = "\n".join(
+            part for part in [fetch_result.stderr, fetch_result.stdout] if part
+        )
+        fixed = handle_error(error_output)
+        if not fixed or attempt == max_retries:
+            return
+        print(f"Retrying git fetch... ({attempt + 1}/{max_retries})")
 
     prune_result = git_prune()
     if prune_result.returncode != 0:

.github/scripts/prepare_test_env.py

@@ -336,8 +336,9 @@ def get_testing_params(self):
             "extra_param", extra_param, env_file=os.getenv("GITHUB_ENV", "")
         )
 
-    def _execute_remote_command(self, host_config, command):
+    def _execute_remote_command(self, host_config, command, timeout=600):
         """Execute a command on remote host via SSH"""
+        host = host_config["host"]
         try:
             import paramiko
 
@@ -352,7 +353,8 @@ def _execute_remote_command(self, host_config, command):
             )
 
             # Execute command
-            stdin, stdout, stderr = ssh.exec_command(command)
+            logger.info(f"[{host}] Executing: {command[:120]}...")
+            stdin, stdout, stderr = ssh.exec_command(command, timeout=timeout)
             stdout_text = stdout.read().decode("utf-8")
             stderr_text = stderr.read().decode("utf-8")
             exit_code = stdout.channel.recv_exit_status()
@@ -361,9 +363,12 @@ def _execute_remote_command(self, host_config, command):
 
             success = exit_code == 0
             output = stdout_text if success else stderr_text
+            if not success:
+                logger.warning(f"[{host}] Command failed (exit {exit_code}): {stderr_text[:200]}")
             return success, output
 
         except Exception as e:
+            logger.error(f"[{host}] Remote command error: {e}")
             return False, str(e)
 
     def _prepare_repositories_remote(self, host_config):