From 7e86f65185fe5e2f94d030280797c20255479763 Mon Sep 17 00:00:00 2001 From: SS-JIA Date: Mon, 19 Oct 2026 14:10:59 -0700 Subject: [PATCH] [ET-VK][CI] Retry rate-limited HF snapshot downloads Summary: The Vulkan LLM job failed on PR #23892 when Hugging Face returned HTTP 429 while listing the pinned Phi-4-mini checkpoint. Qwen and five Llama recipes passed. Hub repository pagination bypasses its existing HTTP backoff helper. Retry snapshot downloads up to five times for HTTP 429. Honor Retry-After (seconds or HTTP-date) and exhausted RateLimit quota resets, with a one-second rounding margin. Use bounded exponential waits when reset headers are absent or invalid. Preserve pinned download arguments, checksum validation, and path-only stdout; retry messages go to stderr. Other errors still fail promptly. Read hf_download.py first, then test_export_vulkan_llm.py. Regression coverage uses the real Hub repository-listing client against a local HTTP server that returns 429 followed by success, and checks retry exhaustion/error propagation. Authored with OpenAI Codex. Test Plan: ## Observed Results | Correctness validation | Environment | Result | |---|---|---| | New retry tests against parent helper | Python 3.11, Hub 1.33.0, local HTTP fixture | 9 expected failures, 4 passed | | Retry/header/error/checksum subset | Python 3.11, Hub 1.33.0 (failed CI version) | 16 passed, 0 failed, 13 deselected | | Same subset with older Hub | Python 3.11, Hub 0.36.2 | 16 passed, 0 failed, 13 deselected | | Lintrunner on both changed files | macOS | Passed | | Seven-model NVIDIA workflow with this fix | GitHub CI | Pending at publication | | Performance validation | Result | |---|---| | Runtime/operator performance and end-to-end A/B | Not run; download retry change, no performance claim | ## Reproduction Guide ### Checkout and dependency setup Install Git and Python 3.11. These focused HTTP/download tests need no GPU, model checkpoint, native build, or full ExecuTorch installation. ```bash git clone https://github.com/pytorch/executorch.git cd executorch git checkout ssjia/vulkan-hf-rate-limit python3.11 -m venv .venv-hf-retry .venv-hf-retry/bin/python -m pip install pytest pyyaml httpx==0.28.1 huggingface_hub==1.33.0 ``` ### Fixed implementation Run the retry tests and existing pinned-download/checksum guards. The conftest cutoff isolates HTTP tests from the repository-wide PyTorch fixture. A local HTTP server emits the protocol responses; sleeps are recorded without waiting. ```bash .venv-hf-retry/bin/python -m pytest --confcutdir=.ci/scripts/tests -q .ci/scripts/tests/vulkan/test_export_vulkan_llm.py -k 'rate_limit or other_http_errors or existing_hf_download' ``` ### Parent reproducer Copy the current tests into a temporary directory and retrieve the parent helper. The retry cases fail because the parent aborts on the first 429. This keeps the checked-out implementation intact. ```bash ET_HF_PARENT=$(mktemp -d) mkdir -p "$ET_HF_PARENT/.ci/scripts/tests/vulkan" "$ET_HF_PARENT/examples/models/llama" cp .ci/scripts/tests/vulkan/test_export_vulkan_llm.py "$ET_HF_PARENT/.ci/scripts/tests/vulkan/test_export_vulkan_llm.py" git show HEAD^:examples/models/llama/hf_download.py > "$ET_HF_PARENT/examples/models/llama/hf_download.py" .venv-hf-retry/bin/python -m pytest --confcutdir="$ET_HF_PARENT/.ci/scripts/tests" -q "$ET_HF_PARENT/.ci/scripts/tests/vulkan/test_export_vulkan_llm.py" -k 'rate_limit or other_http_errors' ``` ### Older Hub compatibility ```bash .venv-hf-retry/bin/python -m pip install huggingface_hub==0.36.2 .venv-hf-retry/bin/python -m pytest --confcutdir=.ci/scripts/tests -q .ci/scripts/tests/vulkan/test_export_vulkan_llm.py -k 'rate_limit or other_http_errors or existing_hf_download' ``` ### Lint ```bash python3.11 -m pip install lintrunner lintrunner-adapters lintrunner init lintrunner lint --output oneline examples/models/llama/hf_download.py .ci/scripts/tests/vulkan/test_export_vulkan_llm.py ``` ### GPU workflow The existing Vulkan workflow installs dependencies, builds the runner, then exports/tests all seven pinned recipes on NVIDIA. Check its job and download the vulkan-llm-e2e artifact to inspect results.tsv and per-model download logs. The regression's failing run is internet-accessible: https://github.com/pytorch/executorch/actions/runs/39289788119/job/118187613858 No runtime performance validation was run. Reviewers: --- .../tests/vulkan/test_export_vulkan_llm.py | 156 ++++++++++++++++++ examples/models/llama/hf_download.py | 56 ++++++- 2 files changed, 205 insertions(+), 7 deletions(-) diff --git a/.ci/scripts/tests/vulkan/test_export_vulkan_llm.py b/.ci/scripts/tests/vulkan/test_export_vulkan_llm.py index 5aadf2572b0..4b4c9ec45a3 100644 --- a/.ci/scripts/tests/vulkan/test_export_vulkan_llm.py +++ b/.ci/scripts/tests/vulkan/test_export_vulkan_llm.py @@ -13,7 +13,11 @@ import shlex import subprocess import sys +import time +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path +from threading import Thread from types import SimpleNamespace import pytest @@ -128,6 +132,158 @@ def test_existing_hf_download_accepts_matching_hash(tmp_path, monkeypatch): ) == str(tmp_path) +@pytest.mark.parametrize( + "headers, expected_wait", + [ + ( + { + "RateLimit": '"api";r=0;t=36', + "RateLimit-Policy": '"fixed window";"api";q=500;w=300', + }, + 37, + ), + ({"Retry-After": "36"}, 37), + ({"Retry-After": "Mon, 19 Oct 2026 17:50:36 GMT"}, 37), + ({"RateLimit": '"api";r=0;t=36', "Retry-After": "50"}, 51), + ({"RateLimit": '"api";r=0;t=36', "Retry-After": "invalid"}, 37), + ({"RateLimit": '"api";r=10;t=900', "Retry-After": "36"}, 37), + ], +) +def test_hf_snapshot_retries_rate_limited_repo_tree( + tmp_path, monkeypatch, capsys, headers, expected_wait +): + from huggingface_hub import HfApi + + module = load_existing_module("examples/models/llama/hf_download.py") + requests = [] + calls = [] + waits = [] + monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost") + monkeypatch.setattr(time, "sleep", waits.append) + monkeypatch.setattr( + time, + "time", + lambda: datetime(2026, 10, 19, 17, 50, tzinfo=timezone.utc).timestamp(), + ) + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + requests.append(self.path) + self.send_response(429 if len(requests) == 1 else 200) + if len(requests) == 1: + for name, value in headers.items(): + self.send_header(name, value) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write( + b'{"error": "Rate limited"}' if len(requests) == 1 else b"[]" + ) + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever) + thread.start() + api = HfApi(endpoint=f"http://127.0.0.1:{server.server_port}") + + def snapshot(**kwargs): + calls.append(kwargs) + list( + api.list_repo_tree( + kwargs["repo_id"], + revision=kwargs["revision"], + token=kwargs["token"], + recursive=True, + ) + ) + return str(tmp_path) + + monkeypatch.setattr(module, "snapshot_download", snapshot) + try: + assert module.download_hf_snapshot("test/model", "r" * 40) == str(tmp_path) + finally: + server.shutdown() + server.server_close() + thread.join() + assert len(requests) == 2 and requests[0] == requests[1] + assert ( + calls + == [ + { + "repo_id": "test/model", + "revision": "r" * 40, + "allow_patterns": ["*.safetensors", "*.json"], + "token": False, + } + ] + * 2 + ) + assert waits == [expected_wait] + output = capsys.readouterr() + assert output.out == "" + assert str(expected_wait) in output.err + + +@pytest.mark.parametrize( + "headers", [{}, {"Retry-After": "invalid"}, {"RateLimit": '"api";r=10;t=900'}] +) +def test_hf_snapshot_rate_limit_retry_exhaustion(monkeypatch, headers): + import httpx + from huggingface_hub.errors import HfHubHTTPError + + module = load_existing_module("examples/models/llama/hf_download.py") + response = httpx.Response( + 429, + headers=headers, + request=httpx.Request("GET", "https://example.com"), + ) + error = HfHubHTTPError("Rate limited", response=response) + calls = [] + waits = [] + + def rate_limited(**kwargs): + calls.append(kwargs) + raise error + + monkeypatch.setattr(module, "snapshot_download", rate_limited) + monkeypatch.setattr(time, "sleep", waits.append) + with pytest.raises(HfHubHTTPError) as raised: + module.download_hf_snapshot("test/model", "r" * 40) + assert raised.value is error + assert len(calls) == 6 + assert waits == [60, 120, 240, 300, 300] + + +@pytest.mark.parametrize("status", [401, 404, 503, None]) +def test_hf_snapshot_does_not_retry_other_http_errors(monkeypatch, status): + import httpx + from huggingface_hub.errors import HfHubHTTPError + + module = load_existing_module("examples/models/llama/hf_download.py") + response = httpx.Response( + status or 429, + request=httpx.Request("GET", "https://example.com"), + ) + error = HfHubHTTPError("Download failed", response=response) + if status is None: + error.response = None + calls = [] + waits = [] + + def failed(**kwargs): + calls.append(kwargs) + raise error + + monkeypatch.setattr(module, "snapshot_download", failed) + monkeypatch.setattr(time, "sleep", waits.append) + with pytest.raises(HfHubHTTPError) as raised: + module.download_hf_snapshot("test/model", "r" * 40) + assert raised.value is error + assert len(calls) == 1 + assert waits == [] + + def test_snapshot_cli_exposes_pinned_download_and_checksum( tmp_path, monkeypatch, capsys ): diff --git a/examples/models/llama/hf_download.py b/examples/models/llama/hf_download.py index 17d234c8ffb..9f46458e837 100644 --- a/examples/models/llama/hf_download.py +++ b/examples/models/llama/hf_download.py @@ -9,14 +9,39 @@ import hashlib import json import os +import re import sys +import time from contextlib import redirect_stdout +from email.utils import parsedate_to_datetime from importlib.metadata import EntryPoint from pathlib import Path -from typing import Callable +from typing import Callable, Mapping from huggingface_hub import snapshot_download +from huggingface_hub.errors import HfHubHTTPError + +_MAX_RATE_LIMIT_RETRIES = 5 + + +def _rate_limit_retry_delay(headers: Mapping[str, str], attempt: int) -> float: + delay = None + retry_after = headers.get("Retry-After", "").strip() + if retry_after.isdigit(): + delay = float(retry_after) + elif retry_after: + try: + delay = max(0, parsedate_to_datetime(retry_after).timestamp() - time.time()) + except (TypeError, ValueError, OverflowError): + pass + rate_limit = re.search(r"\br=0;\s*t=(\d+)\b", headers.get("RateLimit", "")) + if rate_limit: + delay = max(delay or 0, float(rate_limit[1])) + if delay is not None: + # Quota reset times are rounded to whole seconds. + return delay + 1 + return min(60 * 2**attempt, 300) def download_hf_snapshot( @@ -26,12 +51,29 @@ def download_hf_snapshot( metadata_path: Path | None = None, ) -> str: """Download a pinned Safetensors checkpoint and optionally verify its hash.""" - snapshot = snapshot_download( - repo_id=repo_id, - revision=revision, - allow_patterns=["*.safetensors", "*.json"], - token=False, - ) + for attempt in range(_MAX_RATE_LIMIT_RETRIES + 1): + try: + snapshot = snapshot_download( + repo_id=repo_id, + revision=revision, + allow_patterns=["*.safetensors", "*.json"], + token=False, + ) + break + except HfHubHTTPError as error: + if ( + error.response is None + or error.response.status_code != 429 + or attempt == _MAX_RATE_LIMIT_RETRIES + ): + raise + delay = _rate_limit_retry_delay(error.response.headers, attempt) + print( + f"Rate limited downloading {repo_id}; retrying in {delay:g}s " + f"({attempt + 1}/{_MAX_RATE_LIMIT_RETRIES})", + file=sys.stderr, + ) + time.sleep(delay) if expected_sha256: checksum = hashlib.sha256() with (Path(snapshot) / "model.safetensors").open("rb") as checkpoint: