From 73b01a0bb59fb0dcebac30790d05b48cd0a81096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Tue, 6 Oct 2026 22:17:59 +0200 Subject: [PATCH 1/6] refactor: launch sampling in its own self-disabling workflow; collect-stats back to 6-hourly collect-stats.yml returns to its plain 6-hourly schedule. collect-launch.yml samples every ~10 min (best effort) until WINDOW_END (2026-10-20), then runs 'gh workflow disable' on itself, so nothing keeps firing between launches. --- .github/workflows/collect-launch.yml | 69 ++++++++++++++++++++++++++++ .github/workflows/collect-stats.yml | 26 +---------- README.md | 5 +- 3 files changed, 73 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/collect-launch.yml diff --git a/.github/workflows/collect-launch.yml b/.github/workflows/collect-launch.yml new file mode 100644 index 00000000..0fc1b9eb --- /dev/null +++ b/.github/workflows/collect-launch.yml @@ -0,0 +1,69 @@ +# High-resolution samples around a launch, on top of the 6-hourly collect-stats. +# Same collection and files; GitHub runs schedules best-effort (often 10-60 min +# late), and every row carries its own timestamp. After WINDOW_END the workflow +# disables itself, so it never keeps running between launches. To reuse: set a +# new WINDOW_END and `gh workflow enable collect-launch.yml -R ActivityWatch/stats`. + +name: Collect stats (launch window) + +on: + schedule: + - cron: '*/10 * * * *' + workflow_dispatch: + +concurrency: + group: collect-stats + cancel-in-progress: false + +permissions: + contents: write + actions: write + +env: + WINDOW_END: '2026-10-20T00:00:00Z' + +jobs: + sample: + runs-on: ubuntu-latest + steps: + - name: Disable after the launch window + id: window + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ "$(date -u +%s)" -ge "$(date -u -d "$WINDOW_END" +%s)" ]; then + echo "Launch window ended ($WINDOW_END); disabling this workflow." + gh workflow disable collect-launch.yml -R "$GITHUB_REPOSITORY" + echo "open=false" >> "$GITHUB_OUTPUT" + else + echo "open=true" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@v4 + if: steps.window.outputs.open == 'true' + - name: Install uv + if: steps.window.outputs.open == 'true' + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.10" + - name: Collect and append stats + if: steps.window.outputs.open == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + uv sync + uv run python3 collect_stats.py --csv >> data/stats.csv + uv run python3 collect_stats.py --assets --csv >> data/stats-assets.csv + - name: Commit and push + if: steps.window.outputs.open == 'true' + run: | + git config --local user.email "noreply@github.com" + git config --local user.name "GitHub Action" + git add data/stats.csv data/stats-assets.csv + git diff --cached --quiet && exit 0 + git commit -m "chore: update stats (launch window)" + for i in 1 2 3 4 5; do + git push origin HEAD:master && exit 0 + sleep $((i * 5)) + git pull --rebase origin master + done + exit 1 diff --git a/.github/workflows/collect-stats.yml b/.github/workflows/collect-stats.yml index 60e626a9..d740587c 100644 --- a/.github/workflows/collect-stats.yml +++ b/.github/workflows/collect-stats.yml @@ -10,10 +10,7 @@ on: #pull_requests: # branches: [ master ] schedule: - # Every 10 minutes; the "Decide whether to collect" step below only collects - # every 6 hours outside a launch window. GitHub may delay scheduled runs, so - # every row carries its own timestamp. - - cron: '*/10 * * * *' + - cron: '0 */6 * * *' # Never let two collections race on the push. concurrency: @@ -29,32 +26,13 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Decide whether to collect - id: cadence - run: | - # High-resolution window (UTC) around the v0.14.0 launch; 10-minute - # samples correlate downloads with posts (data/launch-events.csv). - # Outside it, collect only if the last sample is ~6 h old. - WINDOW_END=2026-10-27 - now=$(date -u +%s) - last=$(tail -n 1 data/stats.csv | cut -d, -f1) - age=$(( now - $(date -u -d "$last" +%s) )) - if [ "$GITHUB_EVENT_NAME" != "schedule" ] || [ "$now" -lt "$(date -u -d "$WINDOW_END" +%s)" ] || [ "$age" -ge 21000 ]; then - echo "collect=true" >> "$GITHUB_OUTPUT" - else - echo "collect=false" >> "$GITHUB_OUTPUT" - echo "Outside the launch window and last sample is only ${age}s old; skipping." - fi - name: Install uv - if: steps.cadence.outputs.collect == 'true' uses: astral-sh/setup-uv@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies - if: steps.cadence.outputs.collect == 'true' run: uv sync - name: Collect and append stats - if: steps.cadence.outputs.collect == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -69,14 +47,12 @@ jobs: # breakdowns). Appends only assets whose count changed since last logged. uv run python3 collect_stats.py --assets --csv >> data/stats-assets.csv - name: Commit - if: steps.cadence.outputs.collect == 'true' run: | git config --local user.email "noreply@github.com" git config --local user.name "GitHub Action" git add data/stats.csv data/releases.csv data/stats-assets.csv git commit -m "chore: update stats" - name: Push changes - if: steps.cadence.outputs.collect == 'true' run: | # Both stats workflows push to master; retry on a rejected push. for i in 1 2 3 4 5; do diff --git a/README.md b/README.md index 7040dfa2..112b9618 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,9 @@ The following is manually updated: - `firefox-daily-users.csv` - Firefox extension daily active users - `notes.csv` - Manual entries of major/interesting events - `launch-events.csv` - Timestamped launch posts (UTC; channel, url, note), to line up with the - high-resolution download samples. During a launch window (see `WINDOW_END` in - `.github/workflows/collect-stats.yml`) stats are collected every ~10 minutes instead of every 6 hours. + high-resolution download samples. During a launch window, + `.github/workflows/collect-launch.yml` adds best-effort ~10-minute samples on top of the 6-hourly + collection and disables itself after its `WINDOW_END`. ## Android vitals From 1b968badc3d833311e2d8ed3badf59355e91dd72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Tue, 6 Oct 2026 22:23:48 +0200 Subject: [PATCH 2/6] feat: run-anywhere 1-minute sampler (sample_hf.py) instead of a launch workflow Drop collect-launch.yml; collect-stats stays plain 6-hourly. sample_hf.py polls every 60 s with ETag conditional requests, appends change-only rows to data/hf/, resumes from the files, and --trim coarsens before submitting in a PR. --- .github/workflows/collect-launch.yml | 69 --------- README.md | 10 +- sample_hf.py | 213 +++++++++++++++++++++++++++ test_sample_hf.py | 60 ++++++++ 4 files changed, 279 insertions(+), 73 deletions(-) delete mode 100644 .github/workflows/collect-launch.yml create mode 100644 sample_hf.py create mode 100644 test_sample_hf.py diff --git a/.github/workflows/collect-launch.yml b/.github/workflows/collect-launch.yml deleted file mode 100644 index 0fc1b9eb..00000000 --- a/.github/workflows/collect-launch.yml +++ /dev/null @@ -1,69 +0,0 @@ -# High-resolution samples around a launch, on top of the 6-hourly collect-stats. -# Same collection and files; GitHub runs schedules best-effort (often 10-60 min -# late), and every row carries its own timestamp. After WINDOW_END the workflow -# disables itself, so it never keeps running between launches. To reuse: set a -# new WINDOW_END and `gh workflow enable collect-launch.yml -R ActivityWatch/stats`. - -name: Collect stats (launch window) - -on: - schedule: - - cron: '*/10 * * * *' - workflow_dispatch: - -concurrency: - group: collect-stats - cancel-in-progress: false - -permissions: - contents: write - actions: write - -env: - WINDOW_END: '2026-10-20T00:00:00Z' - -jobs: - sample: - runs-on: ubuntu-latest - steps: - - name: Disable after the launch window - id: window - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - if [ "$(date -u +%s)" -ge "$(date -u -d "$WINDOW_END" +%s)" ]; then - echo "Launch window ended ($WINDOW_END); disabling this workflow." - gh workflow disable collect-launch.yml -R "$GITHUB_REPOSITORY" - echo "open=false" >> "$GITHUB_OUTPUT" - else - echo "open=true" >> "$GITHUB_OUTPUT" - fi - - uses: actions/checkout@v4 - if: steps.window.outputs.open == 'true' - - name: Install uv - if: steps.window.outputs.open == 'true' - uses: astral-sh/setup-uv@v5 - with: - python-version: "3.10" - - name: Collect and append stats - if: steps.window.outputs.open == 'true' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - uv sync - uv run python3 collect_stats.py --csv >> data/stats.csv - uv run python3 collect_stats.py --assets --csv >> data/stats-assets.csv - - name: Commit and push - if: steps.window.outputs.open == 'true' - run: | - git config --local user.email "noreply@github.com" - git config --local user.name "GitHub Action" - git add data/stats.csv data/stats-assets.csv - git diff --cached --quiet && exit 0 - git commit -m "chore: update stats (launch window)" - for i in 1 2 3 4 5; do - git push origin HEAD:master && exit 0 - sleep $((i * 5)) - git pull --rebase origin master - done - exit 1 diff --git a/README.md b/README.md index 112b9618..567cca51 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,12 @@ The following is manually updated: - `chrome-weekly-users.csv` - Chrome extension weekly active users - `firefox-daily-users.csv` - Firefox extension daily active users - `notes.csv` - Manual entries of major/interesting events - - `launch-events.csv` - Timestamped launch posts (UTC; channel, url, note), to line up with the - high-resolution download samples. During a launch window, - `.github/workflows/collect-launch.yml` adds best-effort ~10-minute samples on top of the 6-hourly - collection and disables itself after its `WINDOW_END`. + - `launch-events.csv` - Timestamped launch posts (UTC; channel, url, note), to line up with + high-resolution samples. + - `hf/` - High-resolution samples from `sample_hf.py` (same schemas as `stats.csv` / + `stats-assets.csv`, change-only rows). Run it anywhere during a launch and submit the files in a PR: + `GITHUB_TOKEN=$(gh auth token) uv run python3 sample_hf.py --until 2026-10-20` + (default 60 s interval; `--trim MINUTES` coarsens before submitting). ## Android vitals diff --git a/sample_hf.py b/sample_hf.py new file mode 100644 index 00000000..47b19590 --- /dev/null +++ b/sample_hf.py @@ -0,0 +1,213 @@ +""" +High-resolution download sampler for launch windows. + +Run it anywhere (laptop, server, VM) during a launch, then submit the files it +writes in a PR. It polls the GitHub API every `--interval` seconds (default 60) +and appends rows only when something changed: + + data/hf/stats.csv timestamp,downloads,stars (same schema as data/stats.csv) + data/hf/stats-assets.csv timestamp,tag,asset,platform,downloads + +Downloads exclude Tauri updater assets, like collect_stats.downloads(); the +per-asset file keeps them (latest.json fetches = update checks). + +Uses conditional requests (ETag): an unchanged response is a 304, which does +not count against the rate limit. Set GITHUB_TOKEN (e.g. `GITHUB_TOKEN=$(gh auth token)`) +for the 5000/h limit; unauthenticated (60/h) needs --interval >= 120. + + GITHUB_TOKEN=$(gh auth token) uv run python3 sample_hf.py --until 2026-10-20 + +Stop with Ctrl-C; a restart resumes from the files. Coarsen before submitting +with `--trim MINUTES` (keeps the last row per series per MINUTES-wide bucket). +""" + +import argparse +import csv +import os +import time +from datetime import datetime, timezone + +import requests + +from collect_stats import RELEASES_PATH, is_updater_asset, platform + +HF_DIR = "data/hf" +STATS_CSV = os.path.join(HF_DIR, "stats.csv") +ASSETS_CSV = os.path.join(HF_DIR, "stats-assets.csv") +STATS_FIELDS = ["timestamp", "downloads", "stars"] +ASSET_FIELDS = ["timestamp", "tag", "asset", "platform", "downloads"] + + +class ConditionalClient: + """GitHub GET with per-URL ETag caching (304s are free) and Link pagination.""" + + def __init__(self, token: str | None): + self.session = requests.Session() + self.session.headers["Accept"] = "application/vnd.github+json" + if token: + self.session.headers["Authorization"] = f"Bearer {token}" + self.cache: dict[str, tuple[str, object, str | None]] = {} + + def get(self, url: str) -> tuple[object, str | None]: + """Return (json, next_page_url).""" + headers = {} + if url in self.cache: + headers["If-None-Match"] = self.cache[url][0] + r = self.session.get(url, headers=headers, timeout=30) + if r.status_code == 304: + _, data, nxt = self.cache[url] + return data, nxt + r.raise_for_status() + nxt = r.links.get("next", {}).get("url") + self.cache[url] = (r.headers.get("ETag", ""), r.json(), nxt) + return self.cache[url][1], nxt + + def assets(self) -> list[dict]: + rows = [] + url: str | None = f"https://api.github.com{RELEASES_PATH}" + while url: + releases, url = self.get(url) + assert isinstance(releases, list) + for release in releases: + for a in release["assets"]: + rows.append( + { + "tag": release["tag_name"], + "asset": a["name"], + "platform": platform(a["name"]), + "downloads": a["download_count"], + } + ) + return rows + + def stars(self) -> int: + repo, _ = self.get("https://api.github.com/repos/ActivityWatch/activitywatch") + assert isinstance(repo, dict) + return int(repo["stargazers_count"]) + + +def _last_rows(path: str, key_fields: list[str]) -> dict[tuple, dict]: + last: dict[tuple, dict] = {} + if os.path.exists(path): + with open(path, newline="") as f: + for row in csv.DictReader(f): + last[tuple(row[k] for k in key_fields)] = row + return last + + +def _append(path: str, fields: list[str], rows: list[dict]) -> None: + new = not os.path.exists(path) + with open(path, "a", newline="") as f: + w = csv.DictWriter(f, fieldnames=fields) + if new: + w.writeheader() + w.writerows(rows) + + +def sample_once(client: ConditionalClient, state: dict) -> tuple[int, int]: + """Take one sample; append changed rows. Returns (#asset rows, #stats rows).""" + ts = datetime.now(tz=timezone.utc).isoformat() + assets = client.assets() + changed = [ + {"timestamp": ts, **a} + for a in assets + if state["assets"].get((a["tag"], a["asset"])) != a["downloads"] + ] + for a in changed: + state["assets"][(a["tag"], a["asset"])] = a["downloads"] + if changed: + _append(ASSETS_CSV, ASSET_FIELDS, changed) + + total = sum(a["downloads"] for a in assets if not is_updater_asset(a["asset"])) + stars = client.stars() + stats_rows = [] + if (total, stars) != state["stats"]: + state["stats"] = (total, stars) + stats_rows = [{"timestamp": ts, "downloads": total, "stars": stars}] + _append(STATS_CSV, STATS_FIELDS, stats_rows) + return len(changed), len(stats_rows) + + +def load_state() -> dict: + assets = { + k: int(r["downloads"]) + for k, r in _last_rows(ASSETS_CSV, ["tag", "asset"]).items() + } + stats: tuple[int, int] | None = None + if os.path.exists(STATS_CSV): + with open(STATS_CSV, newline="") as f: + rows = list(csv.DictReader(f)) + if rows: + stats = (int(rows[-1]["downloads"]), int(rows[-1]["stars"])) + return {"assets": assets, "stats": stats} + + +def trim(minutes: int) -> None: + """Keep only the last row per series in each `minutes`-wide bucket.""" + bucket_s = minutes * 60 + for path, key in [(STATS_CSV, []), (ASSETS_CSV, ["tag", "asset"])]: + if not os.path.exists(path): + continue + with open(path, newline="") as f: + reader = csv.DictReader(f) + fields = list(reader.fieldnames or []) + rows = list(reader) + kept: dict[tuple, dict] = {} + for r in rows: + b = int(datetime.fromisoformat(r["timestamp"]).timestamp()) // bucket_s + kept[(b, *(r[k] for k in key))] = r # later rows overwrite earlier ones + out = sorted(kept.values(), key=lambda r: r["timestamp"]) + with open(path, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=fields) + w.writeheader() + w.writerows(out) + print(f"{path}: {len(rows)} -> {len(out)} rows") + + +def main() -> None: + p = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + p.add_argument( + "--interval", type=int, default=60, help="seconds between samples (default 60)" + ) + p.add_argument("--until", help="stop at this UTC date/time (ISO), e.g. 2026-10-20") + p.add_argument("--once", action="store_true", help="take one sample and exit") + p.add_argument( + "--trim", type=int, metavar="MINUTES", help="coarsen the files and exit" + ) + args = p.parse_args() + + if args.trim: + trim(args.trim) + return + token = os.getenv("GITHUB_TOKEN") + if not token and args.interval < 120: + p.error( + "set GITHUB_TOKEN for intervals under 120 s (unauthenticated limit is 60/h)" + ) + until = ( + datetime.fromisoformat(args.until).replace(tzinfo=timezone.utc) + if args.until + else None + ) + os.makedirs(HF_DIR, exist_ok=True) + client = ConditionalClient(token) + state = load_state() + while True: + started = time.monotonic() + try: + n_assets, n_stats = sample_once(client, state) + print( + f"{datetime.now(tz=timezone.utc):%Y-%m-%d %H:%M:%S}Z +{n_assets} asset rows, +{n_stats} stats rows", + flush=True, + ) + except requests.RequestException as e: + print(f"sample failed, retrying next interval: {e}", flush=True) + if args.once or (until and datetime.now(tz=timezone.utc) >= until): + return + time.sleep(max(0.0, args.interval - (time.monotonic() - started))) + + +if __name__ == "__main__": + main() diff --git a/test_sample_hf.py b/test_sample_hf.py new file mode 100644 index 00000000..a18827e7 --- /dev/null +++ b/test_sample_hf.py @@ -0,0 +1,60 @@ +import csv + +import sample_hf + + +class FakeClient: + def __init__(self, assets, stars=10): + self._assets = assets + self._stars = stars + + def assets(self): + return [dict(a) for a in self._assets] + + def stars(self): + return self._stars + + +def _rows(path): + with open(path, newline="") as f: + return list(csv.DictReader(f)) + + +def test_only_changes_are_appended(tmp_path, monkeypatch): + monkeypatch.setattr(sample_hf, "STATS_CSV", str(tmp_path / "stats.csv")) + monkeypatch.setattr(sample_hf, "ASSETS_CSV", str(tmp_path / "stats-assets.csv")) + assets = [ + {"tag": "v0.14.0", "asset": "setup.exe", "platform": "windows", "downloads": 5}, + { + "tag": "v0.14.0", + "asset": "latest.json", + "platform": "unknown", + "downloads": 50, + }, + ] + client = FakeClient(assets) + state = sample_hf.load_state() + assert sample_hf.sample_once(client, state) == (2, 1) + assert sample_hf.sample_once(client, state) == (0, 0) # nothing changed + assets[0]["downloads"] = 6 + assert sample_hf.sample_once(client, state) == (1, 1) + stats = _rows(sample_hf.STATS_CSV) + # updater polls (latest.json) are not counted as downloads + assert [r["downloads"] for r in stats] == ["5", "6"] + # a restart resumes from the files + assert sample_hf.sample_once(client, sample_hf.load_state()) == (0, 0) + + +def test_trim_keeps_last_row_per_bucket(tmp_path, monkeypatch): + path = tmp_path / "stats.csv" + monkeypatch.setattr(sample_hf, "STATS_CSV", str(path)) + monkeypatch.setattr(sample_hf, "ASSETS_CSV", str(tmp_path / "missing.csv")) + path.write_text( + "timestamp,downloads,stars\n" + "2026-10-07T10:00:10+00:00,1,1\n" + "2026-10-07T10:05:00+00:00,2,1\n" + "2026-10-07T10:59:59+00:00,3,1\n" + "2026-10-07T11:00:01+00:00,4,1\n" + ) + sample_hf.trim(60) + assert [r["downloads"] for r in _rows(path)] == ["3", "4"] From 424390e7ff0141acdaa90faa6517f7cb7dfb2f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Tue, 6 Oct 2026 22:45:11 +0200 Subject: [PATCH 3/6] fix(sample_hf): lock out --trim while sampling, convert --until offsets to UTC, reject non-positive intervals; test client caching and per-asset trim --- .gitignore | 1 + sample_hf.py | 50 ++++++++++++++++++++++++++++++--------- test_sample_hf.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 4a80915e..0b05a3a1 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ data/sponsors/ data/android/* !data/android/installed.csv data/spreadsheet.csv +data/hf/.lock diff --git a/sample_hf.py b/sample_hf.py index 47b19590..9f13ff70 100644 --- a/sample_hf.py +++ b/sample_hf.py @@ -23,8 +23,10 @@ import argparse import csv +import fcntl import os import time +from contextlib import contextmanager from datetime import datetime, timezone import requests @@ -36,6 +38,20 @@ ASSETS_CSV = os.path.join(HF_DIR, "stats-assets.csv") STATS_FIELDS = ["timestamp", "downloads", "stars"] ASSET_FIELDS = ["timestamp", "tag", "asset", "platform", "downloads"] +LOCK_PATH = os.path.join(HF_DIR, ".lock") + + +@contextmanager +def exclusive_lock(): + """Held by a running sampler for its lifetime, and by --trim, so trimming + can never race with appends.""" + os.makedirs(HF_DIR, exist_ok=True) + with open(LOCK_PATH, "w") as f: + try: + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise SystemExit(f"another sample_hf.py is running ({LOCK_PATH} is locked)") + yield class ConditionalClient: @@ -178,21 +194,33 @@ def main() -> None: ) args = p.parse_args() - if args.trim: - trim(args.trim) + if args.interval <= 0: + p.error("--interval must be positive") + if args.trim is not None: + if args.trim <= 0: + p.error("--trim must be a positive number of minutes") + with exclusive_lock(): + trim(args.trim) return token = os.getenv("GITHUB_TOKEN") if not token and args.interval < 120: p.error( "set GITHUB_TOKEN for intervals under 120 s (unauthenticated limit is 60/h)" ) - until = ( - datetime.fromisoformat(args.until).replace(tzinfo=timezone.utc) - if args.until - else None - ) - os.makedirs(HF_DIR, exist_ok=True) - client = ConditionalClient(token) + until = None + if args.until: + until = datetime.fromisoformat(args.until) + # Naive times are UTC; times with an offset are converted. + until = ( + until.replace(tzinfo=timezone.utc) + if until.tzinfo is None + else until.astimezone(timezone.utc) + ) + with exclusive_lock(): + run(ConditionalClient(token), args.interval, until, args.once) + + +def run(client, interval: int, until, once: bool) -> None: state = load_state() while True: started = time.monotonic() @@ -204,9 +232,9 @@ def main() -> None: ) except requests.RequestException as e: print(f"sample failed, retrying next interval: {e}", flush=True) - if args.once or (until and datetime.now(tz=timezone.utc) >= until): + if once or (until and datetime.now(tz=timezone.utc) >= until): return - time.sleep(max(0.0, args.interval - (time.monotonic() - started))) + time.sleep(max(0.0, interval - (time.monotonic() - started))) if __name__ == "__main__": diff --git a/test_sample_hf.py b/test_sample_hf.py index a18827e7..e364b06b 100644 --- a/test_sample_hf.py +++ b/test_sample_hf.py @@ -58,3 +58,63 @@ def test_trim_keeps_last_row_per_bucket(tmp_path, monkeypatch): ) sample_hf.trim(60) assert [r["downloads"] for r in _rows(path)] == ["3", "4"] + + +class FakeResponse: + def __init__(self, status, data=None, etag="", next_url=None): + self.status_code = status + self._data = data + self.headers = {"ETag": etag} if etag else {} + self.links = {"next": {"url": next_url}} if next_url else {} + + def json(self): + return self._data + + def raise_for_status(self): + assert self.status_code < 400 + + +def test_conditional_client_caches_pages_and_304s(): + page1 = "https://api.github.com" + sample_hf.RELEASES_PATH + page2 = page1 + "&page=2" + + def rel(tag, n): + return { + "tag_name": tag, + "assets": [{"name": f"{tag}-setup.exe", "download_count": n}], + } + + calls = [] + + def fake_get(url, headers=None, timeout=None): + calls.append((url, (headers or {}).get("If-None-Match"))) + if (headers or {}).get("If-None-Match"): + return FakeResponse(304) + if url == page1: + return FakeResponse(200, [rel("v2", 5)], etag="e1", next_url=page2) + return FakeResponse(200, [rel("v1", 3)], etag="e2") + + client = sample_hf.ConditionalClient(None) + client.session.get = fake_get # type: ignore[method-assign] + first = client.assets() + second = client.assets() # both pages answered with 304, served from cache + assert first == second + assert {(a["tag"], a["downloads"]) for a in second} == {("v2", 5), ("v1", 3)} + assert calls[2:] == [(page1, "e1"), (page2, "e2")] + + +def test_trim_keeps_each_asset_series(tmp_path, monkeypatch): + path = tmp_path / "stats-assets.csv" + monkeypatch.setattr(sample_hf, "STATS_CSV", str(tmp_path / "missing.csv")) + monkeypatch.setattr(sample_hf, "ASSETS_CSV", str(path)) + path.write_text( + "timestamp,tag,asset,platform,downloads\n" + "2026-10-07T10:01:00+00:00,v1,a.exe,windows,1\n" + "2026-10-07T10:02:00+00:00,v1,b.dmg,macos,7\n" + "2026-10-07T10:30:00+00:00,v1,a.exe,windows,2\n" + ) + sample_hf.trim(60) + assert [(r["asset"], r["downloads"]) for r in _rows(path)] == [ + ("b.dmg", "7"), + ("a.exe", "2"), + ] From 12c581e636185bf5cdb4bd61d7fd26da8e0fdf02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Tue, 6 Oct 2026 22:57:04 +0200 Subject: [PATCH 4/6] fix(sample_hf): cross-platform lock (msvcrt on Windows, fcntl elsewhere) --- sample_hf.py | 23 ++++++++++++++++++----- test_sample_hf.py | 14 ++++++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/sample_hf.py b/sample_hf.py index 9f13ff70..63a4a57d 100644 --- a/sample_hf.py +++ b/sample_hf.py @@ -23,7 +23,6 @@ import argparse import csv -import fcntl import os import time from contextlib import contextmanager @@ -41,15 +40,29 @@ LOCK_PATH = os.path.join(HF_DIR, ".lock") +def _try_lock(f) -> bool: + """Non-blocking exclusive lock on an open file (fcntl on Unix, msvcrt on Windows).""" + try: + if os.name == "nt": + import msvcrt + + msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + return True + except OSError: + return False + + @contextmanager def exclusive_lock(): """Held by a running sampler for its lifetime, and by --trim, so trimming can never race with appends.""" os.makedirs(HF_DIR, exist_ok=True) - with open(LOCK_PATH, "w") as f: - try: - fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError: + with open(LOCK_PATH, "a+") as f: + if not _try_lock(f): raise SystemExit(f"another sample_hf.py is running ({LOCK_PATH} is locked)") yield diff --git a/test_sample_hf.py b/test_sample_hf.py index e364b06b..2b72d26b 100644 --- a/test_sample_hf.py +++ b/test_sample_hf.py @@ -118,3 +118,17 @@ def test_trim_keeps_each_asset_series(tmp_path, monkeypatch): ("b.dmg", "7"), ("a.exe", "2"), ] + + +def test_lock_excludes_a_second_holder(tmp_path, monkeypatch): + import pytest + + monkeypatch.setattr(sample_hf, "HF_DIR", str(tmp_path)) + monkeypatch.setattr(sample_hf, "LOCK_PATH", str(tmp_path / ".lock")) + with sample_hf.exclusive_lock(): + with open(sample_hf.LOCK_PATH, "a+") as other: + assert not sample_hf._try_lock(other) + with pytest.raises(SystemExit): + with sample_hf.exclusive_lock(): + with sample_hf.exclusive_lock(): + pass From e08bd8d3bb642aff681b980e1ee7edf762137fa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Tue, 6 Oct 2026 22:57:23 +0200 Subject: [PATCH 5/6] fix(sample_hf): narrow the Windows lock branch by sys.platform for mypy --- sample_hf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sample_hf.py b/sample_hf.py index 63a4a57d..9a181032 100644 --- a/sample_hf.py +++ b/sample_hf.py @@ -24,6 +24,7 @@ import argparse import csv import os +import sys import time from contextlib import contextmanager from datetime import datetime, timezone @@ -43,7 +44,7 @@ def _try_lock(f) -> bool: """Non-blocking exclusive lock on an open file (fcntl on Unix, msvcrt on Windows).""" try: - if os.name == "nt": + if sys.platform == "win32": import msvcrt msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1) From 8fbef2921ea963273d95fc785eb251b6f431e903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Bj=C3=A4reholt?= Date: Tue, 6 Oct 2026 23:13:57 +0200 Subject: [PATCH 6/6] fix(sample_hf): skip draft releases (only visible to write-access tokens) --- sample_hf.py | 3 +++ test_sample_hf.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sample_hf.py b/sample_hf.py index 9a181032..275d3f5e 100644 --- a/sample_hf.py +++ b/sample_hf.py @@ -99,6 +99,9 @@ def assets(self) -> list[dict]: releases, url = self.get(url) assert isinstance(releases, list) for release in releases: + if release.get("draft"): + # Only visible to tokens with write access; not public downloads. + continue for a in release["assets"]: rows.append( { diff --git a/test_sample_hf.py b/test_sample_hf.py index 2b72d26b..1bffd5f0 100644 --- a/test_sample_hf.py +++ b/test_sample_hf.py @@ -91,7 +91,8 @@ def fake_get(url, headers=None, timeout=None): if (headers or {}).get("If-None-Match"): return FakeResponse(304) if url == page1: - return FakeResponse(200, [rel("v2", 5)], etag="e1", next_url=page2) + draft = {**rel("v3", 9), "draft": True} + return FakeResponse(200, [draft, rel("v2", 5)], etag="e1", next_url=page2) return FakeResponse(200, [rel("v1", 3)], etag="e2") client = sample_hf.ConditionalClient(None)