diff --git a/.github/workflows/collect-stats.yml b/.github/workflows/collect-stats.yml index 60e626a9..d740587c 100644 --- a/.github/workflows/collect-stats.yml +++ b/.github/workflows/collect-stats.yml @@ -10,10 +10,7 @@ on: #pull_requests: # branches: [ master ] schedule: - # Every 10 minutes; the "Decide whether to collect" step below only collects - # every 6 hours outside a launch window. GitHub may delay scheduled runs, so - # every row carries its own timestamp. - - cron: '*/10 * * * *' + - cron: '0 */6 * * *' # Never let two collections race on the push. concurrency: @@ -29,32 +26,13 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Decide whether to collect - id: cadence - run: | - # High-resolution window (UTC) around the v0.14.0 launch; 10-minute - # samples correlate downloads with posts (data/launch-events.csv). - # Outside it, collect only if the last sample is ~6 h old. - WINDOW_END=2026-10-27 - now=$(date -u +%s) - last=$(tail -n 1 data/stats.csv | cut -d, -f1) - age=$(( now - $(date -u -d "$last" +%s) )) - if [ "$GITHUB_EVENT_NAME" != "schedule" ] || [ "$now" -lt "$(date -u -d "$WINDOW_END" +%s)" ] || [ "$age" -ge 21000 ]; then - echo "collect=true" >> "$GITHUB_OUTPUT" - else - echo "collect=false" >> "$GITHUB_OUTPUT" - echo "Outside the launch window and last sample is only ${age}s old; skipping." - fi - name: Install uv - if: steps.cadence.outputs.collect == 'true' uses: astral-sh/setup-uv@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies - if: steps.cadence.outputs.collect == 'true' run: uv sync - name: Collect and append stats - if: steps.cadence.outputs.collect == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -69,14 +47,12 @@ jobs: # breakdowns). Appends only assets whose count changed since last logged. uv run python3 collect_stats.py --assets --csv >> data/stats-assets.csv - name: Commit - if: steps.cadence.outputs.collect == 'true' run: | git config --local user.email "noreply@github.com" git config --local user.name "GitHub Action" git add data/stats.csv data/releases.csv data/stats-assets.csv git commit -m "chore: update stats" - name: Push changes - if: steps.cadence.outputs.collect == 'true' run: | # Both stats workflows push to master; retry on a rejected push. for i in 1 2 3 4 5; do diff --git a/.gitignore b/.gitignore index 4a80915e..0b05a3a1 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ data/sponsors/ data/android/* !data/android/installed.csv data/spreadsheet.csv +data/hf/.lock diff --git a/README.md b/README.md index 7040dfa2..567cca51 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,12 @@ The following is manually updated: - `chrome-weekly-users.csv` - Chrome extension weekly active users - `firefox-daily-users.csv` - Firefox extension daily active users - `notes.csv` - Manual entries of major/interesting events - - `launch-events.csv` - Timestamped launch posts (UTC; channel, url, note), to line up with the - high-resolution download samples. During a launch window (see `WINDOW_END` in - `.github/workflows/collect-stats.yml`) stats are collected every ~10 minutes instead of every 6 hours. + - `launch-events.csv` - Timestamped launch posts (UTC; channel, url, note), to line up with + high-resolution samples. + - `hf/` - High-resolution samples from `sample_hf.py` (same schemas as `stats.csv` / + `stats-assets.csv`, change-only rows). Run it anywhere during a launch and submit the files in a PR: + `GITHUB_TOKEN=$(gh auth token) uv run python3 sample_hf.py --until 2026-10-20` + (default 60 s interval; `--trim MINUTES` coarsens before submitting). ## Android vitals diff --git a/sample_hf.py b/sample_hf.py new file mode 100644 index 00000000..275d3f5e --- /dev/null +++ b/sample_hf.py @@ -0,0 +1,258 @@ +""" +High-resolution download sampler for launch windows. + +Run it anywhere (laptop, server, VM) during a launch, then submit the files it +writes in a PR. It polls the GitHub API every `--interval` seconds (default 60) +and appends rows only when something changed: + + data/hf/stats.csv timestamp,downloads,stars (same schema as data/stats.csv) + data/hf/stats-assets.csv timestamp,tag,asset,platform,downloads + +Downloads exclude Tauri updater assets, like collect_stats.downloads(); the +per-asset file keeps them (latest.json fetches = update checks). + +Uses conditional requests (ETag): an unchanged response is a 304, which does +not count against the rate limit. Set GITHUB_TOKEN (e.g. `GITHUB_TOKEN=$(gh auth token)`) +for the 5000/h limit; unauthenticated (60/h) needs --interval >= 120. + + GITHUB_TOKEN=$(gh auth token) uv run python3 sample_hf.py --until 2026-10-20 + +Stop with Ctrl-C; a restart resumes from the files. Coarsen before submitting +with `--trim MINUTES` (keeps the last row per series per MINUTES-wide bucket). +""" + +import argparse +import csv +import os +import sys +import time +from contextlib import contextmanager +from datetime import datetime, timezone + +import requests + +from collect_stats import RELEASES_PATH, is_updater_asset, platform + +HF_DIR = "data/hf" +STATS_CSV = os.path.join(HF_DIR, "stats.csv") +ASSETS_CSV = os.path.join(HF_DIR, "stats-assets.csv") +STATS_FIELDS = ["timestamp", "downloads", "stars"] +ASSET_FIELDS = ["timestamp", "tag", "asset", "platform", "downloads"] +LOCK_PATH = os.path.join(HF_DIR, ".lock") + + +def _try_lock(f) -> bool: + """Non-blocking exclusive lock on an open file (fcntl on Unix, msvcrt on Windows).""" + try: + if sys.platform == "win32": + import msvcrt + + msvcrt.locking(f.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + + fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) + return True + except OSError: + return False + + +@contextmanager +def exclusive_lock(): + """Held by a running sampler for its lifetime, and by --trim, so trimming + can never race with appends.""" + os.makedirs(HF_DIR, exist_ok=True) + with open(LOCK_PATH, "a+") as f: + if not _try_lock(f): + raise SystemExit(f"another sample_hf.py is running ({LOCK_PATH} is locked)") + yield + + +class ConditionalClient: + """GitHub GET with per-URL ETag caching (304s are free) and Link pagination.""" + + def __init__(self, token: str | None): + self.session = requests.Session() + self.session.headers["Accept"] = "application/vnd.github+json" + if token: + self.session.headers["Authorization"] = f"Bearer {token}" + self.cache: dict[str, tuple[str, object, str | None]] = {} + + def get(self, url: str) -> tuple[object, str | None]: + """Return (json, next_page_url).""" + headers = {} + if url in self.cache: + headers["If-None-Match"] = self.cache[url][0] + r = self.session.get(url, headers=headers, timeout=30) + if r.status_code == 304: + _, data, nxt = self.cache[url] + return data, nxt + r.raise_for_status() + nxt = r.links.get("next", {}).get("url") + self.cache[url] = (r.headers.get("ETag", ""), r.json(), nxt) + return self.cache[url][1], nxt + + def assets(self) -> list[dict]: + rows = [] + url: str | None = f"https://api.github.com{RELEASES_PATH}" + while url: + releases, url = self.get(url) + assert isinstance(releases, list) + for release in releases: + if release.get("draft"): + # Only visible to tokens with write access; not public downloads. + continue + for a in release["assets"]: + rows.append( + { + "tag": release["tag_name"], + "asset": a["name"], + "platform": platform(a["name"]), + "downloads": a["download_count"], + } + ) + return rows + + def stars(self) -> int: + repo, _ = self.get("https://api.github.com/repos/ActivityWatch/activitywatch") + assert isinstance(repo, dict) + return int(repo["stargazers_count"]) + + +def _last_rows(path: str, key_fields: list[str]) -> dict[tuple, dict]: + last: dict[tuple, dict] = {} + if os.path.exists(path): + with open(path, newline="") as f: + for row in csv.DictReader(f): + last[tuple(row[k] for k in key_fields)] = row + return last + + +def _append(path: str, fields: list[str], rows: list[dict]) -> None: + new = not os.path.exists(path) + with open(path, "a", newline="") as f: + w = csv.DictWriter(f, fieldnames=fields) + if new: + w.writeheader() + w.writerows(rows) + + +def sample_once(client: ConditionalClient, state: dict) -> tuple[int, int]: + """Take one sample; append changed rows. Returns (#asset rows, #stats rows).""" + ts = datetime.now(tz=timezone.utc).isoformat() + assets = client.assets() + changed = [ + {"timestamp": ts, **a} + for a in assets + if state["assets"].get((a["tag"], a["asset"])) != a["downloads"] + ] + for a in changed: + state["assets"][(a["tag"], a["asset"])] = a["downloads"] + if changed: + _append(ASSETS_CSV, ASSET_FIELDS, changed) + + total = sum(a["downloads"] for a in assets if not is_updater_asset(a["asset"])) + stars = client.stars() + stats_rows = [] + if (total, stars) != state["stats"]: + state["stats"] = (total, stars) + stats_rows = [{"timestamp": ts, "downloads": total, "stars": stars}] + _append(STATS_CSV, STATS_FIELDS, stats_rows) + return len(changed), len(stats_rows) + + +def load_state() -> dict: + assets = { + k: int(r["downloads"]) + for k, r in _last_rows(ASSETS_CSV, ["tag", "asset"]).items() + } + stats: tuple[int, int] | None = None + if os.path.exists(STATS_CSV): + with open(STATS_CSV, newline="") as f: + rows = list(csv.DictReader(f)) + if rows: + stats = (int(rows[-1]["downloads"]), int(rows[-1]["stars"])) + return {"assets": assets, "stats": stats} + + +def trim(minutes: int) -> None: + """Keep only the last row per series in each `minutes`-wide bucket.""" + bucket_s = minutes * 60 + for path, key in [(STATS_CSV, []), (ASSETS_CSV, ["tag", "asset"])]: + if not os.path.exists(path): + continue + with open(path, newline="") as f: + reader = csv.DictReader(f) + fields = list(reader.fieldnames or []) + rows = list(reader) + kept: dict[tuple, dict] = {} + for r in rows: + b = int(datetime.fromisoformat(r["timestamp"]).timestamp()) // bucket_s + kept[(b, *(r[k] for k in key))] = r # later rows overwrite earlier ones + out = sorted(kept.values(), key=lambda r: r["timestamp"]) + with open(path, "w", newline="") as f: + w = csv.DictWriter(f, fieldnames=fields) + w.writeheader() + w.writerows(out) + print(f"{path}: {len(rows)} -> {len(out)} rows") + + +def main() -> None: + p = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + p.add_argument( + "--interval", type=int, default=60, help="seconds between samples (default 60)" + ) + p.add_argument("--until", help="stop at this UTC date/time (ISO), e.g. 2026-10-20") + p.add_argument("--once", action="store_true", help="take one sample and exit") + p.add_argument( + "--trim", type=int, metavar="MINUTES", help="coarsen the files and exit" + ) + args = p.parse_args() + + if args.interval <= 0: + p.error("--interval must be positive") + if args.trim is not None: + if args.trim <= 0: + p.error("--trim must be a positive number of minutes") + with exclusive_lock(): + trim(args.trim) + return + token = os.getenv("GITHUB_TOKEN") + if not token and args.interval < 120: + p.error( + "set GITHUB_TOKEN for intervals under 120 s (unauthenticated limit is 60/h)" + ) + until = None + if args.until: + until = datetime.fromisoformat(args.until) + # Naive times are UTC; times with an offset are converted. + until = ( + until.replace(tzinfo=timezone.utc) + if until.tzinfo is None + else until.astimezone(timezone.utc) + ) + with exclusive_lock(): + run(ConditionalClient(token), args.interval, until, args.once) + + +def run(client, interval: int, until, once: bool) -> None: + state = load_state() + while True: + started = time.monotonic() + try: + n_assets, n_stats = sample_once(client, state) + print( + f"{datetime.now(tz=timezone.utc):%Y-%m-%d %H:%M:%S}Z +{n_assets} asset rows, +{n_stats} stats rows", + flush=True, + ) + except requests.RequestException as e: + print(f"sample failed, retrying next interval: {e}", flush=True) + if once or (until and datetime.now(tz=timezone.utc) >= until): + return + time.sleep(max(0.0, interval - (time.monotonic() - started))) + + +if __name__ == "__main__": + main() diff --git a/test_sample_hf.py b/test_sample_hf.py new file mode 100644 index 00000000..1bffd5f0 --- /dev/null +++ b/test_sample_hf.py @@ -0,0 +1,135 @@ +import csv + +import sample_hf + + +class FakeClient: + def __init__(self, assets, stars=10): + self._assets = assets + self._stars = stars + + def assets(self): + return [dict(a) for a in self._assets] + + def stars(self): + return self._stars + + +def _rows(path): + with open(path, newline="") as f: + return list(csv.DictReader(f)) + + +def test_only_changes_are_appended(tmp_path, monkeypatch): + monkeypatch.setattr(sample_hf, "STATS_CSV", str(tmp_path / "stats.csv")) + monkeypatch.setattr(sample_hf, "ASSETS_CSV", str(tmp_path / "stats-assets.csv")) + assets = [ + {"tag": "v0.14.0", "asset": "setup.exe", "platform": "windows", "downloads": 5}, + { + "tag": "v0.14.0", + "asset": "latest.json", + "platform": "unknown", + "downloads": 50, + }, + ] + client = FakeClient(assets) + state = sample_hf.load_state() + assert sample_hf.sample_once(client, state) == (2, 1) + assert sample_hf.sample_once(client, state) == (0, 0) # nothing changed + assets[0]["downloads"] = 6 + assert sample_hf.sample_once(client, state) == (1, 1) + stats = _rows(sample_hf.STATS_CSV) + # updater polls (latest.json) are not counted as downloads + assert [r["downloads"] for r in stats] == ["5", "6"] + # a restart resumes from the files + assert sample_hf.sample_once(client, sample_hf.load_state()) == (0, 0) + + +def test_trim_keeps_last_row_per_bucket(tmp_path, monkeypatch): + path = tmp_path / "stats.csv" + monkeypatch.setattr(sample_hf, "STATS_CSV", str(path)) + monkeypatch.setattr(sample_hf, "ASSETS_CSV", str(tmp_path / "missing.csv")) + path.write_text( + "timestamp,downloads,stars\n" + "2026-10-07T10:00:10+00:00,1,1\n" + "2026-10-07T10:05:00+00:00,2,1\n" + "2026-10-07T10:59:59+00:00,3,1\n" + "2026-10-07T11:00:01+00:00,4,1\n" + ) + sample_hf.trim(60) + assert [r["downloads"] for r in _rows(path)] == ["3", "4"] + + +class FakeResponse: + def __init__(self, status, data=None, etag="", next_url=None): + self.status_code = status + self._data = data + self.headers = {"ETag": etag} if etag else {} + self.links = {"next": {"url": next_url}} if next_url else {} + + def json(self): + return self._data + + def raise_for_status(self): + assert self.status_code < 400 + + +def test_conditional_client_caches_pages_and_304s(): + page1 = "https://api.github.com" + sample_hf.RELEASES_PATH + page2 = page1 + "&page=2" + + def rel(tag, n): + return { + "tag_name": tag, + "assets": [{"name": f"{tag}-setup.exe", "download_count": n}], + } + + calls = [] + + def fake_get(url, headers=None, timeout=None): + calls.append((url, (headers or {}).get("If-None-Match"))) + if (headers or {}).get("If-None-Match"): + return FakeResponse(304) + if url == page1: + draft = {**rel("v3", 9), "draft": True} + return FakeResponse(200, [draft, rel("v2", 5)], etag="e1", next_url=page2) + return FakeResponse(200, [rel("v1", 3)], etag="e2") + + client = sample_hf.ConditionalClient(None) + client.session.get = fake_get # type: ignore[method-assign] + first = client.assets() + second = client.assets() # both pages answered with 304, served from cache + assert first == second + assert {(a["tag"], a["downloads"]) for a in second} == {("v2", 5), ("v1", 3)} + assert calls[2:] == [(page1, "e1"), (page2, "e2")] + + +def test_trim_keeps_each_asset_series(tmp_path, monkeypatch): + path = tmp_path / "stats-assets.csv" + monkeypatch.setattr(sample_hf, "STATS_CSV", str(tmp_path / "missing.csv")) + monkeypatch.setattr(sample_hf, "ASSETS_CSV", str(path)) + path.write_text( + "timestamp,tag,asset,platform,downloads\n" + "2026-10-07T10:01:00+00:00,v1,a.exe,windows,1\n" + "2026-10-07T10:02:00+00:00,v1,b.dmg,macos,7\n" + "2026-10-07T10:30:00+00:00,v1,a.exe,windows,2\n" + ) + sample_hf.trim(60) + assert [(r["asset"], r["downloads"]) for r in _rows(path)] == [ + ("b.dmg", "7"), + ("a.exe", "2"), + ] + + +def test_lock_excludes_a_second_holder(tmp_path, monkeypatch): + import pytest + + monkeypatch.setattr(sample_hf, "HF_DIR", str(tmp_path)) + monkeypatch.setattr(sample_hf, "LOCK_PATH", str(tmp_path / ".lock")) + with sample_hf.exclusive_lock(): + with open(sample_hf.LOCK_PATH, "a+") as other: + assert not sample_hf._try_lock(other) + with pytest.raises(SystemExit): + with sample_hf.exclusive_lock(): + with sample_hf.exclusive_lock(): + pass