From 268e86e13040a416e73ad7eb3e6c94dd6396980a Mon Sep 17 00:00:00 2001 From: Lukas Wallrich Date: Sat, 18 Jul 2026 23:49:17 +0200 Subject: [PATCH 1/2] Fix workflow push races, stale-data commits, and duplicate refresh - refresh-flora.yml, refresh-impact-factor.yml: add `git pull --rebase origin main` before push, mirroring refresh-data.yml, so concurrent pushes to main no longer cause rejected non-fast-forward pushes that discard a run's output. - refresh-impact-factor.yml: drop continue-on-error from the compute_omc step. render_impact_factor.R strictly reads its flora_with_omc.csv output; without the guard a failed enrichment let R commit a "fresh" analysis from stale/absent data. Failing fast is safer than committing misleading output. - refresh-impact-factor.yml: remove the redundant refresh_flora.py step and drop flora.csv / flora_meta.json from the commit; the daily refresh-flora.yml owns that snapshot. - refresh-impact-factor.yml: move cron from Mon 05:00 to Mon 11:00 UTC (after refresh-data's up-to-~6h window) with an explanatory comment. - Delete clean-json.yml: dead safety net. Its targets (meta.json, aggregate.json, originals.json) are all written by refresh_data.py with allow_nan=False via clean_for_json, so no NaN/Infinity tokens are ever emitted. Co-Authored-By: Claude Fable 5 --- .github/workflows/clean-json.yml | 55 --------------------- .github/workflows/refresh-flora.yml | 1 + .github/workflows/refresh-impact-factor.yml | 21 +++++--- 3 files changed, 15 insertions(+), 62 deletions(-) delete mode 100644 .github/workflows/clean-json.yml diff --git a/.github/workflows/clean-json.yml b/.github/workflows/clean-json.yml deleted file mode 100644 index a5a5f74..0000000 --- a/.github/workflows/clean-json.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Clean NaN from JSON - -on: - workflow_dispatch: - -permissions: - contents: write - -jobs: - clean: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Clean JSON files - run: | - python <<'EOF' - import re - from pathlib import Path - - files = [ - "data/meta.json", - "data/aggregate.json", - "data/originals.json", - ] - for f in files: - p = Path(f) - if not p.exists(): - print(f"skip {f} (missing)") - continue - txt = p.read_text() - before = len(re.findall(r"\bNaN\b|\bInfinity\b|\b-Infinity\b", txt)) - txt = re.sub(r"\bNaN\b", "null", txt) - txt = re.sub(r"\b-Infinity\b", "null", txt) - txt = re.sub(r"\bInfinity\b", "null", txt) - p.write_text(txt) - print(f"{f}: replaced {before} invalid tokens") - EOF - - - name: Commit changes - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add data/ - if git diff --cached --quiet; then - echo "Nothing changed." - else - git commit -m "Clean invalid NaN/Infinity tokens from JSON" - git push - fi diff --git a/.github/workflows/refresh-flora.yml b/.github/workflows/refresh-flora.yml index 5e3fbcc..36b7491 100644 --- a/.github/workflows/refresh-flora.yml +++ b/.github/workflows/refresh-flora.yml @@ -39,5 +39,6 @@ jobs: echo "No changes to commit." else git commit -m "Daily FLoRA snapshot: $(date -u +'%Y-%m-%d %H:%M UTC')" + git pull --rebase origin main git push fi diff --git a/.github/workflows/refresh-impact-factor.yml b/.github/workflows/refresh-impact-factor.yml index 2e8dbed..24440cb 100644 --- a/.github/workflows/refresh-impact-factor.yml +++ b/.github/workflows/refresh-impact-factor.yml @@ -2,8 +2,12 @@ name: Refresh Mean Citedness analysis (weekly) on: schedule: - # Monday 05:00 UTC, after the citation refresh - - cron: '0 5 * * 1' + # Monday 11:00 UTC. Deliberately several hours after refresh-data.yml + # (Monday 04:00 UTC, timeout 350 min = up to ~10:00 UTC) so that the + # long citation refresh has finished pushing to main before this job + # commits. Overlapping runs race their pushes to main and a rejected + # non-fast-forward push discards a whole run's output. + - cron: '0 11 * * 1' workflow_dispatch: permissions: @@ -24,11 +28,14 @@ jobs: - name: Install Python dependencies run: pip install -r scripts/requirements.txt - - name: Refresh FLoRA snapshot - run: python scripts/refresh_flora.py + # The daily refresh-flora.yml keeps data/flora.csv fresh, so this job + # consumes the committed snapshot rather than re-downloading it here. - name: Enrich with OpenAlex Mean Citedness - continue-on-error: true + # No continue-on-error: if enrichment fails, flora_with_omc.csv is + # stale or missing, and the R render below would otherwise produce a + # "fresh" impact-factor analysis (new timestamp) from old data. Fail + # fast so nothing misleading gets committed. env: MY_EMAIL: ${{ secrets.MY_EMAIL }} run: python scripts/compute_omc.py @@ -46,13 +53,13 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add data/flora.csv data/flora_meta.json \ - data/flora_with_omc.csv data/flora_with_omc_meta.json \ + git add data/flora_with_omc.csv data/flora_with_omc_meta.json \ data/impact_factor_data.json data/impact_factor_meta.json \ cache/openalex_venues.json || true if git diff --cached --quiet; then echo "No changes to commit." else git commit -m "Mean Citedness refresh: $(date -u +'%Y-%m-%d %H:%M UTC')" + git pull --rebase origin main git push fi From dcbd52d95a80e0855ee80a58854c460ccf359100 Mon Sep 17 00:00:00 2001 From: Lukas Wallrich Date: Sat, 18 Jul 2026 23:52:59 +0200 Subject: [PATCH 2/2] Address codex review: shared concurrency group + snapshot freshness guard - All three data workflows now share concurrency group flora-data-main (queued, not cancelled), giving true mutual exclusion for pushes to main; the rebase-before-push remains as a belt for non-workflow pushes. - The weekly Mean Citedness job fails fast if data/flora.csv is older than 48h, so a broken daily refresh can't feed stale input into a fresh-timestamped analysis. Co-Authored-By: Claude Fable 5 --- .github/workflows/refresh-data.yml | 6 ++++++ .github/workflows/refresh-flora.yml | 6 ++++++ .github/workflows/refresh-impact-factor.yml | 21 +++++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/.github/workflows/refresh-data.yml b/.github/workflows/refresh-data.yml index 91095c4..60aba77 100644 --- a/.github/workflows/refresh-data.yml +++ b/.github/workflows/refresh-data.yml @@ -6,6 +6,12 @@ on: - cron: '0 4 * * 1' workflow_dispatch: +# All data workflows share one concurrency group so their pushes to main are +# serialized (queued, not cancelled) instead of racing. +concurrency: + group: flora-data-main + cancel-in-progress: false + permissions: contents: write diff --git a/.github/workflows/refresh-flora.yml b/.github/workflows/refresh-flora.yml index 36b7491..3747491 100644 --- a/.github/workflows/refresh-flora.yml +++ b/.github/workflows/refresh-flora.yml @@ -6,6 +6,12 @@ on: - cron: '0 3 * * *' workflow_dispatch: +# All data workflows share one concurrency group so their pushes to main are +# serialized (queued, not cancelled) instead of racing. +concurrency: + group: flora-data-main + cancel-in-progress: false + permissions: contents: write diff --git a/.github/workflows/refresh-impact-factor.yml b/.github/workflows/refresh-impact-factor.yml index 24440cb..0b7c9b7 100644 --- a/.github/workflows/refresh-impact-factor.yml +++ b/.github/workflows/refresh-impact-factor.yml @@ -10,6 +10,12 @@ on: - cron: '0 11 * * 1' workflow_dispatch: +# All data workflows share one concurrency group so their pushes to main are +# serialized (queued, not cancelled) instead of racing. +concurrency: + group: flora-data-main + cancel-in-progress: false + permissions: contents: write @@ -30,6 +36,21 @@ jobs: # The daily refresh-flora.yml keeps data/flora.csv fresh, so this job # consumes the committed snapshot rather than re-downloading it here. + # Guard against that assumption failing silently: if the snapshot is + # older than 48h the daily job has been broken, and enriching stale + # input would publish a misleading fresh-timestamped analysis. + - name: Check FLoRA snapshot freshness + run: | + python - <<'EOF' + import json, sys + from datetime import datetime, timezone + stamp = json.load(open("data/flora_meta.json"))["last_updated"] + fetched = datetime.fromisoformat(stamp.replace("Z", "+00:00")) + age_h = (datetime.now(timezone.utc) - fetched).total_seconds() / 3600 + print(f"flora.csv snapshot age: {age_h:.1f} h") + if age_h > 48: + sys.exit("flora.csv snapshot is stale (>48h) - fix refresh-flora.yml first") + EOF - name: Enrich with OpenAlex Mean Citedness # No continue-on-error: if enrichment fails, flora_with_omc.csv is