diff --git a/.github/workflows/clean-json.yml b/.github/workflows/clean-json.yml deleted file mode 100644 index a5a5f74..0000000 --- a/.github/workflows/clean-json.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Clean NaN from JSON - -on: - workflow_dispatch: - -permissions: - contents: write - -jobs: - clean: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Clean JSON files - run: | - python <<'EOF' - import re - from pathlib import Path - - files = [ - "data/meta.json", - "data/aggregate.json", - "data/originals.json", - ] - for f in files: - p = Path(f) - if not p.exists(): - print(f"skip {f} (missing)") - continue - txt = p.read_text() - before = len(re.findall(r"\bNaN\b|\bInfinity\b|\b-Infinity\b", txt)) - txt = re.sub(r"\bNaN\b", "null", txt) - txt = re.sub(r"\b-Infinity\b", "null", txt) - txt = re.sub(r"\bInfinity\b", "null", txt) - p.write_text(txt) - print(f"{f}: replaced {before} invalid tokens") - EOF - - - name: Commit changes - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add data/ - if git diff --cached --quiet; then - echo "Nothing changed." - else - git commit -m "Clean invalid NaN/Infinity tokens from JSON" - git push - fi diff --git a/.github/workflows/refresh-data.yml b/.github/workflows/refresh-data.yml index 91095c4..60aba77 100644 --- a/.github/workflows/refresh-data.yml +++ b/.github/workflows/refresh-data.yml @@ -6,6 +6,12 @@ on: - cron: '0 4 * * 1' workflow_dispatch: +# All data workflows share one concurrency group so their pushes to main are +# serialized (queued, not cancelled) instead of racing. +concurrency: + group: flora-data-main + cancel-in-progress: false + permissions: contents: write diff --git a/.github/workflows/refresh-flora.yml b/.github/workflows/refresh-flora.yml index 5e3fbcc..3747491 100644 --- a/.github/workflows/refresh-flora.yml +++ b/.github/workflows/refresh-flora.yml @@ -6,6 +6,12 @@ on: - cron: '0 3 * * *' workflow_dispatch: +# All data workflows share one concurrency group so their pushes to main are +# serialized (queued, not cancelled) instead of racing. +concurrency: + group: flora-data-main + cancel-in-progress: false + permissions: contents: write @@ -39,5 +45,6 @@ jobs: echo "No changes to commit." else git commit -m "Daily FLoRA snapshot: $(date -u +'%Y-%m-%d %H:%M UTC')" + git pull --rebase origin main git push fi diff --git a/.github/workflows/refresh-impact-factor.yml b/.github/workflows/refresh-impact-factor.yml index 2e8dbed..0b7c9b7 100644 --- a/.github/workflows/refresh-impact-factor.yml +++ b/.github/workflows/refresh-impact-factor.yml @@ -2,10 +2,20 @@ name: Refresh Mean Citedness analysis (weekly) on: schedule: - # Monday 05:00 UTC, after the citation refresh - - cron: '0 5 * * 1' + # Monday 11:00 UTC. Deliberately several hours after refresh-data.yml + # (Monday 04:00 UTC, timeout 350 min = up to ~10:00 UTC) so that the + # long citation refresh has finished pushing to main before this job + # commits. Overlapping runs race their pushes to main and a rejected + # non-fast-forward push discards a whole run's output. + - cron: '0 11 * * 1' workflow_dispatch: +# All data workflows share one concurrency group so their pushes to main are +# serialized (queued, not cancelled) instead of racing. +concurrency: + group: flora-data-main + cancel-in-progress: false + permissions: contents: write @@ -24,11 +34,29 @@ jobs: - name: Install Python dependencies run: pip install -r scripts/requirements.txt - - name: Refresh FLoRA snapshot - run: python scripts/refresh_flora.py + # The daily refresh-flora.yml keeps data/flora.csv fresh, so this job + # consumes the committed snapshot rather than re-downloading it here. + # Guard against that assumption failing silently: if the snapshot is + # older than 48h the daily job has been broken, and enriching stale + # input would publish a misleading fresh-timestamped analysis. + - name: Check FLoRA snapshot freshness + run: | + python - <<'EOF' + import json, sys + from datetime import datetime, timezone + stamp = json.load(open("data/flora_meta.json"))["last_updated"] + fetched = datetime.fromisoformat(stamp.replace("Z", "+00:00")) + age_h = (datetime.now(timezone.utc) - fetched).total_seconds() / 3600 + print(f"flora.csv snapshot age: {age_h:.1f} h") + if age_h > 48: + sys.exit("flora.csv snapshot is stale (>48h) - fix refresh-flora.yml first") + EOF - name: Enrich with OpenAlex Mean Citedness - continue-on-error: true + # No continue-on-error: if enrichment fails, flora_with_omc.csv is + # stale or missing, and the R render below would otherwise produce a + # "fresh" impact-factor analysis (new timestamp) from old data. Fail + # fast so nothing misleading gets committed. env: MY_EMAIL: ${{ secrets.MY_EMAIL }} run: python scripts/compute_omc.py @@ -46,13 +74,13 @@ jobs: run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add data/flora.csv data/flora_meta.json \ - data/flora_with_omc.csv data/flora_with_omc_meta.json \ + git add data/flora_with_omc.csv data/flora_with_omc_meta.json \ data/impact_factor_data.json data/impact_factor_meta.json \ cache/openalex_venues.json || true if git diff --cached --quiet; then echo "No changes to commit." else git commit -m "Mean Citedness refresh: $(date -u +'%Y-%m-%d %H:%M UTC')" + git pull --rebase origin main git push fi