diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 988cc532..3c270207 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -186,3 +186,23 @@ harness once for all of them. that CI never runs. Both were found by mutation-testing the suite and fixed. No mechanizable guard; the durable lesson is: when a test names a narrowing, construct the fixture so the row SURVIVES every other rule, or the assertion proves nothing. + +- [ ] **CE034 — runner-label registry + dogfood runner parity** over + `.github/workflows/*.yml`. Two clauses: (a) every label a job can land on must appear + in `.github/actionlint.yaml`'s `self-hosted-runner.labels` or a stock GitHub-hosted + allowlist — including *both* branches of an expression-valued `runs-on:`, which + actionlint treats as opaque; (b) `action-dogfood`'s label must equal the one the + consumer snippet in `docs/tutorials/02-ci-pipeline.md` advertises. Nothing guards + either today: actionlint is not wired into `make verify` or pre-commit (grep: the + config file is its only mention), and CE026 parses that job's prerequisite *steps* + but never its `runs-on:`. Why it matters: an undeclared label is not a runtime error, + the job queues until GitHub cancels it hours later — indistinguishable from a pool + outage; and a repo-wide `runs-on:` migration has twice swept up `action-dogfood` + (#306, then 027121e in this PR), which exists precisely to prove the published Action + works on the image external integrators use. Implemented and verified once (both + clauses caught their regression class on the real tree) but reverted as out of + proportion to a 16-line runner migration — ~240 lines including tests. Note when + writing it: discriminate labels from expression operands structurally, on the + preceding `&&`/`||`, NOT on the string's shape — a "contains 'ubuntu'" heuristic + silently fails on `uipath-ubunut-latest`, the exact transposition typo the rule is + for. Caught in the multi-model review of PR #86. diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..5a27b8c1 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,11 @@ +# Declares this repo's custom runner labels so actionlint stops reporting them as +# unknown. Advisory only: nothing in `make verify` runs actionlint, so this file helps +# a local run or an editor integration and is not a gate. +# +# These are UiPath's centralized managed GitHub pool labels. Jobs that deliberately +# stay on stock `ubuntu-latest` say why at their own `runs-on:`. +self-hosted-runner: + labels: + - uipath-ubuntu-latest + - uipath-ubuntu-24.04 + - uipath-windows-latest diff --git a/.github/workflows/claude-pr-review.yml b/.github/workflows/claude-pr-review.yml index 45c47087..b9efe76a 100644 --- a/.github/workflows/claude-pr-review.yml +++ b/.github/workflows/claude-pr-review.yml @@ -22,7 +22,7 @@ permissions: {} jobs: claude-review: name: Claude Code Review - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest timeout-minutes: 15 # Trusted triggers only — this repo is public. diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 02ead00b..0d3ac884 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -16,7 +16,7 @@ permissions: jobs: analyze: name: Analyze Python code - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest timeout-minutes: 15 permissions: security-events: write diff --git a/.github/workflows/conventional-commits.yml b/.github/workflows/conventional-commits.yml index 79c8c839..607920a6 100644 --- a/.github/workflows/conventional-commits.yml +++ b/.github/workflows/conventional-commits.yml @@ -16,7 +16,7 @@ permissions: jobs: check-conventional-commits: name: Check PR Title & Commit Messages - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest timeout-minutes: 5 steps: diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 226b70c5..1266c720 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -41,13 +41,13 @@ env: jobs: publish: name: Build and push to GHCR - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest # Skip semantic-release's own commit so we don't double-build on the # version-bump push. if: "!contains(github.event.head_commit.message, 'chore(release):')" timeout-minutes: 30 - env: - SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: "openai-codex-cli-bin,openai-codex" + # No SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS on purpose: nothing installs on the + # host here, and the Docker build doesn't inherit job env (it uses its own ARG). steps: - name: Checkout code @@ -65,17 +65,17 @@ jobs: echo "owner_lc=${OWNER_LC}" >> "$GITHUB_OUTPUT" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Log in to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . file: docker/Dockerfile diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 071cee59..4e9cc15b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,9 +35,12 @@ concurrency: jobs: publish: - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest + # Without a budget, a hang in the unattended `git push --force` below would hold a + # shared-pool runner slot for GitHub's 6h default. + timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Assemble the published tree run: | diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 8692a4da..45496885 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -26,17 +26,26 @@ permissions: # here — TELEMETRY_ENABLED is the single canonical disable gate. env: TELEMETRY_ENABLED: "false" + # The `uipath-*` pool enforces a minimum package-age safe-chain check on installs. + # Workflow-level so every installing job inherits it; per-job copies are how some + # jobs previously ended up with no exclusions at all. The literal is the operative + # value — no secret of that name exists at repo or org level, so the bare `secrets.` + # reference this replaced resolved to an empty list. (Image builds carry their own + # list in docker/Dockerfile; deliberately not the same set.) + SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }} jobs: quality-gate: name: Quality Gate (Format, Lint, Type, Test, Security) - runs-on: ubuntu-latest + # Fork PRs go to stock GitHub-hosted runners: this job runs the PR's own + # `uv.lock` build hooks and test files, and the repo is public, so untrusted code + # must not land on the shared pool image. Everything else uses the pool. + runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ubuntu-latest' || 'uipath-ubuntu-latest' }} timeout-minutes: 10 - # Shared env for all steps: safe-chain min-age exclusions and a dummy - # Anthropic key so CI tests that construct a client don't fail on a missing key. + # A dummy Anthropic key so CI tests that construct a client don't fail on a + # missing key. env: - SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS }} ANTHROPIC_API_KEY: "sk-ant-test-dummy-key-for-ci-tests-only" steps: @@ -172,7 +181,9 @@ jobs: # must trip it. A `evalboard/**`-only filter would skip exactly the change # class this job exists to catch. name: Evalboard (Types, Tests, Build) - runs-on: ubuntu-latest + # Fork-PR carve-out — see `quality-gate`. `pnpm install --frozen-lockfile` runs + # the PR's own lockfile install scripts, same untrusted-code class. + runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ubuntu-latest' || 'uipath-ubuntu-latest' }} timeout-minutes: 15 steps: - name: Checkout code @@ -202,14 +213,15 @@ jobs: # registry validates, and the uipath-specific code paths fail with a # clear hint instead of an import error. name: No-Extra Install (uipath optional) - runs-on: ubuntu-latest + # Fork-PR carve-out — see the comment on `quality-gate` above. + runs-on: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && 'ubuntu-latest' || 'uipath-ubuntu-latest' }} timeout-minutes: 5 steps: - name: Checkout code - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Python 3.13 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.13" @@ -247,7 +259,7 @@ jobs: windows-smoke: name: Windows Smoke Test - runs-on: windows-latest + runs-on: uipath-windows-latest # 15min headroom: the smoke task itself completes in ~7min, but the # actions/cache post-step on Windows is slow when ``.venv`` is large. # We also exclude ``.venv`` from the cached paths (uv re-creates it @@ -261,7 +273,6 @@ jobs: shell: bash env: - SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS }} # Job-level dummy key keeps the unit-test step deterministic and isolated # from real-API leakage. The e2e step below overrides to Bedrock at step scope. ANTHROPIC_API_KEY: "sk-ant-test-dummy-key-for-ci-tests-only" @@ -365,7 +376,7 @@ jobs: e2e-smoke: name: E2E Smoke Tests (Real API) - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest timeout-minutes: 10 # Skip on fork PRs where secrets aren't available if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository @@ -380,7 +391,6 @@ jobs: AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} AWS_REGION: ${{ secrets.AWS_REGION }} BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }} - SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS }} # tasks_run for --tags smoke-pass. 6 task files (hello_date, dataset_example, # smoke_llm_judge, smoke_agent_judge, byod_smoke_test, agentless_smoke_test); # dataset_example fans out to 2 inline rows, so 7 sub-tasks. If you add/remove a @@ -548,7 +558,7 @@ jobs: live-tests: name: Live Integration Tests (Settings Enforcement + Cost Budget) - runs-on: ubuntu-24.04 + runs-on: uipath-ubuntu-24.04 timeout-minutes: 15 # Skip on fork PRs where secrets aren't available if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository @@ -558,7 +568,6 @@ jobs: # settings-enforcement and cost-budget steps use this. The Bedrock # settings-enforcement step adds API_BACKEND=bedrock at step scope only. ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS }} # Bedrock backend for the third settings-enforcement run. Job-level so # the preflight secrets check can validate them; the BedrockRoute is # only engaged via API_BACKEND=bedrock at step scope. @@ -621,10 +630,12 @@ jobs: # `-ra` surfaces skipped tests in the summary so CI logs show what ran vs skipped. # `--strict-markers` rejects unregistered @pytest.mark.* (cheap typo insurance). # JUnit XML feeds the post-run "tests actually passed" assertion below. + # `-n 4` overrides pyproject's `-n auto`: these hit the real API, so + # concurrency must not vary with the runner's vCPU count. 4 = the old shape. run: | mkdir -p tmp .venv/bin/pytest tests/test_claude_settings_enforcement_live.py \ - -m live -v --tb=short --strict-markers -ra -n auto \ + -m live -v --tb=short --strict-markers -ra -n 4 \ --junit-xml=tmp/junit-settings.xml - name: Run claude-settings enforcement live tests (BedrockRoute) @@ -636,7 +647,7 @@ jobs: API_BACKEND: "bedrock" run: | .venv/bin/pytest tests/test_claude_settings_enforcement_live.py \ - -m live -v --tb=short --strict-markers -ra -n auto \ + -m live -v --tb=short --strict-markers -ra -n 4 \ --junit-xml=tmp/junit-settings-bedrock.xml - name: Assert live tests actually ran (not silently skipped) @@ -719,7 +730,7 @@ jobs: codex-live-tests: name: Live Integration Tests (Codex) - runs-on: ubuntu-24.04 + runs-on: uipath-ubuntu-24.04 timeout-minutes: 15 # Skip on fork PRs where secrets aren't available. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository @@ -731,7 +742,6 @@ jobs: CODEX_API_KEY: ${{ secrets.CODEX_API_KEY }} CODEX_BASE_URL: ${{ secrets.CODEX_BASE_URL }} CODEX_MODEL: ${{ secrets.CODEX_MODEL }} - SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS }} steps: - name: Checkout code @@ -795,7 +805,7 @@ jobs: byoa-live-tests: name: Live Integration Tests (BYOA Plugin) - runs-on: ubuntu-24.04 + runs-on: uipath-ubuntu-24.04 timeout-minutes: 15 # Skip on fork PRs where secrets aren't available. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository @@ -803,7 +813,6 @@ jobs: env: # DirectRoute: a plugin agent that subclasses ClaudeCodeAgent uses ANTHROPIC_API_KEY. ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS }} steps: - name: Checkout code @@ -873,6 +882,11 @@ jobs: action-dogfood: name: Action Dogfood (composite action, real API) + # Deliberately NOT on the `uipath-*` pool, and keep it that way: this job is the + # executable proof of the published Action, and docs/tutorials/02-ci-pipeline.md + # (the one consumer snippet naming a runner) says `ubuntu-latest`. Nothing else in + # CI exercises the image integrators actually use. A bulk `runs-on:` migration has + # swept this up twice — check it by hand. runs-on: ubuntu-latest timeout-minutes: 15 # Skip on fork PRs where secrets aren't available (matches e2e-smoke). diff --git a/.github/workflows/publish-testpypi.yml b/.github/workflows/publish-testpypi.yml index 3bf1dbf6..55c3b236 100644 --- a/.github/workflows/publish-testpypi.yml +++ b/.github/workflows/publish-testpypi.yml @@ -34,8 +34,11 @@ permissions: jobs: publish-testpypi: name: Build and publish to TestPyPI - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest timeout-minutes: 10 + env: + # `uv build` resolves build deps under the pool's safe-chain gate. + SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }} environment: name: testpypi url: https://test.pypi.org/project/coder-eval/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3d92f1e2..93003b3d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,21 +49,22 @@ permissions: jobs: release: name: Bump version and publish - # GitHub-hosted so cutting a release does not depend on the self-hosted - # `uipath-ubuntu-latest` pool. semantic-release, uv, twine, and the docker - # buildx -> GHCR push all run fine here. - runs-on: ubuntu-latest + # KNOWINGLY reverses PR #6, which moved this job to a GitHub-hosted runner because + # "cutting a release was blocked whenever that pool was unavailable". The pool's + # availability is what changed; the accepted trade-off is that the release path has + # no GitHub-hosted fallback again. Both jobs here are `workflow_dispatch`-only, so + # no PR check exercises them -- dry-run publish-testpypi.yml after editing this file. + runs-on: uipath-ubuntu-latest timeout-minutes: 15 outputs: # Exposed so the downstream publish-pypi job gates on a version having been # produced (real release on main, or a stamped prerelease on a branch). version: ${{ steps.ver.outputs.version }} env: - # The self-hosted `uipath-ubuntu-latest` runners enforce a minimum - # package-age safe-chain check on uv installs; on GitHub-hosted runners - # this is a no-op. Kept (matching pr-checks.yml) so parity is preserved - # if the job ever moves back to the self-hosted pool. - SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: "openai-codex-cli-bin,openai-codex" + # Load-bearing on the release path: the pool enforces a package-age safe-chain + # check on uv installs. Same expression as pr-checks.yml (see the comment there), + # so a package can't pass PR CI and then fail the release install. + SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS: ${{ secrets.SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS || 'openai-codex-cli-bin,openai-codex' }} steps: # Only a real release (main) needs the app token: semantic-release pushes the @@ -409,14 +410,14 @@ jobs: cache-from: type=registry,ref=ghcr.io/${{ steps.img.outputs.owner_lc }}/coder-eval-agent:buildcache # Publish the wheel+sdist to public PyPI. This runs as its own job so OIDC - # Trusted Publishing is scoped to a dedicated, environment-gated context on - # GitHub-hosted runners -- no PyPI token/secret is stored. Gated on the - # release job having actually cut a version. + # Trusted Publishing is scoped to a dedicated, environment-gated context -- + # no PyPI token/secret is stored. Gated on the release job having actually + # cut a version. publish-pypi: name: Publish to PyPI needs: release if: needs.release.outputs.version != '' - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest timeout-minutes: 10 environment: name: pypi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 991ae30d..62d80cb9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,6 +96,25 @@ after the type/scope (or a `BREAKING CHANGE:` footer) marks a breaking change. Keep PRs small and single-purpose where possible — it makes review faster and bisection easier. +### CI runners + +Workflows run on UiPath's centralized managed GitHub pool, whose labels are listed in +[`.github/actionlint.yaml`](.github/actionlint.yaml). Nothing enforces that list, so +check a new label against it by hand — an unknown label is not a build error, the job +just queues until GitHub cancels it. + +That pool enforces a minimum package-age safe-chain check on installs, so jobs that +install dependencies set `SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS`. The literal in +that expression is the operative value — no secret of that name exists at repo or org +level. `pr-checks.yml` sets it once at the **workflow** level; don't add per-job copies. + +Some jobs deliberately use stock `ubuntu-latest`, each explained at its `runs-on:`: +jobs that execute PR-supplied code (`quality-gate`, `no-uipath-extra`, `evalboard`) +fall back to it **for fork PRs only**, since this repo is public and untrusted code +should not run on the shared pool image — any new job running PR-supplied code needs +the same carve-out. `action-dogfood` always uses it, because it is the executable proof +behind the published Action and must exercise the image integrators actually use. + ## Adding Tasks or Criteria - Task YAMLs live in `tasks/`; see