ci.yml

name: CI

# Public repo → GitHub-hosted Actions are free. Kept fast: Build and Vet share
# one job so `go vet` reuses the compile cache `go build` just populated
# (standalone `go vet ./...` recompiles everything and is slow). Test is sharded
# across a matrix and runs in parallel; the quick checks (Tidy/Format) give
# sub-minute feedback. Every job restores the Go build+module cache. One run per
# PR (+ push to master); superseded runs cancel.
#
# Pipeline-time controls (see #837):
#   * Test is split into a 4-way package-sharded matrix; a "Test" gate job
#     aggregates the shards so the required-check name is preserved.
#   * Race runs a fast subset on PRs and the full `-race ./...` tree nightly on a
#     schedule (advisory — a mid-day race is caught by the nightly run).
#   * A docs-only PR (only `*.md` / `docs/**`) skips the heavy Go jobs (Test,
#     Race, Contrib, Compat). Those are skipped at the JOB level via `if:`, which
#     GitHub reports as a successful check — so required checks are not left
#     pending and docs PRs stay mergeable. The skip is fail-safe: the heavy jobs
#     run unless the classifier explicitly said docs-only (`code=='false'`), so a
#     classifier failure runs the full suite rather than skipping it, and the
#     "Test" gate is fail-closed (it only counts a skip as a pass on a genuine
#     docs-only classifier success).
on:
  pull_request:
  push:
    branches: [master]
  schedule:
    # Nightly full-tree race detector (03:00 UTC). On PRs Race runs a fast
    # subset only; the exhaustive `-race ./...` sweep runs here.
    - cron: "0 3 * * *"

permissions:
  contents: read

concurrency:
  group: ci-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # Classifies the change so the heavy Go jobs can be skipped for a docs-only
  # PR. `code=true` unless EVERY changed file is `*.md` or under `docs/**`
  # (anything else — .go, go.mod, fixtures, workflows — forces the full run, so
  # a non-docs change can never silently skip the tests). push/schedule always
  # run the full pipeline.
  changes:
    name: Detect changes
    runs-on: ubuntu-latest
    outputs:
      code: ${{ steps.classify.outputs.code }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - id: classify
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          set -euo pipefail
          if [ "${{ github.event_name }}" != "pull_request" ]; then
            echo "non-PR event (${{ github.event_name }}) → run full pipeline"
            echo "code=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          files="$(git diff --name-only "$BASE_SHA...$HEAD_SHA")"
          echo "changed files:"
          echo "$files"
          code=false
          while IFS= read -r f; do
            [ -z "$f" ] && continue
            case "$f" in
              *.md) ;;      # markdown anywhere → docs
              docs/*) ;;    # anything under docs/ → docs ('*' matches '/')
              *) code=true ;;
            esac
          done <<EOF
          $files
          EOF
          echo "code=$code"
          echo "code=$code" >> "$GITHUB_OUTPUT"

  build-vet:
    name: Build & Vet
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: false
      # Manual cache with restore-keys so a changed go.sum still restores the
      # previous build+module cache (only the changed deps recompile) instead
      # of a full cold compile of the whole cloud-SDK dependency tree.
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: go-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            go-${{ runner.os }}-
      - name: go build
        run: go build ./...
      - name: go vet (reuses the build cache)
        run: go vet ./...

  contrib-testcontainers:
    name: Contrib (testcontainers)
    runs-on: ubuntu-latest
    needs: changes
    # Fail-safe: run unless the change was a genuine docs-only skip
    # (code=='false'). If the classifier failed/errored, `code` is empty and the
    # job RUNS (a classifier hiccup never silently skips a heavy job).
    if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }}
    # Separate Go module (keeps testcontainers-go out of the core module), so
    # the root build/vet doesn't cover it. Compile + vet only — the container
    # acceptance test needs Docker and is run on demand (go test ./...).
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: false
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: go-${{ runner.os }}-contrib-${{ hashFiles('contrib/testcontainers/go.sum') }}
          restore-keys: |
            go-${{ runner.os }}-contrib-
      - name: build & vet
        working-directory: contrib/testcontainers
        run: |
          go build ./...
          go vet ./...

  contrib-realengine:
    name: Contrib (realengine)
    runs-on: ubuntu-latest
    needs: changes
    # Fail-safe: run unless the change was a genuine docs-only skip
    # (code=='false'). If the classifier failed/errored, `code` is empty and the
    # job RUNS (a classifier hiccup never silently skips a heavy job).
    if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }}
    # Separate Go module (keeps embedded-postgres out of the core module). Build
    # + vet only — the real-Postgres tests fetch a Postgres binary and run it,
    # so they are executed on demand (go test ./...), not on every CI run.
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: false
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: go-${{ runner.os }}-realengine-${{ hashFiles('contrib/realengine/go.sum') }}
          restore-keys: |
            go-${{ runner.os }}-realengine-
      - name: build & vet
        working-directory: contrib/realengine
        run: |
          go build ./...
          go vet ./...

  contrib-dockerengine:
    name: Contrib (dockerengine)
    runs-on: ubuntu-latest
    needs: changes
    # Fail-safe: run unless the change was a genuine docs-only skip
    # (code=='false'). If the classifier failed/errored, `code` is empty and the
    # job RUNS (a classifier hiccup never silently skips a heavy job).
    if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }}
    # Separate Go module (keeps the docker-CLI-oriented deps out of the core
    # module). Build + vet + lint here; the real-container e2es (MySQL, compute,
    # containers, the Azure Functions host) shell out to the docker CLI and pull
    # multi-hundred-MB images, so — like realengine — they run on demand
    # (go test ./...), not on every CI run. ubuntu-latest ships Docker for that.
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: false
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: go-${{ runner.os }}-dockerengine-${{ hashFiles('contrib/dockerengine/go.sum') }}
          restore-keys: |
            go-${{ runner.os }}-dockerengine-
      - name: build & vet
        working-directory: contrib/dockerengine
        run: |
          go build ./...
          go vet ./...
      - name: install golangci-lint v2
        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b "$(go env GOPATH)/bin"
      - name: golangci-lint run (report-only)
        working-directory: contrib/dockerengine
        run: "$(go env GOPATH)/bin/golangci-lint run --timeout=9m ./... || echo '::warning::golangci-lint reported findings (advisory)'"

  contrib-server:
    name: Contrib (server)
    runs-on: ubuntu-latest
    needs: changes
    # Fail-safe: run unless the change was a genuine docs-only skip
    # (code=='false'). If the classifier failed/errored, `code` is empty and the
    # job RUNS (a classifier hiccup never silently skips a heavy job).
    if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }}
    # Separate Go module: the batteries-included wire server that composes the
    # real engines onto the wire servers. Build + vet + lint here; the real-engine
    # e2e (embedded Postgres + miniredis + real AWS SDK) fetches a Postgres binary
    # and runs it, so — like realengine/dockerengine — it runs on demand
    # (go test ./...), not on every CI run.
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: false
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: go-${{ runner.os }}-server-${{ hashFiles('contrib/server/go.sum') }}
          restore-keys: |
            go-${{ runner.os }}-server-
      # Hard failure — keep the contrib module's compile check blocking (a
      # go.sum/compile break here must fail CI, not merge silently).
      - name: build & vet
        working-directory: contrib/server
        run: |
          go build ./...
          go vet ./...
      # embedded-postgres downloads its Postgres binary on first run; cache it so
      # the CI-safe test tier (no docker socket needed — the E2E uses embedded
      # Postgres, the degrade tests run WITHOUT docker) doesn't refetch each run.
      - uses: actions/cache@v4
        with:
          path: ~/.embedded-postgres-go
          key: embedded-postgres-${{ runner.os }}
      # Advisory (report-only), matching the contrib-* convention: these tests
      # need no docker socket but the embedded-Postgres binary download can flake;
      # promote to blocking once proven stable.
      - name: go test (report-only)
        working-directory: contrib/server
        run: "go test -count=1 ./... || echo '::warning::contrib/server go test reported findings (advisory)'"
      - name: install golangci-lint v2
        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b "$(go env GOPATH)/bin"
      - name: golangci-lint run (report-only)
        working-directory: contrib/server
        run: "$(go env GOPATH)/bin/golangci-lint run --timeout=9m ./... || echo '::warning::golangci-lint reported findings (advisory)'"
      # Only run the heavy multi-stage engines image build when the trees it is
      # built from change, so it doesn't fire on every repo PR.
      - name: filter engines-image paths
        id: engines_paths
        uses: dorny/paths-filter@v3
        with:
          filters: |
            engines:
              - 'contrib/server/**'
              - 'contrib/realengine/**'
              - 'contrib/dockerengine/**'
              - 'server/serverkit/**'
      - name: docker smoke build (engines image)
        if: steps.engines_paths.outputs.engines == 'true'
        run: docker build -f contrib/server/Dockerfile -t cloudemu-engines-test .

  contrib-terraform:
    name: Contrib (terraform)
    runs-on: ubuntu-latest
    needs: changes
    # Fail-safe: run unless the change was a genuine docs-only skip
    # (code=='false'). If the classifier failed/errored, `code` is empty and the
    # job RUNS (a classifier hiccup never silently skips a heavy job).
    if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }}
    # Separate Go module: the IaC-compatibility suite. Unlike the other contrib
    # e2es (which need Docker), this one needs only an OpenTofu binary and a
    # provider download, so — as the evidenced-IaC-compat proof — it actually
    # runs the real `tofu init/apply/plan/destroy` loop in CI, not just build+vet.
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: false
      - uses: opentofu/setup-opentofu@v1
        with:
          tofu_wrapper: false
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: go-${{ runner.os }}-terraform-${{ hashFiles('contrib/terraform/go.sum') }}
          restore-keys: |
            go-${{ runner.os }}-terraform-
      - name: build & vet
        working-directory: contrib/terraform
        run: |
          go build ./...
          go vet ./...
      - name: terraform compatibility e2e
        working-directory: contrib/terraform
        run: go test -timeout 15m ./...
      - name: install golangci-lint v2
        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b "$(go env GOPATH)/bin"
      - name: golangci-lint run (report-only)
        working-directory: contrib/terraform
        run: "$(go env GOPATH)/bin/golangci-lint run --timeout=9m ./... || echo '::warning::golangci-lint reported findings (advisory)'"

  test:
    name: Test (${{ matrix.shard }}/4)
    runs-on: ubuntu-latest
    needs: changes
    # Fail-safe: run unless the change was a genuine docs-only skip
    # (code=='false'). If the classifier failed/errored, `code` is empty and the
    # job RUNS (a classifier hiccup never silently skips a heavy job).
    if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }}
    strategy:
      fail-fast: false
      matrix:
        # 4 package shards. `total` in the run step MUST equal this list's
        # length. Shards partition `go list ./...` by line number modulo total,
        # so every package lands in exactly one shard and the union is the whole
        # tree (proof: residues {1,2,3,0} = all classes mod 4).
        shard: [1, 2, 3, 4]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: false
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: go-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            go-${{ runner.os }}-
      - name: go test (shard ${{ matrix.shard }}/4)
        run: |
          set -euo pipefail
          shard=${{ matrix.shard }}
          total=4
          pkgs="$(go list ./... | awk -v s="$shard" -v t="$total" 'NR % t == s % t')"
          if [ -z "$pkgs" ]; then
            echo "shard $shard/$total is empty — nothing to test"
            exit 0
          fi
          echo "shard $shard/$total: $(echo "$pkgs" | wc -l | tr -d ' ') packages"
          # shellcheck disable=SC2086 # word-splitting the newline package list is intended
          go test $pkgs

  # Aggregating gate: keeps the required-check name "Test" stable after
  # sharding. Branch protection should keep requiring "Test" (NOT the per-shard
  # "Test (n/4)" names). This gate is fail-CLOSED — a skipped matrix only passes
  # when it is a *genuine* docs-only skip (classifier succeeded AND said
  # code=='false'). If the `changes` classifier itself failed/was cancelled, or
  # the shards were skipped while code=='true', the gate goes RED — so a
  # classifier hiccup can never green "Test" having run zero tests.
  test-gate:
    name: Test
    runs-on: ubuntu-latest
    needs: [changes, test]
    if: always()
    steps:
      - name: shards result
        run: |
          changes_result="${{ needs.changes.result }}"
          changes_code="${{ needs.changes.outputs.code }}"
          test_result="${{ needs.test.result }}"
          echo "changes.result=$changes_result changes.code=$changes_code test.result=$test_result"
          if [ "$test_result" = "success" ]; then
            echo "all Test shards passed"
            exit 0
          fi
          if [ "$test_result" = "skipped" ] && [ "$changes_result" = "success" ] && [ "$changes_code" = "false" ]; then
            echo "docs-only PR — Test shards legitimately skipped"
            exit 0
          fi
          echo "Test gate FAILED: shards did not pass (shard failure, classifier failure, or an unexpected skip)"
          exit 1

  race:
    name: Race
    runs-on: ubuntu-latest
    needs: changes
    # Fail-safe: run unless the change was a genuine docs-only skip
    # (code=='false'). If the classifier failed/errored, `code` is empty and the
    # job RUNS (a classifier hiccup never silently skips a heavy job).
    if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: false
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: go-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            go-${{ runner.os }}-
      # On PRs: a fast race signal over the concurrency-heavy Bedrock packages
      # (guardrail versions, evaluation-job stop, etc.) — the same subset that
      # previously rode along in the Test job. Nightly (schedule): the full
      # whole-tree race detector. The per-entity-mutex idiom (see SQS
      # queueData.mu / EC2 instanceData.mu; use memstore.Update, never Get+Set)
      # keeps the tree race-clean; -race is 2-10x slower, hence the timeout.
      - name: race (fast subset on PRs, full tree nightly)
        run: |
          set -euo pipefail
          if [ "${{ github.event_name }}" = "schedule" ]; then
            echo "nightly: full-tree race detector"
            go test -race -timeout 20m ./...
          else
            echo "PR: fast race subset (Bedrock)"
            go test -race ./providers/aws/bedrock/... ./server/aws/bedrock/... ./providers/aws/bedrockagent/... ./server/aws/bedrockagent/... ./server/aws/bedrockagentruntime/...
          fi

  tidy:
    name: Tidy
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: true
      - name: go mod tidy is clean
        run: |
          go mod tidy
          git diff --exit-code go.mod go.sum

  compat:
    name: Compat matrix
    runs-on: ubuntu-latest
    needs: changes
    # Fail-safe: run unless the change was a genuine docs-only skip
    # (code=='false'). If the classifier failed/errored, `code` is empty and the
    # job RUNS (a classifier hiccup never silently skips a heavy job).
    if: ${{ !cancelled() && needs.changes.outputs.code != 'false' }}
    # Runs the in-process SDK-compat suite and regenerates docs/compat/ from the
    # results + docs/coverage/coverage.json, then drift-checks it (same pattern
    # as Tidy). A red cell fails the suite; a stale matrix fails the diff.
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: false
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: go-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            go-${{ runner.os }}-
      - name: regenerate compat matrix
        run: go run ./internal/compatgen
      - name: compat matrix is up to date
        run: git diff --exit-code docs/compat/

  structure:
    name: Structure
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: naming & layout convention (docs/STRUCTURE.md)
        run: |
          # No cloud-prefixed directory at ANY depth under providers/<cloud> or
          # server/<cloud> — the parent aws/azure/gcp already encodes the cloud,
          # so the prefix is redundant (docs/STRUCTURE.md §2.2). Recursive so a
          # promoted sub-surface subdir (§4/§5) can't smuggle a prefix back in.
          fail=0
          for cloud in aws azure gcp; do
            for layer in providers server; do
              [ -d "$layer/$cloud" ] || continue
              for d in $(find "$layer/$cloud" -mindepth 1 -type d); do
                case "$(basename "$d")" in
                  "$cloud"?*)
                    echo "structure: '$d' has a redundant '$cloud' prefix — drop it (docs/STRUCTURE.md §2.2)"
                    fail=1 ;;
                esac
              done
            done
          done
          if [ "$fail" -ne 0 ]; then
            echo "Structure check failed. See docs/STRUCTURE.md."
            exit 1
          fi
          echo "structure: no cloud-prefixed provider/wire directories — OK"
      - name: same filename across layers (docs/STRUCTURE.md §3, advisory)
        run: |
          # STRUCTURE.md §3: a feature uses the same filename across the
          # provider and wire layers, so one filename finds both the mock and
          # the wire handler. Checked loosely — anywhere under server/<cloud>/,
          # not necessarily the same <service> subdir, since a sub-surface can
          # legitimately live under a differently-named service on each side
          # (e.g. providers/aws/vpc/traffic_mirror.go /
          # server/aws/ec2/traffic_mirror.go). Exempts each service's own
          # core-CRUD file plus clone.go/tags.go — the per-directory template
          # (§4) documents these as provider-only. Advisory: real pre-existing
          # drift (renamed files, provider-only helpers) means this warns
          # rather than fails a build.
          missing=0
          for cloud in aws azure gcp; do
            provdir="providers/$cloud"
            srvdir="server/$cloud"
            [ -d "$provdir" ] || continue
            [ -d "$srvdir" ] || continue
            for f in $(find "$provdir" -mindepth 2 -name '*.go' ! -name '*_test.go' ! -path '*/driver/*'); do
              base=$(basename "$f")
              svc=$(basename "$(dirname "$f")")
              case "$base" in
                "$svc.go"|clone.go|tags.go) continue ;;
              esac
              if ! find "$srvdir" -name "$base" ! -name '*_test.go' | grep -q .; then
                echo "structure: $f has no same-named file anywhere under $srvdir/ (docs/STRUCTURE.md §3)"
                missing=$((missing + 1))
              fi
            done
          done
          echo "structure: $missing provider file(s) with no same-named wire-layer file"
          if [ "$missing" -gt 0 ]; then
            echo "::warning::$missing provider file(s) lack a same-named file under server/<cloud>/ (docs/STRUCTURE.md §3, advisory)"
          fi

  format:
    name: Format
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: true
      - name: gofmt (report-only)
        run: |
          unformatted="$(gofmt -l .)"
          if [ -n "$unformatted" ]; then
            echo "::warning::gofmt needed on the following files (advisory):"
            echo "$unformatted"
          else
            echo "all files gofmt-clean"
          fi

  lint:
    name: Lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: "1.25"
          cache: true
      - name: install golangci-lint v2
        run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b "$(go env GOPATH)/bin"
      - name: golangci-lint run (report-only)
        run: "$(go env GOPATH)/bin/golangci-lint run --timeout=9m ./... || echo '::warning::golangci-lint reported findings (advisory)'"