tests.yml

name: Tests

on:
  push:
    branches:
      - main
    tags:
      - "v*"
  pull_request:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  validate:
    name: Validate (${{ matrix.os }}, Python ${{ matrix.python-version }})
    strategy:
      fail-fast: false
      matrix:
        os:
          - macos-latest
          - ubuntu-latest
          - windows-latest
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
    runs-on: ${{ matrix.os }}
    timeout-minutes: 10
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install runtime test dependencies
        run: python -m pip install ".[dev,typer]"
      - name: Run platform and Python runtime suite
        run: ./tests/full_validate.sh --gate runtime

  quality:
    name: Quality and security gates
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          fetch-tags: true
      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.13"
      - name: Install quality dependencies
        run: python -m pip install ".[dev,typer,quality]"
      - name: Validate repository baseline
        run: ./tests/full_validate.sh --gate baseline
      - name: Run tests and enforce coverage budgets
        run: ./tests/full_validate.sh --gate coverage
      - name: Enforce strict typing
        run: ./tests/full_validate.sh --gate typing
      - name: Enforce formatting and lint
        run: ./tests/full_validate.sh --gate style
      - name: Validate docs, schemas, contracts, and generated files
        run: ./tests/full_validate.sh --gate contracts
      - name: Run dependency and static security checks
        run: ./tests/full_validate.sh --gate security

  benchmarks:
    name: Benchmark (${{ matrix.profile }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: 15
    strategy:
      fail-fast: false
      matrix:
        include:
          - profile: unix
            os: ubuntu-latest
          - profile: macos
            os: macos-latest
          - profile: windows
            os: windows-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.13"
      - name: Install declared benchmark comparators
        run: python -m pip install ".[typer,benchmark]"
      - name: Run calibrated comparative benchmark
        env:
          BASE_CLI_BENCHMARK_PLATFORM: ${{ matrix.profile }}
          SOURCE_REVISION: ${{ github.sha }}
        run: python scripts/benchmark_runtime.py --check --iterations 31 --output benchmark-results.json
      - name: Retain dated machine-readable benchmark evidence
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: base-cli-benchmark-${{ matrix.profile }}-${{ github.run_id }}
          path: benchmark-results.json
          if-no-files-found: ignore
          retention-days: 90

  linux-distributions:
    name: Validate (${{ matrix.name }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Debian 12
            image: debian:12-slim
            family: debian
          - name: Fedora latest
            image: fedora:latest
            family: fedora
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          fetch-tags: true
      - name: Run distribution validation in Docker
        env:
          DISTRO_IMAGE: ${{ matrix.image }}
          DISTRO_FAMILY: ${{ matrix.family }}
        run: |
          docker run --rm \
            --volume "$GITHUB_WORKSPACE:/workspace" \
            --workdir /workspace \
            --env DISTRO_FAMILY \
            "$DISTRO_IMAGE" \
            sh -lc '
              set -eu
              if [ "$DISTRO_FAMILY" = debian ]; then
                apt-get update
                DEBIAN_FRONTEND=noninteractive apt-get install -y bash git nodejs npm python3 python3-pip python3-venv
              else
                dnf install -y git nodejs npm python3 python3-pip
              fi
              git config --global --add safe.directory /workspace
              python3 -m venv /tmp/base-cli-venv
              /tmp/base-cli-venv/bin/python -m pip install ".[dev,typer]"
              . /tmp/base-cli-venv/bin/activate
              ./tests/full_validate.sh --gate runtime
            '

  wsl:
    name: Validate (WSL)
    runs-on: windows-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
          fetch-tags: true
      - name: Run platform and Python runtime suite inside WSL
        shell: pwsh
        run: |
          $distros = (wsl --list --quiet 2>$null | Out-String)
          if ($distros -notmatch "Ubuntu") {
            wsl --install --distribution Ubuntu --no-launch
          }
          $drive = $env:GITHUB_WORKSPACE.Substring(0, 1).ToLowerInvariant()
          $path = $env:GITHUB_WORKSPACE.Substring(2).Replace('\', '/')
          $linuxWorkspace = "/mnt/$drive$path"
          $revision = $env:GITHUB_SHA
          wsl --distribution Ubuntu --user root -- bash -lc "set -eu; cd '$linuxWorkspace'; sed -i 's/\r$//' tests/full_validate.sh tests/validate.sh; apt-get update -qq; apt-get install -y -qq nodejs npm python3-venv python3.14-venv; python3 -m venv /tmp/base-cli-venv; . /tmp/base-cli-venv/bin/activate; python -m pip install '.[dev,typer,benchmark]'; bash tests/full_validate.sh --gate runtime; BASE_CLI_BENCHMARK_PLATFORM=wsl SOURCE_REVISION='$revision' python scripts/benchmark_runtime.py --check --iterations 31 --output base-cli-benchmark-wsl.json"
      - name: Retain WSL benchmark evidence
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: base-cli-benchmark-wsl-${{ github.run_id }}
          path: base-cli-benchmark-wsl.json
          if-no-files-found: ignore
          retention-days: 90