From a0a7e669502e2991fd75954d6621df363623225b Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 7 Aug 2026 18:43:06 +0100 Subject: [PATCH] Build benchmarks on VMs instead of metal instances which we use for runs Signed-off-by: Adam Gutglick --- .github/workflows/develop-bench.yml | 63 +++++++++++++++++++++++++- .github/workflows/pr-bench-runner.yml | 63 ++++++++++++++++++++++++-- .github/workflows/sql-bench-matrix.yml | 2 +- 3 files changed, 123 insertions(+), 5 deletions(-) diff --git a/.github/workflows/develop-bench.yml b/.github/workflows/develop-bench.yml index 103583bb2aa..c26993088e9 100644 --- a/.github/workflows/develop-bench.yml +++ b/.github/workflows/develop-bench.yml @@ -31,7 +31,53 @@ jobs: bash scripts/commit-json.sh > new-commit.json bash scripts/cat-s3.sh vortex-ci-benchmark-results commits.json new-commit.json + build: + timeout-minutes: 60 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=bench-dedicated/family=c6id.8xlarge/extras=s3-cache/tag=build-{1}', github.run_id, matrix.benchmark.id) + || 'ubuntu-latest' }} + strategy: + fail-fast: false + matrix: + benchmark: + - id: random-access-bench + build_args: "--features lance" + - id: compress-bench + build_args: "--features lance" + steps: + - uses: runs-on/action@v2 + if: github.repository == 'vortex-data/vortex' + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} + - name: Build binary + shell: bash + env: + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" + run: | + cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings + - name: Pre-upload benchmark debuginfo to Polar Signals + uses: ./.github/actions/upload-parca-debuginfo + continue-on-error: true + with: + paths: | + target/release_debug/${{ matrix.benchmark.id }} + polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} + project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + - name: Upload benchmark binary + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ matrix.benchmark.id }}-binary + path: target/release_debug/${{ matrix.benchmark.id }} + bench: + needs: build + if: ${{ !cancelled() }} timeout-minutes: 120 runs-on: >- ${{ github.repository == 'vortex-data/vortex' @@ -57,11 +103,12 @@ jobs: v4_ingest: false steps: - uses: runs-on/action@v2 - if: github.repository == 'vortex-data/vortex' + if: github.repository == 'vortex-data/vortex' && matrix.benchmark.id == 'string-bench' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: ./.github/actions/setup-rust + if: matrix.benchmark.id == 'string-bench' with: repo-token: ${{ secrets.GITHUB_TOKEN }} enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} @@ -75,6 +122,7 @@ jobs: - uses: ./.github/actions/system-info - name: Build binary + if: matrix.benchmark.id == 'string-bench' shell: bash env: RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" @@ -82,6 +130,7 @@ jobs: cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings - name: Pre-upload benchmark debuginfo to Polar Signals + if: matrix.benchmark.id == 'string-bench' uses: ./.github/actions/upload-parca-debuginfo continue-on-error: true with: @@ -90,6 +139,18 @@ jobs: polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + - name: Download benchmark binary + if: matrix.benchmark.id != 'string-bench' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ matrix.benchmark.id }}-binary + path: target/release_debug/ + + - name: Make benchmark binary executable + if: matrix.benchmark.id != 'string-bench' + shell: bash + run: chmod +x target/release_debug/${{ matrix.benchmark.id }} + - name: Setup Polar Signals uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml index 886c8847c6e..82f6c1bf071 100644 --- a/.github/workflows/pr-bench-runner.yml +++ b/.github/workflows/pr-bench-runner.yml @@ -28,7 +28,51 @@ permissions: id-token: write # enables AWS-GitHub OIDC jobs: + build: + if: inputs.benchmark_id != 'string-bench' + timeout-minutes: 60 + runs-on: >- + ${{ github.repository == 'vortex-data/vortex' + && format('runs-on={0}/runner=bench-dedicated/family=c6id.8xlarge/tag=build-{1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '') + || 'ubuntu-latest' }} + steps: + - uses: runs-on/action@v2 + if: github.event.pull_request.head.repo.fork == false + with: + sccache: s3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + - uses: ./.github/actions/setup-rust + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + enable-sccache: ${{ github.event.pull_request.head.repo.fork == false && 'true' || 'false' }} + - name: Build binary + shell: bash + env: + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" + run: | + cargo build --package ${{ inputs.benchmark_id }} --profile release_debug \ + --features ${{ inputs.with_lance && 'lance,' || '' }}unstable_encodings + - name: Pre-upload benchmark debuginfo to Polar Signals + if: github.event.pull_request.head.repo.fork == false + uses: ./.github/actions/upload-parca-debuginfo + continue-on-error: true + with: + paths: | + target/release_debug/${{ inputs.benchmark_id }} + polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} + project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + - name: Upload benchmark binary + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.benchmark_id }}-binary + path: target/release_debug/${{ inputs.benchmark_id }} + bench: + needs: build + if: ${{ !cancelled() && (inputs.benchmark_id == 'string-bench' || needs.build.result == 'success') }} timeout-minutes: 120 runs-on: >- ${{ github.repository == 'vortex-data/vortex' @@ -36,14 +80,14 @@ jobs: || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2 - if: github.event.pull_request.head.repo.fork == false + if: github.event.pull_request.head.repo.fork == false && inputs.benchmark_id == 'string-bench' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - fetch-depth: 0 - uses: ./.github/actions/setup-rust + if: inputs.benchmark_id == 'string-bench' with: repo-token: ${{ secrets.GITHUB_TOKEN }} enable-sccache: ${{ github.event.pull_request.head.repo.fork == false && 'true' || 'false' }} @@ -57,6 +101,7 @@ jobs: - uses: ./.github/actions/system-info - name: Build binary + if: inputs.benchmark_id == 'string-bench' shell: bash env: RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" @@ -65,7 +110,7 @@ jobs: --features ${{ inputs.with_lance && 'lance,' || '' }}unstable_encodings - name: Pre-upload benchmark debuginfo to Polar Signals - if: github.event.pull_request.head.repo.fork == false + if: github.event.pull_request.head.repo.fork == false && inputs.benchmark_id == 'string-bench' uses: ./.github/actions/upload-parca-debuginfo continue-on-error: true with: @@ -74,6 +119,18 @@ jobs: polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + - name: Download benchmark binary + if: inputs.benchmark_id != 'string-bench' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: ${{ inputs.benchmark_id }}-binary + path: target/release_debug/ + + - name: Make benchmark binary executable + if: inputs.benchmark_id != 'string-bench' + shell: bash + run: chmod +x target/release_debug/${{ inputs.benchmark_id }} + - name: Setup Polar Signals if: github.event.pull_request.head.repo.fork == false uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml index 9cd246e4c4f..a2471c084b2 100644 --- a/.github/workflows/sql-bench-matrix.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -50,7 +50,7 @@ jobs: FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=bench-dedicated/family={1}/tag=build{2}', github.run_id, inputs.machine_type, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') + && format('runs-on={0}/runner=bench-dedicated/family=c6id.8xlarge/tag=build{1}', github.run_id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') || 'ubuntu-latest' }} steps: - uses: runs-on/action@v2