diff --git a/.github/actions/check-rebuild/action.yml b/.github/actions/check-rebuild/action.yml index 2d3b6e2df41..1dffe28618e 100644 --- a/.github/actions/check-rebuild/action.yml +++ b/.github/actions/check-rebuild/action.yml @@ -13,15 +13,21 @@ runs: steps: - name: "Build" shell: bash - run: ${{ inputs.command }} + run: bash -c -- "$INPUTS_COMMAND" + env: + INPUTS_COMMAND: ${{ inputs.command }} - name: "Verify rebuild is a no-op" shell: bash run: | - stale=$(${{ inputs.command }} \ - ${{ inputs.message-format-flag }} json 2>/dev/null \ + verify_command=${INPUTS_COMMAND}' "$@"' + stale=$(bash -c -- "$verify_command" _ \ + "$INPUTS_MESSAGE_FORMAT_FLAG" json 2>/dev/null \ | jq -r 'select(.reason == "compiler-artifact" and .fresh == false) | .target.name') if [ -n "$stale" ]; then echo "ERROR: Rebuild recompiled crates that should have been cached:" echo "$stale" exit 1 fi + env: + INPUTS_COMMAND: ${{ inputs.command }} + INPUTS_MESSAGE_FORMAT_FLAG: ${{ inputs.message-format-flag }} diff --git a/.github/actions/setup-flatc/action.yml b/.github/actions/setup-flatc/action.yml index 46d0856a539..4ecaf4e7865 100644 --- a/.github/actions/setup-flatc/action.yml +++ b/.github/actions/setup-flatc/action.yml @@ -11,6 +11,8 @@ runs: id: download-flatc shell: bash run: | - wget -O /tmp/flatc.zip "https://github.com/google/flatbuffers/releases/download/v${{ inputs.flatc_version }}/Linux.flatc.binary.clang++-18.zip" + wget -O /tmp/flatc.zip "https://github.com/google/flatbuffers/releases/download/v${INPUTS_FLATC_VERSION}/Linux.flatc.binary.clang++-18.zip" unzip /tmp/flatc.zip flatc mv flatc /usr/local/bin/ + env: + INPUTS_FLATC_VERSION: ${{ inputs.flatc_version }} diff --git a/.github/actions/setup-protoc/action.yml b/.github/actions/setup-protoc/action.yml index 0eddf14499e..e0ffc4662d2 100644 --- a/.github/actions/setup-protoc/action.yml +++ b/.github/actions/setup-protoc/action.yml @@ -25,7 +25,7 @@ runs: esac fi - PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${{ inputs.protoc_version }}/protoc-${{ inputs.protoc_version }}-${PROTOC_ARCH}.zip" + PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${INPUTS_PROTOC_VERSION}/protoc-${INPUTS_PROTOC_VERSION}-${PROTOC_ARCH}.zip" wget -q -O /tmp/protoc.zip "$PROTOC_URL" unzip -o /tmp/protoc.zip -d /tmp/protoc @@ -41,3 +41,5 @@ runs: rm -rf /tmp/protoc.zip /tmp/protoc protoc --version + env: + INPUTS_PROTOC_VERSION: ${{ inputs.protoc_version }} diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml index 56f14612907..878d60ef4f3 100644 --- a/.github/actions/setup-rust/action.yml +++ b/.github/actions/setup-rust/action.yml @@ -45,7 +45,7 @@ runs: - name: Rust Toolchain id: rust-toolchain - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable + uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 if: steps.check-rustup.outputs.exists != 'true' with: toolchain: "${{ steps.toolchain-config.outputs.toolchain }}" diff --git a/.github/actions/upload-parca-debuginfo/action.yml b/.github/actions/upload-parca-debuginfo/action.yml index 40369a8227e..29423394816 100644 --- a/.github/actions/upload-parca-debuginfo/action.yml +++ b/.github/actions/upload-parca-debuginfo/action.yml @@ -54,8 +54,6 @@ runs: chmod +x parca-debuginfo fi - echo "$PWD" >> "$GITHUB_PATH" - - name: Upload benchmark binary debuginfo to Polar Signals shell: bash env: @@ -84,7 +82,7 @@ runs: fi cmd=( - parca-debuginfo + "$RUNNER_TEMP/parca-debuginfo/parca-debuginfo" upload "--store-address=$STORE_ADDRESS" "--bearer-token-file=$token_file" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86fa85fd4d6..2128a1f1524 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,6 @@ on: permissions: actions: read contents: read - checks: write # audit-check creates checks - issues: write # audit-check creates issues env: CARGO_TERM_COLOR: always @@ -29,7 +27,9 @@ jobs: name: "Mirror DuckDB to R2" if: github.event_name == 'pull_request' uses: ./.github/workflows/duckdb-r2.yml - secrets: inherit + secrets: + DUCKDB_R2_ACCESS_KEY_ID: ${{ secrets.DUCKDB_R2_ACCESS_KEY_ID }} + DUCKDB_R2_SECRET_ACCESS_KEY: ${{ secrets.DUCKDB_R2_SECRET_ACCESS_KEY }} duckdb-ready: name: "DuckDB libraries available in R2" @@ -49,21 +49,10 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: spiraldb/actions/.github/actions/lint-toml@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 - validate-workflow-yaml: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: Validate YAML file - run: | - # Lint the workflows and yamllint's configuration file. - yamllint \ - --strict \ - -c .yamllint.yaml \ - .github/ - python-lint: name: "Python (lint)" runs-on: >- @@ -72,11 +61,13 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 10 steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -104,11 +95,13 @@ jobs: RUST_LOG: "info,maturin=off,uv=debug" MATURIN_PEP517_ARGS: "--profile ci" steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -156,6 +149,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false # sync: false — the test runs via `uv run --no-project` and needs no workspace # packages; the default `uv sync` builds the vortex-data Rust extension (~6 min). - uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 @@ -175,10 +170,12 @@ jobs: RUST_LOG: "info,maturin=off,uv=debug" MATURIN_PEP517_ARGS: "--profile ci" steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -208,11 +205,13 @@ jobs: && format('runs-on={0}/runner=amd64-small/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-docs', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -258,11 +257,13 @@ jobs: rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"unsupported\"'" args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-cuda --exclude vortex-cub --exclude vortex-nvcomp --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd --exclude vortex-test-e2e-cuda --exclude vortex-python-cuda --exclude vortex-sqllogictest --exclude vortex-parquet-variant" steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -285,11 +286,13 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-min-deps', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -304,11 +307,13 @@ jobs: && format('runs-on={0}/runner=amd64-large/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -367,6 +372,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: C/C++ Lint - clang-format run: | git ls-files lang/cpp vortex-cuda vortex-duckdb vortex-ffi \ @@ -384,11 +391,13 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-lint-no-default', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -406,11 +415,13 @@ jobs: && format('runs-on={0}/pool=windows-x64-pre/extras=s3-cache', github.run_id) || 'windows-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Setup (Windows) run: | echo "C:\rust\cargo\bin" >> $env:GITHUB_PATH @@ -448,10 +459,12 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=rust-test-linux-arm64 steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -478,11 +491,13 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=btrblocks-golden', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -510,11 +525,13 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -536,6 +553,8 @@ jobs: continue-on-error: ${{ matrix.checks == 'advisories' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: EmbarkStudios/cargo-deny-action@3c6349835b2b7b196a839186cb8b78e02f7b5f25 # v2 with: command: check ${{ matrix.checks }} @@ -548,11 +567,13 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -597,11 +618,13 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -622,11 +645,13 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=wasm-integration', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -651,11 +676,13 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -692,11 +719,13 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-build', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" diff --git a/.github/workflows/claude-review.yml b/.github/workflows/claude-review.yml index 1d2b546b611..7d50f192b1f 100644 --- a/.github/workflows/claude-review.yml +++ b/.github/workflows/claude-review.yml @@ -173,9 +173,12 @@ jobs: steps: - name: Comment with the refusal reason uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + env: + REFUSAL_REASON: ${{ needs.gate.outputs.reason }} + PULL_NUMBER: ${{ needs.gate.outputs.pull_number }} with: script: | - const reason = ${{ toJSON(needs.gate.outputs.reason) }}; + const reason = process.env.REFUSAL_REASON; const messages = { fork_pr_refused: [ "Claude review automation is disabled for fork pull requests.", @@ -207,10 +210,19 @@ jobs: ].join("\n"), }; + if (!Object.hasOwn(messages, reason)) { + throw new Error(`Unexpected refusal reason: ${reason}`); + } + + const pullNumber = Number(process.env.PULL_NUMBER); + if (!Number.isSafeInteger(pullNumber) || pullNumber <= 0) { + throw new Error(`Invalid pull request number: ${process.env.PULL_NUMBER}`); + } + await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, - issue_number: Number(${{ needs.gate.outputs.pull_number }}), + issue_number: pullNumber, body: messages[reason], }); diff --git a/.github/workflows/close-fixed-fuzzer-issues.yml b/.github/workflows/close-fixed-fuzzer-issues.yml index 2471775759e..4a861988cc6 100644 --- a/.github/workflows/close-fixed-fuzzer-issues.yml +++ b/.github/workflows/close-fixed-fuzzer-issues.yml @@ -31,12 +31,14 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: @@ -45,7 +47,7 @@ jobs: enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Install llvm - uses: aminya/setup-cpp@v1 + uses: aminya/setup-cpp@1f17f92d6a52bfcb1a25348e2c526c2e5cbb1134 # v1.8.0 with: compiler: llvm diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index aeafc73ed0e..16c204e7cb4 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -35,7 +35,9 @@ jobs: run-cuda-benchmarks: ${{ github.event_name != 'pull_request' || steps.filter.outputs.cuda == 'true' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4 + with: + persist-credentials: false + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 id: filter if: github.event_name == 'pull_request' with: @@ -65,11 +67,13 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=bench-codspeed-{1}', github.run_id, matrix.shard) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} @@ -101,10 +105,12 @@ jobs: runs-on: >- runs-on=${{ github.run_id }}/family=g5/cpu=8/image=ubuntu24-gpu-x64/extras=s3-cache/tag=bench-codspeed-cuda-build steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -151,10 +157,12 @@ jobs: runs-on: >- runs-on=${{ github.run_id }}/family=g5/cpu=8/image=ubuntu24-gpu-x64/extras=s3-cache/tag=bench-codspeed-cuda-${{ matrix.shard }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/commit-metadata.yml b/.github/workflows/commit-metadata.yml index 5c136028984..e692035ad4d 100644 --- a/.github/workflows/commit-metadata.yml +++ b/.github/workflows/commit-metadata.yml @@ -21,14 +21,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 2 - - # v4 (Postgres) ingest -- REQUIRED (see develop-bench.yml rationale). Empty records: - # post-ingest.py --postgres upserts the commit row only. Gated on the - # ingest-role ARN var (the assume-role input that MUST exist for OIDC to - # succeed). - # - # `sync: false` -- the ingest runs `uv run --no-project --with`, which needs only - # the uv binary, never the synced workspace (see develop-bench.yml rationale). + persist-credentials: false - name: Install uv for v4 ingest if: vars.GH_BENCH_INGEST_ROLE_ARN != '' uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 diff --git a/.github/workflows/compat-gen-upload.yml b/.github/workflows/compat-gen-upload.yml index bf594626df7..cd72c07e0eb 100644 --- a/.github/workflows/compat-gen-upload.yml +++ b/.github/workflows/compat-gen-upload.yml @@ -34,13 +34,14 @@ jobs: outputs: version: ${{ steps.detect.outputs.version }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} @@ -54,7 +55,7 @@ jobs: - name: Detect version id: detect run: | - GIT_REF="${{ inputs.git_ref }}" + GIT_REF="${INPUTS_GIT_REF}" if [ -n "$GIT_REF" ]; then TAG=$(git describe --tags --abbrev=0 "$GIT_REF") else @@ -63,15 +64,19 @@ jobs: VERSION="${TAG#v}" echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "Detected version: $VERSION (from tag: $TAG)" + env: + INPUTS_GIT_REF: ${{ inputs.git_ref }} - name: Dry run publish run: | - GIT_REF="${{ inputs.git_ref }}" + GIT_REF="${INPUTS_GIT_REF}" CMD="uv run vortex-test/compat-gen/scripts/compat.py publish --dry-run" if [ -n "$GIT_REF" ]; then CMD="$CMD --git-ref $GIT_REF" fi $CMD + env: + INPUTS_GIT_REF: ${{ inputs.git_ref }} upload: needs: dry-run @@ -86,13 +91,14 @@ jobs: id-token: write contents: read steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} @@ -105,9 +111,11 @@ jobs: - name: Upload fixtures for v${{ needs.dry-run.outputs.version }} run: | - GIT_REF="${{ inputs.git_ref }}" + GIT_REF="${INPUTS_GIT_REF}" CMD="uv run vortex-test/compat-gen/scripts/compat.py publish --yes" if [ -n "$GIT_REF" ]; then CMD="$CMD --git-ref $GIT_REF" fi $CMD + env: + INPUTS_GIT_REF: ${{ inputs.git_ref }} diff --git a/.github/workflows/compat-validation.yml b/.github/workflows/compat-validation.yml index d8012193614..2d29af2e8ce 100644 --- a/.github/workflows/compat-validation.yml +++ b/.github/workflows/compat-validation.yml @@ -29,16 +29,19 @@ jobs: || 'ubuntu-latest' }} timeout-minutes: 30 steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Run compat tests + env: + MODE: ${{ inputs.mode || 'last' }} run: | - MODE="${{ inputs.mode || 'last' }}" uv run vortex-test/compat-gen/scripts/compat.py check \ --mode "$MODE" diff --git a/.github/workflows/cuda.yaml b/.github/workflows/cuda.yaml index de081df94f1..5f816b9d307 100644 --- a/.github/workflows/cuda.yaml +++ b/.github/workflows/cuda.yaml @@ -31,7 +31,9 @@ jobs: run-cuda-san: ${{ github.event_name != 'pull_request' || steps.filter.outputs.cuda == 'true' }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4 + with: + persist-credentials: false + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 id: filter if: github.event_name == 'pull_request' with: @@ -56,10 +58,12 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-build steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -91,7 +95,7 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-tests steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 with: sccache: s3 - name: Display NVIDIA SMI details @@ -100,6 +104,8 @@ jobs: nvidia-smi -L nvidia-smi -q -d Memory - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -109,7 +115,7 @@ jobs: with: sync: false - name: Install nextest - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2 + uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: nextest - name: Rust Tests @@ -161,7 +167,7 @@ jobs: - sanitizer: initcheck runner_flags: "--tool initcheck --error-exitcode 1" steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 with: sccache: s3 - name: Display NVIDIA SMI details @@ -170,6 +176,8 @@ jobs: nvidia-smi -L nvidia-smi -q -d Memory - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -190,7 +198,7 @@ jobs: timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/extras=s3-cache/tag=cuda-test-cudf steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 with: sccache: s3 - name: Display NVIDIA SMI details @@ -199,6 +207,8 @@ jobs: nvidia-smi -L nvidia-smi -q -d Memory - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/develop-bench.yml b/.github/workflows/develop-bench.yml index 103583bb2aa..3a04d7e0043 100644 --- a/.github/workflows/develop-bench.yml +++ b/.github/workflows/develop-bench.yml @@ -7,16 +7,19 @@ on: branches: [develop] permissions: - id-token: write # enables AWS-GitHub OIDC - actions: read - contents: write + contents: read jobs: commit-metadata: runs-on: ubuntu-latest timeout-minutes: 10 + permissions: + contents: read + id-token: write # enables AWS-GitHub OIDC steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Setup AWS CLI uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6 with: @@ -33,6 +36,9 @@ jobs: bench: timeout-minutes: 120 + permissions: + contents: read + id-token: write # enables AWS-GitHub OIDC runs-on: >- ${{ github.repository == 'vortex-data/vortex' && format('runs-on={0}/runner=bench-dedicated/family=c6id.metal/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id) @@ -56,11 +62,13 @@ jobs: name: String Encoding v4_ingest: false steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -189,7 +197,15 @@ jobs: sql: uses: ./.github/workflows/sql-bench-matrix.yml - secrets: inherit + permissions: + contents: read + id-token: write # enables AWS-GitHub OIDC + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }} + OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }} + BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }} + INCIDENT_IO_ALERT_TOKEN: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} with: mode: "develop" matrix_preset: "develop" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 99e02bc15b6..40942f78d95 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,22 +9,25 @@ on: branches: [develop] workflow_dispatch: { } -permissions: - # Weirdly, we need write permissions to see draft releases. - # See https://github.com/cli/cli/issues/3037 - contents: write +permissions: { } jobs: build: runs-on: ubuntu-latest timeout-minutes: 30 + permissions: + # Weirdly, we need write permissions to see draft releases. + # See https://github.com/cli/cli/issues/3037 + contents: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up JDK 17 - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5 + uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: java-version: "17" distribution: "temurin" diff --git a/.github/workflows/duckdb-r2.yml b/.github/workflows/duckdb-r2.yml index d7c09085d00..d9df3bf2975 100644 --- a/.github/workflows/duckdb-r2.yml +++ b/.github/workflows/duckdb-r2.yml @@ -5,7 +5,12 @@ name: DuckDB R2 mirror # page and upload it to R2. # If it's a commit, build Duckdb from source and upload it to R2. on: - workflow_call: { } + workflow_call: + secrets: + DUCKDB_R2_ACCESS_KEY_ID: + required: false + DUCKDB_R2_SECRET_ACCESS_KEY: + required: false concurrency: group: duckdb-r2-${{ github.event.pull_request.number || github.ref }} @@ -32,6 +37,8 @@ jobs: any_missing: ${{ steps.resolve.outputs.any_missing }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Resolve version and check R2 id: resolve run: sh scripts/duckdb-r2-resolve.sh @@ -51,6 +58,8 @@ jobs: runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Install build dependencies (Linux) if: needs.check.outputs.release != 'true' && runner.os == 'Linux' @@ -72,12 +81,14 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.DUCKDB_R2_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DUCKDB_R2_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" + AWS_REGION: us-east-1 AWS_ENDPOINT_URL: ${{ env.R2_ENDPOINT_URL }} + NEEDS_CHECK_OUTPUTS_REF_DIR: ${{ needs.check.outputs.ref_dir }} + MATRIX_ARCHIVE: ${{ matrix.archive }} run: | set -Eeuo pipefail python3 scripts/s3-upload.py \ --bucket "$R2_BUCKET" \ - --key "${{ needs.check.outputs.ref_dir }}/${{ matrix.archive }}" \ - --body "${{ matrix.archive }}" \ + --key "${NEEDS_CHECK_OUTPUTS_REF_DIR}/${MATRIX_ARCHIVE}" \ + --body "${MATRIX_ARCHIVE}" \ --checksum-algorithm CRC32 diff --git a/.github/workflows/fuzz-coverage.yml b/.github/workflows/fuzz-coverage.yml index 2b6d3d3fd70..94a7b36fe04 100644 --- a/.github/workflows/fuzz-coverage.yml +++ b/.github/workflows/fuzz-coverage.yml @@ -21,12 +21,14 @@ jobs: && format('runs-on={0}/runner=arm64-medium/disk=large/extras=s3-cache', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 76282a3d272..fff7ee456d3 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -63,7 +63,6 @@ jobs: uses: ./.github/workflows/fuzzer-fix-automation.yml with: issue_number: ${{ needs.report-io-fuzz-failures.outputs.issue_number }} - secrets: inherit # ============================================================================ # Array Operations Fuzzer diff --git a/.github/workflows/fuzzer-fix-automation.yml b/.github/workflows/fuzzer-fix-automation.yml index b813fc1ea1d..5a8fdb5abc8 100644 --- a/.github/workflows/fuzzer-fix-automation.yml +++ b/.github/workflows/fuzzer-fix-automation.yml @@ -73,9 +73,10 @@ jobs: id: fetch_issue env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUTS_ISSUE_NUMBER: ${{ inputs.issue_number }} run: | - ISSUE_DATA=$(gh issue view ${{ inputs.issue_number }} --repo ${{ github.repository }} --json number,title,body,labels) - echo "issue_number=${{ inputs.issue_number }}" >> $GITHUB_OUTPUT + ISSUE_DATA=$(gh issue view ${INPUTS_ISSUE_NUMBER} --repo ${{ github.repository }} --json number,title,body,labels) + echo "issue_number=${INPUTS_ISSUE_NUMBER}" >> $GITHUB_OUTPUT echo "issue_title=$(echo "$ISSUE_DATA" | jq -r '.title')" >> $GITHUB_OUTPUT echo "issue_body<> $GITHUB_OUTPUT echo "$ISSUE_DATA" | jq -r '.body' >> $GITHUB_OUTPUT @@ -88,7 +89,7 @@ jobs: toolchain: ${{ env.NIGHTLY_TOOLCHAIN }} - name: Install llvm - uses: aminya/setup-cpp@v1 + uses: aminya/setup-cpp@1f17f92d6a52bfcb1a25348e2c526c2e5cbb1134 # v1.8.0 with: compiler: llvm @@ -102,9 +103,7 @@ jobs: shell: bash run: | # Extract crash details from the fetched issue body - cat > issue_body.txt <<'ISSUE_EOF' - ${{ steps.fetch_issue.outputs.issue_body }} - ISSUE_EOF + printf '%s\n' "$STEPS_FETCH_ISSUE_OUTPUTS_ISSUE_BODY" > issue_body.txt # Extract target name from issue body TARGET=$(grep -oP '(?<=\*\*Target\*\*: `)[^`]+' issue_body.txt || echo "file_io") @@ -120,13 +119,19 @@ jobs: echo "Extracted: target=$TARGET, crash_file=$CRASH_FILE" rm -f issue_body.txt + env: + STEPS_FETCH_ISSUE_OUTPUTS_ISSUE_BODY: ${{ steps.fetch_issue.outputs.issue_body }} - name: Validate issue details id: validate env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUTS_ISSUE_NUMBER: ${{ inputs.issue_number }} + STEPS_EXTRACT_OUTPUTS_CRASH_FILE: ${{ steps.extract.outputs.crash_file }} + STEPS_EXTRACT_OUTPUTS_ARTIFACT_URL: ${{ steps.extract.outputs.artifact_url }} + STEPS_EXTRACT_OUTPUTS_TARGET: ${{ steps.extract.outputs.target }} run: | - ISSUE_NUM="${{ inputs.issue_number }}" + ISSUE_NUM="${INPUTS_ISSUE_NUMBER}" # Check if issue exists and has fuzzer label ISSUE_LABELS=$(gh issue view "$ISSUE_NUM" --repo ${{ github.repository }} --json labels --jq '.labels[].name') @@ -139,30 +144,33 @@ jobs: echo "✅ Issue #$ISSUE_NUM has 'fuzzer' label" # Check if we have required crash details - if [ -z "${{ steps.extract.outputs.crash_file }}" ]; then + if [ -z "${STEPS_EXTRACT_OUTPUTS_CRASH_FILE}" ]; then echo "❌ Could not extract crash file name from issue" exit 1 fi - if [ -z "${{ steps.extract.outputs.artifact_url }}" ]; then + if [ -z "${STEPS_EXTRACT_OUTPUTS_ARTIFACT_URL}" ]; then echo "❌ Could not extract artifact URL from issue" exit 1 fi - echo "✅ Extracted crash details: target=${{ steps.extract.outputs.target }}, crash_file=${{ steps.extract.outputs.crash_file }}" + echo "✅ Extracted crash details: target=${STEPS_EXTRACT_OUTPUTS_TARGET}, crash_file=${STEPS_EXTRACT_OUTPUTS_CRASH_FILE}" - name: Download and verify crash artifact id: download env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + STEPS_EXTRACT_OUTPUTS_ARTIFACT_URL: ${{ steps.extract.outputs.artifact_url }} + STEPS_EXTRACT_OUTPUTS_TARGET: ${{ steps.extract.outputs.target }} + STEPS_EXTRACT_OUTPUTS_CRASH_FILE: ${{ steps.extract.outputs.crash_file }} run: | # Extract run ID from artifact URL - ARTIFACT_URL="${{ steps.extract.outputs.artifact_url }}" + ARTIFACT_URL="${STEPS_EXTRACT_OUTPUTS_ARTIFACT_URL}" RUN_ID=$(echo "$ARTIFACT_URL" | grep -oP 'runs/\K[0-9]+') ARTIFACT_ID=$(echo "$ARTIFACT_URL" | grep -oP 'artifacts/\K[0-9]+') - # Artifact name matches run-fuzzer.yml upload: ${{ inputs.fuzz_target }}-crash-artifacts - TARGET="${{ steps.extract.outputs.target }}" + # Artifact name matches run-fuzzer.yml upload: -crash-artifacts + TARGET="${STEPS_EXTRACT_OUTPUTS_TARGET}" ARTIFACT_NAME="${TARGET}-crash-artifacts" echo "Downloading artifact $ARTIFACT_NAME (ID: $ARTIFACT_ID) from run $RUN_ID" @@ -171,10 +179,10 @@ jobs: gh run download "$RUN_ID" --name "$ARTIFACT_NAME" --repo ${{ github.repository }} # Verify crash file exists - CRASH_FILE_PATH="${{ steps.extract.outputs.target }}/${{ steps.extract.outputs.crash_file }}" + CRASH_FILE_PATH="${STEPS_EXTRACT_OUTPUTS_TARGET}/${STEPS_EXTRACT_OUTPUTS_CRASH_FILE}" if [ ! -f "$CRASH_FILE_PATH" ]; then echo "❌ Crash file not found: $CRASH_FILE_PATH" - ls -la "${{ steps.extract.outputs.target }}/" || true + ls -la "${STEPS_EXTRACT_OUTPUTS_TARGET}/" || true exit 1 fi @@ -184,10 +192,10 @@ jobs: - name: Build fuzzer target id: build run: | - echo "Building fuzzer target: ${{ steps.extract.outputs.target }} (debug mode for faster build)" + echo "Building fuzzer target: ${STEPS_EXTRACT_OUTPUTS_TARGET} (debug mode for faster build)" # Build the fuzzer target in debug mode (faster than release) - if cargo +$NIGHTLY_TOOLCHAIN fuzz build --dev --sanitizer=none "${{ steps.extract.outputs.target }}" 2>&1 | tee fuzzer_build.log; then + if cargo +$NIGHTLY_TOOLCHAIN fuzz build --dev --sanitizer=none "${STEPS_EXTRACT_OUTPUTS_TARGET}" 2>&1 | tee fuzzer_build.log; then echo "✅ Fuzzer target built successfully" echo "build_success=true" >> $GITHUB_OUTPUT else @@ -199,6 +207,8 @@ jobs: tail -50 fuzzer_build.log exit 1 fi + env: + STEPS_EXTRACT_OUTPUTS_TARGET: ${{ steps.extract.outputs.target }} - name: Reproduce crash id: reproduce @@ -207,7 +217,7 @@ jobs: echo "Attempting to reproduce crash with fuzzer (debug mode)..." # Run fuzzer with crash file (debug mode, no sanitizer, full backtrace) - RUST_BACKTRACE=full timeout 30s cargo +$NIGHTLY_TOOLCHAIN fuzz run --dev --sanitizer=none "${{ steps.extract.outputs.target }}" "${{ steps.download.outputs.crash_file_path }}" -- -runs=1 -rss_limit_mb=0 2>&1 | tee crash_reproduction.log + RUST_BACKTRACE=full timeout 30s cargo +$NIGHTLY_TOOLCHAIN fuzz run --dev --sanitizer=none "${STEPS_EXTRACT_OUTPUTS_TARGET}" "${STEPS_DOWNLOAD_OUTPUTS_CRASH_FILE_PATH}" -- -runs=1 -rss_limit_mb=0 2>&1 | tee crash_reproduction.log FUZZ_EXIT_CODE=${PIPESTATUS[0]} @@ -218,13 +228,19 @@ jobs: echo "✅ Crash reproduced (exit code: $FUZZ_EXIT_CODE)" echo "crash_reproduced=true" >> $GITHUB_OUTPUT fi + env: + STEPS_EXTRACT_OUTPUTS_TARGET: ${{ steps.extract.outputs.target }} + STEPS_DOWNLOAD_OUTPUTS_CRASH_FILE_PATH: ${{ steps.download.outputs.crash_file_path }} - name: Check if crash still exists if: steps.reproduce.outputs.crash_reproduced == 'false' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUTS_ISSUE_NUMBER: ${{ inputs.issue_number }} + STEPS_EXTRACT_OUTPUTS_TARGET: ${{ steps.extract.outputs.target }} + STEPS_DOWNLOAD_OUTPUTS_CRASH_FILE_PATH: ${{ steps.download.outputs.crash_file_path }} run: | - ISSUE_NUM="${{ inputs.issue_number }}" + ISSUE_NUM="${INPUTS_ISSUE_NUMBER}" gh issue comment "$ISSUE_NUM" --repo ${{ github.repository }} --body "## 🤖 Automated Analysis @@ -236,7 +252,7 @@ jobs: I ran: \`\`\`bash - cargo +nightly fuzz run --sanitizer=none ${{ steps.extract.outputs.target }} ${{ steps.download.outputs.crash_file_path }} -- -runs=1 -rss_limit_mb=0 + cargo +nightly fuzz run --sanitizer=none ${STEPS_EXTRACT_OUTPUTS_TARGET} ${STEPS_DOWNLOAD_OUTPUTS_CRASH_FILE_PATH} -- -runs=1 -rss_limit_mb=0 \`\`\` The fuzzer exited with code 0 (success). @@ -272,7 +288,7 @@ jobs: CRASH_FILE: ${{ steps.extract.outputs.crash_file }} CRASH_FILE_PATH: ${{ steps.download.outputs.crash_file_path }} ARTIFACT_URL: ${{ steps.extract.outputs.artifact_url }} - uses: anthropics/claude-code-action@be7b93b1907a4abad570368f3c74b6fe3807510b # v1 + uses: anthropics/claude-code-action@be7b93b1907a4abad570368f3c74b6fe3807510b # v1.0.183 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} # Use the App token (not GITHUB_TOKEN) so the committed fix branch and @@ -408,8 +424,9 @@ jobs: if: steps.reproduce.outputs.crash_reproduced == 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUTS_ISSUE_NUMBER: ${{ inputs.issue_number }} run: | - ISSUE_NUM="${{ inputs.issue_number }}" + ISSUE_NUM="${INPUTS_ISSUE_NUMBER}" # Check for comments from claude-code[bot] COMMENT_COUNT=$(gh api "repos/${{ github.repository }}/issues/$ISSUE_NUM/comments" \ diff --git a/.github/workflows/fuzzer-issue-autofix.yml b/.github/workflows/fuzzer-issue-autofix.yml index a6e992d9ab0..f19f509efb8 100644 --- a/.github/workflows/fuzzer-issue-autofix.yml +++ b/.github/workflows/fuzzer-issue-autofix.yml @@ -106,4 +106,3 @@ jobs: uses: ./.github/workflows/fuzzer-fix-automation.yml with: issue_number: ${{ github.event.issue.number }} - secrets: inherit diff --git a/.github/workflows/minimize_fuzz_corpus_workflow.yml b/.github/workflows/minimize_fuzz_corpus_workflow.yml index af8e270c3c5..fc5b17b374d 100644 --- a/.github/workflows/minimize_fuzz_corpus_workflow.yml +++ b/.github/workflows/minimize_fuzz_corpus_workflow.yml @@ -43,15 +43,17 @@ jobs: - name: Bail if not on develop if: github.ref != 'refs/heads/develop' run: | - echo "::error::Corpus minimization should only run on the develop branch (current: ${{ github.ref }})" + echo "::error::Corpus minimization should only run on the develop branch (current: ${GITHUB_REF})" exit 1 - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: @@ -69,11 +71,12 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" - AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" + AWS_REGION: us-east-1 + AWS_ENDPOINT_URL: https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com + INPUTS_FUZZ_TARGET: ${{ inputs.fuzz_target }} run: | - CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst" - CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}" + CORPUS_KEY="${INPUTS_FUZZ_TARGET}_corpus.tar.zst" + CORPUS_DIR="fuzz/corpus/${INPUTS_FUZZ_TARGET}" if python3 scripts/s3-download.py "s3://vortex-fuzz-corpus/$CORPUS_KEY" "$CORPUS_KEY"; then echo "Downloaded corpus successfully" @@ -87,26 +90,30 @@ jobs: - name: Minimize corpus run: | FEATURES_FLAG="" - if [ -n "${{ inputs.extra_features }}" ]; then - FEATURES_FLAG="--features ${{ inputs.extra_features }}" + if [ -n "${INPUTS_EXTRA_FEATURES}" ]; then + FEATURES_FLAG="--features ${INPUTS_EXTRA_FEATURES}" fi - CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}" + CORPUS_DIR="fuzz/corpus/${INPUTS_FUZZ_TARGET}" MINIMIZED_DIR="${CORPUS_DIR}_minimized" mkdir -p "$MINIMIZED_DIR" cargo +$NIGHTLY_TOOLCHAIN fuzz cmin $FEATURES_FLAG \ - ${{ inputs.fuzz_target }} "$CORPUS_DIR" -- "$MINIMIZED_DIR" + ${INPUTS_FUZZ_TARGET} "$CORPUS_DIR" -- "$MINIMIZED_DIR" rm -rf "$CORPUS_DIR" mv "$MINIMIZED_DIR" "$CORPUS_DIR" + env: + INPUTS_EXTRA_FEATURES: ${{ inputs.extra_features }} + INPUTS_FUZZ_TARGET: ${{ inputs.fuzz_target }} - name: Persist corpus shell: bash env: AWS_ACCESS_KEY_ID: ${{ secrets.R2_FUZZ_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_FUZZ_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" - AWS_ENDPOINT_URL: "https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com" + AWS_REGION: us-east-1 + AWS_ENDPOINT_URL: https://01e9655179bbec953276890b183039bc.r2.cloudflarestorage.com + INPUTS_FUZZ_TARGET: ${{ inputs.fuzz_target }} run: | - CORPUS_KEY="${{ inputs.fuzz_target }}_corpus.tar.zst" - CORPUS_DIR="fuzz/corpus/${{ inputs.fuzz_target }}" + CORPUS_KEY="${INPUTS_FUZZ_TARGET}_corpus.tar.zst" + CORPUS_DIR="fuzz/corpus/${INPUTS_FUZZ_TARGET}" tar -acf "$CORPUS_KEY" "$CORPUS_DIR" python3 scripts/s3-upload.py --bucket vortex-fuzz-corpus --key "$CORPUS_KEY" --body "$CORPUS_KEY" --checksum-algorithm CRC32 diff --git a/.github/workflows/musl.yml b/.github/workflows/musl.yml index c331dbc2859..740542b20ed 100644 --- a/.github/workflows/musl.yml +++ b/.github/workflows/musl.yml @@ -53,6 +53,8 @@ jobs: python3 python3-dev tar tzdata bash - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Install nextest shell: bash diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index cc90e07316f..629c9bfa16e 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -19,7 +19,12 @@ permissions: jobs: sql: uses: ./.github/workflows/sql-bench-matrix.yml - secrets: inherit + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }} + OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }} + BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }} + INCIDENT_IO_ALERT_TOKEN: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} with: mode: "develop" machine_type: ${{ matrix.machine_type.instance_name }} diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 33e9dab920f..5866bbe22e0 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -18,6 +18,8 @@ on: required: true type: string +permissions: { } + jobs: prepare-python: runs-on: ${{ matrix.target.runs-on }} @@ -34,6 +36,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/actions/setup-rust with: @@ -47,7 +50,9 @@ jobs: - name: Cargo Set Version run: | - cargo set-version --workspace ${{ inputs.version }} + cargo set-version --workspace ${INPUTS_VERSION} + env: + INPUTS_VERSION: ${{ inputs.version }} # cargo set-version bumps Cargo.toml crate versions but not the ==0.1.0 pins # between these two wheels in their pyproject.toml. The base and CUDA extension @@ -116,7 +121,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5 + persist-credentials: false + - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: "corretto" java-version: "17" @@ -143,14 +149,15 @@ jobs: runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=prepare-java-linux-arm64 runs-on: ${{ matrix.runner }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5 + persist-credentials: false + - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: "corretto" java-version: "17" diff --git a/.github/workflows/pr-bench-compress.yml b/.github/workflows/pr-bench-compress.yml index 6d7abd7e86f..bdf6cd427d0 100644 --- a/.github/workflows/pr-bench-compress.yml +++ b/.github/workflows/pr-bench-compress.yml @@ -3,7 +3,10 @@ name: PR Compression Benchmark on: - workflow_call: { } + workflow_call: + secrets: + POLAR_SIGNALS_API_KEY: + required: false workflow_dispatch: { } permissions: @@ -14,7 +17,8 @@ permissions: jobs: bench: uses: ./.github/workflows/pr-bench-runner.yml - secrets: inherit + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} with: benchmark_id: compress-bench benchmark_name: Compression diff --git a/.github/workflows/pr-bench-dispatch.yml b/.github/workflows/pr-bench-dispatch.yml index 3cb60c619bf..59f0ed4f3d6 100644 --- a/.github/workflows/pr-bench-dispatch.yml +++ b/.github/workflows/pr-bench-dispatch.yml @@ -13,42 +13,67 @@ on: branches: ["develop"] permissions: - actions: write contents: read - pull-requests: write # for label removal and PR comments - id-token: write # enables AWS-GitHub OIDC jobs: remove-all-label: runs-on: ubuntu-latest timeout-minutes: 10 if: github.event.label.name == 'action/bench-all' + permissions: + pull-requests: write # for label removal steps: - - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 + - name: Remove benchmark label if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' - with: - labels: action/bench-all - fail_on_error: true + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: gh pr edit "$PR_NUMBER" --remove-label "action/bench-all" all-random-access-bench: needs: remove-all-label uses: ./.github/workflows/pr-bench-random-access.yml - secrets: inherit + permissions: + contents: read + pull-requests: write # for PR comments + id-token: write # enables AWS-GitHub OIDC + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} all-compression-bench: needs: remove-all-label uses: ./.github/workflows/pr-bench-compress.yml - secrets: inherit + permissions: + contents: read + pull-requests: write # for PR comments + id-token: write # enables AWS-GitHub OIDC + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} all-string-bench: needs: remove-all-label uses: ./.github/workflows/pr-bench-string.yml - secrets: inherit + permissions: + contents: read + pull-requests: write # for PR comments + id-token: write # enables AWS-GitHub OIDC + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} all-sql-bench: needs: remove-all-label uses: ./.github/workflows/pr-bench-sql.yml - secrets: inherit + permissions: + contents: read + pull-requests: write # for PR comments + id-token: write # enables AWS-GitHub OIDC + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }} + OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }} + BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }} + INCIDENT_IO_ALERT_TOKEN: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} with: matrix_preset: "pr-all" @@ -56,81 +81,127 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 if: github.event.label.name == 'action/bench-random-access' + permissions: + pull-requests: write # for label removal steps: - - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 + - name: Remove benchmark label if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' - with: - labels: action/bench-random-access - fail_on_error: true + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: gh pr edit "$PR_NUMBER" --remove-label "action/bench-random-access" random-access-bench: needs: remove-random-access-label uses: ./.github/workflows/pr-bench-random-access.yml - secrets: inherit + permissions: + contents: read + pull-requests: write # for PR comments + id-token: write # enables AWS-GitHub OIDC + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} remove-compress-label: runs-on: ubuntu-latest timeout-minutes: 10 if: github.event.label.name == 'action/bench-compress' + permissions: + pull-requests: write # for label removal steps: - - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 + - name: Remove benchmark label if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' - with: - labels: action/bench-compress - fail_on_error: true + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: gh pr edit "$PR_NUMBER" --remove-label "action/bench-compress" compression-bench: needs: remove-compress-label uses: ./.github/workflows/pr-bench-compress.yml - secrets: inherit + permissions: + contents: read + pull-requests: write # for PR comments + id-token: write # enables AWS-GitHub OIDC + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} remove-string-label: runs-on: ubuntu-latest timeout-minutes: 10 if: github.event.label.name == 'action/bench-string' + permissions: + pull-requests: write # for label removal steps: - - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 + - name: Remove benchmark label if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' - with: - labels: action/bench-string - fail_on_error: true + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: gh pr edit "$PR_NUMBER" --remove-label "action/bench-string" string-bench: needs: remove-string-label uses: ./.github/workflows/pr-bench-string.yml - secrets: inherit + permissions: + contents: read + pull-requests: write # for PR comments + id-token: write # enables AWS-GitHub OIDC + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} remove-gpu-compress-label: runs-on: ubuntu-latest timeout-minutes: 10 if: github.event.label.name == 'action/bench-gpu-compress' + permissions: + pull-requests: write # for label removal steps: - - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 + - name: Remove benchmark label if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' - with: - labels: action/bench-gpu-compress - fail_on_error: true + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: gh pr edit "$PR_NUMBER" --remove-label "action/bench-gpu-compress" gpu-compress-bench: needs: remove-gpu-compress-label uses: ./.github/workflows/pr-bench-gpu-compress.yml - secrets: inherit + permissions: + contents: read + pull-requests: write # for PR comments remove-sql-label: runs-on: ubuntu-latest timeout-minutes: 10 if: github.event.label.name == 'action/bench-sql' + permissions: + pull-requests: write # for label removal steps: - - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 + - name: Remove benchmark label if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' - with: - labels: action/bench-sql - fail_on_error: true + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: gh pr edit "$PR_NUMBER" --remove-label "action/bench-sql" sql-bench: needs: remove-sql-label uses: ./.github/workflows/pr-bench-sql.yml - secrets: inherit + permissions: + contents: read + pull-requests: write # for PR comments + id-token: write # enables AWS-GitHub OIDC + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }} + OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }} + BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }} + INCIDENT_IO_ALERT_TOKEN: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} with: matrix_preset: "pr" @@ -138,16 +209,29 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 if: github.event.label.name == 'action/bench-sql-compact' + permissions: + pull-requests: write # for label removal steps: - - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 + - name: Remove benchmark label if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' - with: - labels: action/bench-sql-compact - fail_on_error: true + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: gh pr edit "$PR_NUMBER" --remove-label "action/bench-sql-compact" sql-compact-bench: needs: remove-sql-compact-label uses: ./.github/workflows/pr-bench-sql.yml - secrets: inherit + permissions: + contents: read + pull-requests: write # for PR comments + id-token: write # enables AWS-GitHub OIDC + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }} + OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }} + BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }} + INCIDENT_IO_ALERT_TOKEN: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} with: matrix_preset: "pr-compact" diff --git a/.github/workflows/pr-bench-gpu-compress.yml b/.github/workflows/pr-bench-gpu-compress.yml index 757cd1aebd5..097c64fcbc1 100644 --- a/.github/workflows/pr-bench-gpu-compress.yml +++ b/.github/workflows/pr-bench-gpu-compress.yml @@ -22,12 +22,13 @@ jobs: runs-on: >- runs-on=${{ github.run_id }}/family=g5/cpu=8/image=ubuntu24-gpu-x64/extras=s3-cache/tag=gpu-compress-bench steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 with: sccache: s3 - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -72,13 +73,13 @@ jobs: cat comment.md >> "$GITHUB_STEP_SUMMARY" - name: Comment PR if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1 with: file-path: comment.md comment-tag: bench-pr-comment-gpu-compress - name: Comment PR on failure if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1 with: message: | # BENCHMARK FAILED diff --git a/.github/workflows/pr-bench-random-access.yml b/.github/workflows/pr-bench-random-access.yml index 112d4d8687d..f762bc0f3ab 100644 --- a/.github/workflows/pr-bench-random-access.yml +++ b/.github/workflows/pr-bench-random-access.yml @@ -3,7 +3,10 @@ name: PR Random Access Benchmark on: - workflow_call: { } + workflow_call: + secrets: + POLAR_SIGNALS_API_KEY: + required: false workflow_dispatch: { } permissions: @@ -14,7 +17,8 @@ permissions: jobs: bench: uses: ./.github/workflows/pr-bench-runner.yml - secrets: inherit + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} with: benchmark_id: random-access-bench benchmark_name: Random Access diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml index b1b5d8c8799..0c8955fc5db 100644 --- a/.github/workflows/pr-bench-runner.yml +++ b/.github/workflows/pr-bench-runner.yml @@ -21,6 +21,9 @@ on: required: false type: boolean default: false + secrets: + POLAR_SIGNALS_API_KEY: + required: false permissions: contents: read @@ -35,13 +38,14 @@ jobs: && format('runs-on={0}/runner=bench-dedicated/family=c6id.metal/tag={1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '') || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.event.pull_request.head.repo.fork == false with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -58,9 +62,10 @@ jobs: - name: Build binary shell: bash env: - RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" + RUSTFLAGS: -C target-cpu=native -C force-frame-pointers=yes + INPUTS_BENCHMARK_ID: ${{ inputs.benchmark_id }} run: | - cargo build --package ${{ inputs.benchmark_id }} --profile release_debug \ + cargo build --package ${INPUTS_BENCHMARK_ID} --profile release_debug \ --features ${{ inputs.with_lance && 'lance,' || '' }}unstable_encodings - name: Pre-upload benchmark debuginfo to Polar Signals @@ -105,8 +110,9 @@ jobs: RUST_BACKTRACE: full VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" + INPUTS_BENCHMARK_ID: ${{ inputs.benchmark_id }} run: | - bash scripts/bench-taskset.sh target/release_debug/${{ inputs.benchmark_id }} -d gh-json -o results.json + bash scripts/bench-taskset.sh target/release_debug/${INPUTS_BENCHMARK_ID} -d gh-json -o results.json - name: Setup AWS CLI if: github.event.pull_request.head.repo.fork == false @@ -128,20 +134,22 @@ jobs: python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request gzip -d -c data.json.gz > base.json - uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ inputs.benchmark_name }}" \ + uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${INPUTS_BENCHMARK_NAME}" \ > comment.md cat comment.md >> "$GITHUB_STEP_SUMMARY" + env: + INPUTS_BENCHMARK_NAME: ${{ inputs.benchmark_name }} - name: Comment PR if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1 with: file-path: comment.md comment-tag: bench-pr-comment-${{ inputs.benchmark_id }} - name: Comment PR on failure if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1 with: message: | # BENCHMARK FAILED diff --git a/.github/workflows/pr-bench-sql.yml b/.github/workflows/pr-bench-sql.yml index 40789d631a8..5f9b63a8cda 100644 --- a/.github/workflows/pr-bench-sql.yml +++ b/.github/workflows/pr-bench-sql.yml @@ -17,6 +17,17 @@ on: required: false type: string default: "pr" + secrets: + POLAR_SIGNALS_API_KEY: + required: false + OTEL_EXPORTER_OTLP_ENDPOINT: + required: false + OTEL_EXPORTER_OTLP_HEADERS: + required: false + BENCH_REVALIDATE_TOKEN: + required: false + INCIDENT_IO_ALERT_TOKEN: + required: false workflow_dispatch: inputs: matrix_preset: @@ -38,7 +49,12 @@ permissions: jobs: sql: uses: ./.github/workflows/sql-bench-matrix.yml - secrets: inherit + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} + OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }} + OTEL_EXPORTER_OTLP_HEADERS: ${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }} + BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }} + INCIDENT_IO_ALERT_TOKEN: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }} with: mode: "pr" matrix_preset: ${{ inputs.matrix_preset }} diff --git a/.github/workflows/pr-bench-string.yml b/.github/workflows/pr-bench-string.yml index e323f84a11a..7ae66e8e272 100644 --- a/.github/workflows/pr-bench-string.yml +++ b/.github/workflows/pr-bench-string.yml @@ -3,7 +3,10 @@ name: PR String Encoding Benchmark on: - workflow_call: { } + workflow_call: + secrets: + POLAR_SIGNALS_API_KEY: + required: false workflow_dispatch: { } permissions: @@ -14,7 +17,8 @@ permissions: jobs: bench: uses: ./.github/workflows/pr-bench-runner.yml - secrets: inherit + secrets: + POLAR_SIGNALS_API_KEY: ${{ secrets.POLAR_SIGNALS_API_KEY }} with: benchmark_id: string-bench benchmark_name: String Encoding diff --git a/.github/workflows/publish-dry-runs.yml b/.github/workflows/publish-dry-runs.yml index 7bf49513e60..f8ee584352a 100644 --- a/.github/workflows/publish-dry-runs.yml +++ b/.github/workflows/publish-dry-runs.yml @@ -28,8 +28,10 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Rust Dependency Cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: save-if: ${{ github.ref_name == 'develop' }} - uses: ./.github/actions/setup-rust @@ -74,14 +76,15 @@ jobs: runner: runs-on=${{ github.run_id }}/runner=arm64-medium/image=ubuntu24-full-arm64-pre-v2/extras=s3-cache/tag=check-java-publish-build-arm64 runs-on: ${{ matrix.runner }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5 + persist-credentials: false + - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: "corretto" java-version: "17" @@ -109,11 +112,13 @@ jobs: && format('runs-on={0}/runner=amd64-xsmall/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=rust-publish-dry-run', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5b8b1570bde..2d5a0e4d367 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,12 +11,13 @@ on: release: types: [published] +permissions: { } + jobs: package: uses: ./.github/workflows/package.yml with: version: ${{ github.event.release.tag_name }} - secrets: inherit publish-rust: runs-on: ubuntu-latest @@ -26,6 +27,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -36,7 +38,9 @@ jobs: - name: Cargo Set Version run: | - cargo set-version --workspace ${{ github.event.release.tag_name }} + cargo set-version --workspace ${GITHUB_EVENT_RELEASE_TAG_NAME} + env: + GITHUB_EVENT_RELEASE_TAG_NAME: ${{ github.event.release.tag_name }} - name: Release run: | @@ -66,7 +70,7 @@ jobs: - name: Display structure of downloaded files run: ls -R dist/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # release/v1 + uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1 with: attestations: true verbose: true @@ -88,11 +92,15 @@ jobs: working-directory: ./java steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5 + with: + persist-credentials: false + - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 with: distribution: "corretto" java-version: "17" - - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 + with: + cache-disabled: true - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: libvortex_jni_*.zip @@ -119,7 +127,6 @@ jobs: with: git_ref: ${{ github.event.release.tag_name }} confirm_upload: "yes" - secrets: inherit permissions: id-token: write contents: read diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index 7245659e451..0842febf273 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -39,7 +39,7 @@ jobs: fallback_runner: ubuntu-24.04 archive: tgz steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' && contains(matrix.target, 'linux') with: sccache: s3 @@ -47,6 +47,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 + persist-credentials: false - uses: ./.github/actions/setup-rust with: @@ -57,6 +58,8 @@ jobs: - name: Setup Zig if: contains(matrix.target, 'linux') uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 + with: + use-cache: false - name: Install cargo-zigbuild if: contains(matrix.target, 'linux') @@ -85,6 +88,7 @@ jobs: zip ../../../vx-${{ matrix.target }}.zip vx.exe - name: Upload release asset - run: gh release upload "${{ github.event.release.tag_name }}" vx-${{ matrix.target }}.${{ matrix.archive == 'tgz' && 'tar.gz' || 'zip' }} + run: gh release upload "${GITHUB_EVENT_RELEASE_TAG_NAME}" vx-${{ matrix.target }}.${{ matrix.archive == 'tgz' && 'tar.gz' || 'zip' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_EVENT_RELEASE_TAG_NAME: ${{ github.event.release.tag_name }} diff --git a/.github/workflows/report-fuzz-crash.yml b/.github/workflows/report-fuzz-crash.yml index 3467d0cdcce..02b8b3a93c5 100644 --- a/.github/workflows/report-fuzz-crash.yml +++ b/.github/workflows/report-fuzz-crash.yml @@ -50,6 +50,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Download fuzzer logs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -68,11 +70,13 @@ jobs: - name: Extract crash info id: extract + env: + INPUTS_CRASH_FILE: ${{ inputs.crash_file }} run: | python3 -m fuzz_report extract \ logs/fuzz_output.log \ --crash-dir crash_artifacts \ - --crash-name "${{ inputs.crash_file }}" \ + --crash-name "${INPUTS_CRASH_FILE}" \ -o crash_info.json # Validate that extraction found a real crash. @@ -118,7 +122,7 @@ jobs: steps.dedup.outputs.duplicate != 'true' || steps.dedup.outputs.confidence != 'exact' continue-on-error: true - uses: anthropics/claude-code-action@be7b93b1907a4abad570368f3c74b6fe3807510b # v1 + uses: anthropics/claude-code-action@be7b93b1907a4abad570368f3c74b6fe3807510b # v1.0.183 with: claude_code_oauth_token: ${{ secrets.claude_code_oauth_token }} github_token: ${{ secrets.gh_token }} @@ -144,18 +148,24 @@ jobs: id: report env: GH_TOKEN: ${{ secrets.gh_token }} + INPUTS_FUZZ_TARGET: ${{ inputs.fuzz_target }} + INPUTS_FUZZ_NAME: ${{ inputs.fuzz_name || inputs.fuzz_target }} + INPUTS_CRASH_FILE: ${{ inputs.crash_file }} + INPUTS_BRANCH: ${{ inputs.branch }} + INPUTS_COMMIT: ${{ inputs.commit }} + INPUTS_ARTIFACT_URL: ${{ inputs.artifact_url }} run: | python3 -m fuzz_report report \ crash_info.json \ --repo "${{ github.repository }}" \ --dedup-result dedup_result.json \ --claude-analysis claude_analysis.txt \ - -v "FUZZ_TARGET=${{ inputs.fuzz_target }}" \ - -v "FUZZ_NAME=${{ inputs.fuzz_name || inputs.fuzz_target }}" \ - -v "CRASH_FILE=${{ inputs.crash_file }}" \ - -v "BRANCH=${{ inputs.branch }}" \ - -v "COMMIT=${{ inputs.commit }}" \ - -v "ARTIFACT_URL=${{ inputs.artifact_url }}" + -v "FUZZ_TARGET=${INPUTS_FUZZ_TARGET}" \ + -v "FUZZ_NAME=${INPUTS_FUZZ_NAME}" \ + -v "CRASH_FILE=${INPUTS_CRASH_FILE}" \ + -v "BRANCH=${INPUTS_BRANCH}" \ + -v "COMMIT=${INPUTS_COMMIT}" \ + -v "ARTIFACT_URL=${INPUTS_ARTIFACT_URL}" - name: Alert incident.io on validation failure if: failure() && steps.report.outputs.validation_failed == 'true' diff --git a/.github/workflows/reuse.yml b/.github/workflows/reuse.yml index e4bfefe128a..143fe855e48 100644 --- a/.github/workflows/reuse.yml +++ b/.github/workflows/reuse.yml @@ -12,11 +12,15 @@ on: branches: [develop] pull_request: { } +permissions: { } + jobs: reuse-check: runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: REUSE Compliance Check uses: fsfe/reuse-action@676e2d560c9a403aa252096d99fcab3e1132b0f5 # v6 diff --git a/.github/workflows/run-fuzzer.yml b/.github/workflows/run-fuzzer.yml index fd26140bc1e..0c754882236 100644 --- a/.github/workflows/run-fuzzer.yml +++ b/.github/workflows/run-fuzzer.yml @@ -71,12 +71,14 @@ jobs: first_crash_name: ${{ steps.check.outputs.first_crash_name }} artifact_url: ${{ steps.upload_artifacts.outputs.artifact-url }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: @@ -85,7 +87,7 @@ jobs: enable-sccache: ${{ github.repository == 'vortex-data/vortex' && 'true' || 'false' }} - name: Install llvm - uses: aminya/setup-cpp@v1 + uses: aminya/setup-cpp@1f17f92d6a52bfcb1a25348e2c526c2e5cbb1134 # v1.8.0 with: compiler: llvm @@ -119,27 +121,48 @@ jobs: run: | CORPUS_DIR="fuzz/corpus/${FUZZ_NAME}" FEATURES_FLAG="" - if [ -n "${{ inputs.extra_features }}" ]; then - FEATURES_FLAG="--features ${{ inputs.extra_features }}" + if [ -n "${INPUTS_EXTRA_FEATURES}" ]; then + FEATURES_FLAG="--features ${INPUTS_EXTRA_FEATURES}" fi FORK_FLAG="" if [ "${{ inputs.jobs }}" -gt 1 ]; then FORK_FLAG="-fork=${{ inputs.jobs }}" fi - ${{ inputs.extra_env }} RUST_BACKTRACE=1 \ + EXTRA_ENV=() + if [ -n "$INPUTS_EXTRA_ENV" ]; then + read -r -a EXTRA_ENV <<< "$INPUTS_EXTRA_ENV" + fi + for assignment in "${EXTRA_ENV[@]}"; do + if [[ ! "$assignment" =~ ^[A-Za-z_][A-Za-z0-9_]*= ]]; then + echo "::error::Invalid extra_env assignment: $assignment" + exit 1 + fi + done + env "${EXTRA_ENV[@]}" RUST_BACKTRACE=1 \ cargo +$NIGHTLY_TOOLCHAIN fuzz run --release --debug-assertions \ $FEATURES_FLAG \ - ${{ inputs.fuzz_target }} "$CORPUS_DIR" -- \ + "$INPUTS_FUZZ_TARGET" "$CORPUS_DIR" -- \ $FORK_FLAG -max_total_time=${{ inputs.max_time }} -rss_limit_mb=0 \ 2>&1 | tee fuzz_output.log continue-on-error: true + env: + INPUTS_EXTRA_FEATURES: ${{ inputs.extra_features }} + INPUTS_EXTRA_ENV: ${{ inputs.extra_env }} + INPUTS_FUZZ_TARGET: ${{ inputs.fuzz_target }} - name: Check for crashes id: check + env: + FUZZ_OUTCOME: ${{ steps.fuzz.outcome }} run: | # Find actual crash files, not just the directory structure FIRST_CRASH=$(find fuzz/artifacts -type f \( -name "crash-*" -o -name "leak-*" -o -name "timeout-*" -o -name "oom-*" \) 2>/dev/null | head -1 || true) + if [ "$FUZZ_OUTCOME" != "success" ] && [ -z "$FIRST_CRASH" ]; then + echo "::error::Fuzzer failed without producing a crash artifact" + exit 1 + fi + if [ -n "$FIRST_CRASH" ]; then echo "crashes_found=true" >> $GITHUB_OUTPUT echo "first_crash=$FIRST_CRASH" >> $GITHUB_OUTPUT @@ -162,15 +185,19 @@ jobs: # fuzz_output.log. Replay the crashing input in single-process mode to get # the full output for the crash reporting pipeline. FEATURES_FLAG="" - if [ -n "${{ inputs.extra_features }}" ]; then - FEATURES_FLAG="--features ${{ inputs.extra_features }}" + if [ -n "${INPUTS_EXTRA_FEATURES}" ]; then + FEATURES_FLAG="--features ${INPUTS_EXTRA_FEATURES}" fi RUST_BACKTRACE=1 \ cargo +$NIGHTLY_TOOLCHAIN fuzz run --release --debug-assertions \ $FEATURES_FLAG \ - ${{ inputs.fuzz_target }} \ - "${{ steps.check.outputs.first_crash }}" \ + "$INPUTS_FUZZ_TARGET" \ + "$STEPS_CHECK_OUTPUTS_FIRST_CRASH" \ 2>&1 | tee fuzz_output.log || true + env: + INPUTS_EXTRA_FEATURES: ${{ inputs.extra_features }} + INPUTS_FUZZ_TARGET: ${{ inputs.fuzz_target }} + STEPS_CHECK_OUTPUTS_FIRST_CRASH: ${{ steps.check.outputs.first_crash }} - name: Archive crash artifacts id: upload_artifacts diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index de135445176..22a5e211fea 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -26,7 +26,9 @@ jobs: name: "Mirror DuckDB to R2" if: github.event_name == 'pull_request' uses: ./.github/workflows/duckdb-r2.yml - secrets: inherit + secrets: + DUCKDB_R2_ACCESS_KEY_ID: ${{ secrets.DUCKDB_R2_ACCESS_KEY_ID }} + DUCKDB_R2_SECRET_ACCESS_KEY: ${{ secrets.DUCKDB_R2_SECRET_ACCESS_KEY }} duckdb-ready: name: "DuckDB libraries available in R2" @@ -61,11 +63,13 @@ jobs: LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw" GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov" steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -126,11 +130,13 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-coverage', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -182,11 +188,13 @@ jobs: # unset -Zsanitizer flag and we should allow that. RUSTFLAGS: "-A warnings -Cunsafe-allow-abi-mismatch=sanitizer -C debuginfo=2 -C opt-level=0 -C strip=none" steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -251,11 +259,13 @@ jobs: && format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/tag=rust-ffi-test-sanitizer', github.run_id) || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" @@ -303,11 +313,13 @@ jobs: RUSTFLAGS: "-A warnings" RUST_BACKTRACE: full steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: github.repository == 'vortex-data/vortex' with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-prebuild with: enable-sccache: "true" diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml index 0db85313641..1f3173039f8 100644 --- a/.github/workflows/sql-bench-matrix.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -14,6 +14,17 @@ on: required: false type: string default: c6id.metal + secrets: + POLAR_SIGNALS_API_KEY: + required: false + OTEL_EXPORTER_OTLP_ENDPOINT: + required: false + OTEL_EXPORTER_OTLP_HEADERS: + required: false + BENCH_REVALIDATE_TOKEN: + required: false + INCIDENT_IO_ALERT_TOKEN: + required: false jobs: resolve-matrix: @@ -53,13 +64,14 @@ jobs: && format('runs-on={0}/runner=bench-dedicated/family={1}/tag=build{2}', github.run_id, inputs.machine_type, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -67,10 +79,11 @@ jobs: - name: Build binaries shell: bash env: - RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" + RUSTFLAGS: -C target-cpu=native -C force-frame-pointers=yes + INPUTS_MODE: ${{ inputs.mode }} run: | packages=(--bin data-gen --bin datafusion-bench --bin duckdb-bench) - if [ "${{ inputs.mode }}" != "pr" ]; then + if [ "${INPUTS_MODE}" != "pr" ]; then packages+=(--bin lance-bench) fi cargo build "${packages[@]}" --profile release_debug --features unstable_encodings @@ -114,13 +127,14 @@ jobs: && format('runs-on={0}/runner=bench-dedicated/family={1}/tag={2}{3}', github.run_id, inputs.machine_type, matrix.id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '') || 'ubuntu-latest' }} steps: - - uses: runs-on/action@v2 + - uses: runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2 if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -255,7 +269,7 @@ jobs: - name: Comment PR if: inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1 with: file-path: comment.md # There is exactly one comment per comment-tag. If a comment with this tag already exists, @@ -266,7 +280,7 @@ jobs: - name: Comment PR on failure if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false - uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 + uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3.0.1 with: message: | # 🚨🚨🚨❌❌❌ SQL BENCHMARK FAILED ❌❌❌🚨🚨🚨 diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml index 2fcf11658f5..3cd976104be 100644 --- a/.github/workflows/typos.yml +++ b/.github/workflows/typos.yml @@ -20,5 +20,7 @@ jobs: steps: - name: Checkout Actions Repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Spell Check Repo uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # v1.47.2 diff --git a/.github/workflows/wasm-fuzz.yml b/.github/workflows/wasm-fuzz.yml index ac655147074..9ff0c86953f 100644 --- a/.github/workflows/wasm-fuzz.yml +++ b/.github/workflows/wasm-fuzz.yml @@ -26,6 +26,8 @@ jobs: timeout-minutes: 240 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 108d1044629..7ba951a0897 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -29,7 +29,9 @@ jobs: web: ${{ steps.filter.outputs.web }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4 + with: + persist-credentials: false + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 id: filter with: filters: | @@ -45,6 +47,8 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -71,6 +75,8 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - uses: ./.github/actions/setup-rust with: repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml new file mode 100644 index 00000000000..84aca4a570c --- /dev/null +++ b/.github/workflows/workflow-lint.yml @@ -0,0 +1,35 @@ +name: Workflow Lint + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }} +on: + push: + branches: [develop] + pull_request: { } + workflow_dispatch: { } + +permissions: { } + +jobs: + validate-workflow-yaml: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + security-events: write # Required for zizmor's usage of advanced security + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Validate YAML files + run: | + # Lint the workflows and yamllint's configuration file. + yamllint \ + --strict \ + -c .yamllint.yaml \ + .github/ + - name: Run zizmor 🌈 + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 + with: + min-severity: medium + min-confidence: medium