From d5f3f20521f0f617483f3d2bc196531b3dfd4f8c Mon Sep 17 00:00:00 2001 From: "paul@iqmo.com" Date: Fri, 28 Aug 2026 16:45:11 -0400 Subject: [PATCH 1/4] ci: use oidc for sccache s3 short term credentials --- compose.yaml | 1 + dev/tasks/docker-tests/github.linux.yml | 6 ++++++ dev/tasks/macros.jinja | 16 ++++++++++++++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/compose.yaml b/compose.yaml index 00e791ccea18..48a1bb060a17 100644 --- a/compose.yaml +++ b/compose.yaml @@ -73,6 +73,7 @@ x-cpp: &cpp x-sccache: &sccache AWS_ACCESS_KEY_ID: AWS_SECRET_ACCESS_KEY: + AWS_SESSION_TOKEN: SCCACHE_BUCKET: SCCACHE_REGION: SCCACHE_S3_KEY_PREFIX: ${SCCACHE_S3_KEY_PREFIX:-sccache} diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml index a893432cf941..afb0e1569eea 100644 --- a/dev/tasks/docker-tests/github.linux.yml +++ b/dev/tasks/docker-tests/github.linux.yml @@ -19,6 +19,11 @@ {{ macros.github_header() }} +permissions: + contents: read + # For OIDC token for sccache + id-token: write + jobs: test: name: | @@ -31,6 +36,7 @@ jobs: {{ macros.github_free_space()|indent }} {{ macros.github_install_archery()|indent }} + {{ macros.github_configure_aws_credentials()|indent }} - name: Execute Docker Build shell: bash env: diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 8e9a41b46b75..db550024b8e3 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -26,6 +26,7 @@ on: env: ARCHERY_DEBUG: 1 + SCCACHE_OIDC_ROLE: {{ '${{ secrets.AWS_ROLE_ARN }}' }} {% endmacro %} {%- macro github_checkout_arrow(fetch_depth=1, submodules="recursive", action_v="4") -%} @@ -287,10 +288,21 @@ env: {% endif %} {%- endmacro -%} +{%- macro github_configure_aws_credentials() -%} + - name: Configure AWS credentials for sccache + if: env.SCCACHE_OIDC_ROLE != '' + uses: aws-actions/configure-aws-credentials@v6 + with: + aws-region: {{ '${{ secrets.SCCACHE_REGION }}' }} + role-to-assume: {{ '${{ env.SCCACHE_OIDC_ROLE }}' }} + role-duration-seconds: 14400 +{% endmacro %} + {% macro github_set_sccache_envvars(sccache_key_prefix = "sccache") %} {% set sccache_vars = { - "AWS_ACCESS_KEY_ID": '${{ secrets.AWS_ACCESS_KEY_ID }}', - "AWS_SECRET_ACCESS_KEY": '${{ secrets.AWS_SECRET_ACCESS_KEY }}', + "AWS_ACCESS_KEY_ID": '${{ env.AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }}', + "AWS_SECRET_ACCESS_KEY": '${{ env.AWS_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }}', + "AWS_SESSION_TOKEN": '${{ env.AWS_SESSION_TOKEN }}', "SCCACHE_BUCKET": '${{ secrets.SCCACHE_BUCKET }}', "SCCACHE_REGION": '${{ secrets.SCCACHE_REGION }}', "SCCACHE_S3_KEY_PREFIX": sccache_key_prefix From a6306645bec9da971cce8038c112d76f55db7dca Mon Sep 17 00:00:00 2001 From: "paul@iqmo.com" Date: Mon, 31 Aug 2026 22:28:00 -0400 Subject: [PATCH 2/4] ci: use SCCACHE_ENABLED env var, rather than copying AWS_ROLE_ARN to env. Remove references to unneeded secrets. --- dev/tasks/macros.jinja | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index db550024b8e3..386abe62098c 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -26,7 +26,7 @@ on: env: ARCHERY_DEBUG: 1 - SCCACHE_OIDC_ROLE: {{ '${{ secrets.AWS_ROLE_ARN }}' }} + SCCACHE_ENABLED: {{ "${{ secrets.SCCACHE_REGION != '' }}" }} {% endmacro %} {%- macro github_checkout_arrow(fetch_depth=1, submodules="recursive", action_v="4") -%} @@ -290,18 +290,18 @@ env: {%- macro github_configure_aws_credentials() -%} - name: Configure AWS credentials for sccache - if: env.SCCACHE_OIDC_ROLE != '' + if: env.SCCACHE_ENABLED == 'true' uses: aws-actions/configure-aws-credentials@v6 with: aws-region: {{ '${{ secrets.SCCACHE_REGION }}' }} - role-to-assume: {{ '${{ env.SCCACHE_OIDC_ROLE }}' }} + role-to-assume: {{ '${{ secrets.AWS_ROLE_ARN }}' }} role-duration-seconds: 14400 {% endmacro %} {% macro github_set_sccache_envvars(sccache_key_prefix = "sccache") %} {% set sccache_vars = { - "AWS_ACCESS_KEY_ID": '${{ env.AWS_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }}', - "AWS_SECRET_ACCESS_KEY": '${{ env.AWS_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }}', + "AWS_ACCESS_KEY_ID": '${{ env.AWS_ACCESS_KEY_ID }}', + "AWS_SECRET_ACCESS_KEY": '${{ env.AWS_SECRET_ACCESS_KEY }}', "AWS_SESSION_TOKEN": '${{ env.AWS_SESSION_TOKEN }}', "SCCACHE_BUCKET": '${{ secrets.SCCACHE_BUCKET }}', "SCCACHE_REGION": '${{ secrets.SCCACHE_REGION }}', From a8808dd14a5250ec551aa964633784375cc61cfa Mon Sep 17 00:00:00 2001 From: "paul@iqmo.com" Date: Fri, 4 Sep 2026 01:07:47 -0400 Subject: [PATCH 3/4] apply github_configure_aws_credentials macro to all github.*.yml workflows --- dev/tasks/matlab/github.yml | 8 ++++++++ dev/tasks/python-wheels/github.linux.yml | 3 +++ dev/tasks/r/github.linux.arrow.version.back.compat.yml | 6 ++++++ dev/tasks/r/github.linux.cran.yml | 6 ++++++ dev/tasks/r/github.linux.offline.build.yml | 6 ++++++ dev/tasks/r/github.linux.sanitizers.yml | 6 ++++++ dev/tasks/r/github.linux.versions.yml | 6 ++++++ dev/tasks/r/github.macos-linux.local.yml | 6 ++++++ dev/tasks/r/github.packages.yml | 9 +++++++++ 9 files changed, 56 insertions(+) diff --git a/dev/tasks/matlab/github.yml b/dev/tasks/matlab/github.yml index 77f3056c362a..39e79ec15199 100644 --- a/dev/tasks/matlab/github.yml +++ b/dev/tasks/matlab/github.yml @@ -19,6 +19,11 @@ {{ macros.github_header() }} +permissions: + contents: write + # For OIDC token for sccache + id-token: write + jobs: ubuntu: @@ -32,6 +37,7 @@ jobs: uses: matlab-actions/setup-matlab@v2 with: release: R2025b + {{ macros.github_configure_aws_credentials()|indent }} - name: Build MATLAB Interface env: {{ macros.github_set_sccache_envvars()|indent(8) }} @@ -74,6 +80,7 @@ jobs: uses: matlab-actions/setup-matlab@v2 with: release: R2025b + {{ macros.github_configure_aws_credentials()|indent }} - name: Build MATLAB Interface env: {{ macros.github_set_sccache_envvars()|indent(8) }} @@ -103,6 +110,7 @@ jobs: - name: Install sccache shell: bash run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache + {{ macros.github_configure_aws_credentials()|indent }} - name: Build MATLAB Interface shell: cmd env: diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml index e2a74eb841ee..16ee668bff43 100644 --- a/dev/tasks/python-wheels/github.linux.yml +++ b/dev/tasks/python-wheels/github.linux.yml @@ -21,6 +21,8 @@ permissions: packages: write + # For OIDC token for sccache + id-token: write jobs: build: @@ -65,6 +67,7 @@ jobs: fi echo "TEST_IMAGE_PREFIX=${test_image_prefix}" >> ${GITHUB_ENV} + {{ macros.github_configure_aws_credentials()|indent }} - name: Build wheel shell: bash env: diff --git a/dev/tasks/r/github.linux.arrow.version.back.compat.yml b/dev/tasks/r/github.linux.arrow.version.back.compat.yml index bbe88d205d11..6478ab9c99ea 100644 --- a/dev/tasks/r/github.linux.arrow.version.back.compat.yml +++ b/dev/tasks/r/github.linux.arrow.version.back.compat.yml @@ -19,6 +19,11 @@ {{ macros.github_header() }} +permissions: + contents: read + # For OIDC token for sccache + id-token: write + jobs: write-files: name: "Write files" @@ -44,6 +49,7 @@ jobs: shell: Rscript {0} - name: Install sccache run: arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin + {{ macros.github_configure_aws_credentials()|indent }} - name: Install Arrow env: {{ macros.github_set_sccache_envvars()|indent(8) }} diff --git a/dev/tasks/r/github.linux.cran.yml b/dev/tasks/r/github.linux.cran.yml index a136e1a6be90..0c1ef5ae857e 100644 --- a/dev/tasks/r/github.linux.cran.yml +++ b/dev/tasks/r/github.linux.cran.yml @@ -19,6 +19,11 @@ {{ macros.github_header() }} +permissions: + contents: read + # For OIDC token for sccache + id-token: write + jobs: as-cran: name: "rhub/{{ '${{ matrix.config.r_image }}' }}" @@ -43,6 +48,7 @@ jobs: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_install_archery()|indent }} + {{ macros.github_configure_aws_credentials()|indent }} - name: Docker Run shell: bash env: diff --git a/dev/tasks/r/github.linux.offline.build.yml b/dev/tasks/r/github.linux.offline.build.yml index ac4ad1013739..fb5862ccb778 100644 --- a/dev/tasks/r/github.linux.offline.build.yml +++ b/dev/tasks/r/github.linux.offline.build.yml @@ -19,6 +19,11 @@ {{ macros.github_header() }} +permissions: + contents: read + # For OIDC token for sccache + id-token: write + jobs: grab-dependencies: name: "Download thirdparty dependencies" @@ -77,6 +82,7 @@ jobs: install.packages(c("remotes", "glue", "sys")) remotes::install_deps("arrow/r", dependencies = TRUE) shell: Rscript {0} + {{ macros.github_configure_aws_credentials()|indent }} - name: Install env: ARROW_OFFLINE_BUILD: true diff --git a/dev/tasks/r/github.linux.sanitizers.yml b/dev/tasks/r/github.linux.sanitizers.yml index ed31b13a08b5..af810cd47a83 100644 --- a/dev/tasks/r/github.linux.sanitizers.yml +++ b/dev/tasks/r/github.linux.sanitizers.yml @@ -19,6 +19,11 @@ {{ macros.github_header() }} +permissions: + contents: read + # For OIDC token for sccache + id-token: write + jobs: r-linux-sanitizers: name: "rhub/{{ '${{ matrix.config.r_image }}' }}" @@ -41,6 +46,7 @@ jobs: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_install_archery()|indent }} + {{ macros.github_configure_aws_credentials()|indent }} - name: Docker Run shell: bash env: diff --git a/dev/tasks/r/github.linux.versions.yml b/dev/tasks/r/github.linux.versions.yml index 644494bfbae6..a597d1dcb41c 100644 --- a/dev/tasks/r/github.linux.versions.yml +++ b/dev/tasks/r/github.linux.versions.yml @@ -19,6 +19,11 @@ {{ macros.github_header() }} +permissions: + contents: read + # For OIDC token for sccache + id-token: write + jobs: r-versions: name: "posit/r-base:{{ MATRIX }}-jammy" @@ -43,6 +48,7 @@ jobs: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_install_archery()|indent }} + {{ macros.github_configure_aws_credentials()|indent }} - name: Docker Run shell: bash env: diff --git a/dev/tasks/r/github.macos-linux.local.yml b/dev/tasks/r/github.macos-linux.local.yml index ada1c7642d75..2391c97b0f91 100644 --- a/dev/tasks/r/github.macos-linux.local.yml +++ b/dev/tasks/r/github.macos-linux.local.yml @@ -19,6 +19,11 @@ {{ macros.github_header() }} +permissions: + contents: read + # For OIDC token for sccache + id-token: write + jobs: local: name: "install from local source" @@ -78,6 +83,7 @@ jobs: extra-packages: | any::rcmdcheck any::sys + {{ macros.github_configure_aws_credentials()|indent }} - name: Install env: _R_CHECK_CRAN_INCOMING_: false diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index feeac3dba47b..05da643306e0 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -21,6 +21,11 @@ {{ macros.github_header() }} +permissions: + contents: write + # For OIDC token for sccache + id-token: write + jobs: source: # This job will change the version to either the custom_version param or YMD format. @@ -77,6 +82,7 @@ jobs: brew install sccache ninja brew install openssl@3.0 brew install libxml2 + {{ macros.github_configure_aws_credentials()|indent }} - name: Build libarrow shell: bash env: @@ -132,6 +138,7 @@ jobs: {{ macros.github_checkout_arrow()|indent }} {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} {{ macros.github_install_archery()|indent }} + {{ macros.github_configure_aws_credentials()|indent }} - name: Build libarrow shell: bash env: @@ -182,6 +189,7 @@ jobs: - name: Install sccache shell: bash run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache + {{ macros.github_configure_aws_credentials()|indent }} - name: Build Arrow C++ with rtools40 shell: bash env: @@ -387,6 +395,7 @@ jobs: run: | cores=`nproc || sysctl -n hw.logicalcpu` echo "MAKEFLAGS=-j$cores" >> $GITHUB_ENV + {{ macros.github_configure_aws_credentials()|indent }} - name: Install arrow source package env: # Test source build so be sure not to download a binary From 14976326137c05c97e3938234de47e00d8da635c Mon Sep 17 00:00:00 2001 From: "paul@iqmo.com" Date: Fri, 4 Sep 2026 01:08:09 -0400 Subject: [PATCH 4/4] continue-on-error: so failure to acquire credentials isn't fatal --- dev/tasks/macros.jinja | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 386abe62098c..5b74999d958c 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -291,6 +291,8 @@ env: {%- macro github_configure_aws_credentials() -%} - name: Configure AWS credentials for sccache if: env.SCCACHE_ENABLED == 'true' + # don't fail the build if AWS credentials are unavailable + continue-on-error: true uses: aws-actions/configure-aws-credentials@v6 with: aws-region: {{ '${{ secrets.SCCACHE_REGION }}' }}