From 8c8e3d997a5abaed8272b11c44357bb073d18866 Mon Sep 17 00:00:00 2001 From: Victor Lin Date: Wed, 18 Mar 2026 13:23:54 -0700 Subject: [PATCH 1/4] Test AWS Batch with --detach Without this, the workflow execution happens in the run-build job instead of the wait-N jobs. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dedba21..6568f41 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -71,7 +71,7 @@ jobs: with: repo: nextstrain/zika-tutorial runtime: aws-batch - run: nextstrain build --env BOGUS_ENVVAR . + run: nextstrain build --detach --no-download --env BOGUS_ENVVAR . artifact-name: build-output-aws-batch env: | BOGUS_ENVVAR: BOGUS_ENVVAR From e716626c575ce457cfc694fa245d3205de7dc94e Mon Sep 17 00:00:00 2001 From: Victor Lin Date: Wed, 18 Mar 2026 13:08:33 -0700 Subject: [PATCH 2/4] Support artifact-paths with AWS Batch This is now possible with the wait-N jobs. --- .github/workflows/pathogen-repo-build.yaml | 104 +++++++++++++++--- .github/workflows/pathogen-repo-build.yaml.in | 42 +++++-- 2 files changed, 119 insertions(+), 27 deletions(-) diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index 24c8181..c53b027 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -111,8 +111,6 @@ on: .snakemake/log/ The "build.log" contains log messages from the `nextstrain build` command. The other paths are common output paths for Nextstrain builds. If a path does not exist in your build, then the action will still succeed and will print out a warning for the non-existent file(s). Use an exclude pattern for any of the default paths that you would like to exclude from the artifact (e.g. !build.log). - - This is not supported for builds on AWS Batch because the workflow detaches from the build. Please use the `nextstrain build` command locally to reattach to AWS Batch builds to download outputs. type: string required: false outputs: @@ -217,8 +215,6 @@ on: .snakemake/log/ The "build.log" contains log messages from the `nextstrain build` command. The other paths are common output paths for Nextstrain builds. If a path does not exist in your build, then the action will still succeed and will print out a warning for the non-existent file(s). Use an exclude pattern for any of the default paths that you would like to exclude from the artifact (e.g. !build.log). - - This is not supported for builds on AWS Batch because the workflow detaches from the build. Please use the `nextstrain build` command locally to reattach to AWS Batch builds to download outputs. type: string required: false repo: @@ -231,6 +227,13 @@ env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d + NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS: | + build.log + auspice/ + results/ + benchmarks/ + logs/ + .snakemake/log/ permissions: id-token: write jobs: @@ -348,18 +351,13 @@ jobs: run: | "$NEXTSTRAIN_GITHUB_DIR"/bin/interpolate-env < "$NEXTSTRAIN_GITHUB_DIR"/text-templates/attach-aws-batch.md \ > "$GITHUB_STEP_SUMMARY" - - if: always() + - if: ${{ always() && !env.AWS_BATCH_JOB_ID }} uses: actions/upload-artifact@v7 with: if-no-files-found: warn name: ${{ inputs.artifact-name }} path: | - ${{ env.NEXTSTRAIN_BUILD_LOG }} - auspice/ - results/ - benchmarks/ - logs/ - .snakemake/log/ + ${{ env.NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS }} ${{ inputs.artifact-paths }} outputs: AWS_BATCH_JOB_ID: ${{ env.AWS_BATCH_JOB_ID }} @@ -402,6 +400,7 @@ jobs: name: Attach to AWS Batch job env: AWS_BATCH_JOB_ID: ${{ needs.run-build.outputs.AWS_BATCH_JOB_ID }} + ARTIFACT_PATHS: ${{ inputs.artifact-paths }} run: | # Using timeout to detach from build before we hit the job.timeout-minutes # to avoid cancelling the job in order to avoid a "Cancelled" status @@ -409,13 +408,23 @@ jobs: # cleanly detaches from the build without cancelling it.² # ¹ # ² + + # Set download args for default and custom artifact paths, skipping + # blank lines. Append directories with ** to match files within. + download_args=() + while IFS= read -r line; do + [[ -z "$line" ]] && continue + [[ "$line" == */ ]] && line+='**' + download_args+=(--download "$line") + done <<< "${NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS}${ARTIFACT_PATHS}" + exit_status=0 timeout --signal INT 359m \ nextstrain build \ --aws-batch \ --attach "$AWS_BATCH_JOB_ID" \ --detach-on-interrupt \ - --no-download || exit_status=$? + "${download_args[@]}" . || exit_status=$? if [[ $exit_status == 0 ]]; then conclusion="success" @@ -431,6 +440,14 @@ jobs: echo "conclusion=$conclusion" | tee -a "$GITHUB_OUTPUT" exit $exit_status + - if: ${{ always() && steps.attach.outputs.conclusion != 'timeout' }} + uses: actions/upload-artifact@v7 + with: + if-no-files-found: warn + name: ${{ inputs.artifact-name }} + path: | + ${{ env.NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS }} + ${{ inputs.artifact-paths }} # Emit a "conclusion" output for the job that's based on the conclusion # (success, timeout, failure) of the "attach" step above. # This is the conclusion we care about for the job. @@ -470,6 +487,7 @@ jobs: name: Attach to AWS Batch job env: AWS_BATCH_JOB_ID: ${{ needs.run-build.outputs.AWS_BATCH_JOB_ID }} + ARTIFACT_PATHS: ${{ inputs.artifact-paths }} run: | # Using timeout to detach from build before we hit the job.timeout-minutes # to avoid cancelling the job in order to avoid a "Cancelled" status @@ -477,13 +495,23 @@ jobs: # cleanly detaches from the build without cancelling it.² # ¹ # ² + + # Set download args for default and custom artifact paths, skipping + # blank lines. Append directories with ** to match files within. + download_args=() + while IFS= read -r line; do + [[ -z "$line" ]] && continue + [[ "$line" == */ ]] && line+='**' + download_args+=(--download "$line") + done <<< "${NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS}${ARTIFACT_PATHS}" + exit_status=0 timeout --signal INT 359m \ nextstrain build \ --aws-batch \ --attach "$AWS_BATCH_JOB_ID" \ --detach-on-interrupt \ - --no-download || exit_status=$? + "${download_args[@]}" . || exit_status=$? if [[ $exit_status == 0 ]]; then conclusion="success" @@ -499,6 +527,14 @@ jobs: echo "conclusion=$conclusion" | tee -a "$GITHUB_OUTPUT" exit $exit_status + - if: ${{ always() && steps.attach.outputs.conclusion != 'timeout' }} + uses: actions/upload-artifact@v7 + with: + if-no-files-found: warn + name: ${{ inputs.artifact-name }} + path: | + ${{ env.NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS }} + ${{ inputs.artifact-paths }} # Emit a "conclusion" output for the job that's based on the conclusion # (success, timeout, failure) of the "attach" step above. # This is the conclusion we care about for the job. @@ -539,6 +575,7 @@ jobs: name: Attach to AWS Batch job env: AWS_BATCH_JOB_ID: ${{ needs.run-build.outputs.AWS_BATCH_JOB_ID }} + ARTIFACT_PATHS: ${{ inputs.artifact-paths }} run: | # Using timeout to detach from build before we hit the job.timeout-minutes # to avoid cancelling the job in order to avoid a "Cancelled" status @@ -546,13 +583,23 @@ jobs: # cleanly detaches from the build without cancelling it.² # ¹ # ² + + # Set download args for default and custom artifact paths, skipping + # blank lines. Append directories with ** to match files within. + download_args=() + while IFS= read -r line; do + [[ -z "$line" ]] && continue + [[ "$line" == */ ]] && line+='**' + download_args+=(--download "$line") + done <<< "${NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS}${ARTIFACT_PATHS}" + exit_status=0 timeout --signal INT 359m \ nextstrain build \ --aws-batch \ --attach "$AWS_BATCH_JOB_ID" \ --detach-on-interrupt \ - --no-download || exit_status=$? + "${download_args[@]}" . || exit_status=$? if [[ $exit_status == 0 ]]; then conclusion="success" @@ -568,6 +615,14 @@ jobs: echo "conclusion=$conclusion" | tee -a "$GITHUB_OUTPUT" exit $exit_status + - if: ${{ always() && steps.attach.outputs.conclusion != 'timeout' }} + uses: actions/upload-artifact@v7 + with: + if-no-files-found: warn + name: ${{ inputs.artifact-name }} + path: | + ${{ env.NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS }} + ${{ inputs.artifact-paths }} # Emit a "conclusion" output for the job that's based on the conclusion # (success, timeout, failure) of the "attach" step above. # This is the conclusion we care about for the job. @@ -608,6 +663,7 @@ jobs: name: Attach to AWS Batch job env: AWS_BATCH_JOB_ID: ${{ needs.run-build.outputs.AWS_BATCH_JOB_ID }} + ARTIFACT_PATHS: ${{ inputs.artifact-paths }} run: | # Using timeout to detach from build before we hit the job.timeout-minutes # to avoid cancelling the job in order to avoid a "Cancelled" status @@ -615,13 +671,23 @@ jobs: # cleanly detaches from the build without cancelling it.² # ¹ # ² + + # Set download args for default and custom artifact paths, skipping + # blank lines. Append directories with ** to match files within. + download_args=() + while IFS= read -r line; do + [[ -z "$line" ]] && continue + [[ "$line" == */ ]] && line+='**' + download_args+=(--download "$line") + done <<< "${NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS}${ARTIFACT_PATHS}" + exit_status=0 timeout --signal INT 359m \ nextstrain build \ --aws-batch \ --attach "$AWS_BATCH_JOB_ID" \ --detach-on-interrupt \ - --no-download || exit_status=$? + "${download_args[@]}" . || exit_status=$? if [[ $exit_status == 0 ]]; then conclusion="success" @@ -637,6 +703,14 @@ jobs: echo "conclusion=$conclusion" | tee -a "$GITHUB_OUTPUT" exit $exit_status + - if: ${{ always() && steps.attach.outputs.conclusion != 'timeout' }} + uses: actions/upload-artifact@v7 + with: + if-no-files-found: warn + name: ${{ inputs.artifact-name }} + path: | + ${{ env.NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS }} + ${{ inputs.artifact-paths }} # Emit a "conclusion" output for the job that's based on the conclusion # (success, timeout, failure) of the "attach" step above. # This is the conclusion we care about for the job. diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index 52a0f6c..d2b174b 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -155,10 +155,6 @@ on: succeed and will print out a warning for the non-existent file(s). Use an exclude pattern for any of the default paths that you would like to exclude from the artifact (e.g. !build.log). - - This is not supported for builds on AWS Batch because the workflow - detaches from the build. Please use the `nextstrain build` command - locally to reattach to AWS Batch builds to download outputs. type: string required: false @@ -189,6 +185,13 @@ env: NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github NEXTSTRAIN_BUILD_LOG: build.log NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d + NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS: | + build.log + auspice/ + results/ + benchmarks/ + logs/ + .snakemake/log/ permissions: id-token: write @@ -329,18 +332,13 @@ jobs: "$NEXTSTRAIN_GITHUB_DIR"/bin/interpolate-env < "$NEXTSTRAIN_GITHUB_DIR"/text-templates/attach-aws-batch.md \ > "$GITHUB_STEP_SUMMARY" - - if: always() + - if: ${{ always() && !env.AWS_BATCH_JOB_ID }} uses: actions/upload-artifact@v7 with: if-no-files-found: warn name: ${{ inputs.artifact-name }} path: | - ${{ env.NEXTSTRAIN_BUILD_LOG }} - auspice/ - results/ - benchmarks/ - logs/ - .snakemake/log/ + ${{ env.NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS }} ${{ inputs.artifact-paths }} outputs: @@ -366,6 +364,7 @@ jobs: name: Attach to AWS Batch job env: AWS_BATCH_JOB_ID: ${{ needs.run-build.outputs.AWS_BATCH_JOB_ID }} + ARTIFACT_PATHS: ${{ inputs.artifact-paths }} run: | # Using timeout to detach from build before we hit the job.timeout-minutes # to avoid cancelling the job in order to avoid a "Cancelled" status @@ -373,13 +372,23 @@ jobs: # cleanly detaches from the build without cancelling it.² # ¹ # ² + + # Set download args for default and custom artifact paths, skipping + # blank lines. Append directories with ** to match files within. + download_args=() + while IFS= read -r line; do + [[ -z "$line" ]] && continue + [[ "$line" == */ ]] && line+='**' + download_args+=(--download "$line") + done <<< "${NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS}${ARTIFACT_PATHS}" + exit_status=0 timeout --signal INT 359m \ nextstrain build \ --aws-batch \ --attach "$AWS_BATCH_JOB_ID" \ --detach-on-interrupt \ - --no-download || exit_status=$? + "${download_args[@]}" . || exit_status=$? if [[ $exit_status == 0 ]]; then conclusion="success" @@ -396,6 +405,15 @@ jobs: echo "conclusion=$conclusion" | tee -a "$GITHUB_OUTPUT" exit $exit_status + - if: ${{ always() && steps.attach.outputs.conclusion != 'timeout' }} + uses: actions/upload-artifact@v7 + with: + if-no-files-found: warn + name: ${{ inputs.artifact-name }} + path: | + ${{ env.NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS }} + ${{ inputs.artifact-paths }} + # Emit a "conclusion" output for the job that's based on the conclusion # (success, timeout, failure) of the "attach" step above. # This is the conclusion we care about for the job. From 69361614d5a0967760425aac9359c06fdca6d2f2 Mon Sep 17 00:00:00 2001 From: Victor Lin Date: Wed, 18 Mar 2026 14:05:27 -0700 Subject: [PATCH 3/4] Create build.log in wait-N jobs This allows the full build log to be uploaded upon completion, matching behavior in run-build with local runtimes. --- .github/workflows/pathogen-repo-build.yaml | 20 +++++++++++++++---- .github/workflows/pathogen-repo-build.yaml.in | 5 ++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index c53b027..3e0c9b0 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -424,7 +424,10 @@ jobs: --aws-batch \ --attach "$AWS_BATCH_JOB_ID" \ --detach-on-interrupt \ - "${download_args[@]}" . || exit_status=$? + "${download_args[@]}" . \ + |& tee .git/"$NEXTSTRAIN_BUILD_LOG" || exit_status=$? + + mv .git/"$NEXTSTRAIN_BUILD_LOG" "$NEXTSTRAIN_BUILD_LOG" if [[ $exit_status == 0 ]]; then conclusion="success" @@ -511,7 +514,10 @@ jobs: --aws-batch \ --attach "$AWS_BATCH_JOB_ID" \ --detach-on-interrupt \ - "${download_args[@]}" . || exit_status=$? + "${download_args[@]}" . \ + |& tee .git/"$NEXTSTRAIN_BUILD_LOG" || exit_status=$? + + mv .git/"$NEXTSTRAIN_BUILD_LOG" "$NEXTSTRAIN_BUILD_LOG" if [[ $exit_status == 0 ]]; then conclusion="success" @@ -599,7 +605,10 @@ jobs: --aws-batch \ --attach "$AWS_BATCH_JOB_ID" \ --detach-on-interrupt \ - "${download_args[@]}" . || exit_status=$? + "${download_args[@]}" . \ + |& tee .git/"$NEXTSTRAIN_BUILD_LOG" || exit_status=$? + + mv .git/"$NEXTSTRAIN_BUILD_LOG" "$NEXTSTRAIN_BUILD_LOG" if [[ $exit_status == 0 ]]; then conclusion="success" @@ -687,7 +696,10 @@ jobs: --aws-batch \ --attach "$AWS_BATCH_JOB_ID" \ --detach-on-interrupt \ - "${download_args[@]}" . || exit_status=$? + "${download_args[@]}" . \ + |& tee .git/"$NEXTSTRAIN_BUILD_LOG" || exit_status=$? + + mv .git/"$NEXTSTRAIN_BUILD_LOG" "$NEXTSTRAIN_BUILD_LOG" if [[ $exit_status == 0 ]]; then conclusion="success" diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index d2b174b..ba5d900 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -388,7 +388,10 @@ jobs: --aws-batch \ --attach "$AWS_BATCH_JOB_ID" \ --detach-on-interrupt \ - "${download_args[@]}" . || exit_status=$? + "${download_args[@]}" . \ + |& tee .git/"$NEXTSTRAIN_BUILD_LOG" || exit_status=$? + + mv .git/"$NEXTSTRAIN_BUILD_LOG" "$NEXTSTRAIN_BUILD_LOG" if [[ $exit_status == 0 ]]; then conclusion="success" From 337b4a978a11f6de94c1a9d123c5873c1558bed5 Mon Sep 17 00:00:00 2001 From: Victor Lin Date: Fri, 20 Mar 2026 09:58:19 -0700 Subject: [PATCH 4/4] Only upload logs and benchmarks for AWS Batch runtime Unlike local runtimes, where uploaded artifacts are the only copy of generated files, AWS Batch instances temporarily persist files after the run finishes. This means it's not important to upload the auspice/ and results/ folders, which could contain private or large data. --- .github/workflows/pathogen-repo-build.yaml | 12 ++++++------ .github/workflows/pathogen-repo-build.yaml.in | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pathogen-repo-build.yaml b/.github/workflows/pathogen-repo-build.yaml index 3e0c9b0..e4511ec 100644 --- a/.github/workflows/pathogen-repo-build.yaml +++ b/.github/workflows/pathogen-repo-build.yaml @@ -104,11 +104,11 @@ on: build.log - auspice/ - results/ benchmarks/ logs/ .snakemake/log/ + auspice/ (not included for `aws-batch` runtime) + results/ (not included for `aws-batch` runtime) The "build.log" contains log messages from the `nextstrain build` command. The other paths are common output paths for Nextstrain builds. If a path does not exist in your build, then the action will still succeed and will print out a warning for the non-existent file(s). Use an exclude pattern for any of the default paths that you would like to exclude from the artifact (e.g. !build.log). type: string @@ -208,11 +208,11 @@ on: build.log - auspice/ - results/ benchmarks/ logs/ .snakemake/log/ + auspice/ (not included for `aws-batch` runtime) + results/ (not included for `aws-batch` runtime) The "build.log" contains log messages from the `nextstrain build` command. The other paths are common output paths for Nextstrain builds. If a path does not exist in your build, then the action will still succeed and will print out a warning for the non-existent file(s). Use an exclude pattern for any of the default paths that you would like to exclude from the artifact (e.g. !build.log). type: string @@ -229,11 +229,11 @@ env: NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS: | build.log - auspice/ - results/ benchmarks/ logs/ .snakemake/log/ + ${{ inputs.runtime != 'aws-batch' && 'auspice/' || '' }} + ${{ inputs.runtime != 'aws-batch' && 'results/' || '' }} permissions: id-token: write jobs: diff --git a/.github/workflows/pathogen-repo-build.yaml.in b/.github/workflows/pathogen-repo-build.yaml.in index ba5d900..cef0f5c 100644 --- a/.github/workflows/pathogen-repo-build.yaml.in +++ b/.github/workflows/pathogen-repo-build.yaml.in @@ -142,11 +142,11 @@ on: The default paths included in the artifact are: build.log - auspice/ - results/ benchmarks/ logs/ .snakemake/log/ + auspice/ (not included for `aws-batch` runtime) + results/ (not included for `aws-batch` runtime) The "build.log" contains log messages from the `nextstrain build` command. @@ -187,11 +187,11 @@ env: NEXTSTRAIN_RUNTIME_ENVDIR: .git/nextstrain/env.d NEXTSTRAIN_DEFAULT_ARTIFACT_PATHS: | build.log - auspice/ - results/ benchmarks/ logs/ .snakemake/log/ + ${{ inputs.runtime != 'aws-batch' && 'auspice/' || '' }} + ${{ inputs.runtime != 'aws-batch' && 'results/' || '' }} permissions: id-token: write