Use workflow job matrix for builds

per1234 · per1234 · commit d7c4d47f01fc · 2022-10-14T06:03:12.000-07:00
Previously, a single GitHub Actions workflow job was used to make tester, nighly, and release builds of the application.

These builds were done serially. A more efficient approach is to run the builds in parallel. This is accomplished by
using a job matrix, where each target has its own generated job in the workflow.

Previously, checksum generation code was present both in the taskfile and in the workflows (due to the need to update
the macOS checksum after the notarization process done in the workflow). The checksum generation is now done exclusively
in the build workflows.
diff --git a/.github/workflows/publish-go-nightly-task.yml b/.github/workflows/publish-go-nightly-task.yml
@@ -22,6 +22,19 @@ jobs:
   create-nightly-artifacts:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        os:
+          - Windows_32bit
+          - Windows_64bit
+          - Linux_32bit
+          - Linux_64bit
+          - Linux_ARMv6
+          - Linux_ARMv7
+          - Linux_ARM64
+          - macOS_64bit
+          - macOS_ARM64
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -35,7 +48,7 @@ jobs:
       - name: Build
         env:
           NIGHTLY: true
-        run: task dist:all
+        run: task dist:${{ matrix.os }}
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -128,14 +141,10 @@ jobs:
         run: |
           gon "${{ env.GON_CONFIG_PATH }}"
 
-      - name: Re-package binary and output checksum
+      - name: Re-package binary
         id: re-package
         working-directory: ${{ env.DIST_DIR }}
-        # This step performs the following:
-        # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
-        # 2. Recalculate package checksum
-        # 3. Output the new checksum to include in the nnnnnn-checksums.txt file
-        #    (it cannot be done there because of workflow job parallelization)
+        # Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
         run: |
           # GitHub's upload/download-artifact actions don't preserve file permissions,
           # so we need to add execution permission back until the action is made to do this.
@@ -145,11 +154,9 @@ jobs:
           tar -czvf "$PACKAGE_FILENAME" \
             -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \
             -C ../../ LICENSE.txt
-          CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)"
           echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV
-          echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE"
 
-      - name: Upload artifacts
+      - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
           if-no-files-found: error
@@ -167,15 +174,11 @@ jobs:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ env.DIST_DIR }}
 
-      - name: Update checksum
+      - name: Create checksum file
+        working-directory: ${{ env.DIST_DIR}}
         run: |
-          declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}")
-          for checksum_line in "${checksum_lines[@]}"
-          do
-            CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1)
-            PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2)
-            perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM}  ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
-          done
+          TAG="nightly-$(date -u +"%Y%m%d")"
+          sha256sum ${{ env.PROJECT_NAME }}_${TAG}* > ${TAG}-checksums.txt
 
       - name: Upload release files on Arduino downloads servers
         uses: docker://plugins/s3
diff --git a/.github/workflows/publish-go-tester-task.yml b/.github/workflows/publish-go-tester-task.yml
@@ -24,9 +24,10 @@ on:
   repository_dispatch:
 
 env:
+  # As defined by the Taskfile's PROJECT_NAME variable
+  PROJECT_NAME: arduino-lint
   # As defined by the Taskfile's DIST_DIR variable
   DIST_DIR: dist
-  BUILDS_ARTIFACT: build-artifacts
 
 jobs:
   run-determination:
@@ -52,86 +53,107 @@ jobs:
 
           echo "::set-output name=result::$RESULT"
 
-  build:
+  package-name-prefix:
     needs: run-determination
     if: needs.run-determination.outputs.result == 'true'
     runs-on: ubuntu-latest
-
+    outputs:
+      prefix: ${{ steps.calculation.outputs.prefix }}
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Install Task
-        uses: arduino/setup-task@v1
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          version: 3.x
-
-      - name: Build
+      - name: package name prefix calculation
+        id: calculation
         run: |
           PACKAGE_NAME_PREFIX="test"
           if [ "${{ github.event_name }}" = "pull_request" ]; then
             PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.event.number }}"
           fi
           PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-"
-          export PACKAGE_NAME_PREFIX
-          task dist:all
 
-      # Transfer builds to artifacts job
-      - name: Upload combined builds artifact
-        uses: actions/upload-artifact@v3
-        with:
-          path: ${{ env.DIST_DIR }}
-          name: ${{ env.BUILDS_ARTIFACT }}
+          echo "::set-output name=prefix::$PACKAGE_NAME_PREFIX"
 
-  artifacts:
-    name: ${{ matrix.artifact.name }} artifact
-    needs: build
+  build:
+    needs: package-name-prefix
+    name: Build ${{ matrix.os.name }}
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        artifact:
-          - path: "*checksums.txt"
-            name: checksums
-          - path: "*Linux_32bit.tar.gz"
+        os:
+          - task: Windows_32bit
+            path: "*Windows_32bit.zip"
+            name: Windows_X86-32
+          - task: Windows_64bit
+            path: "*Windows_64bit.zip"
+            name: Windows_X86-64
+          - task: Linux_32bit
+            path: "*Linux_32bit.tar.gz"
             name: Linux_X86-32
-          - path: "*Linux_64bit.tar.gz"
+          - task: Linux_64bit
+            path: "*Linux_64bit.tar.gz"
             name: Linux_X86-64
-          - path: "*Linux_ARM64.tar.gz"
-            name: Linux_ARM64
-          - path: "*Linux_ARMv6.tar.gz"
+          - task: Linux_ARMv6
+            path: "*Linux_ARMv6.tar.gz"
             name: Linux_ARMv6
-          - path: "*Linux_ARMv7.tar.gz"
+          - task: Linux_ARMv7
+            path: "*Linux_ARMv7.tar.gz"
             name: Linux_ARMv7
-          - path: "*macOS_64bit.tar.gz"
+          - task: Linux_ARM64
+            path: "*Linux_ARM64.tar.gz"
+            name: Linux_ARM64
+          - task: macOS_64bit
+            path: "*macOS_64bit.tar.gz"
             name: macOS_64
-          - path: "*macOS_ARM64.tar.gz"
+          - task: macOS_ARM64
+            path: "*macOS_ARM64.tar.gz"
             name: macOS_ARM64
-          - path: "*Windows_32bit.zip"
-            name: Windows_X86-32
-          - path: "*Windows_64bit.zip"
-            name: Windows_X86-64
 
     steps:
-      - name: Download combined builds artifact
-        uses: actions/download-artifact@v3
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install Task
+        uses: arduino/setup-task@v1
         with:
-          name: ${{ env.BUILDS_ARTIFACT }}
-          path: ${{ env.BUILDS_ARTIFACT }}
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          version: 3.x
 
-      - name: Upload individual build artifact
+      - name: Build
+        run: |
+          PACKAGE_NAME_PREFIX=${{ needs.package-name-prefix.outputs.prefix }}
+          export PACKAGE_NAME_PREFIX
+          task dist:${{ matrix.os.task }}
+
+      # Transfer builds to artifacts job
+      - name: Upload build artifact
         uses: actions/upload-artifact@v3
         with:
-          path: ${{ env.BUILDS_ARTIFACT }}/${{ matrix.artifact.path }}
-          name: ${{ matrix.artifact.name }}
+          path: ${{ env.DIST_DIR }}/${{ matrix.os.path }}
+          name: ${{ matrix.os.name }}
 
-  clean:
-    needs: artifacts
+  checksums:
+    needs:
+      - build
+      - package-name-prefix
     runs-on: ubuntu-latest
 
     steps:
-      - name: Remove unneeded combined builds artifact
-        uses: geekyeggo/delete-artifact@v2
+      - name: Download build artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Create checksum file
+        run: |
+          TAG="${{ needs.package-name-prefix.outputs.prefix }}git-snapshot"
+          declare -a artifacts=($(ls -d */))
+          for artifact in ${artifacts[@]}
+          do
+            cd $artifact
+            checksum=$(sha256sum ${{ env.PROJECT_NAME }}_${TAG}*)
+            cd ..
+            echo $checksum >> ${TAG}-checksums.txt
+          done
+
+      - name: Upload checksum artifact
+        uses: actions/upload-artifact@v3
         with:
-          name: ${{ env.BUILDS_ARTIFACT }}
+          path: ./*checksums.txt
+          name: checksums
diff --git a/.github/workflows/release-go-task.yml b/.github/workflows/release-go-task.yml
@@ -19,13 +19,28 @@ jobs:
   create-release-artifacts:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        os:
+          - Windows_32bit
+          - Windows_64bit
+          - Linux_32bit
+          - Linux_64bit
+          - Linux_ARMv6
+          - Linux_ARMv7
+          - Linux_ARM64
+          - macOS_64bit
+          - macOS_ARM64
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Create changelog
+        # Avoid creating the same changelog for each os
+        if: matrix.os == 'Windows_32bit'
         uses: arduino/create-changelog@v1
         with:
           tag-regex: '^[0-9]+\.[0-9]+\.[0-9]+.*$'
@@ -40,7 +55,7 @@ jobs:
           version: 3.x
 
       - name: Build
-        run: task dist:all
+        run: task dist:${{ matrix.os }}
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -132,14 +147,10 @@ jobs:
         run: |
           gon "${{ env.GON_CONFIG_PATH }}"
 
-      - name: Re-package binary and output checksum
+      - name: Re-package binary
         id: re-package
         working-directory: ${{ env.DIST_DIR }}
-        # This step performs the following:
-        # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
-        # 2. Recalculate package checksum
-        # 3. Output the new checksum to include in the nnnnnn-checksums.txt file
-        #    (it cannot be done there because of workflow job parallelization)
+        # Repackage the signed binary replaced in place by Gon (ignoring the output zip file)
         run: |
           # GitHub's upload/download-artifact actions don't preserve file permissions,
           # so we need to add execution permission back until the action is made to do this.
@@ -149,11 +160,9 @@ jobs:
           tar -czvf "$PACKAGE_FILENAME" \
           -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \
           -C ../../ LICENSE.txt
-          CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)"
           echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV
-          echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE"
 
-      - name: Upload artifacts
+      - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
           if-no-files-found: error
@@ -171,15 +180,11 @@ jobs:
           name: ${{ env.ARTIFACT_NAME }}
           path: ${{ env.DIST_DIR }}
 
-      - name: Update checksum
+      - name: Create checksum file
+        working-directory: ${{ env.DIST_DIR}}
         run: |
-          declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}")
-          for checksum_line in "${checksum_lines[@]}"
-          do
-            CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1)
-            PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2)
-            perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM}  ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt
-          done
+          TAG="${GITHUB_REF/refs\/tags\//}"
+          sha256sum ${{ env.PROJECT_NAME }}_${TAG}* > ${TAG}-checksums.txt
 
       - name: Identify Prerelease
         # This is a workaround while waiting for create-release action
diff --git a/DistTasks.yml b/DistTasks.yml
