Reuse build comparison baselines and summarize CI mismatches

.gitattributes

@@ -2,6 +2,7 @@
 # Set default behavior to automatically normalize line endings.
 ###############################################################################
 * text=auto
+spec/*.sh text eol=lf
 
 ###############################################################################
 # Set default behavior for command prompt diff.
@@ -66,4 +67,4 @@
 # Executable files - preserve execution permissions on Unix systems
 # (https://git-scm.com/docs/gitattributes#_executable)
 ###############################################################################
-runtime/*.exe binary
+runtime/*.exe binary

.github/workflows/buildtest.yml

@@ -1,93 +1,96 @@
----
-name: Run Tests
+name: Run Build Diff Tests
 on:
   pull_request:
-    branches:
-      - dev
+    branches: [dev]
+  schedule:
+    - cron: '17 4 * * *'
   workflow_dispatch:
+    inputs:
+      mode:
+        description: Compare builds or prepare the shared baseline
+        type: choice
+        options: [compare, prepare]
+        default: compare
+      base_ref:
+        description: Base branch or commit
+        default: dev
+permissions:
+  contents: read
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: build-diff-${{ github.event_name == 'pull_request' && github.ref || 'baseline' }}
   cancel-in-progress: true
 jobs:
   run_build_diff:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      BASE_ONLY: ${{ (github.event_name == 'schedule' || inputs.mode == 'prepare') && '1' || '0' }}
     steps:
-      - name: Checkout HEAD
+      - name: Checkout test code
         uses: actions/checkout@v4
-      - name: Fetch Dev branch
-        id: get-dev-ref
-        run: |
-            git fetch --depth=1 origin dev
-            echo "devref=$(git rev-parse origin/dev)" >> $GITHUB_OUTPUT
-      - name: Download Dev branch cache
-        id: download-dev-ref-cache
-        uses: dawidd6/action-download-artifact@3ecf4024886f219d9290351234889bfb45d1b9da
-        with:
-          name: cache-devref-${{ steps.get-dev-ref.outputs.devref }}
-          path: /tmp/cache/
-          if_no_artifact_found: warn
-          search_artifacts: true
-      # Dev ref cache contains the build list and build xmls. Use that one to keep tests reproducible
-      - name: Update static builds list from cache
-        if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'true' }}
-        run: cat /tmp/cache/builds.txt > spec/builds.txt
-      - name: Download latest build list
-        if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' }}
-        id: download-build-list
-        uses: dawidd6/action-download-artifact@3ecf4024886f219d9290351234889bfb45d1b9da
-        with:
-          name: builds.txt
-          path: /tmp/latestbuildlist/
-          workflow: updatebuildlist.yml
-          if_no_artifact_found: warn
-          search_artifacts: true
-      - name: Update static builds list
-        if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' && steps.download-build-list.outputs.found_artifact == 'true' }}
-        run: cat /tmp/latestbuildlist/builds.txt > spec/builds.txt
-      - name: Download latest build xmls
-        if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' }}
-        uses: dawidd6/action-download-artifact@3ecf4024886f219d9290351234889bfb45d1b9da
         with:
-          name: build-xmls
-          path: /tmp/cache/
-          if_no_artifact_found: warn
-          search_artifacts: true
-      - name: Calculate build xmls and differences between them
+          persist-credentials: false
+      - name: Select base and download the corpus once
+        id: inputs
+        env:
+          BASE_REF: ${{ github.event.pull_request.base.sha || inputs.base_ref || 'dev' }}
+        shell: bash
         run: |
-            mkdir /tmp/cache || true # Make sure /tmp/cache exists. Ignore exit code
-            chmod -R 777 /tmp/cache && docker compose run -v '/tmp/cache/:/cache' -e 'CACHEDIR=/cache' busted-diff | tee /tmp/dockerlog
-      - name: Generate artefact
-        run: |
-          sed -n '/Runtime comparison for/,/Savefile Diff for/{/Savefile Diff for/!p;}' /tmp/dockerlog > /tmp/artefact
-          sed -n '/Savefile Diff for/, $p' /tmp/dockerlog >> /tmp/artefact
-          [ -s /tmp/artefact ] || rm /tmp/artefact
-      - name: Upload artefact
-        uses: actions/upload-artifact@v4
+          git fetch --depth=1 origin "$BASE_REF"
+          export DEVREF=$(git rev-parse FETCH_HEAD)
+          source spec/BuildCache.sh
+          echo "base=$DEV_SHA" >> "$GITHUB_OUTPUT"
+          echo "key=$CACHE_KEY" >> "$GITHUB_OUTPUT"
+          echo "corpus=$CORPUS_FILE" >> "$GITHUB_OUTPUT"
+          echo "Baseline cache: $CACHE_KEY"
+      - name: Find or restore the calculated base
+        id: cache
+        uses: actions/cache/restore@v4
+        continue-on-error: true
         with:
-          name: build-diff-output
-          path: /tmp/artefact
-      - name: Save used build list into cache
-        if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' }}
-        run: cp spec/builds.txt /tmp/cache/
-      - name: Move xmls found in builds.txt to a new directory
-        if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' && steps.download-build-list.outputs.found_artifact == 'true' }}
+          key: ${{ steps.inputs.outputs.key }}
+          path: /tmp/pob-cache/${{ steps.inputs.outputs.key }}
+          lookup-only: ${{ env.BASE_ONLY == '1' }}
+      - name: Calculate builds
+        id: calculate
+        if: env.BASE_ONLY != '1' || steps.cache.outputs.cache-hit != 'true'
+        env:
+          DEVREF: ${{ steps.inputs.outputs.base }}
+          CORPUS_FILE: ${{ steps.inputs.outputs.corpus }}
+          CACHE_KEY: ${{ steps.inputs.outputs.key }}
+        shell: bash
         run: |
-          mkdir new-build-xmls
-          while IFS= read -r line; do
-              FILENAME="/tmp/cache/${line//[^a-zA-Z0-9]/}.xml"
-              if [ -f "$FILENAME" ]; then
-                  mv "$FILENAME" "./new-build-xmls/"
-              fi
-          done < "spec/builds.txt"
-      - name: Upload new build xmls
-        if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' && steps.download-build-list.outputs.found_artifact == 'true' }}
-        uses: actions/upload-artifact@v4
+          mkdir -p /tmp/pob-cache
+          result=0
+          docker compose run --rm -v /tmp/pob-cache:/cache \
+            -v "$CORPUS_FILE:/corpus.json:ro" -e CORPUS_FILE=/corpus.json \
+            busted-diff | tee /tmp/dockerlog || result=$?
+          test -f "/tmp/pob-cache/$CACHE_KEY/$DEVREF"
+          echo "base_complete=true" >> "$GITHUB_OUTPUT"
+          exit "$result"
+      - name: Save the completed base
+        if: ${{ !cancelled() && steps.calculate.outputs.base_complete == 'true' && steps.cache.outputs.cache-hit != 'true' }}
+        uses: actions/cache/save@v4
+        continue-on-error: true
         with:
-          name: build-xmls
-          path: './new-build-xmls/*'
-      - name: Upload dev ref cache
-        if: ${{ steps.download-dev-ref-cache.outputs.found_artifact == 'false' }}
+          key: ${{ steps.inputs.outputs.key }}
+          path: /tmp/pob-cache/${{ steps.inputs.outputs.key }}
+      - name: Collect comparison output
+        if: always() && env.BASE_ONLY != '1' && steps.calculate.outcome != 'skipped'
+        shell: bash
+        run: |
+          if [ -f /tmp/dockerlog ]; then
+            sed -n '/^## /,$p' /tmp/dockerlog > /tmp/build-diff-output
+            [ -s /tmp/build-diff-output ] || rm -f /tmp/build-diff-output
+            sed -n '/^## Build comparison summary$/,$p' /tmp/dockerlog >> "$GITHUB_STEP_SUMMARY"
+          fi
+          if [ ! -s "$GITHUB_STEP_SUMMARY" ]; then
+            printf '## Build comparison summary\n\nCalculation or comparison did not finish. See the **Calculate builds** log for the error.\n' >> "$GITHUB_STEP_SUMMARY"
+          fi
+      - name: Upload comparison output
+        if: always() && env.BASE_ONLY != '1'
         uses: actions/upload-artifact@v4
         with:
-          name: cache-devref-${{ steps.get-dev-ref.outputs.devref }}
-          path: /tmp/cache/
+          name: build-diff-output
+          path: /tmp/build-diff-output
+          if-no-files-found: ignore

CONTRIBUTING.md

@@ -305,7 +305,7 @@ While both can be ran locally it's recommended to use the provided docker image
 PoB uses the [Busted](https://lunarmodules.github.io/busted/) framework to run its tests. Tests are stored under `spec/System` and run automatically when a PR is modified.
 More tests can be added to this folder to test specific functionality, or new test builds can be added to ensure nothing changed that wasn't intended. 
 
-### Running tests
+To run the build difference tests, first fetch `origin/dev`, then run `docker compose run --rm busted-diff`. Set `DEVREF` to compare against another branch or commit. The test uses the same PoB Codes builds and checked-in fixtures on both revisions, and reports stat, saved-XML and timing differences. Calculated-stat differences fail the check and appear in the GitHub run summary; the full report remains in the log and `build-diff-output` artifact. Base results are reused from the Docker `build-cache` volume when the base, corpus and test tools match; `docker compose down --volumes` clears that cache. GitHub Actions prepares the shared baseline nightly, while PRs calculate both sides themselves if it is unavailable.
 
 1. Install [Docker](https://www.docker.com/get-started)
 2. Run `docker-compose up` from the command line

docker-compose.yml

@@ -1,9 +1,24 @@
 services:
   busted-tests:
     #build: .
-    image: ghcr.io/pathofbuildingcommunity/pathofbuilding-tests:latest
+    image: &test-image ghcr.io/pathofbuildingcommunity/pathofbuilding-tests@sha256:171dc3da232b8c874882e4ae3b3aa4a6e130a6c9450a31904b312435a6bf5daf
     environment:
       HOME: /tmp
+      LUA_PATH: "/usr/share/lua/5.1/?.lua;/usr/share/lua/5.1/?/init.lua;;"
+      LUA_CPATH: "/usr/lib/lua/5.1/?.so;;"
+    # The official image supplies LuaJIT; these are the existing script dependencies.
+    entrypoint: &test-entrypoint
+      - /bin/sh
+      - -ec
+      - |
+        apk add --no-cache git parallel libxml2-utils lua5.1-posix lua5.1-curl zlib-dev
+        luarocks install lua-zlib 1.4-0
+        if [ -n "$${CACHEDIR:-}" ]; then
+          mkdir -p "$$CACHEDIR"
+          chown -R nobody:nobody "$$CACHEDIR"
+        fi
+        exec su -p nobody -s /bin/sh -c "$$*"
+      - --
     container_name: pathofbuilding-tests
     command: busted --lua=luajit
     security_opt:
@@ -13,3 +28,27 @@ services:
     working_dir: /workdir
     volumes:
       - ./:/workdir:ro
+  busted-diff:
+    #build: .
+    image: *test-image
+    environment: #Where in the container the folders are stored
+      WORKDIR: /workdir
+      HOME: /tmp
+      CACHEDIR: /cache
+      BASE_ONLY: ${BASE_ONLY:-0}
+      BUILD_JOBS: ${BUILD_JOBS:-2}
+      DEVREF: ${DEVREF:-origin/dev}
+      HEADREF: ${HEADREF:-}
+      LUA_PATH: "/usr/share/lua/5.1/?.lua;/usr/share/lua/5.1/?/init.lua;;"
+      LUA_CPATH: "/usr/lib/lua/5.1/?.so;;"
+    container_name: busted-diff
+    entrypoint: *test-entrypoint
+    command: ["dos2unix < /workdir/spec/BuildDiff.sh | /bin/sh"]
+    security_opt:
+      - no-new-privileges:true
+    working_dir: /workdir
+    volumes:
+      - ./:/workdir:ro
+      - build-cache:/cache
+volumes:
+  build-cache:

spec/BuildCache.sh

@@ -0,0 +1,14 @@
+# Shared by GitHub Actions and the local runner so cache identity is identical.
+DEV_SHA=$(git rev-parse "${DEVREF:-origin/dev}^{commit}")
+if [ -z "${CORPUS_FILE:-}" ]; then
+    CORPUS_FILE=$(mktemp)
+    curl --fail --show-error --silent --max-time 60 \
+        https://api.pob.codes/test-builds/corpus -o "$CORPUS_FILE"
+    chmod a+r "$CORPUS_FILE" # The container reads this public feed as nobody.
+fi
+corpus_hash=$(sha256sum "$CORPUS_FILE" | cut -d ' ' -f 1)
+test_hash=$(git ls-files -z -- .busted docker-compose.yml src/HeadlessWrapper.lua \
+    spec/BuildCache.sh spec/BuildDiff.sh spec/BuildStats.lua spec/FetchTestBuilds.lua \
+    spec/GenerateBuilds.lua spec/TestBuilds | xargs -0 git hash-object -- | sha256sum | cut -d ' ' -f 1)
+CACHE_KEY="pob-builds-v1-$DEV_SHA-$corpus_hash-$test_hash"
+export DEV_SHA CORPUS_FILE CACHE_KEY