docs+ci: report the GNU test suite pass rate in the README

.github/workflows/ci.yml

@@ -131,3 +131,55 @@ jobs:
         run: uptime --version
       - name: Run tests
         run: cd coreutils && GNU_COREUTILS_INSTALLED=1 make test
+
+  # Run GNU's own test suite against our binaries. This is a different
+  # measurement from `make test`: make test runs the unit tests this repo
+  # wrote, while this runs the tests GNU writes for its own implementation,
+  # against ours. The number it produces is the pass rate of the real suite.
+  #
+  # It fetches the GNU release tarball rather than the git sources, because the
+  # tarball ships its configure and tests/ pre-generated, and the only workaround
+  # the build needs is copying lib/stdckdint.in.h into place. See the comment at
+  # the top of util/gnu_test_suite.vsh.
+  ubuntu-gnu-test-suite:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout Latest V
+        uses: actions/checkout@v6
+        with:
+          repository: vlang/v
+          path: v
+      - name: Build V
+        run: cd v && make && sudo ./v symlink && cd -
+      - name: Checkout Coreutils
+        uses: actions/checkout@v6
+        with:
+          path: coreutils
+      - name: V doctor
+        run: v doctor
+      - name: Build all our tools
+        run: cd coreutils && v run build.vsh
+      - name: Cache the GNU test suite build
+        uses: actions/cache@v4
+        with:
+          path: coreutils/.gnu
+          key: gnu-suite-${{ runner.os }}-${{ hashFiles('util/gnu_test_suite.vsh') }}
+      - name: Run the GNU coreutils test suite against our binaries
+        run: |
+          cd coreutils
+          v run util/gnu_test_suite.vsh --bin-dir bin --jobs $(nproc)
+          status=$?
+          if [ $status -ne 0 ]; then
+            echo "::warning::the GNU suite reported failing tests; see the log below"
+          fi
+          exit 0
+      - name: Upload the full suite log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gnu-test-suite-log
+          path: |
+            coreutils/.gnu/coreutils-9.4/tests/test-suite.log
+            coreutils/.gnu/coreutils-9.4/check.log
+          if-no-files-found: warn

.github/workflows/gnu-suite-report.yml

@@ -0,0 +1,126 @@
+# Report the GNU coreutils test suite pass rate in the README.
+#
+# The suite job in ci.yml measures us on every push and PR. This job runs the
+# same measurement on a schedule and writes the number into the README, so the
+# headline figure is the real pass rate rather than a count someone typed by
+# hand. Both run util/gnu_test_suite.vsh, so they cannot disagree about the
+# measurement itself.
+name: GNU test suite report
+
+on:
+  schedule:
+    # Weekly, Monday 06:00 UTC. Chosen so a busy day does not race it.
+    - cron: '0 6 * * 1'
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+concurrency:
+  group: gnu-suite-report
+  cancel-in-progress: false
+
+jobs:
+  report:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - name: Checkout Latest V
+        uses: actions/checkout@v6
+        with:
+          repository: vlang/v
+          path: v
+      - name: Build V
+        run: cd v && make && sudo ./v symlink && cd -
+
+      - name: Checkout Coreutils
+        uses: actions/checkout@v6
+        with:
+          path: coreutils
+          # The default token, so the commit below is attributed to Actions.
+          persist-credentials: true
+
+      - name: Build all our tools
+        run: cd coreutils && v run build.vsh
+
+      - name: Restore the GNU suite build cache
+        uses: actions/cache@v4
+        with:
+          path: coreutils/.gnu
+          key: gnu-suite-report-${{ runner.os }}-${{ hashFiles('coreutils/util/gnu_test_suite.vsh') }}
+          restore-keys: |
+            gnu-suite-report-${{ runner.os }}-
+
+      - name: Run the GNU coreutils test suite
+        id: suite
+        run: |
+          cd coreutils
+          # The harness prints a machine-readable GNU_SUITE_RESULT line; capture
+          # the whole run for the log, and the result line for the README.
+          v run util/gnu_test_suite.vsh --bin-dir bin --jobs $(nproc) 2>&1 | tee ../suite-output.txt
+          result=$(grep '^GNU_SUITE_RESULT ' ../suite-output.txt | tail -1)
+          if [ -z "$result" ]; then
+            echo "::error::the harness produced no GNU_SUITE_RESULT line"
+            exit 1
+          fi
+          echo "result=$result" >> "$GITHUB_OUTPUT"
+          # shellcheck disable=SC2001
+          rate=$(echo "$result" | sed 's/.*rate=\([0-9.]*\).*/\1/')
+          runnable=$(echo "$result" | sed 's/.*runnable=\([0-9]*\).*/\1/')
+          echo "pass_rate=${rate}" >> "$GITHUB_OUTPUT"
+          echo "runnable_tests=${runnable}" >> "$GITHUB_OUTPUT"
+          echo "GNU test suite: ${rate}% of ${runnable} runnable tests"
+        continue-on-error: true
+
+      - name: Upload the full suite log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gnu-suite-report-log
+          path: |
+            coreutils/.gnu/coreutils-9.4/tests/test-suite.log
+            suite-output.txt
+          if-no-files-found: warn
+
+      - name: Update the README with the measured rate
+        run: |
+          cd coreutils
+          rate="${{ steps.suite.outputs.pass_rate }}"
+          runnable="${{ steps.suite.outputs.runnable_tests }}"
+          if [ -z "$rate" ]; then
+            echo "::warning::no rate to write, skipping the README update"
+            exit 0
+          fi
+          # Replace the two markers so that the surrounding prose and the
+          # date line stay under our control and the number is the only part
+          # a machine writes.
+          date=$(date -u '+%Y-%m-%d')
+          python3 - "$rate" "$runnable" "$date" <<'PY'
+          import re, sys
+          rate, runnable, date = sys.argv[1], sys.argv[2], sys.argv[3]
+          path = "README.md"
+          text = open(path).read()
+          text = re.sub(r"(<!-- GNU_SUITE_RATE -->).*?(<!-- /GNU_SUITE_RATE -->)",
+                        r"\g<1>**%s%%** (%s runnable tests)\g<2>" % (rate, runnable),
+                        text, flags=re.S)
+          text = re.sub(r"(<!-- GNU_SUITE_UPDATED -->).*?(<!-- /GNU_SUITE_UPDATED -->)",
+                        r"\g<1>%s\g<2>" % date, text, flags=re.S)
+          open(path, "w").write(text)
+          PY
+          grep -n "GNU_SUITE_RATE" README.md || true
+
+      - name: Commit the updated README
+        run: |
+          cd coreutils
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          if git diff --quiet README.md; then
+            echo "README already up to date"
+            exit 0
+          fi
+          git add README.md
+          git commit -m "docs: update the GNU test suite pass rate
+
+Autocommitted by the GNU test suite report workflow, which runs
+util/gnu_test_suite.vsh on a schedule and writes the measured number."
+          git push

.gitignore

@@ -18,3 +18,6 @@ src/touch/touch
 # Reference GNU coreutils archives, fetched locally for testing
 coreutils-*-win.zip
 
+# The GNU coreutils test suite that util/gnu_test_suite.vsh fetches and builds
+.gnu/
+

README.md

@@ -49,6 +49,22 @@ The completed count is updated automatically by a GitHub Action after merge.
 GNU coreutils. They are not 100% compatiable. If you encounter different behaviors,
 compare against the true GNU coreutils version on the Linux-based tests first.
 
+### Compatibility with real GNU coreutils
+
+The unit tests in this repository are tests we wrote for ourselves, so a green run
+says our code does what we intended, not that it matches GNU. To answer that
+question, `util/gnu_test_suite.vsh` runs *GNU's own test suite* against the
+binaries this repo builds: it fetches the GNU release tarball, builds it, points
+`tests/local.mk` and every test's `path_prepend_ ./src` at our `bin/` instead of
+GNU's own `src/`, and runs `make check`.
+
+<!-- GNU_SUITE_RATE -->not yet measured<!-- /GNU_SUITE_RATE -->
+of the runnable tests pass, as of <!-- GNU_SUITE_UPDATED -->never run<!-- /GNU_SUITE_UPDATED -->.
+The figure is written by the `GNU test suite report` workflow, which runs the
+suite on a schedule, so it is a measurement rather than a claim. Tests for tools
+that are not built are reported as `not-run` and excluded from the rate, and the
+full `test-suite.log` is uploaded as a build artifact.
+
 ## Completed (82/109) - 75% done!
 
 |  Done   | Cmd       | Descripton                                       | Windows |