improved testing and CI (#278)

Author: firewave

.github/workflows/CI-unixish.yml

@@ -17,7 +17,7 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Install missing software on ubuntu
-      if: matrix.os == 'ubuntu-20.04'
+      if: matrix.os == 'ubuntu-22.04'
       run: |
           sudo apt-get update
           sudo apt-get install valgrind
@@ -28,31 +28,43 @@ jobs:
     - name: make test
       run: make -j$(nproc) test CXX=${{ matrix.compiler }}
 
-    - name: ensure that simplecpp.cpp uses c++03
-      run: CXX=${{ matrix.compiler }} ; $CXX -fsyntax-only -std=c++98 simplecpp.cpp
-
     - name: Run valgrind
-      if: matrix.os == 'ubuntu-20.04'
+      if: matrix.os == 'ubuntu-22.04'
       run: |
+          make clean
+          # this valgrind version doesn't support DWARF 5 yet
+          make -j$(nproc) CXX=${{ matrix.compiler }} CXXFLAGS="-gdwarf-4"
           valgrind --leak-check=full --num-callers=50 --show-reachable=yes --track-origins=yes --gen-suppressions=all --error-exitcode=42 ./testrunner
 
+    - name: Run with libstdc++ debug mode
+      if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'gcc'
+      run: |
+        make clean
+        make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-g3 -D_GLIBCXX_DEBUG"
+
+    - name: Run with libc++ debug mode
+      if: matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++'
+      run: |
+        make clean
+        make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-stdlib=libc++ -g3 -DLIBCXX_ENABLE_DEBUG_MODE" LDFLAGS="-lc++"
+
     - name: Run AddressSanitizer
-      if: matrix.os == 'ubuntu-20.04'
+      if: matrix.os == 'ubuntu-22.04'
       run: |
         make clean
         make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=address" LDFLAGS="-fsanitize=address"
       env:
         ASAN_OPTIONS: detect_stack_use_after_return=1
 
     - name: Run UndefinedBehaviorSanitizer
-      if: matrix.os == 'ubuntu-20.04'
+      if: matrix.os == 'ubuntu-22.04'
       run: |
         make clean
         make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -fsanitize=undefined -fno-sanitize=signed-integer-overflow" LDFLAGS="-fsanitize=undefined -fno-sanitize=signed-integer-overflow"
 
-    # TODO: enable when false positives are fixed
+    # TODO: requires instrumented libc++
     - name: Run MemorySanitizer
-      if: false && matrix.os == 'ubuntu-20.04' && matrix.compiler == 'clang++'
+      if: false && matrix.os == 'ubuntu-22.04' && matrix.compiler == 'clang++'
       run: |
         make clean
         make -j$(nproc) test CXX=${{ matrix.compiler }} CXXFLAGS="-O2 -g3 -stdlib=libc++ -fsanitize=memory" LDFLAGS="-lc++ -fsanitize=memory"

.github/workflows/CI-windows.yml

@@ -16,6 +16,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-2019, windows-2022]
+        config: [Release, Debug]
       fail-fast: false
 
     runs-on: ${{ matrix.os }}
@@ -40,9 +41,9 @@ jobs:
       
       - name: Build
         run: |
-           msbuild -m simplecpp.sln /p:Configuration=Release /p:Platform=x64 || exit /b !errorlevel!
+           msbuild -m simplecpp.sln /p:Configuration=${{ matrix.config }} /p:Platform=x64 || exit /b !errorlevel!
            
       - name: Test
         run: |
-          .\Release\testrunner.exe || exit /b !errorlevel!
+          .\${{ matrix.config }}\testrunner.exe || exit /b !errorlevel!
         

run-tests.py

@@ -2,6 +2,7 @@
 import glob
 import os
 import subprocess
+import sys
 
 def cleanup(out):
   ret = ''
@@ -80,6 +81,7 @@ def cleanup(out):
 
 numberOfSkipped = 0
 numberOfFailed = 0
+numberOfFixed = 0
 
 usedTodos = []
 
@@ -101,10 +103,13 @@ def cleanup(out):
   gcc_output = cleanup(comm[0])
 
   simplecpp_cmd = ['./simplecpp']
-  simplecpp_cmd.extend(cmd.split(' '))
+  # -E is not supported and we bail out on unknown options
+  simplecpp_cmd.extend(cmd.replace('-E ', '', 1).split(' '))
   p = subprocess.Popen(simplecpp_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
   comm = p.communicate()
+  simplecpp_ec = p.returncode
   simplecpp_output = cleanup(comm[0])
+  simplecpp_err = comm[0].decode('utf-8').strip()
 
   if simplecpp_output != clang_output and simplecpp_output != gcc_output:
     filename = cmd[cmd.rfind('/')+1:]
@@ -113,14 +118,21 @@ def cleanup(out):
       usedTodos.append(filename)
     else:
       print('FAILED ' + cmd)
+      if simplecpp_ec:
+          print('simplecpp failed - ' + simplecpp_err)
       numberOfFailed = numberOfFailed + 1
 
 for filename in todo:
     if not filename in usedTodos:
         print('FIXED ' + filename)
+        numberOfFixed = numberOfFixed + 1
 
 print('Number of tests: ' + str(len(commands)))
 print('Number of skipped: ' + str(numberOfSkipped))
-print('Number of todos: ' + str(len(usedTodos)))
+print('Number of todos (fixed): ' + str(len(usedTodos)) + ' (' + str(numberOfFixed) + ')')
 print('Number of failed: ' + str(numberOfFailed))
 
+if numberOfFailed or numberOfFixed:
+    sys.exit(1)
+
+sys.exit(0)