testrunner: added option -t to print timing information about the tests [skip ci]

Author: firewave

Makefile

@@ -722,7 +722,7 @@ cli/stacktrace.o: cli/stacktrace.cpp cli/stacktrace.h lib/config.h lib/utils.h
 cli/threadexecutor.o: cli/threadexecutor.cpp cli/executor.h cli/threadexecutor.h lib/addoninfo.h lib/check.h lib/checkers.h lib/config.h lib/cppcheck.h lib/errorlogger.h lib/errortypes.h lib/filesettings.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/timer.h lib/utils.h
 	$(CXX) ${INCLUDE_FOR_CLI} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ cli/threadexecutor.cpp
 
-test/fixture.o: test/fixture.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/check.h lib/checkers.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/standards.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/xml.h test/fixture.h test/helpers.h test/options.h test/redirect.h
+test/fixture.o: test/fixture.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/check.h lib/checkers.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/standards.h lib/timer.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/xml.h test/fixture.h test/helpers.h test/options.h test/redirect.h
 	$(CXX) ${INCLUDE_FOR_TEST} ${CFLAGS_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/fixture.cpp
 
 test/helpers.o: test/helpers.cpp cli/filelister.h externals/simplecpp/simplecpp.h externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/checkers.h lib/config.h lib/errortypes.h lib/filesettings.h lib/library.h lib/mathlib.h lib/path.h lib/pathmatch.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/xml.h test/helpers.h
@@ -731,7 +731,7 @@ test/helpers.o: test/helpers.cpp cli/filelister.h externals/simplecpp/simplecpp.
 test/main.o: test/main.cpp externals/simplecpp/simplecpp.h lib/addoninfo.h lib/check.h lib/checkers.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/preprocessor.h lib/settings.h lib/standards.h lib/utils.h test/fixture.h test/options.h
 	$(CXX) ${INCLUDE_FOR_TEST} ${CFLAGS_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/main.cpp
 
-test/options.o: test/options.cpp test/options.h
+test/options.o: test/options.cpp lib/config.h lib/timer.h test/options.h
 	$(CXX) ${INCLUDE_FOR_TEST} ${CFLAGS_FOR_TEST} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ test/options.cpp
 
 test/test64bit.o: test/test64bit.cpp lib/addoninfo.h lib/check.h lib/check64bit.h lib/checkers.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/standards.h lib/tokenize.h lib/tokenlist.h lib/utils.h test/fixture.h test/helpers.h

test/fixture.cpp

@@ -23,6 +23,7 @@
 #include "library.h"
 #include "options.h"
 #include "redirect.h"
+#include "timer.h"
 
 #include <algorithm>
 #include <cstdio>
@@ -88,6 +89,7 @@ TestFixture::TestFixture(const char * const _name)
     : classname(_name)
 {}
 
+TestFixture::~TestFixture() = default;
 
 bool TestFixture::prepareTest(const char testname[])
 {
@@ -101,12 +103,14 @@ bool TestFixture::prepareTest(const char testname[])
         // Tests will be executed - prepare them
         mTestname = testname;
         ++countTests;
+        std::string fullTestName =  classname + "::" + mTestname;
         if (quiet_tests) {
             std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr
             std::fflush(stdout);
         } else {
-            std::cout << classname << "::" << mTestname << std::endl;
+            std::cout << fullTestName << std::endl;
         }
+        mTimer.reset(new Timer(fullTestName, ShowTime::TOP5_SUMMARY, timerResults));
         return !dry_run;
     }
     return false;
@@ -116,6 +120,10 @@ void TestFixture::teardownTest()
 {
     teardownTestInternal();
 
+    // TODO: print more detailed data
+    if (mTimer)
+        mTimer->stop();
+
     {
         const std::string s = errout_str();
         if (!s.empty())
@@ -381,6 +389,7 @@ void TestFixture::processOptions(const options& args)
     quiet_tests = args.quiet();
     dry_run = args.dry_run();
     exename = args.exe();
+    timerResults = args.timer_results();
 }
 
 std::size_t TestFixture::runTests(const options& args)
@@ -405,10 +414,12 @@ std::size_t TestFixture::runTests(const options& args)
                     continue;
             }
 
-            TestFixture* fixture = test->create();
-            fixture->processOptions(args);
-            fixture->run(testname);
-        }
+        TestFixture* fixture;
+        Timer::run(test->classname + " - create", ShowTime::TOP5_SUMMARY, args.timer_results(), [&](){
+            fixture = test->create();
+        });
+        fixture->processOptions(args);
+        fixture->run(testname);
     }
 
     if (args.summary() && !args.dry_run()) {

test/fixture.h

@@ -42,6 +42,8 @@
 
 class options;
 class Tokenizer;
+class Timer;
+class TimerResultsIntf;
 
 class TestFixture : public ErrorLogger {
 private:
@@ -60,6 +62,7 @@ class TestFixture : public ErrorLogger {
     bool quiet_tests{};
     bool dry_run{};
     bool mNewTemplate{};
+    TimerResultsIntf* timerResults{};
 
     virtual void run() = 0;
 
@@ -284,6 +287,8 @@ class TestFixture : public ErrorLogger {
     std::ostringstream mOutput;
     std::ostringstream mErrout;
 
+    std::unique_ptr<Timer> mTimer;
+
     void reportOut(const std::string &outmsg, Color c = Color::Reset) override;
     void reportErr(const ErrorMessage &msg) override;
     void reportMetric(const std::string &metric) override
@@ -297,6 +302,7 @@ class TestFixture : public ErrorLogger {
     const std::string classname;
 
     explicit TestFixture(const char * _name);
+    ~TestFixture() override;
 
     static std::size_t runTests(const options& args);
 };

test/options.cpp

@@ -16,6 +16,9 @@
 
 #include "options.h"
 
+#include "timer.h"
+
+// TODO: bailout on unknown arguments
 options::options(int argc, const char* const argv[])
     : mWhichTests(argv + 1, argv + argc)
     ,mQuiet(mWhichTests.count("-q") != 0)
@@ -25,6 +28,9 @@ options::options(int argc, const char* const argv[])
     ,mExcludeTests(mWhichTests.count("-x") != 0)
     ,mExe(argv[0])
 {
+    if (mArgs.count("-t") != 0)
+        mTimerResults.reset(new TimerResults);
+
     for (auto it = mWhichTests.cbegin(); it != mWhichTests.cend();) {
         if (!it->empty() && (((*it)[0] == '-') || (it->find("::") != std::string::npos && mWhichTests.count(it->substr(0, it->find("::"))))))
             it = mWhichTests.erase(it);
@@ -37,6 +43,12 @@ options::options(int argc, const char* const argv[])
     }
 }
 
+options::~options()
+{
+    if (mTimerResults)
+        mTimerResults->showResults(ShowTime::TOP5_FILE);
+}
+
 bool options::quiet() const
 {
     return mQuiet;
@@ -71,3 +83,8 @@ bool options::exclude_tests() const
 {
     return mExcludeTests;
 }
+
+TimerResultsIntf* options::timer_results() const
+{
+    return mTimerResults.get();
+}

test/options.h

@@ -17,9 +17,13 @@
 #ifndef OPTIONS_H
 #define OPTIONS_H
 
+#include <memory>
 #include <set>
 #include <string>
 
+class TimerResultsIntf;
+class TimerResults;
+
 /**
  * @brief Class to parse command-line parameters for ./testrunner .
  * Has getters for available switches and parameters.
@@ -29,6 +33,7 @@ class options {
 public:
     /** Call from main() to populate object */
     options(int argc, const char* const argv[]);
+    ~options();
     /** Don't print the name of each method being tested. */
     bool quiet() const;
     /** Print help. */
@@ -39,6 +44,8 @@ class options {
     bool dry_run() const;
     /** Exclude provided lists of tests. */
     bool exclude_tests() const;
+    /** The timer results. */
+    TimerResultsIntf* timer_results() const;
     /** Which test should be run. Empty string means 'all tests' */
     const std::set<std::string>& which_test() const;
 
@@ -55,6 +62,7 @@ class options {
     const bool mSummary;
     const bool mDryRun;
     const bool mExcludeTests;
+    std::unique_ptr<TimerResults> mTimerResults;
     std::string mExe;
 };