Tokenizer: remove simplifyTokenList2

danmar · danmar · commit db155a59c119 · 2022-07-18T16:20:36.000+02:00
diff --git a/cli/main.cpp b/cli/main.cpp
@@ -52,8 +52,6 @@
  *   - Macros are expanded
  * -# Tokenize the file (see Tokenizer)
  * -# Run the runChecks of all check classes.
- * -# Simplify the tokenlist (Tokenizer::simplifyTokenList2)
- * -# Run the runSimplifiedChecks of all check classes
  *
  * When errors are found, they are reported back to the CppCheckExecutor through the ErrorLogger interface.
  */
diff --git a/gui/test/benchmark/simple/benchmarksimple.cpp b/gui/test/benchmark/simple/benchmarksimple.cpp
@@ -44,38 +44,4 @@ void BenchmarkSimple::tokenize()
     }
 }
 
-void BenchmarkSimple::simplify()
-{
-    QFile file(QString(SRCDIR) + "/../../data/benchmark/simple.cpp");
-    QByteArray data = file.readAll();
-
-    Settings settings;
-    settings.debugwarnings = true;
-
-    // tokenize..
-    Tokenizer tokenizer(&settings, this);
-    std::istringstream istr(data.constData());
-    tokenizer.tokenize(istr, "test.cpp");
-    QBENCHMARK {
-        tokenizer.simplifyTokenList2();
-    }
-}
-
-void BenchmarkSimple::tokenizeAndSimplify()
-{
-    QFile file(QString(SRCDIR) + "/../../data/benchmark/simple.cpp");
-    QByteArray data = file.readAll();
-
-    Settings settings;
-    settings.debugwarnings = true;
-
-    // tokenize..
-    Tokenizer tokenizer(&settings, this);
-    std::istringstream istr(data.constData());
-    QBENCHMARK {
-        tokenizer.tokenize(istr, "test.cpp");
-        tokenizer.simplifyTokenList2();
-    }
-}
-
 QTEST_MAIN(BenchmarkSimple)
diff --git a/gui/test/benchmark/simple/benchmarksimple.h b/gui/test/benchmark/simple/benchmarksimple.h
@@ -28,8 +28,6 @@ class BenchmarkSimple : public QObject, public ErrorLogger {
 
 private slots:
     void tokenize();
-    void simplify();
-    void tokenizeAndSimplify();
 
 private:
     // Empty implementations of ErrorLogger methods.
diff --git a/lib/cppcheck.cpp b/lib/cppcheck.cpp
@@ -893,11 +893,8 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
                     checkUnusedFunctions.parseTokens(tokenizer, filename.c_str(), &mSettings);
 
                 // handling of "simple" rules has been removed.
-                if (mSimplify && hasRule("simple")) {
-                    // FIXME Remove this function
-                    tokenizer.simplifyTokenList2();
+                if (mSimplify && hasRule("simple"))
                     throw InternalError(nullptr, "Handling of \"simple\" rules has been removed in Cppcheck. Use --addon instead.");
-                }
 
             } catch (const simplecpp::Output &o) {
                 // #error etc during preprocessing
diff --git a/lib/token.cpp b/lib/token.cpp
@@ -2411,10 +2411,11 @@ TokenImpl::~TokenImpl()
     delete mValueType;
     delete mValues;
 
-    if (mTemplateSimplifierPointers)
+    if (mTemplateSimplifierPointers) {
         for (auto *templateSimplifierPointer : *mTemplateSimplifierPointers) {
             templateSimplifierPointer->token(nullptr);
         }
+    }
     delete mTemplateSimplifierPointers;
 
     while (mCppcheckAttributes) {
diff --git a/lib/token.h b/lib/token.h
@@ -1395,10 +1395,6 @@ class CPPCHECKLIB Token {
      */
     bool isCalculation() const;
 
-    void clearAst() {
-        mImpl->mAstOperand1 = mImpl->mAstOperand2 = mImpl->mAstParent = nullptr;
-    }
-
     void clearValueFlow() {
         delete mImpl->mValues;
         mImpl->mValues = nullptr;
diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp
@@ -5150,38 +5150,6 @@ bool Tokenizer::simplifyTokenList1(const char FileName[])
 
     return true;
 }
-
-bool Tokenizer::simplifyTokenList2()
-{
-    // clear the _functionList so it can't contain dead pointers
-    deleteSymbolDatabase();
-
-    // Clear AST,ValueFlow. These will be created again at the end of this function.
-    for (Token *tok = list.front(); tok; tok = tok->next()) {
-        tok->clearAst();
-        tok->clearValueFlow();
-    }
-
-    if (Settings::terminated())
-        return false;
-
-
-    bool modified = true;
-    while (modified) {
-        if (Settings::terminated())
-            return false;
-
-        modified = false;
-        modified |= simplifyConstTernaryOp();
-        validate();
-    }
-
-    simplifyComma();
-
-    removeRedundantSemicolons();
-
-    return true;
-}
 //---------------------------------------------------------------------------
 
 void Tokenizer::printDebugOutput(int simplification) const
@@ -5873,87 +5841,6 @@ Token *Tokenizer::simplifyAddBracesPair(Token *tok, bool commandWithCondition)
     return tokBracesEnd;
 }
 
-bool Tokenizer::simplifyConstTernaryOp()
-{
-    bool ret = false;
-    const Token *templateParameterEnd = nullptr; // The end of the current template parameter list, if any
-    for (Token *tok = list.front(); tok; tok = tok->next()) {
-        if (tok->str() == "<" && TemplateSimplifier::templateParameters(tok))
-            templateParameterEnd = tok->findClosingBracket();
-        if (tok == templateParameterEnd)
-            templateParameterEnd = nullptr; // End of the current template parameter list
-        if (tok->str() != "?")
-            continue;
-
-        if (!Token::Match(tok->tokAt(-2), "<|=|,|(|[|{|}|;|case|return %bool%|%num%") &&
-            !Token::Match(tok->tokAt(-4), "<|=|,|(|[|{|}|;|case|return ( %bool%|%num% )"))
-            continue;
-
-        const int offset = (tok->previous()->str() == ")") ? 2 : 1;
-
-        if (tok->strAt(-2*offset) == "<") {
-            if (isC() || !TemplateSimplifier::templateParameters(tok->tokAt(-2*offset)))
-                continue; // '<' is less than; the condition is not a constant
-        }
-
-        // Find the token ":" then go to the next token
-        Token *colon = skipTernaryOp(tok);
-        if (!colon || colon->previous()->str() != ":" || !colon->next())
-            continue;
-
-        //handle the GNU extension: "x ? : y" <-> "x ? x : y"
-        if (colon->previous() == tok->next())
-            tok->insertToken(tok->strAt(-offset));
-
-        // go back before the condition, if possible
-        tok = tok->tokAt(-2);
-        if (offset == 2) {
-            // go further back before the "("
-            tok = tok->tokAt(-2);
-            //simplify the parentheses
-            tok->deleteNext();
-            tok->next()->deleteNext();
-        }
-
-        if (Token::Match(tok->next(), "false|0")) {
-            // Use code after colon, remove code before it.
-            Token::eraseTokens(tok, colon);
-
-            tok = tok->next();
-            ret = true;
-        }
-
-        // The condition is true. Delete the operator after the ":"..
-        else {
-            // delete the condition token and the "?"
-            tok->deleteNext(2);
-
-            int ternaryOplevel = 0;
-            for (const Token *endTok = colon; endTok; endTok = endTok->next()) {
-                if (Token::Match(endTok, "(|[|{"))
-                    endTok = endTok->link();
-                else if (endTok->str() == "<" && (endTok->strAt(1) == ">" || TemplateSimplifier::templateParameters(endTok)))
-                    endTok = endTok->findClosingBracket();
-                else if (endTok->str() == "?")
-                    ++ternaryOplevel;
-                else if (Token::Match(endTok, ")|}|]|;|,|:|>")) {
-                    if (endTok->str() == ":" && ternaryOplevel)
-                        --ternaryOplevel;
-                    else if (endTok->str() == ">" && !templateParameterEnd)
-                        ;
-                    else {
-                        Token::eraseTokens(colon->tokAt(-2), endTok);
-                        ret = true;
-                        break;
-                    }
-                }
-            }
-        }
-    }
-    return ret;
-}
-
-
 void Tokenizer::simplifyFunctionParameters()
 {
     for (Token *tok = list.front(); tok; tok = tok->next()) {
@@ -7352,133 +7239,7 @@ bool Tokenizer::isOneNumber(const std::string &s)
         return false;
     return isNumberOneOf(s, 1L, "1.0");
 }
-
-void Tokenizer::simplifyComma()
-{
-    bool inReturn = false;
-
-    for (Token *tok = list.front(); tok; tok = tok->next()) {
-
-        // skip enums
-        if (Token::Match(tok, "enum class|struct| %name%| :|{")) {
-            skipEnumBody(&tok);
-        }
-        if (!tok)
-            syntaxError(nullptr); // invalid code like in #4195
-
-        if (Token::Match(tok, "(|[") || Token::Match(tok->previous(), "%name%|= {")) {
-            tok = tok->link();
-            continue;
-        }
-
-        if (Token::simpleMatch(tok, "= (") && Token::simpleMatch(tok->linkAt(1), ") {")) {
-            tok = tok->linkAt(1)->linkAt(1);
-            continue;
-        }
-
-        // Skip unhandled template specifiers..
-        if (tok->link() && tok->str() == "<")
-            tok = tok->link();
-
-        if (tok->str() == "return" && Token::Match(tok->previous(), "[;{}]"))
-            inReturn = true;
-
-        if (inReturn && Token::Match(tok, "[;{}?:]"))
-            inReturn = false;
-
-        if (!tok->next() || tok->str() != ",")
-            continue;
-
-        // We must not accept just any keyword, e.g. accepting int
-        // would cause function parameters to corrupt.
-        if (isCPP() && tok->strAt(1) == "delete") {
-            // Handle "delete a, delete b;"
-            tok->str(";");
-        }
-
-        if (isCPP() && Token::Match(tok->tokAt(-2), "delete %name% , %name% ;") &&
-            tok->next()->varId() != 0) {
-            // Handle "delete a, b;" - convert to delete a; b;
-            tok->str(";");
-        } else if (!inReturn && tok->tokAt(-2)) {
-            bool replace = false;
-            for (Token *tok2 = tok->previous(); tok2; tok2 = tok2->previous()) {
-                if (tok2->str() == "=") {
-                    // Handle "a = 0, b = 0;"
-                    replace = true;
-                } else if (isCPP() && (Token::Match(tok2, "delete %name%") ||
-                                       Token::Match(tok2, "delete [ ] %name%"))) {
-                    // Handle "delete a, a = 0;"
-                    replace = true;
-                } else if (Token::Match(tok2, "[?:;,{}()]")) {
-                    if (replace && Token::Match(tok2, "[;{}]"))
-                        tok->str(";");
-                    break;
-                }
-            }
-        }
-
-        // find token where return ends and also count commas
-        if (inReturn) {
-            Token *startFrom = nullptr;    // "[;{}]" token before "return"
-            Token *endAt = nullptr;        // first ";" token after "[;{}] return"
-
-            // find "; return" pattern before comma
-            for (Token *tok2 = tok->previous(); tok2; tok2 = tok2->previous()) {
-                if (tok2->str() == "return") {
-                    startFrom = tok2->previous();
-                    break;
-                }
-            }
-            if (!startFrom)
-                // to be very sure...
-                return;
-            int commaCounter = 0;
-            for (Token *tok2 = startFrom->next(); tok2; tok2 = tok2->next()) {
-                if (tok2->str() == ";") {
-                    endAt = tok2;
-                    break;
-
-                } else if (Token::Match(tok2, "(|[") ||
-                           (tok2->str() == "{" && tok2->previous() && tok2->previous()->str() == "=")) {
-                    tok2 = tok2->link();
-
-                } else if (tok2->str() == ",") {
-                    ++commaCounter;
-                }
-            }
-
-            if (!endAt)
-                //probably a syntax error
-                return;
-
-            if (commaCounter) {
-                // change tokens:
-                // "; return a ( ) , b ( ) , c ;"
-                // to
-                // "; a ( ) ; b ( ) ; return c ;"
-
-                // remove "return"
-                startFrom->deleteNext();
-                for (Token *tok2 = startFrom->next(); tok2 != endAt; tok2 = tok2->next()) {
-                    if (Token::Match(tok2, "(|[") ||
-                        (tok2->str() == "{" && tok2->previous() && tok2->previous()->str() == "=")) {
-                        tok2 = tok2->link();
-
-                    } else if (tok2->str() == ",") {
-                        tok2->str(";");
-                        --commaCounter;
-                        if (commaCounter == 0) {
-                            tok2->insertToken("return");
-                        }
-                    }
-                }
-                tok = endAt;
-            }
-        }
-    }
-}
-
+// ------------------------------------------------------------------------
 void Tokenizer::checkConfiguration() const
 {
     if (!mSettings->checkConfiguration)
@@ -9352,12 +9113,6 @@ void Tokenizer::createSymbolDatabase()
     mSymbolDatabase->validate();
 }
 
-void Tokenizer::deleteSymbolDatabase()
-{
-    delete mSymbolDatabase;
-    mSymbolDatabase = nullptr;
-}
-
 bool Tokenizer::operatorEnd(const Token * tok) const
 {
     if (tok && tok->str() == ")") {
diff --git a/lib/tokenize.h b/lib/tokenize.h
diff --git a/test/testsimplifytokens.cpp b/test/testsimplifytokens.cpp

Original file line number	Diff line number	Diff line change
`@@ -52,8 +52,6 @@`
`52`	`52`	`* - Macros are expanded`
`53`	`53`	`* -# Tokenize the file (see Tokenizer)`
`54`	`54`	`* -# Run the runChecks of all check classes.`
`55`		`- * -# Simplify the tokenlist (Tokenizer::simplifyTokenList2)`
`56`		`- * -# Run the runSimplifiedChecks of all check classes`
`57`	`55`	`*`
`58`	`56`	`* When errors are found, they are reported back to the CppCheckExecutor through the ErrorLogger interface.`
`59`	`57`	`*/`