/*

* Cppcheck - A tool for static C/C++ code analysis

* Copyright (C) 2007-2020 Cppcheck team.

*

* This program is free software: you can redistribute it and/or modify

* it under the terms of the GNU General Public License as published by

* the Free Software Foundation, either version 3 of the License, or

* (at your option) any later version.

*

* This program is distributed in the hope that it will be useful,

* but WITHOUT ANY WARRANTY; without even the implied warranty of

* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

* GNU General Public License for more details.

*

* You should have received a copy of the GNU General Public License

* along with this program. If not, see <http://www.gnu.org/licenses/>.

*/

#include "config.h"

#include "platform.h"

#include "preprocessor.h" // usually tests here should not use preprocessor...

#include "settings.h"

#include "standards.h"

#include "testsuite.h"

#include "token.h"

#include "tokenize.h"

#include "tokenlist.h"

#include <list>

#include <set>

#include <sstream>

#include <string>

struct InternalError;

class TestTokenizer : public TestFixture {

public:

TestTokenizer() : TestFixture("TestTokenizer") {

}

private:

Settings settings0;

Settings settings1;

Settings settings2;

Settings settings_windows;

void run() OVERRIDE {

LOAD_LIB_2(settings_windows.library, "windows.cfg");

TEST_CASE(tokenize1);

TEST_CASE(tokenize2);

TEST_CASE(tokenize4);

TEST_CASE(tokenize5);

TEST_CASE(tokenize6); // array access. replace "*(p+1)" => "p[1]"

TEST_CASE(tokenize7);

TEST_CASE(tokenize8);

TEST_CASE(tokenize9);

TEST_CASE(tokenize11);

TEST_CASE(tokenize13); // bailout if the code contains "@" - that is not handled well.

TEST_CASE(tokenize14); // tokenize "0X10" => 16

TEST_CASE(tokenizeHexWithSuffix); // tokenize 0xFFFFFFul

TEST_CASE(tokenize15); // tokenize ".123"

TEST_CASE(tokenize17); // #2759

TEST_CASE(tokenize18); // tokenize "(X&&Y)" into "( X && Y )" instead of "( X & & Y )"

TEST_CASE(tokenize19); // #3006 (segmentation fault)

TEST_CASE(tokenize21); // tokenize 0x0E-7

TEST_CASE(tokenize22); // special marker $ from preprocessor

TEST_CASE(tokenize25); // #4239 (segmentation fault)

TEST_CASE(tokenize26); // #4245 (segmentation fault)

TEST_CASE(tokenize27); // #4525 (segmentation fault)

TEST_CASE(tokenize31); // #3503 (Wrong handling of member function taking function pointer as argument)

TEST_CASE(tokenize32); // #5884 (fsanitize=undefined: left shift of negative value -10000 in lib/templatesimplifier.cpp:852:46)

TEST_CASE(tokenize33); // #5780 Various crashes on valid template code

TEST_CASE(tokenize34); // #8031

TEST_CASE(tokenize35); // #8361

TEST_CASE(tokenize36); // #8436

TEST_CASE(tokenize37); // #8550

TEST_CASE(tokenize38); // #9569

TEST_CASE(validate);

TEST_CASE(objectiveC); // Syntax error should be written for objective C/C++ code.

TEST_CASE(syntax_case_default);

TEST_CASE(foreach); // #3690

TEST_CASE(ifconstexpr);

TEST_CASE(combineOperators);

TEST_CASE(concatenateNegativeNumber);

TEST_CASE(longtok);

TEST_CASE(removeUnusedTemplates);

TEST_CASE(simplifyCasts1);

TEST_CASE(simplifyCasts2);

TEST_CASE(simplifyCasts3);

TEST_CASE(simplifyCasts4);

TEST_CASE(simplifyCasts5);

TEST_CASE(simplifyCasts7);

TEST_CASE(simplifyCasts8);

TEST_CASE(simplifyCasts9);

TEST_CASE(simplifyCasts10);

TEST_CASE(simplifyCasts11);

TEST_CASE(simplifyCasts12);

TEST_CASE(simplifyCasts13);

TEST_CASE(simplifyCasts14);

TEST_CASE(simplifyCasts15); // #5996 - don't remove cast in 'a+static_cast<int>(b?60:0)'

TEST_CASE(simplifyCasts16); // #6278

TEST_CASE(simplifyCasts17); // #6110 - don't remove any parentheses in 'a(b)(c)'

TEST_CASE(simplifyAt);

TEST_CASE(inlineasm);

TEST_CASE(simplifyAsm2); // #4725 (writing asm() around "^{}")

TEST_CASE(ifAddBraces1);

TEST_CASE(ifAddBraces2);

TEST_CASE(ifAddBraces3);

TEST_CASE(ifAddBraces4);

TEST_CASE(ifAddBraces5);

TEST_CASE(ifAddBraces7);

TEST_CASE(ifAddBraces9);

TEST_CASE(ifAddBraces11);

TEST_CASE(ifAddBraces12);

TEST_CASE(ifAddBraces13);

TEST_CASE(ifAddBraces15); // #2616 - unknown macro before if

TEST_CASE(ifAddBraces16);

TEST_CASE(ifAddBraces17); // '} else' should be in the same line

TEST_CASE(ifAddBraces18); // #3424 - if if { } else else

TEST_CASE(ifAddBraces19); // #3928 - if for if else

TEST_CASE(ifAddBraces20); // #5012 - syntax error 'else }'

TEST_CASE(ifAddBraces21); // #5332 - if (x) label: {} ..

TEST_CASE(whileAddBraces);

TEST_CASE(doWhileAddBraces);

TEST_CASE(forAddBraces1);

TEST_CASE(forAddBraces2); // #5088

TEST_CASE(simplifyKnownVariables1);

TEST_CASE(simplifyKnownVariables2);

TEST_CASE(simplifyKnownVariables3);

TEST_CASE(simplifyKnownVariables4);

TEST_CASE(simplifyKnownVariables5);

TEST_CASE(simplifyKnownVariables6);

TEST_CASE(simplifyKnownVariables7);

TEST_CASE(simplifyKnownVariables8);

TEST_CASE(simplifyKnownVariables9);

TEST_CASE(simplifyKnownVariables10);

TEST_CASE(simplifyKnownVariables11);

TEST_CASE(simplifyKnownVariables13);

TEST_CASE(simplifyKnownVariables14);

TEST_CASE(simplifyKnownVariables15);

TEST_CASE(simplifyKnownVariables16);

TEST_CASE(simplifyKnownVariables17);

TEST_CASE(simplifyKnownVariables18);

TEST_CASE(simplifyKnownVariables19);

TEST_CASE(simplifyKnownVariables20);

TEST_CASE(simplifyKnownVariables21);

TEST_CASE(simplifyKnownVariables22);

TEST_CASE(simplifyKnownVariables23);

TEST_CASE(simplifyKnownVariables25);

TEST_CASE(simplifyKnownVariables27);

TEST_CASE(simplifyKnownVariables28);

TEST_CASE(simplifyKnownVariables29); // ticket #1811

TEST_CASE(simplifyKnownVariables30);

TEST_CASE(simplifyKnownVariables31);

TEST_CASE(simplifyKnownVariables32); // const

TEST_CASE(simplifyKnownVariables33); // struct variable

TEST_CASE(simplifyKnownVariables34);

TEST_CASE(simplifyKnownVariables35); // ticket #2353 - False positive: Division by zero 'if (x == 0) return 0; return 10 / x;'

TEST_CASE(simplifyKnownVariables36); // ticket #2304 - known value for strcpy parameter

TEST_CASE(simplifyKnownVariables37); // ticket #2398 - false positive caused by no simplification in for loop

TEST_CASE(simplifyKnownVariables38); // ticket #2399 - simplify conditions

TEST_CASE(simplifyKnownVariables39);

TEST_CASE(simplifyKnownVariables40);

TEST_CASE(simplifyKnownVariables41); // p=&x; if (p) ..

TEST_CASE(simplifyKnownVariables42); // ticket #2031 - known string value after strcpy

TEST_CASE(simplifyKnownVariables43);

TEST_CASE(simplifyKnownVariables44); // ticket #3117 - don't simplify static variables

TEST_CASE(simplifyKnownVariables45); // ticket #3281 - static constant variable not simplified

TEST_CASE(simplifyKnownVariables46); // ticket #3587 - >>

TEST_CASE(simplifyKnownVariables47); // ticket #3627 - >>

TEST_CASE(simplifyKnownVariables48); // ticket #3754 - wrong simplification in for loop header

TEST_CASE(simplifyKnownVariables49); // #3691 - continue in switch

TEST_CASE(simplifyKnownVariables50); // #4066 sprintf changes

TEST_CASE(simplifyKnownVariables51); // #4409 hang

TEST_CASE(simplifyKnownVariables52); // #4728 "= x %cop%"

TEST_CASE(simplifyKnownVariables53); // references

TEST_CASE(simplifyKnownVariables54); // #4913 'x' is not 0 after *--x=0;

TEST_CASE(simplifyKnownVariables55); // pointer alias

TEST_CASE(simplifyKnownVariables56); // ticket #5301 - >>

TEST_CASE(simplifyKnownVariables57); // ticket #4724

TEST_CASE(simplifyKnownVariables58); // ticket #5268

TEST_CASE(simplifyKnownVariables59); // skip for header

TEST_CASE(simplifyKnownVariables60); // #6829

TEST_CASE(simplifyKnownVariables61); // #7805

TEST_CASE(simplifyKnownVariables62); // #5666 - p=&str[0]

TEST_CASE(simplifyKnownVariablesBailOutAssign1);

TEST_CASE(simplifyKnownVariablesBailOutAssign2);

TEST_CASE(simplifyKnownVariablesBailOutAssign3); // #4395 - nested assignments

TEST_CASE(simplifyKnownVariablesBailOutFor1);

TEST_CASE(simplifyKnownVariablesBailOutFor2);

TEST_CASE(simplifyKnownVariablesBailOutFor3);

TEST_CASE(simplifyKnownVariablesBailOutMemberFunction);

TEST_CASE(simplifyKnownVariablesBailOutConditionalIncrement);

TEST_CASE(simplifyKnownVariablesBailOutSwitchBreak); // ticket #2324

TEST_CASE(simplifyKnownVariablesFloat); // #2454 - float variable

TEST_CASE(simplifyKnownVariablesClassMember); // #2815 - value of class member may be changed by function call

TEST_CASE(simplifyKnownVariablesFunctionCalls); // Function calls (don't assume pass by reference)

TEST_CASE(simplifyKnownVariablesGlobalVars);

TEST_CASE(simplifyKnownVariablesReturn); // 3500 - return

TEST_CASE(simplifyKnownVariablesPointerAliasFunctionCall); // #7440

TEST_CASE(simplifyExternC);

TEST_CASE(simplifyKeyword); // #5842 - remove C99 static keyword between []

TEST_CASE(isZeroNumber);

TEST_CASE(isOneNumber);

TEST_CASE(isTwoNumber);

TEST_CASE(simplifyFunctionParameters);

TEST_CASE(simplifyFunctionParameters1); // #3721

TEST_CASE(simplifyFunctionParameters2); // #4430

TEST_CASE(simplifyFunctionParameters3); // #4436

TEST_CASE(simplifyFunctionParameters4); // #9421

TEST_CASE(simplifyFunctionParametersMultiTemplate);

TEST_CASE(simplifyFunctionParametersErrors);

TEST_CASE(removeParentheses1); // Ticket #61

TEST_CASE(removeParentheses3);

TEST_CASE(removeParentheses4); // Ticket #390

TEST_CASE(removeParentheses5); // Ticket #392

TEST_CASE(removeParentheses6);

TEST_CASE(removeParentheses7);

TEST_CASE(removeParentheses8); // Ticket #1865

TEST_CASE(removeParentheses9); // Ticket #1962

TEST_CASE(removeParentheses10); // Ticket #2320

TEST_CASE(removeParentheses11); // Ticket #2505

TEST_CASE(removeParentheses12); // Ticket #2760 ',(b)='

TEST_CASE(removeParentheses13);

TEST_CASE(removeParentheses14); // Ticket #3309

TEST_CASE(removeParentheses15); // Ticket #4142

TEST_CASE(removeParentheses16); // Ticket #4423 '*(x.y)='

TEST_CASE(removeParentheses17); // Don't remove parentheses in 'a ? b : (c>0 ? d : e);'

TEST_CASE(removeParentheses18); // 'float(*a)[2]' => 'float *a[2]'

TEST_CASE(removeParentheses19); // ((typeof(x) *)0)

TEST_CASE(removeParentheses20); // Ticket #5479: a<b<int>>(2);

TEST_CASE(removeParentheses21); // Don't "simplify" casts

TEST_CASE(removeParentheses22);

TEST_CASE(removeParentheses23); // Ticket #6103 - Infinite loop upon valid input

TEST_CASE(removeParentheses24); // Ticket #7040

TEST_CASE(tokenize_double);

TEST_CASE(tokenize_strings);

TEST_CASE(simplify_constants);

TEST_CASE(simplify_constants2);

TEST_CASE(simplify_constants3);

TEST_CASE(simplify_constants4);

TEST_CASE(simplify_constants5);

TEST_CASE(simplify_constants6); // Ticket #5625: Ternary operator as template parameter

TEST_CASE(simplifyMulAndParens); // Ticket #2784 + #3184

TEST_CASE(simplifyStructDecl);

TEST_CASE(vardecl1);

TEST_CASE(vardecl2);

TEST_CASE(vardecl3);

TEST_CASE(vardecl4);

TEST_CASE(vardecl5); // #7048

TEST_CASE(vardec_static);

TEST_CASE(vardecl6);

TEST_CASE(vardecl7);

TEST_CASE(vardecl8);

TEST_CASE(vardecl9);

TEST_CASE(vardecl10);

TEST_CASE(vardecl11);

TEST_CASE(vardecl12);

TEST_CASE(vardecl13);

TEST_CASE(vardecl14);

TEST_CASE(vardecl15);

TEST_CASE(vardecl16);

TEST_CASE(vardecl17);

TEST_CASE(vardecl18);

TEST_CASE(vardecl19);

TEST_CASE(vardecl20); // #3700 - register const int H = 0;

TEST_CASE(vardecl21); // #4042 - a::b const *p = 0;

TEST_CASE(vardecl22); // #4211 - segmentation fault

TEST_CASE(vardecl23); // #4276 - segmentation fault

TEST_CASE(vardecl24); // #4187 - variable declaration within lambda function

TEST_CASE(vardecl25); // #4799 - segmentation fault

TEST_CASE(vardecl26); // #5907 - incorrect handling of extern declarations

TEST_CASE(vardecl27); // #7850 - crash on valid C code

TEST_CASE(vardecl_stl_1);

TEST_CASE(vardecl_stl_2);

TEST_CASE(vardecl_template_1);

TEST_CASE(vardecl_template_2);

TEST_CASE(vardecl_union);

TEST_CASE(vardecl_par); // #2743 - set links if variable type contains parentheses

TEST_CASE(vardecl_par2); // #3912 - set correct links

TEST_CASE(vardecl_par3); // #6556 - Fred x1(a), x2(b);

TEST_CASE(vardecl_class_ref);

TEST_CASE(volatile_variables);

// unsigned i; => unsigned int i;

TEST_CASE(unsigned1);

TEST_CASE(unsigned2);

TEST_CASE(unsigned3); // template arguments

TEST_CASE(simplifyStdType); // #4947, #4950, #4951

TEST_CASE(createLinks);

TEST_CASE(createLinks2);

TEST_CASE(signed1);

TEST_CASE(simplifyString);

TEST_CASE(simplifyConst);

TEST_CASE(switchCase);

TEST_CASE(simplifyPointerToStandardType);

TEST_CASE(functionpointer1);

TEST_CASE(functionpointer2);

TEST_CASE(functionpointer3);

TEST_CASE(functionpointer4);

TEST_CASE(functionpointer5);

TEST_CASE(functionpointer6);

TEST_CASE(functionpointer7);

TEST_CASE(functionpointer8); // #7410 - throw

TEST_CASE(functionpointer9); // #6113 - function call with function pointer

TEST_CASE(removeRedundantAssignment);

TEST_CASE(removedeclspec);

TEST_CASE(removeattribute);

TEST_CASE(functionAttributeBefore);

TEST_CASE(functionAttributeAfter);

TEST_CASE(cpp03template1);

TEST_CASE(cpp0xtemplate1);

TEST_CASE(cpp0xtemplate2);

TEST_CASE(cpp0xtemplate3);

TEST_CASE(cpp0xtemplate4); // Ticket #6181: Mishandled C++11 syntax

TEST_CASE(cpp0xtemplate5); // Ticket #9154 change >> to > >

TEST_CASE(cpp14template); // Ticket #6708

TEST_CASE(arraySize);

TEST_CASE(labels);

TEST_CASE(simplifyInitVar);

TEST_CASE(simplifyInitVar2);

TEST_CASE(simplifyInitVar3);

TEST_CASE(bitfields1);

TEST_CASE(bitfields2);

TEST_CASE(bitfields3);

TEST_CASE(bitfields4); // ticket #1956

TEST_CASE(bitfields5); // ticket #1956

TEST_CASE(bitfields6); // ticket #2595

TEST_CASE(bitfields7); // ticket #1987

TEST_CASE(bitfields8);

TEST_CASE(bitfields9); // ticket #2706

TEST_CASE(bitfields10);

TEST_CASE(bitfields12); // ticket #3485 (segmentation fault)

TEST_CASE(bitfields13); // ticket #3502 (segmentation fault)

TEST_CASE(bitfields14); // ticket #4561 (segfault for 'class a { signals: };')

TEST_CASE(bitfields15); // ticket #7747 (enum Foo {A,B}:4;)

TEST_CASE(bitfields16); // Save bitfield bit count

TEST_CASE(simplifyNamespaceStd);

TEST_CASE(microsoftMemory);

TEST_CASE(microsoftString);

TEST_CASE(borland);

TEST_CASE(simplifyQtSignalsSlots1);

TEST_CASE(simplifyQtSignalsSlots2);

TEST_CASE(simplifySQL);

TEST_CASE(simplifyCAlternativeTokens);

TEST_CASE(simplifyCalculations);

// x = ({ 123; }); => { x = 123; }

TEST_CASE(simplifyRoundCurlyParentheses);

TEST_CASE(simplifyOperatorName1);

TEST_CASE(simplifyOperatorName2);

TEST_CASE(simplifyOperatorName3);

TEST_CASE(simplifyOperatorName4);

TEST_CASE(simplifyOperatorName5);

TEST_CASE(simplifyOperatorName6); // ticket #3194

TEST_CASE(simplifyOperatorName7); // ticket #4619

TEST_CASE(simplifyOperatorName8); // ticket #5706

TEST_CASE(simplifyOperatorName9); // ticket #5709 - comma operator not properly tokenized

TEST_CASE(simplifyOperatorName10); // #8746 - using a::operator=

TEST_CASE(simplifyOperatorName11); // #8889

TEST_CASE(simplifyOperatorName12); // #9110

TEST_CASE(simplifyOperatorName13); // user defined literal

TEST_CASE(simplifyOperatorName14); // std::complex operator "" if

TEST_CASE(simplifyOperatorName15); // ticket #9468 syntaxError

TEST_CASE(simplifyOperatorName16); // ticket #9472

TEST_CASE(simplifyOperatorName17);

TEST_CASE(simplifyOperatorName18); // global namespace

TEST_CASE(simplifyOperatorName19);

TEST_CASE(simplifyOperatorName20);

TEST_CASE(simplifyOperatorName21);

TEST_CASE(simplifyOperatorName22);

TEST_CASE(simplifyOperatorName23);

TEST_CASE(simplifyOperatorName24);

TEST_CASE(simplifyOperatorName25);

TEST_CASE(simplifyOperatorName26);

TEST_CASE(simplifyNullArray);

// Some simple cleanups of unhandled macros in the global scope

TEST_CASE(removeMacrosInGlobalScope);

TEST_CASE(removeMacroInVarDecl);

TEST_CASE(addSemicolonAfterUnknownMacro);

// a = b = 0;

TEST_CASE(multipleAssignment);

TEST_CASE(sizeOfCharLiteral);

TEST_CASE(platformWin);

TEST_CASE(platformWin32);

TEST_CASE(platformWin32A);

TEST_CASE(platformWin32W);

TEST_CASE(platformWin64);

TEST_CASE(platformUnix32);

TEST_CASE(platformUnix64);

TEST_CASE(platformWin32AStringCat); // ticket #5015

TEST_CASE(platformWin32WStringCat); // ticket #5015

TEST_CASE(platformWinWithNamespace);

TEST_CASE(simplifyStaticConst);

TEST_CASE(simplifyCPPAttribute);

TEST_CASE(simplifyCaseRange);

TEST_CASE(simplifyEmptyNamespaces);

TEST_CASE(compileLimits); // #5592 crash: gcc: testsuit: gcc.c-torture/compile/limits-declparen.c

TEST_CASE(prepareTernaryOpForAST);

// AST data

TEST_CASE(astexpr);

TEST_CASE(astexpr2); // limit large expressions

TEST_CASE(astpar);

TEST_CASE(astnewdelete);

TEST_CASE(astbrackets);

TEST_CASE(astunaryop);

TEST_CASE(astfunction);

TEST_CASE(asttemplate);

TEST_CASE(astcast);

TEST_CASE(astlambda);

TEST_CASE(astcase);

TEST_CASE(astrefqualifier);

TEST_CASE(astvardecl);

TEST_CASE(astnewscoped);

TEST_CASE(startOfExecutableScope);

TEST_CASE(removeMacroInClassDef); // #6058

TEST_CASE(sizeofAddParentheses);

TEST_CASE(reportUnknownMacros);

// Make sure the Tokenizer::findGarbageCode() does not have false positives

// The TestGarbage ensures that there are true positives

TEST_CASE(findGarbageCode);

TEST_CASE(checkEnableIf);

TEST_CASE(checkTemplates);

TEST_CASE(checkNamespaces);

TEST_CASE(checkLambdas);

TEST_CASE(checkIfCppCast);

TEST_CASE(checkRefQualifiers);

TEST_CASE(checkConditionBlock);

// #9052

TEST_CASE(noCrash1);

TEST_CASE(noCrash2);

// --check-config

TEST_CASE(checkConfiguration);

TEST_CASE(unknownType); // #8952

TEST_CASE(unknownMacroBeforeReturn);

TEST_CASE(cppcast);

}

std::string tokenizeAndStringify(const char code[], bool simplify = false, bool expand = true, Settings::PlatformType platform = Settings::Native, const char* filename = "test.cpp", bool cpp11 = true) {

errout.str("");

settings1.debugwarnings = true;

settings1.platform(platform);

settings1.standards.cpp = cpp11 ? Standards::CPP11 : Standards::CPP03;

// tokenize..

Tokenizer tokenizer(&settings1, this);

std::istringstream istr(code);

tokenizer.tokenize(istr, filename);

if (simplify)

tokenizer.simplifyTokenList2();

// filter out ValueFlow messages..

const std::string debugwarnings = errout.str();

errout.str("");

std::istringstream istr2(debugwarnings);

std::string line;

while (std::getline(istr2,line)) {

if (line.find("valueflow.cpp") == std::string::npos)

errout << line << "\n";

}

if (tokenizer.tokens())

return tokenizer.tokens()->stringifyList(false, expand, false, true, false, nullptr, nullptr);

else

return "";

}

std::string tokenizeAndStringifyWindows(const char code[], bool simplify = false, bool expand = true, Settings::PlatformType platform = Settings::Native, const char* filename = "test.cpp", bool cpp11 = true) {

errout.str("");

settings_windows.debugwarnings = true;

settings_windows.platform(platform);

settings_windows.standards.cpp = cpp11 ? Standards::CPP11 : Standards::CPP03;

// tokenize..

Tokenizer tokenizer(&settings_windows, this);

std::istringstream istr(code);

tokenizer.tokenize(istr, filename);

if (simplify)

tokenizer.simplifyTokenList2();

// filter out ValueFlow messages..

const std::string debugwarnings = errout.str();

errout.str("");

std::istringstream istr2(debugwarnings);

std::string line;

while (std::getline(istr2,line)) {

if (line.find("valueflow.cpp") == std::string::npos)

errout << line << "\n";

}

if (tokenizer.tokens())

return tokenizer.tokens()->stringifyList(false, expand, false, true, false, nullptr, nullptr);

else

return "";

}

std::string tokenizeAndStringify(const char code[], const Settings &settings) {

errout.str("");

// tokenize..

Tokenizer tokenizer(&settings, this);

std::istringstream istr(code);

tokenizer.tokenize(istr, "test.cpp");

if (!tokenizer.tokens())

return "";

return tokenizer.tokens()->stringifyList(false, true, false, true, false, nullptr, nullptr);

}

std::string tokenizeDebugListing(const char code[], bool simplify = false, const char filename[] = "test.cpp") {

errout.str("");

settings2.standards.c = Standards::C89;

settings2.standards.cpp = Standards::CPP03;

Tokenizer tokenizer(&settings2, this);

std::istringstream istr(code);

tokenizer.tokenize(istr, filename);

if (simplify)

tokenizer.simplifyTokenList2();

// result..

return tokenizer.tokens()->stringifyList(true,true,true,true,false);

}

void tokenize1() {

const char code[] = "void f ( )\n"

"{ if ( p . y ( ) > yof ) { } }";

ASSERT_EQUALS(code, tokenizeAndStringify(code));

}

void tokenize2() {

const char code[] = "{ sizeof a, sizeof b }";

ASSERT_EQUALS("{ sizeof ( a ) , sizeof ( b ) }", tokenizeAndStringify(code));

}

void tokenize4() {

const char code[] = "class foo\n"

"{\n"

"public:\n"

" const int i;\n"

"}";

ASSERT_EQUALS("class foo\n"

"{\n"

"public:\n"

"const int i ;\n"

"}", tokenizeAndStringify(code));

ASSERT_EQUALS("", errout.str());

}

void tokenize5() {

// Tokenize values

ASSERT_EQUALS("; + 1E3 ;", tokenizeAndStringify("; +1E3 ;"));

ASSERT_EQUALS("; 1E-2 ;", tokenizeAndStringify("; 1E-2 ;"));

}

void tokenize6() {

// "&p[1]" => "p+1"

/*

ASSERT_EQUALS("; x = p + n ;", tokenizeAndStringify("; x = & p [ n ] ;", true));

ASSERT_EQUALS("; x = ( p + n ) [ m ] ;", tokenizeAndStringify("; x = & p [ n ] [ m ] ;", true));

ASSERT_EQUALS("; x = y & p [ n ] ;", tokenizeAndStringify("; x = y & p [ n ] ;", true));

ASSERT_EQUALS("; x = 10 & p [ n ] ;", tokenizeAndStringify("; x = 10 & p [ n ] ;", true));

ASSERT_EQUALS("; x = y [ 10 ] & p [ n ] ;", tokenizeAndStringify("; x = y [ 10 ] & p [ n ] ;", true));

ASSERT_EQUALS("; x = ( a + m ) & p [ n ] ;", tokenizeAndStringify("; x = ( a + m ) & p [ n ] ;", true));*/

// "*(p+1)" => "p[1]"

ASSERT_EQUALS("; x = p [ 1 ] ;", tokenizeAndStringify("; x = * ( p + 1 ) ;", true));

ASSERT_EQUALS("; x = p [ 0xA ] ;", tokenizeAndStringify("; x = * ( p + 0xA ) ;", true));

ASSERT_EQUALS("; x = p [ n ] ;", tokenizeAndStringify("; x = * ( p + n ) ;", true));

ASSERT_EQUALS("; x = y * ( p + n ) ;", tokenizeAndStringify("; x = y * ( p + n ) ;", true));

ASSERT_EQUALS("; x = 10 * ( p + n ) ;", tokenizeAndStringify("; x = 10 * ( p + n ) ;", true));

ASSERT_EQUALS("; x = y [ 10 ] * ( p + n ) ;", tokenizeAndStringify("; x = y [ 10 ] * ( p + n ) ;", true));

ASSERT_EQUALS("; x = ( a + m ) * ( p + n ) ;", tokenizeAndStringify("; x = ( a + m ) * ( p + n ) ;", true));

// "*(p-1)" => "p[-1]" and "*(p-n)" => "p[-n]"

ASSERT_EQUALS("; x = p [ -1 ] ;", tokenizeAndStringify("; x = *(p - 1);", true));

ASSERT_EQUALS("; x = p [ -0xA ] ;", tokenizeAndStringify("; x = *(p - 0xA);", true));

ASSERT_EQUALS("; x = p [ - n ] ;", tokenizeAndStringify("; x = *(p - n);", true));

ASSERT_EQUALS("; x = y * ( p - 1 ) ;", tokenizeAndStringify("; x = y * (p - 1);", true));

ASSERT_EQUALS("; x = 10 * ( p - 1 ) ;", tokenizeAndStringify("; x = 10 * (p - 1);", true));

ASSERT_EQUALS("; x = y [ 10 ] * ( p - 1 ) ;", tokenizeAndStringify("; x = y[10] * (p - 1);", true));

ASSERT_EQUALS("; x = ( a - m ) * ( p - n ) ;", tokenizeAndStringify("; x = (a - m) * (p - n);", true));

// Test that the array-index simplification is not applied when there's no dereference:

// "(x-y)" => "(x-y)" and "(x+y)" => "(x+y)"

ASSERT_EQUALS("; a = b * ( x - y ) ;", tokenizeAndStringify("; a = b * (x - y);", true));

ASSERT_EQUALS("; a = b * x [ - y ] ;", tokenizeAndStringify("; a = b * *(x - y);", true));

ASSERT_EQUALS("; a = a * ( x - y ) ;", tokenizeAndStringify("; a *= (x - y);", true));

ASSERT_EQUALS("; z = a ++ * ( x - y ) ;", tokenizeAndStringify("; z = a++ * (x - y);", true));

ASSERT_EQUALS("; z = a ++ * ( x + y ) ;", tokenizeAndStringify("; z = a++ * (x + y);", true));

ASSERT_EQUALS("; z = a -- * ( x - y ) ;", tokenizeAndStringify("; z = a-- * (x - y);", true));

ASSERT_EQUALS("; z = a -- * ( x + y ) ;", tokenizeAndStringify("; z = a-- * (x + y);", true));

ASSERT_EQUALS("; z = 'a' * ( x - y ) ;", tokenizeAndStringify("; z = 'a' * (x - y);", true));

ASSERT_EQUALS("; z = \"a\" * ( x - y ) ;", tokenizeAndStringify("; z = \"a\" * (x - y);", true));

ASSERT_EQUALS("; z = 'a' * ( x + y ) ;", tokenizeAndStringify("; z = 'a' * (x + y);", true));

ASSERT_EQUALS("; z = \"a\" * ( x + y ) ;", tokenizeAndStringify("; z = \"a\" * (x + y);", true));

ASSERT_EQUALS("; z = foo ( ) * ( x + y ) ;", tokenizeAndStringify("; z = foo() * (x + y);", true));

}

void tokenize7() {

const char code[] = "void f() {\n"

" int x1 = 1;\n"

" int x2(x1);\n"

"}\n";

ASSERT_EQUALS("void f ( ) {\nint x1 ; x1 = 1 ;\nint x2 ; x2 = x1 ;\n}",

tokenizeAndStringify(code, false));

}

void tokenize8() {

const char code[] = "void f() {\n"

" int x1(g());\n"

" int x2(x1);\n"

"}\n";

ASSERT_EQUALS("1: void f ( ) {\n"

"2: int x1@1 ; x1@1 = g ( ) ;\n"

"3: int x2@2 ; x2@2 = x1@1 ;\n"

"4: }\n",

tokenizeDebugListing(code, false));

}

void tokenize9() {

const char code[] = "typedef void (*fp)();\n"

"typedef fp (*fpp)();\n"

"void f() {\n"

" fpp x = (fpp)f();\n"

"}";

tokenizeAndStringify(code, false);

ASSERT_EQUALS("", errout.str());

}

void tokenize11() {

ASSERT_EQUALS("X * sizeof ( Y ( ) ) ;", tokenizeAndStringify("X * sizeof(Y());", false));

}

// bailout if there is "@" - it is not handled well

void tokenize13() {

const char code[] = "@implementation\n"

"-(Foo *)foo: (Bar *)bar\n"

"{ }\n"

"@end\n";

ASSERT_THROW(tokenizeAndStringify(code), InternalError);

}

// Ticket #2361: 0X10 => 16

void tokenize14() {

ASSERT_EQUALS("; 0x10 ;", tokenizeAndStringify(";0x10;"));

ASSERT_EQUALS("; 0X10 ;", tokenizeAndStringify(";0X10;"));

ASSERT_EQUALS("; 0444 ;", tokenizeAndStringify(";0444;"));

}

// Ticket #8050

void tokenizeHexWithSuffix() {

ASSERT_EQUALS("; 0xFFFFFF ;", tokenizeAndStringify(";0xFFFFFF;"));

ASSERT_EQUALS("; 0xFFFFFFu ;", tokenizeAndStringify(";0xFFFFFFu;"));

ASSERT_EQUALS("; 0xFFFFFFul ;", tokenizeAndStringify(";0xFFFFFFul;"));

// Number of digits decides about internal representation...

ASSERT_EQUALS("; 0xFFFFFFFF ;", tokenizeAndStringify(";0xFFFFFFFF;"));

ASSERT_EQUALS("; 0xFFFFFFFFu ;", tokenizeAndStringify(";0xFFFFFFFFu;"));

ASSERT_EQUALS("; 0xFFFFFFFFul ;", tokenizeAndStringify(";0xFFFFFFFFul;"));

}

// Ticket #2429: 0.125

void tokenize15() {

ASSERT_EQUALS("0.125 ;", tokenizeAndStringify(".125;"));

ASSERT_EQUALS("005.125 ;", tokenizeAndStringify("005.125;")); // Don't confuse with octal values

}

void tokenize17() { // #2759

ASSERT_EQUALS("class B : private :: A { } ;", tokenizeAndStringify("class B : private ::A { };"));

}

void tokenize18() { // tokenize "(X&&Y)" into "( X && Y )" instead of "( X & & Y )"

ASSERT_EQUALS("( X && Y ) ;", tokenizeAndStringify("(X&&Y);"));

}

void tokenize19() {

// #3006 - added hasComplicatedSyntaxErrorsInTemplates to avoid segmentation fault

ASSERT_THROW(tokenizeAndStringify("x < () <"), InternalError);

// #3496 - make sure hasComplicatedSyntaxErrorsInTemplates works

ASSERT_EQUALS("void a ( Fred * f ) { for ( ; n < f . x ( ) ; ) { } }",

tokenizeAndStringify("void a(Fred* f) MACRO { for (;n < f->x();) {} }"));

// #6216 - make sure hasComplicatedSyntaxErrorsInTemplates works

ASSERT_EQUALS("C :: C ( )\n"

": v { }\n"

"{\n"

"for ( int dim = 0 ; dim < v . size ( ) ; ++ dim ) {\n"

"v [ dim ] . f ( ) ;\n"

"}\n"

"} ;",

tokenizeAndStringify("C::C()\n"

":v{}\n"

"{\n"

" for (int dim = 0; dim < v.size(); ++dim) {\n"

" v[dim]->f();\n"

" }\n"

"};"));

}

void tokenize21() { // tokenize 0x0E-7

ASSERT_EQUALS("0x0E - 7 ;", tokenizeAndStringify("0x0E-7;"));

}

void tokenize22() { // tokenize special marker $ from preprocessor

ASSERT_EQUALS("a$b", tokenizeAndStringify("a$b"));

ASSERT_EQUALS("a $b\nc", tokenizeAndStringify("a $b\nc"));

ASSERT_EQUALS("a = $0 ;", tokenizeAndStringify("a = $0;"));

ASSERT_EQUALS("a$ ++ ;", tokenizeAndStringify("a$++;"));

ASSERT_EQUALS("$if ( ! p )", tokenizeAndStringify("$if(!p)"));

}

// #4239 - segfault for "f ( struct { int typedef T x ; } ) { }"

void tokenize25() {

ASSERT_THROW(tokenizeAndStringify("f ( struct { int typedef T x ; } ) { }"), InternalError);

}

// #4245 - segfault

void tokenize26() {

ASSERT_THROW(tokenizeAndStringify("class x { protected : template < int y = } ;"), InternalError); // Garbage code

}

void tokenize27() {

// #4525 - segfault

tokenizeAndStringify("struct except_spec_d_good : except_spec_a, except_spec_b {\n"

"~except_spec_d_good();\n"

"};\n"

"struct S { S(); };\n"

"S::S() __attribute((pure)) = default;"

);

// original code: glibc-2.18/posix/bug-regex20.c

tokenizeAndStringify("static unsigned int re_string_context_at (const re_string_t *input, int idx, int eflags) internal_function __attribute__ ((pure));");

}

// #3503 - don't "simplify" SetFunction member function to a variable

void tokenize31() {

ASSERT_EQUALS("struct TTestClass { TTestClass ( ) { }\n"

"void SetFunction ( Other ( * m_f ) ( ) ) { }\n"

"} ;",

tokenizeAndStringify("struct TTestClass { TTestClass() { }\n"

" void SetFunction(Other(*m_f)()) { }\n"

"};"));

ASSERT_EQUALS("struct TTestClass { TTestClass ( ) { }\n"

"void SetFunction ( Other ( * m_f ) ( ) ) ;\n"

"} ;",

tokenizeAndStringify("struct TTestClass { TTestClass() { }\n"

" void SetFunction(Other(*m_f)());\n"

"};"));

}

// #5884 - Avoid left shift of negative integer value.

void tokenize32() {

// Do not simplify negative integer left shifts.

const char * code = "void f ( ) { int max_x ; max_x = -10000 << 16 ; }";

ASSERT_EQUALS(code, tokenizeAndStringify(code));

}

// #5780 Various crashes on valid template code in Tokenizer::setVarId()

void tokenize33() {

const char * code = "template<typename T, typename A = Alloc<T>> struct vector {};\n"

"void z() {\n"

" vector<int> VI;\n"

"}\n";

tokenizeAndStringify(code, true);

}

void tokenize34() { // #8031

{

const char code[] = "struct Container {\n"

" Container();\n"

" int* mElements;\n"

"};\n"

"Container::Container() : mElements(nullptr) {}\n"

"Container intContainer;";

const char exp [] = "1: struct Container {\n"

"2: Container ( ) ;\n"

"3: int * mElements@1 ;\n"

"4: } ;\n"

"5: Container :: Container ( ) : mElements@1 ( nullptr ) { }\n"

"6: Container intContainer@2 ;\n";

ASSERT_EQUALS(exp, tokenizeDebugListing(code));

}

{

const char code[] = "template<class T> struct Container {\n"

" Container();\n"

" int* mElements;\n"

"};\n"

"template <class T> Container<T>::Container() : mElements(nullptr) {}\n"

"Container<int> intContainer;";

const char exp [] = "1: struct Container<int> ;\n"

"2:\n"

"|\n"

"5:\n"

"6: Container<int> intContainer@1 ; struct Container<int> {\n"

"2: Container<int> ( ) ;\n"

"3: int * mElements@2 ;\n"

"4: } ;\n"

"5: Container<int> :: Container<int> ( ) : mElements@2 ( nullptr ) { }\n";

ASSERT_EQUALS(exp, tokenizeDebugListing(code));

}

}

void tokenize35() { // #8361

tokenizeAndStringify("typedef int CRCWord; "

"template<typename T> ::CRCWord const Compute(T const t) { return 0; }");

}

void tokenize36() { // #8436

const char code[] = "int foo ( int i ) { return i ? * new int { 5 } : int { i ? 0 : 1 } ; }";

ASSERT_EQUALS(code, tokenizeAndStringify(code));

}

void tokenize37() { // #8550

const char codeC[] = "class name { public: static void init ( ) {} } ; "

"typedef class name N; "

"void foo ( ) { return N :: init ( ) ; }";

const char expC [] = "class name { public: static void init ( ) { } } ; "

"void foo ( ) { return name :: init ( ) ; }";

ASSERT_EQUALS(expC, tokenizeAndStringify(codeC));

const char codeS[] = "class name { public: static void init ( ) {} } ; "

"typedef struct name N; "

"void foo ( ) { return N :: init ( ) ; }";

const char expS [] = "class name { public: static void init ( ) { } } ; "

"void foo ( ) { return name :: init ( ) ; }";

ASSERT_EQUALS(expS, tokenizeAndStringify(codeS));

}

void tokenize38() { // #9569

const char code[] = "using Binary = std::vector<char>; enum Type { Binary };";

const char exp[] = "enum Type { Binary } ;";

ASSERT_EQUALS(exp, tokenizeAndStringify(code));

}

void validate() {

// C++ code in C file

ASSERT_THROW(tokenizeAndStringify(";using namespace std;",false,false,Settings::Native,"test.c"), InternalError);

ASSERT_THROW(tokenizeAndStringify(";std::map<int,int> m;",false,false,Settings::Native,"test.c"), InternalError);

ASSERT_THROW(tokenizeAndStringify(";template<class T> class X { };",false,false,Settings::Native,"test.c"), InternalError);

ASSERT_THROW(tokenizeAndStringify("int X<Y>() {};",false,false,Settings::Native,"test.c"), InternalError);

ASSERT_THROW(tokenizeAndStringify("void foo(int i) { reinterpret_cast<char>(i) };",false,false,Settings::Native,"test.h"), InternalError);

}

void objectiveC() {

ASSERT_THROW(tokenizeAndStringify("void f() { [foo bar]; }"), InternalError);

}

void syntax_case_default() { // correct syntax

tokenizeAndStringify("void f() {switch (n) { case 0: z(); break;}}");

ASSERT_EQUALS("", errout.str());

tokenizeAndStringify("void f() {switch (n) { case 0:; break;}}");

ASSERT_EQUALS("", errout.str());

// TODO: Do not throw AST validation exception

TODO_ASSERT_THROW(tokenizeAndStringify("void f() {switch (n) { case 0?1:2 : z(); break;}}"), InternalError);

//ASSERT_EQUALS("", errout.str());

// TODO: Do not throw AST validation exception

TODO_ASSERT_THROW(tokenizeAndStringify("void f() {switch (n) { case 0?(1?3:4):2 : z(); break;}}"), InternalError);

ASSERT_EQUALS("", errout.str());

//allow GCC '({ %name%|%num%|%bool% ; })' statement expression extension

// TODO: Do not throw AST validation exception

TODO_ASSERT_THROW(tokenizeAndStringify("void f() {switch (n) { case 0?({0;}):1: z(); break;}}"), InternalError);

ASSERT_EQUALS("", errout.str());

//'b' can be or a macro or an undefined enum

tokenizeAndStringify("void f() {switch (n) { case b: z(); break;}}");

ASSERT_EQUALS("", errout.str());

//valid, when there's this declaration: 'constexpr int g() { return 2; }'

tokenizeAndStringify("void f() {switch (n) { case g(): z(); break;}}");

ASSERT_EQUALS("", errout.str());

//valid, when there's also this declaration: 'constexpr int g[1] = {0};'

tokenizeAndStringify("void f() {switch (n) { case g[0]: z(); break;}}");

ASSERT_EQUALS("", errout.str());

//valid, similar to above case

tokenizeAndStringify("void f() {switch (n) { case *g: z(); break;}}");

ASSERT_EQUALS("", errout.str());

//valid, when 'x' and 'y' are constexpr.

tokenizeAndStringify("void f() {switch (n) { case sqrt(x+y): z(); break;}}");

ASSERT_EQUALS("", errout.str());

}

void foreach () {

// #3690,#5154

const char code[] ="void f() { for each ( char c in MyString ) { Console::Write(c); } }";

ASSERT_EQUALS("void f ( ) { asm ( \"char c in MyString\" ) { Console :: Write ( c ) ; } }", tokenizeAndStringify(code));

}

void ifconstexpr() {

ASSERT_EQUALS("void f ( ) { if ( FOO ) { bar ( c ) ; } }", tokenizeAndStringify("void f() { if constexpr ( FOO ) { bar(c); } }"));

}

void combineOperators() {

ASSERT_EQUALS("; private: ;", tokenizeAndStringify(";private:;", false));

ASSERT_EQUALS("; protected: ;", tokenizeAndStringify(";protected:;", false));

ASSERT_EQUALS("; public: ;", tokenizeAndStringify(";public:;", false));

ASSERT_EQUALS("; __published: ;", tokenizeAndStringify(";__published:;", false));

ASSERT_EQUALS("a . public : ;", tokenizeAndStringify("a.public:;", false));

ASSERT_EQUALS("void f ( x & = 2 ) ;", tokenizeAndStringify("void f(x &= 2);", false));

ASSERT_EQUALS("const_cast < a * > ( & e )", tokenizeAndStringify("const_cast<a*>(&e)", false));

}

void concatenateNegativeNumber() {

ASSERT_EQUALS("i = -12 ;", tokenizeAndStringify("i = -12;"));

ASSERT_EQUALS("1 - 2 ;", tokenizeAndStringify("1-2;"));

ASSERT_EQUALS("foo ( -1 ) - 2 ;", tokenizeAndStringify("foo(-1)-2;"));

ASSERT_EQUALS("int f ( ) { return -2 ; }", tokenizeAndStringify("int f(){return -2;}"));

ASSERT_EQUALS("int x [ 2 ] = { -2 , 1 }", tokenizeAndStringify("int x[2] = {-2,1}"));

ASSERT_EQUALS("f ( 123 )", tokenizeAndStringify("f(+123)"));

}

void longtok() {

const std::string filedata(10000, 'a');

ASSERT_EQUALS(filedata, tokenizeAndStringify(filedata.c_str(), true));

}

void removeUnusedTemplates() {

Settings s;

s.checkUnusedTemplates = false;

ASSERT_EQUALS(";",

tokenizeAndStringify("; template <typename... a> uint8_t b(std::tuple<uint8_t> d) {\n"

" std::tuple<a...> c{std::move(d)};\n"

" return std::get<0>(c);\n"

"}", s));