From 17dc849f19c5f326e42ca4e13a9d9c45755f23d5 Mon Sep 17 00:00:00 2001 From: chrchr-github Date: Sat, 11 Mar 2023 12:41:17 +0100 Subject: [PATCH 1/6] Partial fix for #11378 internalAstError regressions (iscpp11init) --- lib/tokenlist.cpp | 2 +- test/testtokenize.cpp | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index 6f12d39e5ca..0379f0dcc04 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -560,7 +560,7 @@ static bool iscpp11init_impl(const Token * const tok) nameToken = nameToken->link()->previous(); if (nameToken->str() == "]") { const Token* newTok = nameToken->link()->previous(); - while (Token::Match(newTok, "%type%") && !newTok->isKeyword()) + while (Token::Match(newTok, "%type%|::") && !newTok->isKeyword()) newTok = newTok->previous(); if (Token::simpleMatch(newTok, "new")) return true; diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index ee255ad166b..ff11e8bb537 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -7469,7 +7469,7 @@ class TestTokenizer : public TestFixture { ASSERT_LOC(tok, file, line); ASSERT_LOC(tok->isCpp11init() == expected, file, line); }; - + /* testIsCpp11init("class X : public A, C::D {};", "D {", TokenImpl::Cpp11init::NOINIT); @@ -7559,7 +7559,14 @@ class TestTokenizer : public TestFixture { " void operator()(Args...) {}\n" "};\n", "{ void", - TokenImpl::Cpp11init::NOINIT); + TokenImpl::Cpp11init::NOINIT);*/ + + testIsCpp11init("struct S {\n" + " std::uint8_t * p;\n" + " S() : p{ new std::uint8_t[1]{} } {}\n" + "};\n", + "{ new", + TokenImpl::Cpp11init::CPP11INIT); ASSERT_NO_THROW(tokenizeAndStringify("template struct X {};\n" // don't crash "template auto f(T t) -> X {}\n")); From db77c272bdf00331f8bc21325fe70245add2086e Mon Sep 17 00:00:00 2001 From: chrchr-github Date: Sat, 11 Mar 2023 12:42:38 +0100 Subject: [PATCH 2/6] Undo --- test/testtokenize.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index ff11e8bb537..cc07e30639d 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -7469,7 +7469,7 @@ class TestTokenizer : public TestFixture { ASSERT_LOC(tok, file, line); ASSERT_LOC(tok->isCpp11init() == expected, file, line); }; - /* + testIsCpp11init("class X : public A, C::D {};", "D {", TokenImpl::Cpp11init::NOINIT); @@ -7559,7 +7559,7 @@ class TestTokenizer : public TestFixture { " void operator()(Args...) {}\n" "};\n", "{ void", - TokenImpl::Cpp11init::NOINIT);*/ + TokenImpl::Cpp11init::NOINIT); testIsCpp11init("struct S {\n" " std::uint8_t * p;\n" From 5cf8a52eb0307aac564b020bf191df57c5ad48e1 Mon Sep 17 00:00:00 2001 From: chrchr-github Date: Sat, 11 Mar 2023 12:45:20 +0100 Subject: [PATCH 3/6] Fix test --- test/testtokenize.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index cc07e30639d..217e5a50f7c 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -7562,10 +7562,10 @@ class TestTokenizer : public TestFixture { TokenImpl::Cpp11init::NOINIT); testIsCpp11init("struct S {\n" - " std::uint8_t * p;\n" + " std::uint8_t* p;\n" " S() : p{ new std::uint8_t[1]{} } {}\n" "};\n", - "{ new", + "{ } } {", TokenImpl::Cpp11init::CPP11INIT); ASSERT_NO_THROW(tokenizeAndStringify("template struct X {};\n" // don't crash From 63243cd6a6d731810044c2d546bfa3f92281e550 Mon Sep 17 00:00:00 2001 From: chrchr-github Date: Sat, 11 Mar 2023 13:52:18 +0100 Subject: [PATCH 4/6] Fix internalAstError with placement new --- lib/tokenize.cpp | 5 ++++- lib/tokenlist.cpp | 2 +- test/testtokenize.cpp | 7 +++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp index c69beb68014..a3f341835bd 100644 --- a/lib/tokenize.cpp +++ b/lib/tokenize.cpp @@ -5552,7 +5552,10 @@ void Tokenizer::removeMacrosInGlobalScope() if (tok->str() == "(") { tok = tok->link(); if (Token::Match(tok, ") %type% {") && - !Token::Match(tok->next(), "const|namespace|class|struct|union|noexcept|override|final|volatile|mutable")) + !tok->next()->isStandardType() && + !tok->next()->isKeyword() && + !Token::Match(tok->next(), "override|final") && + tok->next()->isUpperCaseName()) tok->deleteNext(); } diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index 0379f0dcc04..df4e4e50123 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -558,7 +558,7 @@ static bool iscpp11init_impl(const Token * const tok) return true; if (nameToken->str() == ">" && nameToken->link()) nameToken = nameToken->link()->previous(); - if (nameToken->str() == "]") { + if (nameToken->str() == "]") {//if (Token::Match(nameToken, "]|)")) { const Token* newTok = nameToken->link()->previous(); while (Token::Match(newTok, "%type%|::") && !newTok->isKeyword()) newTok = newTok->previous(); diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index 217e5a50f7c..e9706914137 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -7568,6 +7568,13 @@ class TestTokenizer : public TestFixture { "{ } } {", TokenImpl::Cpp11init::CPP11INIT); + testIsCpp11init("struct S {\n" + " S() : p{new (malloc(4)) int{}} {}\n" + " int* p;\n" + "};\n", + "{ } } {", + TokenImpl::Cpp11init::CPP11INIT); + ASSERT_NO_THROW(tokenizeAndStringify("template struct X {};\n" // don't crash "template auto f(T t) -> X {}\n")); #undef testIsCpp11init From 8c684e370b85d417bacd3cffbe0631400b06ffa0 Mon Sep 17 00:00:00 2001 From: chrchr-github Date: Sat, 11 Mar 2023 13:55:39 +0100 Subject: [PATCH 5/6] Remove comment --- lib/tokenlist.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp index df4e4e50123..0379f0dcc04 100644 --- a/lib/tokenlist.cpp +++ b/lib/tokenlist.cpp @@ -558,7 +558,7 @@ static bool iscpp11init_impl(const Token * const tok) return true; if (nameToken->str() == ">" && nameToken->link()) nameToken = nameToken->link()->previous(); - if (nameToken->str() == "]") {//if (Token::Match(nameToken, "]|)")) { + if (nameToken->str() == "]") { const Token* newTok = nameToken->link()->previous(); while (Token::Match(newTok, "%type%|::") && !newTok->isKeyword()) newTok = newTok->previous(); From a29eff0d0b9779a37c99212f7cf917fd134d0490 Mon Sep 17 00:00:00 2001 From: chrchr-github Date: Sat, 11 Mar 2023 14:11:06 +0100 Subject: [PATCH 6/6] Add test --- test/testtokenize.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp index e9706914137..fbc1efeafee 100644 --- a/test/testtokenize.cpp +++ b/test/testtokenize.cpp @@ -5256,6 +5256,15 @@ class TestTokenizer : public TestFixture { "\n" "BOOL CSetProgsAdvDlg::OnInitDialog() {}"), InternalError); + + ASSERT_EQUALS("struct S {\n" + "S ( ) : p { new ( malloc ( 4 ) ) int { } } { }\n" + "int * p ;\n" + "} ;", + tokenizeAndStringify("struct S {\n" + " S() : p{new (malloc(4)) int{}} {}\n" + " int* p;\n" + "};\n")); } void addSemicolonAfterUnknownMacro() {