diff --git a/simplecpp.cpp b/simplecpp.cpp index 47e67483..3e572fc0 100644 --- a/simplecpp.cpp +++ b/simplecpp.cpp @@ -3,18 +3,12 @@ * Copyright (C) 2016-2023 simplecpp team */ +// needs to be specified here otherwise _mingw.h will define it as 0x0601 +// causing FileIdInfo not to be available #if defined(_WIN32) # ifndef _WIN32_WINNT # define _WIN32_WINNT 0x0602 # endif -# ifndef NOMINMAX -# define NOMINMAX -# endif -# ifndef WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN -# endif -# include -# undef ERROR #endif #include "simplecpp.h" @@ -51,10 +45,19 @@ #include #include -#ifdef _WIN32 +#if defined(_WIN32) +# ifndef NOMINMAX +# define NOMINMAX +# endif +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# undef ERROR # include #else # include +# include #endif static bool isHex(const std::string &s) @@ -658,8 +661,6 @@ static const std::string COMMENT_END("*/"); void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, OutputList *outputList) { - std::stack loc; - unsigned int multiline = 0U; const Token *oldLastToken = nullptr; @@ -698,59 +699,44 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, if (oldLastToken != cback()) { oldLastToken = cback(); - const Token * const llTok = isLastLinePreprocessor(); - if (!llTok) + + // #line 3 + // #line 3 "file.c" + // #3 + // #3 "file.c" + const Token * ppTok = isLastLinePreprocessor(); + if (!ppTok) continue; - const Token * const llNextToken = llTok->next; - if (!llTok->next) + + const auto advanceAndSkipComments = [](const Token* tok) { + do { + tok = tok->next; + } while (tok && tok->comment); + return tok; + }; + + // skip # + ppTok = advanceAndSkipComments(ppTok); + if (!ppTok) continue; - if (llNextToken->next) { - // #file "file.c" - if (llNextToken->str() == "file" && - llNextToken->next->str()[0] == '\"') - { - const Token *strtok = cback(); - while (strtok->comment) - strtok = strtok->previous; - loc.push(location); - location.fileIndex = fileIndex(strtok->str().substr(1U, strtok->str().size() - 2U)); - location.line = 1U; - } - // TODO: add support for "# 3" - // #3 "file.c" - // #line 3 "file.c" - else if ((llNextToken->number && - llNextToken->next->str()[0] == '\"') || - (llNextToken->str() == "line" && - llNextToken->next->number && - llNextToken->next->next && - llNextToken->next->next->str()[0] == '\"')) - { - const Token *strtok = cback(); - while (strtok->comment) - strtok = strtok->previous; - const Token *numtok = strtok->previous; - while (numtok->comment) - numtok = numtok->previous; - lineDirective(fileIndex(replaceAll(strtok->str().substr(1U, strtok->str().size() - 2U),"\\\\","\\")), - std::atol(numtok->str().c_str()), location); - } - // #line 3 - else if (llNextToken->str() == "line" && - llNextToken->next->number) - { - const Token *numtok = cback(); - while (numtok->comment) - numtok = numtok->previous; - lineDirective(location.fileIndex, std::atol(numtok->str().c_str()), location); - } - } - // #endfile - else if (llNextToken->str() == "endfile" && !loc.empty()) - { - location = loc.top(); - loc.pop(); - } + + if (ppTok->str() == "line") + ppTok = advanceAndSkipComments(ppTok); + + if (!ppTok || !ppTok->number) + continue; + + const unsigned int line = std::atol(ppTok->str().c_str()); + ppTok = advanceAndSkipComments(ppTok); + + unsigned int fileindex; + + if (ppTok && ppTok->str()[0] == '\"') + fileindex = fileIndex(replaceAll(ppTok->str().substr(1U, ppTok->str().size() - 2U),"\\\\","\\")); + else + fileindex = location.fileIndex; + + lineDirective(fileindex, line, location); } continue; @@ -1040,7 +1026,7 @@ void simplecpp::TokenList::combineOperators() continue; } const Token *prev = tok->previous; - while (prev && prev->isOneOf(";{}()")) + while (prev && prev->isOneOf(";{}(")) prev = prev->previous; executableScope.push(prev && prev->op == ')'); continue; @@ -2332,9 +2318,6 @@ namespace simplecpp { const Token *nextTok = B->next; if (canBeConcatenatedStringOrChar) { - if (unexpectedA) - throw invalidHashHash::unexpectedToken(tok->location, name(), A); - // It seems clearer to handle this case separately even though the code is similar-ish, but we don't want to merge here. // TODO The question is whether the ## or varargs may still apply, and how to provoke? if (expandArg(tokensB, B, parametertokens)) { @@ -3090,6 +3073,65 @@ static std::string openHeader(std::ifstream &f, const simplecpp::DUI &dui, const return ""; } +namespace { + struct FileID { +#ifdef _WIN32 + struct { + std::uint64_t VolumeSerialNumber; + struct { + std::uint64_t IdentifierHi; + std::uint64_t IdentifierLo; + } FileId; + } fileIdInfo; + + bool operator==(const FileID &that) const noexcept { + return fileIdInfo.VolumeSerialNumber == that.fileIdInfo.VolumeSerialNumber && + fileIdInfo.FileId.IdentifierHi == that.fileIdInfo.FileId.IdentifierHi && + fileIdInfo.FileId.IdentifierLo == that.fileIdInfo.FileId.IdentifierLo; + } +#else + dev_t dev; + ino_t ino; + + bool operator==(const FileID& that) const noexcept { + return dev == that.dev && ino == that.ino; + } +#endif + struct Hasher { + std::size_t operator()(const FileID &id) const { +#ifdef _WIN32 + return static_cast(id.fileIdInfo.FileId.IdentifierHi ^ id.fileIdInfo.FileId.IdentifierLo ^ + id.fileIdInfo.VolumeSerialNumber); +#else + return static_cast(id.dev) ^ static_cast(id.ino); +#endif + } + }; + }; +} + +struct simplecpp::FileDataCache::Impl +{ + void clear() + { + mIdMap.clear(); + } + + using id_map_type = std::unordered_map; + + id_map_type mIdMap; +}; + +simplecpp::FileDataCache::FileDataCache() + : mImpl(new Impl) +{} + +simplecpp::FileDataCache::~FileDataCache() = default; +simplecpp::FileDataCache::FileDataCache(FileDataCache &&) noexcept = default; +simplecpp::FileDataCache &simplecpp::FileDataCache::operator=(simplecpp::FileDataCache &&) noexcept = default; + +static bool getFileId(const std::string &path, FileID &id); + std::pair simplecpp::FileDataCache::tryload(FileDataCache::name_map_type::iterator &name_it, const simplecpp::DUI &dui, std::vector &filenames, simplecpp::OutputList *outputList) { const std::string &path = name_it->first; @@ -3098,8 +3140,8 @@ std::pair simplecpp::FileDataCache::tryload(FileDat if (!getFileId(path, fileId)) return {nullptr, false}; - const auto id_it = mIdMap.find(fileId); - if (id_it != mIdMap.end()) { + const auto id_it = mImpl->mIdMap.find(fileId); + if (id_it != mImpl->mIdMap.end()) { name_it->second = id_it->second; return {id_it->second, false}; } @@ -3110,9 +3152,12 @@ std::pair simplecpp::FileDataCache::tryload(FileDat data->tokens.removeComments(); name_it->second = data; - mIdMap.emplace(fileId, data); + mImpl->mIdMap.emplace(fileId, data); mData.emplace_back(data); + if (mLoadCallback) + mLoadCallback(*data); + return {data, true}; } @@ -3162,7 +3207,14 @@ std::pair simplecpp::FileDataCache::get(const std:: return {nullptr, false}; } -bool simplecpp::FileDataCache::getFileId(const std::string &path, FileID &id) +void simplecpp::FileDataCache::clear() +{ + mImpl->clear(); + mNameMap.clear(); + mData.clear(); +} + +static bool getFileId(const std::string &path, FileID &id) { #ifdef _WIN32 HANDLE hFile = CreateFileA(path.c_str(), 0, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); @@ -3349,20 +3401,20 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL std::map sizeOfType(rawtokens.sizeOfType); sizeOfType.insert(std::make_pair("char", sizeof(char))); sizeOfType.insert(std::make_pair("short", sizeof(short))); - sizeOfType.insert(std::make_pair("short int", sizeOfType["short"])); + sizeOfType.insert(std::make_pair("short int", sizeof(short))); sizeOfType.insert(std::make_pair("int", sizeof(int))); sizeOfType.insert(std::make_pair("long", sizeof(long))); - sizeOfType.insert(std::make_pair("long int", sizeOfType["long"])); + sizeOfType.insert(std::make_pair("long int", sizeof(long))); sizeOfType.insert(std::make_pair("long long", sizeof(long long))); sizeOfType.insert(std::make_pair("float", sizeof(float))); sizeOfType.insert(std::make_pair("double", sizeof(double))); sizeOfType.insert(std::make_pair("long double", sizeof(long double))); sizeOfType.insert(std::make_pair("char *", sizeof(char *))); sizeOfType.insert(std::make_pair("short *", sizeof(short *))); - sizeOfType.insert(std::make_pair("short int *", sizeOfType["short *"])); + sizeOfType.insert(std::make_pair("short int *", sizeof(short *))); sizeOfType.insert(std::make_pair("int *", sizeof(int *))); sizeOfType.insert(std::make_pair("long *", sizeof(long *))); - sizeOfType.insert(std::make_pair("long int *", sizeOfType["long *"])); + sizeOfType.insert(std::make_pair("long int *", sizeof(long *))); sizeOfType.insert(std::make_pair("long long *", sizeof(long long *))); sizeOfType.insert(std::make_pair("float *", sizeof(float *))); sizeOfType.insert(std::make_pair("double *", sizeof(double *))); diff --git a/simplecpp.h b/simplecpp.h index f29166ff..3d6fc526 100644 --- a/simplecpp.h +++ b/simplecpp.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -41,10 +42,6 @@ # define SIMPLECPP_LIB #endif -#ifndef _WIN32 -# include -#endif - #if defined(_MSC_VER) # pragma warning(push) // suppress warnings about "conversion from 'type1' to 'type2', possible loss of data" @@ -452,13 +449,14 @@ namespace simplecpp { class SIMPLECPP_LIB FileDataCache { public: - FileDataCache() = default; + FileDataCache(); + ~FileDataCache(); FileDataCache(const FileDataCache &) = delete; - FileDataCache(FileDataCache &&) = default; + FileDataCache(FileDataCache &&) noexcept; FileDataCache &operator=(const FileDataCache &) = delete; - FileDataCache &operator=(FileDataCache &&) = default; + FileDataCache &operator=(FileDataCache &&) noexcept; /** Get the cached data for a file, or load and then return it if it isn't cached. * returns the file data and true if the file was loaded, false if it was cached. */ @@ -472,11 +470,7 @@ namespace simplecpp { mNameMap.emplace(newdata->filename, newdata); } - void clear() { - mNameMap.clear(); - mIdMap.clear(); - mData.clear(); - } + void clear(); using container_type = std::vector>; using iterator = container_type::iterator; @@ -505,52 +499,23 @@ namespace simplecpp { return mData.cend(); } - private: - struct FileID { -#ifdef _WIN32 - struct { - std::uint64_t VolumeSerialNumber; - struct { - std::uint64_t IdentifierHi; - std::uint64_t IdentifierLo; - } FileId; - } fileIdInfo; - - bool operator==(const FileID &that) const noexcept { - return fileIdInfo.VolumeSerialNumber == that.fileIdInfo.VolumeSerialNumber && - fileIdInfo.FileId.IdentifierHi == that.fileIdInfo.FileId.IdentifierHi && - fileIdInfo.FileId.IdentifierLo == that.fileIdInfo.FileId.IdentifierLo; - } -#else - dev_t dev; - ino_t ino; + using load_callback_type = std::function; - bool operator==(const FileID& that) const noexcept { - return dev == that.dev && ino == that.ino; - } -#endif - struct Hasher { - std::size_t operator()(const FileID &id) const { -#ifdef _WIN32 - return static_cast(id.fileIdInfo.FileId.IdentifierHi ^ id.fileIdInfo.FileId.IdentifierLo ^ - id.fileIdInfo.VolumeSerialNumber); -#else - return static_cast(id.dev) ^ static_cast(id.ino); -#endif - } - }; - }; + void set_load_callback(load_callback_type cb) { + mLoadCallback = std::move(cb); + } - using name_map_type = std::unordered_map; - using id_map_type = std::unordered_map; + private: + struct Impl; + std::unique_ptr mImpl; - static bool getFileId(const std::string &path, FileID &id); + using name_map_type = std::unordered_map; std::pair tryload(name_map_type::iterator &name_it, const DUI &dui, std::vector &filenames, OutputList *outputList); container_type mData; name_map_type mNameMap; - id_map_type mIdMap; + load_callback_type mLoadCallback; }; /** Converts character literal (including prefix, but not ud-suffix) to long long value. diff --git a/test.cpp b/test.cpp index e45cbe39..2250ecf1 100644 --- a/test.cpp +++ b/test.cpp @@ -493,6 +493,7 @@ static void combineOperators_andequal() ASSERT_EQUALS("x &= 2 ;", preprocess("x &= 2;")); ASSERT_EQUALS("void f ( x & = 2 ) ;", preprocess("void f(x &= 2);")); ASSERT_EQUALS("f ( x &= 2 ) ;", preprocess("f(x &= 2);")); + ASSERT_EQUALS("f ( ) { return new int ( i &= 1 ) ; }", preprocess("f () { return new int(i &= 1); }")); } static void combineOperators_ellipsis() @@ -1874,6 +1875,16 @@ static void hashhash_empty_va_args() ASSERT_EQUALS("\n\n\n( 2 )", preprocess(code)); } +static void hashhash_va_args_unexpected() +{ + const char code[] = + "#define C(...)!##__VA_ARGS__\n" + "C(1)"; + simplecpp::OutputList outputList; + preprocess(code, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'C', Invalid ## usage when expanding 'C': Unexpected token '!'\n", toString(outputList)); +} + static void hashhash_universal_character() { const char code[] = @@ -1883,6 +1894,16 @@ static void hashhash_universal_character() ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Combining '\\u01' and '04' yields universal character '\\u0104'. This is undefined behavior according to C standard chapter 5.1.1.2, paragraph 4.\n", toString(outputList)); } +static void hashhash_universal_character_2() +{ + const char code[] = + "#define A(x,y) x##y\nint A(\\U0104, 0104);"; + simplecpp::OutputList outputList; + preprocess(code, simplecpp::DUI(), &outputList); + ASSERT_EQUALS("file0,1,syntax_error,failed to expand 'A', Invalid ## usage when expanding 'A': Combining '\\U0104' and '0104' yields universal character '\\U01040104'. This is undefined behavior according to C standard chapter 5.1.1.2, paragraph 4.\n", toString(outputList)); +} + + static void has_include_1() { const char code[] = "#ifdef __has_include\n" @@ -2412,7 +2433,8 @@ static void location8() "# 3\n" "__LINE__ __FILE__\n"; ASSERT_EQUALS("\n" - "2 \"\"", // TODO: should say 3 + "\n" + "3 \"\"", preprocess(code)); } @@ -2453,7 +2475,16 @@ static void location11() preprocess(code)); } -// TODO: test #file/#endfile +static void location12() +{ + const char code[] = + "/**//**/#/**//**/line/**//**/3/**//**/\"file.c\"/**/\n" + "__LINE__ __FILE__\n"; + ASSERT_EQUALS("\n" + "#line 3 \"file.c\"\n" + "3 \"file.c\"", + preprocess(code)); +} static void missingHeader1() { @@ -4008,11 +4039,14 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(hashhash_invalid_missing_args); TEST_CASE(hashhash_null_stmt); TEST_CASE(hashhash_empty_va_args); + TEST_CASE(hashhash_va_args_unexpected); + // C standard, 5.1.1.2, paragraph 4: // If a character sequence that matches the syntax of a universal // character name is produced by token concatenation (6.10.3.3), // the behavior is undefined." TEST_CASE(hashhash_universal_character); + TEST_CASE(hashhash_universal_character_2); // c++17 __has_include TEST_CASE(has_include_1); @@ -4061,6 +4095,7 @@ static void runTests(int argc, char **argv, Input input) TEST_CASE(location9); TEST_CASE(location10); TEST_CASE(location11); + TEST_CASE(location12); TEST_CASE(missingHeader1); TEST_CASE(missingHeader2);