Merge remote-tracking branch 'upstream/master'

larslj · larslj · commit 0c806cfea8e5 · 2017-09-13T22:18:52.000+02:00
diff --git a/simplecpp.cpp b/simplecpp.cpp
@@ -401,6 +401,20 @@ void simplecpp::TokenList::readfile(std::istream &istr, const std::string &filen
         if (ch < ' ' && ch != '\t' && ch != '\n' && ch != '\r')
             ch = ' ';
 
+        if (ch >= 0x80) {
+            if (outputList) {
+                simplecpp::Output err(files);
+                err.type = simplecpp::Output::UNHANDLED_CHAR_ERROR;
+                err.location = location;
+                std::ostringstream s;
+                s << (int)ch;
+                err.msg = "The code contains unhandled character(s) (character code=" + s.str() + "). Neither unicode nor extended ascii is supported.";
+                outputList->push_back(err);
+            }
+            clear();
+            return;
+        }
+
         if (ch == '\n') {
             if (cback() && cback()->op == '\\') {
                 if (location.col > cback()->location.col + 1U)
@@ -1220,6 +1234,7 @@ namespace simplecpp {
                 }
                 if (!sameline(nametoken, argtok)) {
                     endToken = argtok ? argtok->previous : argtok;
+                    valueToken = NULL;
                     return false;
                 }
                 valueToken = argtok ? argtok->next : NULL;
@@ -1286,8 +1301,8 @@ namespace simplecpp {
                 } else {
                     if (!expandArg(tokens, tok, tok->location, macros, expandedmacros, parametertokens)) {
                         bool expanded = false;
-                        if (macros.find(tok->str) != macros.end() && expandedmacros.find(tok->str) == expandedmacros.end()) {
-                            const std::map<TokenString, Macro>::const_iterator it = macros.find(tok->str);
+                        const std::map<TokenString, Macro>::const_iterator it = macros.find(tok->str);
+                        if (it != macros.end() && expandedmacros.find(tok->str) == expandedmacros.end()) {
                             const Macro &m = it->second;
                             if (!m.functionLike()) {
                                 m.expand(tokens, tok, macros, files);
diff --git a/simplecpp.h b/simplecpp.h
@@ -164,7 +164,8 @@ namespace simplecpp {
             MISSING_HEADER,
             INCLUDE_NESTED_TOO_DEEPLY,
             SYNTAX_ERROR,
-            PORTABILITY_BACKSLASH
+            PORTABILITY_BACKSLASH,
+            UNHANDLED_CHAR_ERROR
         } type;
         Location location;
         std::string msg;
diff --git a/test.cpp b/test.cpp
@@ -90,6 +90,8 @@ static std::string toString(const simplecpp::OutputList &outputList)
         case simplecpp::Output::Type::PORTABILITY_BACKSLASH:
             ostr << "portability_backslash,";
             break;
+        case simplecpp::Output::Type::UNHANDLED_CHAR_ERROR:
+            ostr << "unhandled_char_error,";
         }
 
         ostr << output.msg << '\n';
@@ -1086,6 +1088,17 @@ static void readfile_cpp14_number()
     ASSERT_EQUALS("A = 12345 ;", readfile("A = 12\'345;"));
 }
 
+static void readfile_unhandled_chars()
+{
+    simplecpp::OutputList outputList;
+    readfile("// 你好世界", -1, &outputList);
+    ASSERT_EQUALS("", toString(outputList));
+    readfile("s=\"你好世界\"", -1, &outputList);
+    ASSERT_EQUALS("", toString(outputList));
+    readfile("int 你好世界=0;", -1, &outputList);
+    ASSERT_EQUALS("file0,1,unhandled_char_error,The code contains unhandled character(s) (character code=228). Neither unicode nor extended ascii is supported.\n", toString(outputList));
+}
+
 static void stringify1()
 {
     const char code_c[] = "#include \"A.h\"\n"
@@ -1443,6 +1456,7 @@ int main(int argc, char **argv)
     TEST_CASE(readfile_string);
     TEST_CASE(readfile_rawstring);
     TEST_CASE(readfile_cpp14_number);
+    TEST_CASE(readfile_unhandled_chars);
 
     TEST_CASE(stringify1);
 
