Adding validating step.

lemire · lemire · commit 93f4e1d9b433 · 2018-07-25T22:59:40.000-04:00
diff --git a/Makefile b/Makefile
@@ -14,8 +14,9 @@ EXECUTABLES=parse
 
 
 all: $(EXECUTABLES)
+	-./parse
 
-parse: main.cpp common_defs.h vecdecode.h linux-perf-events.h
+parse: main.cpp common_defs.h linux-perf-events.h
 	$(CXX) $(CXXFLAGS) -o parse main.cpp
 
 
diff --git a/jsonchecker/fail01_EXCLUDE.json b/jsonchecker/fail01_EXCLUDE.json
@@ -0,0 +1 @@
+"A JSON payload should be an object or array, not a string."
diff --git a/jsonchecker/fail02.json b/jsonchecker/fail02.json
@@ -0,0 +1 @@
+["Unclosed array"
diff --git a/jsonchecker/fail03.json b/jsonchecker/fail03.json
@@ -0,0 +1 @@
+{unquoted_key: "keys must be quoted"}
diff --git a/jsonchecker/fail04.json b/jsonchecker/fail04.json
@@ -0,0 +1 @@
+["extra comma",]
diff --git a/jsonchecker/fail05.json b/jsonchecker/fail05.json
@@ -0,0 +1 @@
+["double extra comma",,]
diff --git a/jsonchecker/fail06.json b/jsonchecker/fail06.json
@@ -0,0 +1 @@
+[   , "<-- missing value"]
diff --git a/jsonchecker/fail07.json b/jsonchecker/fail07.json
@@ -0,0 +1 @@
+["Comma after the close"],
diff --git a/jsonchecker/fail08.json b/jsonchecker/fail08.json
@@ -0,0 +1 @@
+["Extra close"]]
diff --git a/jsonchecker/fail09.json b/jsonchecker/fail09.json
@@ -0,0 +1 @@
+{"Extra comma": true,}
diff --git a/jsonchecker/fail10.json b/jsonchecker/fail10.json
@@ -0,0 +1 @@
+{"Extra value after close": true} "misplaced quoted value"
diff --git a/jsonchecker/fail11.json b/jsonchecker/fail11.json
@@ -0,0 +1 @@
+{"Illegal expression": 1 + 2}
diff --git a/jsonchecker/fail12.json b/jsonchecker/fail12.json
@@ -0,0 +1 @@
+{"Illegal invocation": alert()}
diff --git a/jsonchecker/fail13.json b/jsonchecker/fail13.json
@@ -0,0 +1 @@
+{"Numbers cannot have leading zeroes": 013}
diff --git a/jsonchecker/fail14.json b/jsonchecker/fail14.json
@@ -0,0 +1 @@
+{"Numbers cannot be hex": 0x14}
diff --git a/jsonchecker/fail15.json b/jsonchecker/fail15.json
@@ -0,0 +1 @@
+["Illegal backslash escape: \x15"]
diff --git a/jsonchecker/fail16.json b/jsonchecker/fail16.json
@@ -0,0 +1 @@
+[\naked]
diff --git a/jsonchecker/fail17.json b/jsonchecker/fail17.json
@@ -0,0 +1 @@
+["Illegal backslash escape: \017"]
diff --git a/jsonchecker/fail18_EXCLUDE.json b/jsonchecker/fail18_EXCLUDE.json
@@ -0,0 +1 @@
+[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]
diff --git a/jsonchecker/fail19.json b/jsonchecker/fail19.json
@@ -0,0 +1 @@
+{"Missing colon" null}
diff --git a/jsonchecker/fail20.json b/jsonchecker/fail20.json
@@ -0,0 +1 @@
+{"Double colon":: null}
diff --git a/jsonchecker/fail21.json b/jsonchecker/fail21.json
@@ -0,0 +1 @@
+{"Comma instead of colon", null}
diff --git a/jsonchecker/fail22.json b/jsonchecker/fail22.json
@@ -0,0 +1 @@
+["Colon instead of comma": false]
diff --git a/jsonchecker/fail23.json b/jsonchecker/fail23.json
@@ -0,0 +1 @@
+["Bad value", truth]
diff --git a/jsonchecker/fail24.json b/jsonchecker/fail24.json
@@ -0,0 +1 @@
+['single quote']
diff --git a/jsonchecker/fail25.json b/jsonchecker/fail25.json
@@ -0,0 +1 @@
+["	tab	character	in	string	"]
diff --git a/jsonchecker/fail26.json b/jsonchecker/fail26.json
@@ -0,0 +1 @@
+["tab\   character\   in\  string\  "]
diff --git a/jsonchecker/fail27.json b/jsonchecker/fail27.json
@@ -0,0 +1,2 @@
+["line
+break"]
diff --git a/jsonchecker/fail28.json b/jsonchecker/fail28.json
@@ -0,0 +1,2 @@
+["line\
+break"]
diff --git a/jsonchecker/fail29.json b/jsonchecker/fail29.json
@@ -0,0 +1 @@
+[0e]
diff --git a/jsonchecker/fail30.json b/jsonchecker/fail30.json
@@ -0,0 +1 @@
+[0e+]
diff --git a/jsonchecker/fail31.json b/jsonchecker/fail31.json
@@ -0,0 +1 @@
+[0e+-1]
diff --git a/jsonchecker/fail32.json b/jsonchecker/fail32.json
@@ -0,0 +1 @@
+{"Comma instead if closing brace": true,
diff --git a/jsonchecker/fail33.json b/jsonchecker/fail33.json
@@ -0,0 +1 @@
+["mismatch"}
diff --git a/jsonchecker/pass01.json b/jsonchecker/pass01.json
@@ -0,0 +1,58 @@
+[
+    "JSON Test Pattern pass1",
+    {"object with 1 member":["array with 1 element"]},
+    {},
+    [],
+    -42,
+    true,
+    false,
+    null,
+    {
+        "integer": 1234567890,
+        "real": -9876.543210,
+        "e": 0.123456789e-12,
+        "E": 1.234567890E+34,
+        "":  23456789012E66,
+        "zero": 0,
+        "one": 1,
+        "space": " ",
+        "quote": "\"",
+        "backslash": "\\",
+        "controls": "\b\f\n\r\t",
+        "slash": "/ & \/",
+        "alpha": "abcdefghijklmnopqrstuvwyz",
+        "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ",
+        "digit": "0123456789",
+        "0123456789": "digit",
+        "special": "`1~!@#$%^&*()_+-={':[,]}|;.</>?",
+        "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A",
+        "true": true,
+        "false": false,
+        "null": null,
+        "array":[  ],
+        "object":{  },
+        "address": "50 St. James Street",
+        "url": "http://www.JSON.org/",
+        "comment": "// /* <!-- --",
+        "# -- --> */": " ",
+        " s p a c e d " :[1,2 , 3
+
+,
+
+4 , 5        ,          6           ,7        ],"compact":[1,2,3,4,5,6,7],
+        "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}",
+        "quotes": "&#34; \u0022 %22 0x22 034 &#x22;",
+        "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?"
+: "A key can be any string"
+    },
+    0.5 ,98.6
+,
+99.44
+,
+
+1066,
+1e1,
+0.1e1,
+1e-1,
+1e00,2e+00,2e-00
+,"rosebud"]
diff --git a/jsonchecker/pass02.json b/jsonchecker/pass02.json
@@ -0,0 +1 @@
+[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]]
diff --git a/jsonchecker/pass03.json b/jsonchecker/pass03.json
@@ -0,0 +1,6 @@
+{
+    "JSON Test Pattern pass3": {
+        "The outermost value": "must be an object or array.",
+        "In this test": "It is an object."
+    }
+}
diff --git a/main.cpp b/main.cpp
@@ -1,3 +1,10 @@
+#include <dirent.h>
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
 #include <iostream>
 #include <iomanip>
 #include <chrono>
@@ -1160,9 +1167,81 @@ void colorfuldisplay(ParsedJson & pj, const u8 * buf) {
     }
     std::cout << std::endl;
 }
+
+
+/**
+ * Does the file filename ends with the given extension.
+ */
+static bool hasExtension(const char *filename, const char *extension) {
+    const char *ext = strrchr(filename, '.');
+    return (ext && !strcmp(ext, extension));
+}
+
+bool startsWith(const char *pre, const char *str) {
+    size_t lenpre = strlen(pre),
+           lenstr = strlen(str);
+    return lenstr < lenpre ? false : strncmp(pre, str, lenpre) == 0;
+}
+
+void validate() {
+  init_state_machine();// to be safe
+  const char *dirname = "jsonchecker/"; // ugly, hardcoded, brittle
+  const char *extension = ".json";
+  size_t dirlen = strlen(dirname);
+  struct dirent **entry_list;
+  int c = scandir(dirname, &entry_list, 0, alphasort);
+  if (c < 0) {
+    printf("error accessing %s \n", dirname);
+    return;
+  }
+  if (c == 0) {
+    printf("nothing in dir %s \n", dirname);
+    return;
+  }
+  for (int i = 0; i < c; i++) {
+    const char *name = entry_list[i]->d_name;
+    if (hasExtension(name, extension)) {
+      size_t filelen = strlen(name);
+      char *fullpath = (char *)malloc(dirlen + filelen + 1);
+      strcpy(fullpath, dirname);
+      strcpy(fullpath + dirlen, name);
+      pair<u8 *, size_t> p = get_corpus(fullpath);
+      ParsedJson pj;
+      if (posix_memalign( (void **)&pj.structurals, 8, ROUNDUP_N(p.second, 64)/8)) {
+              cerr << "Could not allocate memory\n";
+              return;
+      };
+      pj.n_structural_indexes = 0;
+      u32 max_structures = ROUNDUP_N(p.second, 64) + 2 + 7;
+      pj.structural_indexes = new u32[max_structures];
+      pj.nodes = new JsonNode[max_structures];
+      find_structural_bits(p.first, p.second, pj);
+      flatten_indexes(p.second, pj);
+      bool isok = ape_machine(p.first, p.second, pj);
+      if(isok)
+       isok = shovel_machine(p.first, p.second, pj);
+      if(startsWith("pass",name)) {
+        if(!isok) printf("warning: file %s should pass but it fails.\n",name);
+      }
+      if(startsWith("fail",name)) {
+        if(isok) printf("warning: file %s should fail but it passes.\n",name);
+      }
+      free(pj.structurals);
+      free(p.first);
+      delete[] pj.structural_indexes;
+      delete[] pj.nodes;
+      free(fullpath);
+    }
+  }
+  for (int i = 0; i < c; ++i) free(entry_list[i]);
+  free(entry_list);
+}
+
 int main(int argc, char * argv[]) {
     if (argc != 2) {
         cerr << "Usage: " << argv[0] << " <jsonfile>\n";
+        cout << "We are going to validate:\n" << std::endl;
+        validate();
         exit(1);
     }
     pair<u8 *, size_t> p = get_corpus(argv[1]);

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+"A JSON payload should be an object or array, not a string."`