Skip to content

Commit 6cc5131

Browse files
lemirelemire
committed
Adding an allparserscheckfile program.
1 parent 0453d54 commit 6cc5131

4 files changed

Lines changed: 74 additions & 2 deletions

File tree

‎LIMITATIONS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
To simplify the engineering, we make some assumptions that can be lifted with some effort:
22

3+
- We support UTF-8 (and thus ASCII), nothing else (no Latin, no UTF-16).
34
- We assume AVX2 support. No support for non-x86 processors is included.
45
- This library cannot parse JSON document of size 16MB or more.
56
- We expect the input memory pointer to 256-bit aligned and to be padded (e.g., with spaces) so that it can be read entirely in blocks of 256 bits.

‎Makefile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
.PHONY: clean cleandist
88

99
CXXFLAGS = -std=c++11 -g2 -O3 -march=native -Wall -Wextra -Wshadow -Iinclude -Ibenchmark/linux -Idependencies/rapidjson/include -Idependencies/sajson/include
10-
EXECUTABLES=parse jsoncheck numberparsingcheck minifiercompetition parsingcompetition minify
10+
EXECUTABLES=parse jsoncheck numberparsingcheck minifiercompetition parsingcompetition minify allparserscheckfile
1111

1212
HEADERS= include/jsonparser/stringparsing.h include/jsonparser/numberparsing.h include/jsonparser/jsonparser.h include/jsonparser/common_defs.h include/jsonparser/jsonioutil.h benchmark/benchmark.h benchmark/linux/linux-perf-events.h include/jsonparser/simdjson_internal.h include/jsonparser/stage1_find_marks.h include/jsonparser/stage2_flatten.h include/jsonparser/stage34_unified.h
1313
LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/stage1_find_marks.cpp src/stage2_flatten.cpp src/stage34_unified.cpp
@@ -57,7 +57,8 @@ minify: tools/minify.cpp $(HEADERS) $(MINIFIERHEADERS) $(LIBFILES) $(MINIFIERLIB
5757
parsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES)
5858
$(CXX) $(CXXFLAGS) -o parsingcompetition $(LIBFILES) benchmark/parsingcompetition.cpp -I. $(LIBFLAGS)
5959

60-
60+
allparserscheckfile: tests/allparserscheckfile.cpp $(HEADERS) $(LIBFILES)
61+
$(CXX) $(CXXFLAGS) -o allparserscheckfile $(LIBFILES) tests/allparserscheckfile.cpp -I. $(LIBFLAGS)
6162

6263
parsehisto: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
6364
$(CXX) $(CXXFLAGS) -o parsehisto benchmark/parse.cpp $(LIBFILES) $(LIBFLAGS) -DBUILDHISTOGRAM

‎benchmark/parsingcompetition.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ int main(int argc, char *argv[]) {
6262
false, memcpy(buffer, p.first, p.second), repeat, volume, true);
6363
BEST_TIME(d.Parse((const char *)buffer).HasParseError(), false,
6464
memcpy(buffer, p.first, p.second), repeat, volume, true);
65+
BEST_TIME(d.ParseInsitu<kParseValidateEncodingFlag>(buffer).HasParseError(), false,
66+
memcpy(buffer, p.first, p.second), repeat, volume, true);
6567
BEST_TIME(d.ParseInsitu(buffer).HasParseError(), false,
6668
memcpy(buffer, p.first, p.second), repeat, volume, true);
6769

‎tests/allparserscheckfile.cpp‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
2+
#include "jsonparser/jsonparser.h"
3+
4+
5+
// #define RAPIDJSON_SSE2 // bad
6+
// #define RAPIDJSON_SSE42 // bad
7+
#include "rapidjson/document.h"
8+
#include "rapidjson/reader.h" // you have to check in the submodule
9+
#include "rapidjson/stringbuffer.h"
10+
#include "rapidjson/writer.h"
11+
12+
#include "sajson.h"
13+
14+
15+
16+
17+
using namespace rapidjson;
18+
using namespace std;
19+
20+
int main(int argc, char *argv[]) {
21+
if (argc < 2) {
22+
cerr << "Usage: " << argv[0] << " <jsonfile>\n";
23+
cerr << "Or " << argv[0] << " -v <jsonfile>\n";
24+
exit(1);
25+
}
26+
bool verbose = false;
27+
if (argc > 2) {
28+
if (strcmp(argv[1], "-v"))
29+
verbose = true;
30+
}
31+
pair<u8 *, size_t> p = get_corpus(argv[argc - 1]);
32+
if (verbose) {
33+
std::cout << "Input has ";
34+
if (p.second > 1024 * 1024)
35+
std::cout << p.second / (1024 * 1024) << " MB ";
36+
else if (p.second > 1024)
37+
std::cout << p.second / 1024 << " KB ";
38+
else
39+
std::cout << p.second << " B ";
40+
std::cout << std::endl;
41+
}
42+
ParsedJson *pj_ptr = allocate_ParsedJson(p.second);
43+
if (pj_ptr == NULL) {
44+
std::cerr << "can't allocate memory" << std::endl;
45+
return EXIT_FAILURE;
46+
}
47+
ParsedJson &pj(*pj_ptr);
48+
49+
bool ours_correct = json_parse(p.first, p.second, pj);
50+
51+
rapidjson::Document d;
52+
53+
char *buffer = (char *)malloc(p.second + 1);
54+
memcpy(buffer, p.first, p.second);
55+
buffer[p.second] = '\0';
56+
bool rapid_correct = (d.Parse((const char *)buffer).HasParseError() == false);
57+
bool rapid_correct_checkencoding = (d.Parse<kParseValidateEncodingFlag>((const char *)buffer).HasParseError() == false);
58+
bool sajson_correct = sajson::parse(sajson::dynamic_allocation(), sajson::mutable_string_view(p.second, buffer)).is_valid();
59+
printf("our parser : %s \n", ours_correct ? "correct":"invalid");
60+
printf("rapid : %s \n", rapid_correct ? "correct":"invalid");
61+
printf("rapid (check encoding) : %s \n", rapid_correct_checkencoding ? "correct":"invalid");
62+
printf("sajson : %s \n", sajson_correct ? "correct":"invalid");
63+
64+
free(buffer);
65+
free(p.first);
66+
deallocate_ParsedJson(pj_ptr);
67+
return EXIT_SUCCESS;
68+
}

0 commit comments

Comments
 (0)