Skip to content

Commit 52c4b65

Browse files
lemirelemire
committed
Progress validating the API.
1 parent a56e92a commit 52c4b65

4 files changed

Lines changed: 137 additions & 15 deletions

File tree

‎include/simdjson/jsonformatutils.h‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#pragma once
22

33
#include <stdio.h>
4+
#include <iostream>
5+
#include <iomanip>
46

57
static inline void print_with_escapes(const unsigned char *src) {
68
while (*src) {
@@ -35,3 +37,41 @@ static inline void print_with_escapes(const unsigned char *src) {
3537
}
3638
}
3739

40+
static inline void print_with_escapes(const unsigned char *src, std::ostream &os) {
41+
while (*src) {
42+
switch (*src) {
43+
case '\n':
44+
os << '\\';
45+
os << 'n';
46+
break;
47+
case '\r':
48+
os << '\\';
49+
os << 'r';
50+
break;
51+
case '\"':
52+
os << '\\';
53+
os << '"';
54+
break;
55+
case '\t':
56+
os << '\\';
57+
os << 't';
58+
break;
59+
case '\\':
60+
os << '\\';
61+
os << '\\';
62+
break;
63+
default:
64+
if (*src <= 0x1F) {
65+
std::ios::fmtflags f(os.flags());
66+
os << std::hex << std::setw(4) << std::setfill('0') << (int) *src;
67+
os.flags(f);
68+
} else
69+
os << *src;
70+
}
71+
src++;
72+
}
73+
}
74+
75+
static inline void print_with_escapes(const char *src, std::ostream &os) {
76+
print_with_escapes((const unsigned char *)src, os);
77+
}

‎include/simdjson/parsedjson.h‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -264,16 +264,16 @@ struct ParsedJson {
264264
printf("false\n");
265265
break;
266266
case '{': // we have an object
267-
printf("{\n");
267+
printf("{\t// pointing to next tape location %llu \n", payload);
268268
break;
269269
case '}': // we end an object
270-
printf("}\n");
270+
printf("}\t// pointing to previous tape location %llu \n", payload);
271271
break;
272272
case '[': // we start an array
273-
printf("[\n");
273+
printf("[\t// pointing to next tape location %llu \n", payload);
274274
break;
275275
case ']': // we end an array
276-
printf("]\n");
276+
printf("]\t// pointing to previous tape location %llu \n", payload);
277277
break;
278278
case 'r': // we start and end with the root node
279279
printf("end of root\n");
@@ -368,7 +368,7 @@ struct ParsedJson {
368368
current_val(o.current_val), depthindex(o.depthindex) {
369369
o.depthindex = NULL;// we take ownship
370370
}
371-
371+
WARN_UNUSED
372372
bool isOk() const {
373373
return location < tape_length;
374374
}
@@ -384,7 +384,9 @@ struct ParsedJson {
384384
// return true if we can do the navigation, false
385385
// otherwise
386386

387-
// valid if we're not at the end of a scope
387+
// withing a give scope, we move forward
388+
// valid if we're not at the end of a scope (returns true)
389+
WARN_UNUSED
388390
really_inline bool next() {
389391
if ((current_type == '[') || (current_type == '{')){
390392
// we need to jump
@@ -417,6 +419,7 @@ struct ParsedJson {
417419
}
418420

419421
// valid if we're not at the start of a scope
422+
WARN_UNUSED
420423
really_inline bool prev() {
421424
if(location - 1 < depthindex[depth]) return false;
422425
location -= 1;
@@ -437,6 +440,7 @@ struct ParsedJson {
437440

438441

439442
// valid unless we are at the first level of the document
443+
WARN_UNUSED
440444
really_inline bool up() {
441445
if(depth == 1) {
442446
return false; // don't allow moving back to root
@@ -451,11 +455,16 @@ struct ParsedJson {
451455
}
452456

453457

454-
// valid if we're at a [ or { call site; moves us to start of
455-
// that scope
458+
// valid if we're at a [ or { and it starts a non-empty scope; moves us to start of
459+
// that deeper scope if it not empty
460+
WARN_UNUSED
456461
really_inline bool down() {
457462
if(location + 1 >= tape_length) return false;
458463
if ((current_type == '[') || (current_type == '{')) {
464+
size_t npos = (current_val & JSONVALUEMASK);
465+
if(npos == location + 2) {
466+
return false; // we have an empty scope
467+
}
459468
depth++;
460469
location = location + 1;
461470
depthindex[depth] = location;
@@ -477,17 +486,24 @@ struct ParsedJson {
477486
// the start of our current scope; always succeeds
478487

479488
// print the thing we're currently pointing at
480-
bool print(std::ostream &os) const {
489+
bool print(std::ostream &os, bool escape_strings = true) const {
481490
if(!isOk()) return false;
482491
switch (current_type) {
483492
case '"': // we have a string
484-
os << '"' << get_string() << '"';
493+
os << '"';
494+
if(escape_strings) {
495+
print_with_escapes(get_string(), os);
496+
} else {
497+
os << get_string();
498+
}
499+
os << '"';
485500
break;
486501
case 'l': // we have a long int
487502
os << get_integer();
488503
break;
489504
case 'd':
490-
os << get_double();
505+
os << get_double();
506+
break;
491507
case 'n': // we have a null
492508
os << "null";
493509
break;

‎scripts/testjson2json.sh‎

Lines changed: 61 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@
22

33
TMPDIR1=$(mktemp -d -t simdjsonXXXXXXXX)
44
TMPDIR2=$(mktemp -d -t simdjsonXXXXXXXX)
5-
5+
TMPDIR3=$(mktemp -d -t simdjsonXXXXXXXX)
6+
TMPDIR4=$(mktemp -d -t simdjsonXXXXXXXX)
67
trap "exit 1" HUP INT PIPE QUIT TERM
7-
trap "rm -rf $TMPDIR1 $TMPDIR2" EXIT
8+
trap "rm -rf $TMPDIR1 $TMPDIR2 $TMPDIR3 $TMPDIR4" EXIT
89

910
function founderror() {
1011
echo "code is wrong"
@@ -15,26 +16,84 @@ make minify json2json
1516
for i in `cd jsonexamples && ls -1 *.json`; do
1617
echo $i
1718
./json2json jsonexamples/$i > $TMPDIR1/$i
19+
./json2json -a jsonexamples/$i > $TMPDIR3/$i
1820
./json2json $TMPDIR1/$i > $TMPDIR2/$i
21+
./json2json -a $TMPDIR3/$i > $TMPDIR4/$i
1922
cmp $TMPDIR1/$i $TMPDIR2/$i
2023
retVal=$?
2124
if [ $retVal -ne 0 ]; then
2225
founderror
2326
fi
27+
cmp $TMPDIR3/$i $TMPDIR4/$i
28+
retVal=$?
29+
if [ $retVal -ne 0 ]; then
30+
founderror
31+
fi
2432
./minify $TMPDIR1/$i > $TMPDIR1/minify$i
2533
./minify $TMPDIR2/$i > $TMPDIR2/minify$i
34+
./minify $TMPDIR3/$i > $TMPDIR3/minify$i
35+
./minify $TMPDIR4/$i > $TMPDIR4/minify$i
2636
cmp $TMPDIR1/minify$i $TMPDIR2/minify$i
2737
retVal=$?
2838
if [ $retVal -ne 0 ]; then
2939
founderror
3040
fi
41+
cmp $TMPDIR3/minify$i $TMPDIR4/minify$i
42+
retVal=$?
43+
if [ $retVal -ne 0 ]; then
44+
founderror
45+
fi
3146
./json2json $TMPDIR1/minify$i > $TMPDIR2/bisminify$i
3247
cmp $TMPDIR1/$i $TMPDIR2/bisminify$i
3348
retVal=$?
3449
if [ $retVal -ne 0 ]; then
3550
founderror
3651
fi
3752
done
53+
54+
for i in `cd jsonchecker && ls -1 pass*.json`; do
55+
echo $i
56+
./json2json jsonchecker/$i > $TMPDIR1/$i
57+
./json2json -a jsonchecker/$i > $TMPDIR3/$i
58+
./json2json $TMPDIR1/$i > $TMPDIR2/$i
59+
./json2json -a $TMPDIR3/$i > $TMPDIR4/$i
60+
cmp $TMPDIR1/$i $TMPDIR2/$i
61+
retVal=$?
62+
if [ $retVal -ne 0 ]; then
63+
echo "reg failure"
64+
founderror
65+
fi
66+
cmp $TMPDIR3/$i $TMPDIR4/$i
67+
retVal=$?
68+
if [ $retVal -ne 0 ]; then
69+
echo "-a failure"
70+
founderror
71+
fi
72+
./minify $TMPDIR1/$i > $TMPDIR1/minify$i
73+
./minify $TMPDIR2/$i > $TMPDIR2/minify$i
74+
./minify $TMPDIR3/$i > $TMPDIR3/minify$i
75+
./minify $TMPDIR4/$i > $TMPDIR4/minify$i
76+
cmp $TMPDIR1/minify$i $TMPDIR2/minify$i
77+
retVal=$?
78+
if [ $retVal -ne 0 ]; then
79+
echo "reg failure, step 2"
80+
founderror
81+
fi
82+
cmp $TMPDIR3/minify$i $TMPDIR4/minify$i
83+
retVal=$?
84+
if [ $retVal -ne 0 ]; then
85+
echo "-a failure, step 2"
86+
founderror
87+
fi
88+
./json2json $TMPDIR1/minify$i > $TMPDIR2/bisminify$i
89+
cmp $TMPDIR1/$i $TMPDIR2/bisminify$i
90+
retVal=$?
91+
if [ $retVal -ne 0 ]; then
92+
founderror
93+
fi
94+
done
95+
96+
3897
echo "test successful"
3998

4099
exit 0

‎tools/json2json.cpp‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,14 @@ void compute_dump(ParsedJson::iterator &pjh) {
1414
return; // we are done
1515
}
1616
// we have either an array or an object
17-
pjh.down();
17+
bool goingdown = pjh.down();
18+
if(!goingdown) {
19+
// we have an empty scope
20+
if(inobject) std::cout<<"{}";
21+
else std::cout<<"[]";
22+
return;
23+
}
24+
// we have a non-empty scope and we are at the beginning of it
1825
if (inobject) {
1926
std::cout << "{";
2027
assert(pjh.get_type() == '"');
@@ -40,7 +47,7 @@ void compute_dump(ParsedJson::iterator &pjh) {
4047
}
4148
std::cout << "]";
4249
}
43-
pjh.up();
50+
assert(pjh.up());
4451
}
4552

4653
int main(int argc, char *argv[]) {

0 commit comments

Comments
 (0)