Skip to content

Commit 176d2cc

Browse files
lemirelemire
committed
Tweaking.
1 parent 52c4b65 commit 176d2cc

6 files changed

Lines changed: 26 additions & 42 deletions

File tree

‎benchmark/parse.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ int main(int argc, char *argv[]) {
181181
}
182182
ParsedJson pj = build_parsed_json(p); // do the parsing again to get the stats
183183
if( ! pj.isValid() ) {
184-
std::cerr << "could not parse again? " << std::endl;
184+
std::cerr << "Could not parse. " << std::endl;
185185
return EXIT_FAILURE;
186186
}
187187
#ifndef SQUASH_COUNTERS

‎include/simdjson/numberparsing.h‎

Lines changed: 12 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -159,17 +159,16 @@ static inline uint32_t parse_eight_digits_unrolled(const char *chars) {
159159
// Note: a redesign could avoid this function entirely.
160160
//
161161
static never_inline bool
162-
parse_highprecision_float(const u8 *const buf,
162+
parse_float(const u8 *const buf,
163163
ParsedJson &pj, const u32 offset,
164164
bool found_minus) {
165165
const char *p = (const char *)(buf + offset);
166-
167166
bool negative = false;
168167
if (found_minus) {
169168
++p;
170169
negative = true;
171170
}
172-
long double i;
171+
double i;
173172
if (*p == '0') { // 0 cannot be followed by an integer
174173
++p;
175174
i = 0;
@@ -183,26 +182,15 @@ parse_highprecision_float(const u8 *const buf,
183182
++p;
184183
}
185184
}
186-
187-
int64_t exponent = 0;
188-
189185
if ('.' == *p) {
190186
++p;
191-
const char *const firstafterperiod = p;
192-
#ifdef SWAR_NUMBER_PARSING
193-
// this helps if we have lots of decimals!
194-
// this turns out to be frequent enough.
195-
if (is_made_of_eight_digits_fast(p)) {
196-
i = i * 100000000 + parse_eight_digits_unrolled(p);
197-
p += 8;
198-
}
199-
#endif
187+
double fractionalweight = 1;
200188
while (is_integer(*p)) {
201189
unsigned char digit = *p - '0';
202190
++p;
203-
i = i * 10 + digit;
191+
fractionalweight *= 0.1;
192+
i = i + digit * fractionalweight;
204193
}
205-
exponent = firstafterperiod - p;
206194
}
207195
if (('e' == *p) || ('E' == *p)) {
208196
++p;
@@ -244,29 +232,21 @@ parse_highprecision_float(const u8 *const buf,
244232
#endif
245233
return false;
246234
}
247-
exponent += (negexp ? -expnumber : expnumber);
248-
}
249-
if (i == 0) {
250-
pj.write_tape_double(0.0);
251-
#ifdef JSON_TEST_NUMBERS // for unit testing
252-
foundFloat(0.0, buf + offset);
253-
#endif
254-
} else {
235+
int exponent = (negexp ? -expnumber : expnumber);
255236
if ((exponent > 308) || (exponent < -308)) {
256237
// we refuse to parse this
257238
#ifdef JSON_TEST_NUMBERS // for unit testing
258239
foundInvalidNumber(buf + offset);
259240
#endif
260241
return false;
261242
}
262-
double d = i;
263-
d *= power_of_ten[308 + exponent];
264-
d = negative ? -d : d;
265-
pj.write_tape_double(d);
243+
i *= power_of_ten[308 + exponent];
244+
}
245+
double d = negative ? -i : i;
246+
pj.write_tape_double(d);
266247
#ifdef JSON_TEST_NUMBERS // for unit testing
267-
foundFloat(d, buf + offset);
248+
foundFloat(d, buf + offset);
268249
#endif
269-
}
270250
return true;
271251
}
272252

@@ -472,7 +452,7 @@ static really_inline bool parse_number(const u8 *const buf,
472452
if (unlikely(digitcount >= 19)) { // this is uncommon!!!
473453
// this is almost never going to get called!!!
474454
// we start anew, going slowly!!!
475-
return parse_highprecision_float(buf, pj, offset,
455+
return parse_float(buf, pj, offset,
476456
found_minus);
477457
}
478458
///////////

‎jsonchecker/fail37.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[12a]

‎jsonchecker/pass05.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
12345678900000002170460276904689664.000000

‎src/stage2_flatten.cpp‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,7 @@ bool flatten_indexes(size_t len, ParsedJson &pj) {
119119
}
120120
pj.n_structural_indexes = base;
121121
if(len != base_ptr[pj.n_structural_indexes-1]) {
122-
// can happen with malformed JSON such as unclosed quotes (["this is an unclosed string ])
123-
return false;
122+
base_ptr[pj.n_structural_indexes++] = len;
124123
}
125124
base_ptr[pj.n_structural_indexes] = 0; // make it safe to dereference one beyond this array
126125
return true;

‎tests/numberparsingcheck.cpp‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,17 @@ inline void foundFloat(double result, const u8 *buf) {
7171
printf(" while parsing %s \n", fullpath);
7272
parse_error |= PARSE_ERROR;
7373
}
74+
if( fpclassify(expected) != fpclassify(result) ) {
75+
printf("floats not in the same category expected: %f observed: %f \n", expected, result);
76+
printf("%.128s\n", buf);
77+
parse_error |= PARSE_ERROR;
78+
}
7479
// we want to get some reasonable relative accuracy
75-
if (fabs(expected - result) / fmin(fabs(expected), fabs(result)) >
76-
0.000000000000001) {
77-
printf("parsed %.32f from \n", result);
78-
printf(" %.32s whereas strtod gives\n", buf);
79-
printf(" %.32f,", expected);
80+
else if (fabs(expected - result) / fmin(fabs(expected), fabs(result)) >
81+
1e-14) {
82+
printf("parsed %.128e from \n", result);
83+
printf(" %.200s whereas strtod gives\n", buf);
84+
printf(" %.128e,", expected);
8085
printf(" while parsing %s \n", fullpath);
8186
parse_error |= PARSE_ERROR;
8287
}
@@ -96,8 +101,6 @@ static bool hasExtension(const char *filename, const char *extension) {
96101
bool validate(const char *dirname) {
97102
parse_error = 0;
98103
size_t total_count = 0;
99-
100-
// init_state_machine(); // no longer necessary
101104
const char *extension = ".json";
102105
size_t dirlen = strlen(dirname);
103106
struct dirent **entry_list;

0 commit comments

Comments
 (0)