Skip to content

Isolated jkeiser fix for issue 1632: make it so that INCORRECT_TYPE is a recoverable condition in On Demand #1663

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lemire merged 18 commits into from
Jul 23, 2021
Merged

Isolated jkeiser fix for issue 1632: make it so that INCORRECT_TYPE is a recoverable condition in On Demand #1663

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
4b118bd
Verifying issue.
lemire Jul 18, 2021
267e70d
Adding another case.
lemire Jul 18, 2021
3300d84
Adding type test.
lemire Jul 19, 2021
3c5c2d2
This backports (isolate) a fix by jkeiser of issue 1632.
lemire Jul 19, 2021
f364e35
Minor fixes.
lemire Jul 19, 2021
e8e731a
Merge branch 'master' into dlemire/backport_jkeiser_fix_for_issue1632
lemire Jul 19, 2021
a349969
Adding a duplicate of the existing test.
lemire Jul 19, 2021
8b2bff9
Messy temporary work.
lemire Jul 19, 2021
df24da3
Cleaning.
lemire Jul 19, 2021
4501aeb
Disabling a couple of asserts.
lemire Jul 19, 2021
2e688a2
Removing two other asserts.
lemire Jul 19, 2021
9cace04
Adding more tests.
lemire Jul 19, 2021
d0e2a7f
Minor fixes
lemire Jul 19, 2021
c3f7eff
Adding some commentary
lemire Jul 19, 2021
24068a0
Merge branch 'master' into dlemire/backport_jkeiser_fix_for_issue1632
lemire Jul 22, 2021
f2d419d
Basic guarding.
lemire Jul 22, 2021
1fe50cb
Trimming.
lemire Jul 22, 2021
d7770e9
Disabling 'json[N-1]' test.
lemire Jul 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Minor fixes
  • Loading branch information
@lemire
lemire committed Jul 19, 2021
commit d0e2a7fb005af1b7d149a818c084ded18c2cdef6
2 changes: 1 addition & 1 deletion include/simdjson/common_defs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ namespace std {
#endif

// Feature flag for partially-implemented "don't require padding" feature
// TODO remove before 1.0
// TODO remove once feature complete.
#ifndef __SIMDJSON_CHECK_EOF
# define __SIMDJSON_CHECK_EOF 1
#endif
Expand Down
8 changes: 8 additions & 0 deletions include/simdjson/generic/ondemand/json_iterator-inl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,12 +189,16 @@ simdjson_really_inline void json_iterator::abandon() noexcept {
simdjson_really_inline const uint8_t *json_iterator::return_current_and_advance() noexcept {
// The following assert_more_tokens is currently disabled because rely on end-of-file buffering.
// assert_more_tokens();
// This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
// is ON by default, we have no choice but to disable it for real with a comment.
return token.return_current_and_advance();
}

simdjson_really_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept {
// The following assert_more_tokens is currently disabled because rely on end-of-file buffering.
// assert_more_tokens(delta+1);
// This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
// is ON by default, we have no choice but to disable it for real with a comment.
return token.peek(delta);
}

Expand All @@ -207,13 +211,17 @@ simdjson_really_inline const uint8_t *json_iterator::peek(token_position positio
// todo: currently we require end-of-string buffering, but the following
// assert_valid_position should be turned on if/when we lift that condition.
// assert_valid_position(position);
// This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
// is ON by default, we have no choice but to disable it for real with a comment.
return token.peek(position);
}

simdjson_really_inline uint32_t json_iterator::peek_length(token_position position) const noexcept {
// todo: currently we require end-of-string buffering, but the following
// assert_valid_position should be turned on if/when we lift that condition.
// assert_valid_position(position);
// This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
// is ON by default, we have no choice but to disable it for real with a comment.
return token.peek_length(position);
}

Expand Down
19 changes: 6 additions & 13 deletions tests/ondemand/ondemand_array_error_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ namespace array_error_tests {
TEST_START();
ONDEMAND_SUBTEST("missing comma", "[1 1]", assert_iterate(doc, { int64_t(1) }, { TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[1,,1]", assert_iterate(doc, { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[,]", assert_iterate(doc, { INCORRECT_TYPE }));
ONDEMAND_SUBTEST("extra comma ", "[,,]", assert_iterate(doc, { INCORRECT_TYPE, INCORRECT_TYPE, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[,]", assert_iterate(doc, { INCORRECT_TYPE, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", "[,,]", assert_iterate(doc, { INCORRECT_TYPE, TAPE_ERROR }));
TEST_SUCCEED();
}
bool top_level_array_iterate_unclosed_error() {
Expand All @@ -81,9 +81,9 @@ namespace array_error_tests {
TEST_START();
ONDEMAND_SUBTEST("missing comma", R"({ "a": [1 1] })", assert_iterate(doc["a"], { int64_t(1) }, { TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [1,,1] })", assert_iterate(doc["a"], { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [1,,] })", assert_iterate(doc["a"], { int64_t(1) }, { INCORRECT_TYPE }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [,] })", assert_iterate(doc["a"], { INCORRECT_TYPE }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [,,] })", assert_iterate(doc["a"], { INCORRECT_TYPE, INCORRECT_TYPE, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [1,,] })", assert_iterate(doc["a"], { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR }));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [,] })", assert_iterate(doc["a"], { INCORRECT_TYPE, TAPE_ERROR}));
ONDEMAND_SUBTEST("extra comma ", R"({ "a": [,,] })", assert_iterate(doc["a"], { INCORRECT_TYPE, TAPE_ERROR }));
TEST_SUCCEED();
}
bool array_iterate_unclosed_error() {
Expand All @@ -100,17 +100,10 @@ namespace array_error_tests {
}
bool array_iterate_incomplete_error() {
TEST_START();
#if __SIMDJSON_CHECK_EOF
ONDEMAND_SUBTEST("unclosed after array", R"([ [1] )", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT }));
ONDEMAND_SUBTEST("unclosed after array", R"([ [1,])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed after array", R"([ [1,])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCORRECT_TYPE, INCOMPLETE_ARRAY_OR_OBJECT }));
ONDEMAND_SUBTEST("unclosed after array", R"([ [1])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT }));
ONDEMAND_SUBTEST("unclosed after array", R"([ [])", assert_iterate(doc.get_array().at(0), { INCOMPLETE_ARRAY_OR_OBJECT }));
#else
ONDEMAND_SUBTEST("unclosed after array", R"({ "a": [1] )", assert_iterate(doc["a"], { int64_t(1) } ));
ONDEMAND_SUBTEST("unclosed after array", R"({ "a": [1,])", assert_iterate(doc["a"], { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR }));
ONDEMAND_SUBTEST("unclosed after array", R"({ "a": [1])", assert_iterate(doc["a"], { int64_t(1) } ));
ONDEMAND_SUBTEST("unclosed after array", R"({ "a": [])", assert_iterate(doc["a"], ));
#endif
TEST_SUCCEED();
}

Expand Down