Merge branch 'master' into dlemire/backport_jkeiser_fix_for_issue1632

simdjson · lemire · Jul 23, 2021 · Jul 18, 2021 · Jul 18, 2021 · Jul 19, 2021
commit e8e731a31123503435f5418dda2cb404d2a88feb
diff --git a/include/simdjson/generic/ondemand/json_iterator-inl.h b/include/simdjson/generic/ondemand/json_iterator-inl.h
@@ -106,7 +106,7 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child
 
   // Now that we've considered the first value, we only increment/decrement for arrays/objects
   while (position() < end_position()) {
-    switch (*advance()) {
+    switch (*return_current_and_advance()) {
       case '[': case '{':
         logger::log_start_value(*this, "skip");
         _depth++;
@@ -186,9 +186,9 @@ simdjson_really_inline void json_iterator::abandon() noexcept {
   _depth = 0;
 }
 
-simdjson_really_inline const uint8_t *json_iterator::advance() noexcept {
+simdjson_really_inline const uint8_t *json_iterator::return_current_and_advance() noexcept {
   assert_more_tokens();
-  return token.advance();
+  return token.return_current_and_advance();
 }
 
 simdjson_really_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept {

diff --git a/include/simdjson/generic/ondemand/object-inl.h b/include/simdjson/generic/ondemand/object-inl.h
@@ -41,10 +41,45 @@ simdjson_really_inline simdjson_result<object> object::start_root(value_iterator
   SIMDJSON_TRY( iter.start_root_object().error() );
   return object(iter);
 }
+simdjson_really_inline error_code object::consume() noexcept {
+  if(iter.is_at_key()) {
+    /**
+     * whenever you are pointing at a key, calling skip_child() is
+     * unsafe because you will hit a string and you will assume that
+     * it is string value, and this mistake will lead you to make bad
+     * depth computation.
+     */
+    /**
+     * We want to 'consume' the key. We could really
+     * just do _json_iter->return_current_and_advance(); at this
+     * point, but, for clarity, we will use the high-level API to
+     * eat the key. We assume that the compiler optimizes away
+     * most of the work.
+     */
+    simdjson_unused raw_json_string actual_key;
+    auto error = iter.field_key().get(actual_key);
+    if (error) { iter.abandon(); return error; };
+    // Let us move to the value while we are at it.
+    if ((error = iter.field_value())) { iter.abandon(); return error; }
+  }
+  auto error_skip = iter.json_iter().skip_child(iter.depth()-1);
+  if(error_skip) { iter.abandon(); }
+  return error_skip;
+}
+
+simdjson_really_inline simdjson_result<std::string_view> object::raw_json() noexcept {
+  const uint8_t * starting_point{iter.peek_start()};
+  auto error = consume();
+  if(error) { return error; }
+  const uint8_t * final_point{iter._json_iter->peek(0)};
+  return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
+}
+
 simdjson_really_inline simdjson_result<object> object::started(value_iterator &iter) noexcept {
   SIMDJSON_TRY( iter.started_object().error() );
   return object(iter);
 }
+
 simdjson_really_inline object object::resume(const value_iterator &iter) noexcept {
   return iter;
 }

diff --git a/include/simdjson/generic/ondemand/token_iterator-inl.h b/include/simdjson/generic/ondemand/token_iterator-inl.h
@@ -14,7 +14,7 @@ simdjson_really_inline uint32_t token_iterator::current_offset() const noexcept
 }
 
 
-simdjson_really_inline const uint8_t *token_iterator::advance() noexcept {
+simdjson_really_inline const uint8_t *token_iterator::return_current_and_advance() noexcept {
   return &buf[*(_position++)];
 }
 

diff --git a/include/simdjson/generic/ondemand/value_iterator-inl.h b/include/simdjson/generic/ondemand/value_iterator-inl.h
@@ -27,7 +27,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
 #endif
   if (*_json_iter->peek() == '}') {
     logger::log_value(*_json_iter, "empty object");
-    _json_iter->advance();
+    _json_iter->return_current_and_advance();
     end_container();
     return false;
   }
@@ -55,7 +55,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
 
   // It's illegal to call this unless there are more tokens: anything that ends in } or ] is
   // obligated to verify there are more tokens if they are not the top level.
-  switch (*_json_iter->advance()) {
+  switch (*_json_iter->return_current_and_advance()) {
     case '}':
       logger::log_end_value(*_json_iter, "object");
       SIMDJSON_TRY( end_container() );
@@ -351,15 +351,15 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
 simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::field_key() noexcept {
   assert_at_next();
 
-  const uint8_t *key = _json_iter->advance();
+  const uint8_t *key = _json_iter->return_current_and_advance();
   if (*(key++) != '"') { return report_error(TAPE_ERROR, "Object key is not a string"); }
   return raw_json_string(key);
 }
 
 simdjson_warn_unused simdjson_really_inline error_code value_iterator::field_value() noexcept {
   assert_at_next();
 
-  if (*_json_iter->advance() != ':') { return report_error(TAPE_ERROR, "Missing colon in object field"); }
+  if (*_json_iter->return_current_and_advance() != ':') { return report_error(TAPE_ERROR, "Missing colon in object field"); }
   _json_iter->descend_to(depth()+1);
   return SUCCESS;
 }
@@ -385,7 +385,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   assert_at_container_start();
   if (*_json_iter->peek() == ']') {
     logger::log_value(*_json_iter, "empty array");
-    _json_iter->advance();
+    _json_iter->return_current_and_advance();
     SIMDJSON_TRY( end_container() );
     return false;
   }
@@ -407,7 +407,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
   assert_at_next();
 
   logger::log_event(*this, "has_next_element");
-  switch (*_json_iter->advance()) {
+  switch (*_json_iter->return_current_and_advance()) {
     case ']':
       logger::log_end_value(*_json_iter, "array");
       SIMDJSON_TRY( end_container() );
@@ -595,10 +595,9 @@ simdjson_really_inline error_code value_iterator::start_container(uint8_t start_
     json = peek_start();
   } else {
     assert_at_start();
-    json = _json_iter->advance();
+    json = _json_iter->return_current_and_advance();
   }
 
-  // Get the JSON and advance the cursor
   if (*json != start_char) { return incorrect_type_error(incorrect_type_message); }
   return SUCCESS;
 }

diff --git a/tests/ondemand/ondemand_parse_api_tests.cpp b/tests/ondemand/ondemand_parse_api_tests.cpp
@@ -201,8 +201,7 @@ namespace parse_api_tests {
 
     std::cout << "truncated document " << std::endl;
     ASSERT_SUCCESS( parser.iterate(jsonbad).get(doc) );
-
-    ASSERT_EQUAL( simdjson::to_string(doc).get(output), INCOMPLETE_ARRAY_OR_OBJECT );
+    ASSERT_EQUAL( simdjson::to_json_string(doc).get(output), TAPE_ERROR );
 
     std::cout << "correct document with new doc" << std::endl;
     ondemand::document doc2;
@@ -221,7 +220,7 @@ namespace parse_api_tests {
 
     std::cout << "unclosed string document " << std::endl;
     ASSERT_SUCCESS( parser.iterate(jsonbad).get(doc) );
-    ASSERT_EQUAL( simdjson::to_string(doc).get(output), INCOMPLETE_ARRAY_OR_OBJECT );
+    ASSERT_EQUAL( simdjson::to_json_string(doc).get(output), TAPE_ERROR );
 
     // next two lines are terrible code.
     doc.~document();
-Original file line number
+Diff line change
@@ Expand Up @@
     }
-    simdjson_really_inline const uint8_t *token_iterator::advance() noexcept {
+    simdjson_really_inline const uint8_t *token_iterator::return_current_and_advance() noexcept {
       return &buf[*(_position++)];
     }
@@ Expand Down @@