Ok, the new code seems quite fast.

lemire · lemire · commit 12b518578d95 · 2018-11-29T22:15:02.000-05:00
diff --git a/benchmark/parsingcompetition.cpp b/benchmark/parsingcompetition.cpp
@@ -121,5 +121,6 @@ int main(int argc, char *argv[]) {
   BEST_TIME("memcpy            ", (memcpy(buffer, p.first, p.second) == buffer), true, , repeat, volume, true);
   free(p.first);
   free(ast_buffer);
+  free(buffer);
   deallocate_ParsedJson(pj_ptr);
 }
diff --git a/include/jsonparser/simdjson_internal.h b/include/jsonparser/simdjson_internal.h
@@ -64,7 +64,8 @@ struct ParsedJson {
   size_t bytecapacity;  // indicates how many bits are meant to be supported by
                         // structurals
   size_t depthcapacity; // how deep we can go
-
+  size_t tapecapacity;
+  size_t stringcapacity; 
   u32 current_loc;
   u8 *structurals;
   u32 n_structural_indexes;
@@ -77,19 +78,15 @@ struct ParsedJson {
   u8 *string_buf; // should be at least bytecapacity
   u8 *current_string_buf_loc;
 
-  u8 *number_buf; // holds either doubles or longs, really // should be at least
-                  // 4 * bytecapacity
-  u8 *current_number_buf_loc;
-
   // this should be called when parsing (right before writing the tapes)
   void init() {
     current_string_buf_loc = string_buf;
-    current_number_buf_loc = number_buf;
     current_loc = 0;
   }
 
   // print the json to stdout (should be valid)
-  void printjson() {
+  // return false if the tape is likely wrong (e.g., you did not parse a valid JSON).
+  bool printjson() {
     size_t tapeidx = 0;
     u64 tape_val = tape[tapeidx];
     u8 type = (tape_val >> 56);
@@ -98,15 +95,17 @@ struct ParsedJson {
       howmany = tape_val & JSONVALUEMASK;
     } else {
       printf("Error: no starting root node?");
-      abort();
+      return false;
+    }
+    if(howmany > tapecapacity)  {
+      printf("We may be exceeding the tape capacity. Is this a valid document?\n");
+      return false;
     }
     tapeidx++;
     bool *inobject = new bool[depthcapacity];
     size_t *inobjectidx = new size_t[depthcapacity];
     int depth = 1; // only root at level 0
     inobjectidx[depth] = 0;
-    int64_t intval;
-    double doubleval;
     for (; tapeidx < howmany; tapeidx++) {
       tape_val = tape[tapeidx];
       u64 payload = tape_val & JSONVALUEMASK;
@@ -130,12 +129,12 @@ struct ParsedJson {
         putchar('"');
         break;
       case 'l': // we have a long int
-        memcpy(&intval, number_buf + payload, sizeof(intval));
-        printf("%" PRId64, intval);
+        if(tapeidx + 1 >= howmany) return false;
+        printf("%" PRId64, (int64_t) tape[tapeidx++]);
         break;
       case 'd': // we have a double
-        memcpy(&doubleval, number_buf + payload, sizeof(doubleval));
-        printf("%f", doubleval);
+        if(tapeidx + 1 >= howmany) return false;
+        printf("%f", *((double * )& tape[tapeidx++]));
         break;
       case 'n': // we have a null
         printf("null");
@@ -170,11 +169,13 @@ struct ParsedJson {
         break;
       case 'r': // we start and end with the root node
         printf("should we be hitting the root node?\n");
+        return false;
       default:
         printf("bug %c\n", type);
-        abort();
+        return false;
       }
     }
+    return true;
   }
 
   // all elements are stored on the tape using a 64-bit word.
@@ -197,15 +198,13 @@ struct ParsedJson {
   }
 
   really_inline void write_tape_s64(s64 i) {
-    write_tape(current_number_buf_loc - number_buf, 'l');
-    memcpy(current_number_buf_loc, &i, sizeof(s64));
-    current_number_buf_loc += sizeof(s64);
+    write_tape(0, 'l');
+    tape[current_loc++] =*( (u64*) &i);
   }
 
   really_inline void write_tape_double(double d) {
-    write_tape(current_number_buf_loc - number_buf, 'd');
-    memcpy(current_number_buf_loc, &d, sizeof(double));
-    current_number_buf_loc += sizeof(double);
+    write_tape(0, 'd');
+    tape[current_loc++] =*( (u64*) &d);
   }
 
   really_inline u32 get_current_loc() { return current_loc; }
diff --git a/src/jsonparser.cpp b/src/jsonparser.cpp
@@ -33,20 +33,20 @@ ParsedJson *allocate_ParsedJson(size_t len, size_t maxdepth) {
     delete pj_ptr;
     return NULL;
   }
-  pj.string_buf = new u8[ROUNDUP_N(len, 64)];
-  pj.number_buf = new u8[4 * ROUNDUP_N(len, 64)];
-  pj.tape = new u64[ROUNDUP_N(len, 64)];
+  size_t tapecapacity = ROUNDUP_N(len, 64); 
+  size_t stringcapacity = ROUNDUP_N(len, 64);
+  pj.string_buf = new u8[stringcapacity];
+  pj.tape = new u64[tapecapacity];
   pj.containing_scope_offset = new u32[maxdepth];
   pj.ret_address = new void*[maxdepth];
 
-  if ((pj.string_buf == NULL) || (pj.number_buf == NULL) || (pj.tape == NULL) 
+  if ((pj.string_buf == NULL) || (pj.tape == NULL) 
     || (pj.containing_scope_offset == NULL) || (pj.ret_address == NULL) )  {
     std::cerr << "Could not allocate memory"
               << std::endl;
     delete[] pj.ret_address;
     delete[] pj.containing_scope_offset;
     delete[] pj.tape;
-    delete[] pj.number_buf;
     delete[] pj.string_buf;
     delete[] pj.structural_indexes;
     delete[] pj.structurals;
@@ -56,6 +56,8 @@ ParsedJson *allocate_ParsedJson(size_t len, size_t maxdepth) {
 
   pj.bytecapacity = len;
   pj.depthcapacity =  maxdepth;
+  pj.tapecapacity = tapecapacity;
+  pj.stringcapacity = stringcapacity;
   return pj_ptr;
 }
 
@@ -65,7 +67,6 @@ void deallocate_ParsedJson(ParsedJson *pj_ptr) {
   delete[] pj_ptr->ret_address;
   delete[] pj_ptr->containing_scope_offset;
   delete[] pj_ptr->tape;
-  delete[] pj_ptr->number_buf;
   delete[] pj_ptr->string_buf;
   delete[] pj_ptr->structural_indexes;
   free(pj_ptr->structurals);

Original file line number	Diff line number	Diff line change
`@@ -121,5 +121,6 @@ int main(int argc, char *argv[]) {`
`121`	`121`	`BEST_TIME("memcpy ", (memcpy(buffer, p.first, p.second) == buffer), true, , repeat, volume, true);`
`122`	`122`	`free(p.first);`
`123`	`123`	`free(ast_buffer);`
	`124`	`+ free(buffer);`
`124`	`125`	`deallocate_ParsedJson(pj_ptr);`
`125`	`126`	`}`