Still working

lemire · lemire · commit 5fae7b2100f8 · 2018-11-27T10:10:39.000-05:00
diff --git a/benchmark/minifiercompetition.cpp b/benchmark/minifiercompetition.cpp
@@ -113,11 +113,11 @@ int main(int argc, char *argv[]) {
 
   BEST_TIME("sajson despaced", sajson::parse(sajson::bounded_allocation(ast_buffer, astbuffersize), sajson::mutable_string_view(minisize, buffer)).is_valid(), true, memcpy(buffer, minibuffer, p.second), repeat, volume, true);
 
-  ParsedJson *pj_ptr = allocate_ParsedJson(p.second);
+  ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024);
   ParsedJson &pj(*pj_ptr);
   BEST_TIME("json_parse orig", json_parse((const u8*)buffer, p.second, pj), true, memcpy(buffer, p.first, p.second), repeat, volume, true);
   
-  ParsedJson *pj_ptr2 = allocate_ParsedJson(p.second);
+  ParsedJson *pj_ptr2 = allocate_ParsedJson(p.second, 1024);
   ParsedJson &pj2(*pj_ptr2);
 
 
diff --git a/benchmark/parse.cpp b/benchmark/parse.cpp
@@ -126,7 +126,7 @@ int main(int argc, char *argv[]) {
   if(verbose) cout << "[verbose] loading " << filename << endl;
   pair<u8 *, size_t> p = get_corpus(filename);
   if(verbose) cout << "[verbose] loaded " << filename << " ("<< p.second << " bytes)" << endl;
-  ParsedJson *pj_ptr = allocate_ParsedJson(p.second);
+  ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024);
   ParsedJson &pj(*pj_ptr);
   if(verbose) cout << "[verbose] allocated memory for parsed JSON " << endl;
 
diff --git a/benchmark/parsingcompetition.cpp b/benchmark/parsingcompetition.cpp
@@ -73,7 +73,7 @@ int main(int argc, char *argv[]) {
       std::cout << p.second << " B ";
     std::cout << std::endl;
   }
-  ParsedJson *pj_ptr = allocate_ParsedJson(p.second);
+  ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024);
   if (pj_ptr == NULL) {
     std::cerr << "can't allocate memory" << std::endl;
     return EXIT_FAILURE;
diff --git a/include/jsonparser/jsonparser.h b/include/jsonparser/jsonparser.h
@@ -12,7 +12,7 @@
 // Return NULL if memory cannot be allocated.
 // This structure is meant to be reused from document to document, as needed.
 // you can use deallocate_ParsedJson to deallocate the memory.
-ParsedJson *allocate_ParsedJson(size_t len);
+ParsedJson *allocate_ParsedJson(size_t len, size_t maxdepth);
 
 // deallocate a ParsedJson struct (see allocate_ParsedJson)
 void deallocate_ParsedJson(ParsedJson *pj_ptr);
diff --git a/include/jsonparser/numberparsing.h b/include/jsonparser/numberparsing.h
@@ -237,7 +237,7 @@ parse_highprecision_float(const u8 *const buf, UNUSED size_t len,
     exponent += (negexp ? -expnumber : expnumber);
   }
   if (i == 0) {
-    pj.write_tape_double(depth, 0.0);
+    pj.write_tape_double(0.0);
 #ifdef JSON_TEST_NUMBERS // for unit testing
     foundFloat(0.0, buf + offset);
 #endif
@@ -252,7 +252,7 @@ parse_highprecision_float(const u8 *const buf, UNUSED size_t len,
     double d = i;
     d *= power_of_ten[308 + exponent];
     d = negative ? -d : d;
-    pj.write_tape_double(depth, d);
+    pj.write_tape_double(d);
 #ifdef JSON_TEST_NUMBERS // for unit testing
     foundFloat(d, buf + offset);
 #endif
@@ -325,7 +325,7 @@ static never_inline bool parse_large_integer(const u8 *const buf,
     }
   }
   int64_t signed_answer = negative ? -i : i;
-  pj.write_tape_s64(depth, signed_answer);
+  pj.write_tape_s64(signed_answer);
 #ifdef JSON_TEST_NUMBERS // for unit testing
   foundInteger(signed_answer, buf + offset);
 #endif
@@ -468,7 +468,7 @@ static really_inline bool parse_number(const u8 *const buf, UNUSED size_t len,
     // We want 0.1e1 to be a float.
     //////////
     if (i == 0) {
-      pj.write_tape_double(depth, 0.0);
+      pj.write_tape_double(0.0);
 #ifdef JSON_TEST_NUMBERS // for unit testing
       foundFloat(0.0, buf + offset);
 #endif
@@ -483,7 +483,7 @@ static really_inline bool parse_number(const u8 *const buf, UNUSED size_t len,
       double d = i;
       d *= power_of_ten[308 + exponent];
       // d = negative ? -d : d;
-      pj.write_tape_double(depth, d);
+      pj.write_tape_double(d);
 #ifdef JSON_TEST_NUMBERS // for unit testing
       foundFloat(d, buf + offset);
 #endif
@@ -493,7 +493,7 @@ static really_inline bool parse_number(const u8 *const buf, UNUSED size_t len,
       return parse_large_integer(buf, len, pj, depth, offset, found_zero,
                                  found_minus);
     }
-    pj.write_tape_s64(depth, i);
+    pj.write_tape_s64(i);
 #ifdef JSON_TEST_NUMBERS // for unit testing
     foundInteger(i, buf + offset);
 #endif
diff --git a/include/jsonparser/simdjson_internal.h b/include/jsonparser/simdjson_internal.h
@@ -32,20 +32,26 @@ struct ParsedJson {
   size_t bytecapacity; // indicates how many bits are meant to be supported by
                        // structurals
   size_t depthcapacity; // how deep we can go
+
+  u32 current_loc;
   u8 *structurals;
   u32 n_structural_indexes;
   u32 *structural_indexes;
 
   u64 * tape;//[MAX_TAPE];
-  u32 * tape_locs;
+  u32 * containing_scope_offset;
+  void * * ret_address;
+
   u8 * string_buf;// should be at least bytecapacity
+
   u8 *current_string_buf_loc;
   u8 * number_buf;// holds either doubles or longs, really // should be at least 4 * bytecapacity
   u8 *current_number_buf_loc;
     
     void init() {
         current_string_buf_loc = string_buf;
         current_number_buf_loc = number_buf;
+        current_loc = 0;
 
         //for (u32 i = 0; i < MAX_DEPTH; i++) {
         //    tape_locs[i] = i * MAX_TAPE_ENTRIES;
@@ -75,33 +81,53 @@ struct ParsedJson {
             }
         }*/
     }
-    // TODO: will need a way of saving strings that's a bit more encapsulated
 
-    void write_tape(u32 depth, u64 val, u8 c) {
-        tape[tape_locs[depth]] = val | (((u64)c) << 56);
-        tape_locs[depth]++;
+    // all elements are stored on the tape using a 64-bit word.
+    //
+    // strings, double and ints are stored as 
+    //  a 64-bit word with a pointer to the actual value
+    // 
+    //
+    // 
+    // for objects or arrays, store [ or {  at the beginning and } and ] at the end.
+    // For the openings ([ or {), we annotate them with a reference to the location on the tape of
+    // the end, and for then closings (} and ]), we annotate them with a reference to the 
+    // location of the opening
+    //
+    // 
+
+    // this should be considered a private function
+    void write_tape(u64 val, u8 c) {
+        tape[current_loc++] =  val | (((u64)c) << 56);
+        //tape[tape_locs[depth]] = val | (((u64)c) << 56);
+        //tape_locs[depth]++;
     }
 
-    void write_tape_s64(u32 depth, s64 i) {
+
+    void write_tape_s64(s64 i) {
         *((s64 *)current_number_buf_loc) = i;
         current_number_buf_loc += 8;
-        write_tape(depth, current_number_buf_loc - number_buf, 'l');
+        write_tape(current_number_buf_loc - number_buf, 'l');
     }
 
-    void write_tape_double(u32 depth, double d) {
+    void write_tape_double(double d) {
         *((double *)current_number_buf_loc) = d;
         current_number_buf_loc += 8;
-        write_tape(depth, current_number_buf_loc - number_buf, 'd');
+        write_tape(current_number_buf_loc - number_buf, 'd');
     }
 
-    u32 save_loc(u32 depth) {
-        return tape_locs[depth];
+    u32 get_current_loc() {
+        return current_loc;
     }
 
-    void write_saved_loc(u32 saved_loc, u64 val, u8 c) {
-        tape[saved_loc] = val | (((u64)c) << 56);
+    void annotate_previousloc(u32 saved_loc,u64 val) {
+        tape[saved_loc] |= val;
     }
 
+    /*void write_saved_loc(u32 saved_loc, u64 val, u8 c) {
+        tape[saved_loc] = val | (((u64)c) << 56);
+    }*/
+
     // public interface
 #if 1
     
@@ -121,13 +147,13 @@ struct ParsedJson {
         bool prev();                      // valid if we're not at the start of a scope
         bool up();                        // valid if we are at depth != 0
         bool down();                      // valid if we're at a [ or { call site; moves us to header of that scope
-        void to_start_scope();            // move us to the start of our current scope; always succeeds
-        void to_end_scope();              // move us to the start of our current scope; always succeeds
+        //void to_start_scope();            // move us to the start of our current scope; always succeeds
+        //void to_end_scope();              // move us to the start of our current scope; always succeeds
 
         // these navigation elements move us across scope if need be, so allow us to iterate over
         // everything at a given depth
-        bool next_flat();                 // valid if we're not at the end of a tape
-        bool prev_flat();                 // valid if we're not at the start of a tape
+        //bool next_flat();                 // valid if we're not at the end of a tape
+        //bool prev_flat();                 // valid if we're not at the start of a tape
 
         void print(std::ostream & os);    // print the thing we're currently pointing at
         u8 get_type();                    // retrieve the character code of what we're looking at: [{"sltfn are the possibilities
diff --git a/include/jsonparser/stringparsing.h b/include/jsonparser/stringparsing.h
@@ -114,7 +114,7 @@ really_inline  bool parse_string(const u8 *buf, UNUSED size_t len,
       // we encountered quotes first. Move dst to point to quotes and exit
       dst[quote_dist] = 0; // null terminate and get out
 
-      pj.write_tape(depth, pj.current_string_buf_loc - pj.string_buf, '"');
+      pj.write_tape(pj.current_string_buf_loc - pj.string_buf, '"');
 
       pj.current_string_buf_loc = dst + quote_dist + 1; // the +1 is due to the 0 value
 #ifdef CHECKUNESCAPED
diff --git a/src/jsonioutil.cpp b/src/jsonioutil.cpp
@@ -23,6 +23,7 @@ std::pair<u8 *, size_t> get_corpus(std::string filename) {
     size_t length = buffer.str().size(); // +1 for null
     u8* aligned_buffer = (u8 *)allocate_aligned_buffer(length);
     memcpy(aligned_buffer, buffer.str().c_str(), length);
+    aligned_buffer[length] = '\0';
     is.close();
     return std::make_pair((u8 *)aligned_buffer, length);
   }
diff --git a/src/jsonparser.cpp b/src/jsonparser.cpp
@@ -5,7 +5,11 @@
 // returns NULL if memory cannot be allocated
 // This structure is meant to be reused from document to document, as needed.
 // you can use deallocate_ParsedJson to deallocate the memory.
-ParsedJson *allocate_ParsedJson(size_t len) {
+ParsedJson *allocate_ParsedJson(size_t len, size_t maxdepth) {
+  if((maxdepth == 0) || (len == 0)) {
+    std::cerr << "capacities must be non-zero " << std::endl;
+    return NULL;
+  }
   ParsedJson *pj_ptr = new ParsedJson;
   if (pj_ptr == NULL) {
     std::cerr << "Could not allocate memory for core struct." << std::endl;
@@ -32,13 +36,15 @@ ParsedJson *allocate_ParsedJson(size_t len) {
   pj.string_buf = new u8[ROUNDUP_N(len, 64)];
   pj.number_buf = new u8[4 * ROUNDUP_N(len, 64)];
   pj.tape = new u64[ROUNDUP_N(len, 64)];
-  size_t depthcapacity = ROUNDUP_N(len, 64);
-  pj.tape_locs = new u32[depthcapacity];
+  pj.containing_scope_offset = new u32[maxdepth];
+  pj.ret_address = new void*[maxdepth];
 
-  if ((pj.string_buf == NULL) || (pj.number_buf == NULL) || (pj.tape == NULL) || (pj.tape_locs == NULL))  {
+  if ((pj.string_buf == NULL) || (pj.number_buf == NULL) || (pj.tape == NULL) 
+    || (pj.containing_scope_offset == NULL) || (pj.ret_address == NULL) )  {
     std::cerr << "Could not allocate memory"
               << std::endl;
-    delete[] pj.tape_locs;
+    delete[] pj.ret_address;
+    delete[] pj.containing_scope_offset;
     delete[] pj.tape;
     delete[] pj.number_buf;
     delete[] pj.string_buf;
@@ -49,14 +55,15 @@ ParsedJson *allocate_ParsedJson(size_t len) {
   }
 
   pj.bytecapacity = len;
-  pj.depthcapacity =  depthcapacity;
+  pj.depthcapacity =  maxdepth;
   return pj_ptr;
 }
 
 void deallocate_ParsedJson(ParsedJson *pj_ptr) {
   if (pj_ptr == NULL)
     return;
-  delete[] pj_ptr->tape_locs;
+  delete[] pj_ptr->ret_address;
+  delete[] pj_ptr->containing_scope_offset;
   delete[] pj_ptr->tape;
   delete[] pj_ptr->number_buf;
   delete[] pj_ptr->string_buf;
diff --git a/src/stage1_find_marks.cpp b/src/stage1_find_marks.cpp
@@ -251,7 +251,7 @@ WARN_UNUSED
     *(u64 *)(pj.structurals + idx / 8) = structurals;
   }
   if(buf[len] != '\0') {
-      std::cerr << "Your string should NULL terminated." << std::endl;
+      std::cerr << "Your string should be NULL terminated." << std::endl;
       return false;
 
   }
diff --git a/src/stage34_unified.cpp b/src/stage34_unified.cpp
diff --git a/tests/allparserscheckfile.cpp b/tests/allparserscheckfile.cpp
diff --git a/tests/jsoncheck.cpp b/tests/jsoncheck.cpp
diff --git a/tests/numberparsingcheck.cpp b/tests/numberparsingcheck.cpp
diff --git a/tests/stringparsingcheck.cpp b/tests/stringparsingcheck.cpp

Original file line number	Diff line number	Diff line change
`@@ -73,7 +73,7 @@ int main(int argc, char *argv[]) {`
`73`	`73`	`std::cout << p.second << " B ";`
`74`	`74`	`std::cout << std::endl;`
`75`	`75`	`}`
`76`		`- ParsedJson *pj_ptr = allocate_ParsedJson(p.second);`
	`76`	`+ ParsedJson *pj_ptr = allocate_ParsedJson(p.second, 1024);`
`77`	`77`	`if (pj_ptr == NULL) {`
`78`	`78`	`std::cerr << "can't allocate memory" << std::endl;`
`79`	`79`	`return EXIT_FAILURE;`
Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,7 @@ std::pair<u8 *, size_t> get_corpus(std::string filename) {`
`23`	`23`	`size_t length = buffer.str().size(); // +1 for null`
`24`	`24`	`u8* aligned_buffer = (u8 *)allocate_aligned_buffer(length);`
`25`	`25`	`memcpy(aligned_buffer, buffer.str().c_str(), length);`
	`26`	`+ aligned_buffer[length] = '\0';`
`26`	`27`	`is.close();`
`27`	`28`	`return std::make_pair((u8 *)aligned_buffer, length);`
`28`	`29`	`}`
Original file line number	Diff line number	Diff line change
`@@ -251,7 +251,7 @@ WARN_UNUSED`
`251`	`251`	`(u64 )(pj.structurals + idx / 8) = structurals;`
`252`	`252`	`}`
`253`	`253`	`if(buf[len] != '\0') {`
`254`		`- std::cerr << "Your string should NULL terminated." << std::endl;`
	`254`	`+ std::cerr << "Your string should be NULL terminated." << std::endl;`
`255`	`255`	`return false;`
`256`	`256`
`257`	`257`	`}`