JavaScriptExpert
diff --git a/‎Makefile‎
Lines changed: 12 additions & 6 deletions b/‎Makefile‎
Lines changed: 12 additions & 6 deletions
diff --git a/‎amalgamation.sh‎
Lines changed: 28 additions & 7 deletions b/‎amalgamation.sh‎
Lines changed: 28 additions & 7 deletions
diff --git a/‎benchmark/CMakeLists.txt‎
Lines changed: 1 addition & 0 deletions b/‎benchmark/CMakeLists.txt‎
Lines changed: 1 addition & 0 deletions
diff --git a/‎benchmark/parse_stream.cpp‎
Lines changed: 157 additions & 0 deletions b/‎benchmark/parse_stream.cpp‎
Lines changed: 157 additions & 0 deletions
diff --git a/‎include/simdjson/jsonstream.h‎
Lines changed: 43 additions & 0 deletions b/‎include/simdjson/jsonstream.h‎
Lines changed: 43 additions & 0 deletions
diff --git a/‎include/simdjson/simdjson.h‎
Lines changed: 1 addition & 0 deletions b/‎include/simdjson/simdjson.h‎
Lines changed: 1 addition & 0 deletions
diff --git a/‎include/simdjson/stage1_find_marks.h‎
100644100755
Lines changed: 12 additions & 2 deletions b/‎include/simdjson/stage1_find_marks.h‎
100644100755
Lines changed: 12 additions & 2 deletions
diff --git a/‎include/simdjson/stage2_build_tape.h‎
Lines changed: 13 additions & 0 deletions b/‎include/simdjson/stage2_build_tape.h‎
Lines changed: 13 additions & 0 deletions
@@ -59,16 +59,16 @@ endif # ifeq ($(SANITIZE),1)
 endif # ifeq ($(MEMSANITIZE),1)
 
 MAINEXECUTABLES=parse minify json2json jsonstats statisticalmodel jsonpointer
-TESTEXECUTABLES=jsoncheck jsoncheck_noavx integer_tests numberparsingcheck stringparsingcheck pointercheck
+TESTEXECUTABLES=jsoncheck jsoncheck_noavx integer_tests numberparsingcheck stringparsingcheck pointercheck jsonstream_test
 COMPARISONEXECUTABLES=minifiercompetition parsingcompetition parseandstatcompetition distinctuseridcompetition allparserscheckfile allparsingcompetition
 SUPPLEMENTARYEXECUTABLES=parse_noutf8validation parse_nonumberparsing parse_nostringparsing
 
 # Load headers and sources
-LIBHEADERS=src/simdprune_tables.h src/numberparsing.h src/jsoncharutils.h src/arm64/bitmask.h src/arm64/simd.h src/arm64/stage1_find_marks.h src/arm64/stage2_build_tape.h src/arm64/stringparsing.h src/generic/stage1_find_marks.h src/generic/stage2_build_tape.h src/generic/stringparsing.h src/haswell/bitmask.h src/haswell/simd.h src/generic/simdutf8check.h src/haswell/stage1_find_marks.h src/haswell/stage2_build_tape.h src/haswell/stringparsing.h src/westmere/bitmask.h src/westmere/simd.h src/westmere/stage1_find_marks.h src/westmere/stage2_build_tape.h src/westmere/stringparsing.h
+LIBHEADERS=src/simdprune_tables.h src/numberparsing.h src/jsoncharutils.h src/arm64/bitmask.h src/arm64/simd.h src/arm64/stage1_find_marks.h src/arm64/stage2_build_tape.h src/arm64/stringparsing.h src/generic/stage1_find_marks.h src/generic/stage2_build_tape.h src/generic/stringparsing.h src/haswell/bitmask.h src/haswell/simd.h src/generic/simdutf8check.h src/haswell/stage1_find_marks.h src/haswell/stage2_build_tape.h src/haswell/stringparsing.h src/westmere/bitmask.h src/westmere/simd.h src/westmere/stage1_find_marks.h src/westmere/stage2_build_tape.h src/westmere/stringparsing.h src/generic/stage2_streaming_build_tape.h
 PUBHEADERS=include/simdjson/common_defs.h include/simdjson/isadetection.h include/simdjson/jsonformatutils.h include/simdjson/jsonioutil.h include/simdjson/jsonminifier.h include/simdjson/jsonparser.h include/simdjson/padded_string.h include/simdjson/parsedjson.h include/simdjson/parsedjsoniterator.h include/simdjson/portability.h include/simdjson/simdjson.h include/simdjson/simdjson_version.h include/simdjson/stage1_find_marks.h include/simdjson/stage2_build_tape.h
 HEADERS=$(PUBHEADERS) $(LIBHEADERS)
 
-LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/simdjson.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/parsedjson.cpp src/parsedjsoniterator.cpp
+LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/jsonstream.cpp src/simdjson.cpp src/stage1_find_marks.cpp src/stage2_build_tape.cpp src/parsedjson.cpp src/parsedjsoniterator.cpp
 MINIFIERHEADERS=include/simdjson/jsonminifier.h
 MINIFIERLIBFILES=src/jsonminifier.cpp
 
@@ -111,6 +111,9 @@ run_stringparsingcheck: stringparsingcheck
 run_jsoncheck: jsoncheck
 	./jsoncheck
 
+run_jsonstream_test: jsonstream_test
+	./jsonstream_test
+
 run_jsoncheck_noavx: jsoncheck_noavx
 	./jsoncheck_noavx
 
@@ -123,12 +126,12 @@ run_issue150_sh: allparserscheckfile
 run_testjson2json_sh: minify json2json
 	./scripts/testjson2json.sh
 
-test: run_basictests run_jsoncheck run_numberparsingcheck run_integer_tests run_stringparsingcheck run_pointercheck run_testjson2json_sh run_issue150_sh run_jsoncheck_noavx
+test: run_basictests run_jsoncheck run_numberparsingcheck run_integer_tests run_stringparsingcheck  run_jsonstream_test run_pointercheck run_testjson2json_sh run_issue150_sh run_jsoncheck_noavx
 	@echo "It looks like the code is good!"
 
-quiettest: run_basictests run_jsoncheck run_numberparsingcheck run_integer_tests run_stringparsingcheck run_pointercheck run_testjson2json_sh run_issue150_sh run_jsoncheck_noavx
+quiettest: run_basictests run_jsoncheck run_numberparsingcheck run_integer_tests run_stringparsingcheck run_jsoncheck run_jsonstream_test run_pointercheck run_testjson2json_sh run_issue150_sh run_jsoncheck_noavx
 
-quicktests: run_basictests run_jsoncheck run_numberparsingcheck run_integer_tests run_stringparsingcheck run_pointercheck run_jsoncheck_noavx
+quicktests: run_basictests run_jsoncheck run_numberparsingcheck run_integer_tests run_stringparsingcheck run_jsoncheck run_jsonstream_test run_pointercheck run_jsoncheck_noavx
 
 slowtests: run_testjson2json_sh run_issue150_sh
 
@@ -168,6 +171,9 @@ parse_nostringparsing: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
 jsoncheck:tests/jsoncheck.cpp $(HEADERS) $(LIBFILES)
 	$(CXX) $(CXXFLAGS) -o jsoncheck $(LIBFILES) tests/jsoncheck.cpp -I. $(LIBFLAGS)
 
+jsonstream_test:tests/jsonstream_test.cpp $(HEADERS) $(LIBFILES)
+	$(CXX) $(CXXFLAGS) -o jsonstream_test $(LIBFILES) tests/jsonstream_test.cpp -I. $(LIBFLAGS)
+
 
 jsoncheck_noavx:tests/jsoncheck.cpp $(HEADERS) $(LIBFILES)
 	$(CXX) $(CXXFLAGS) -o jsoncheck_noavx $(LIBFILES) tests/jsoncheck.cpp -I. $(LIBFLAGS) -DSIMDJSON_DISABLE_AVX2_DETECTION
 
@@ -20,6 +20,7 @@ $SCRIPTPATH/src/simdjson.cpp
 $SCRIPTPATH/src/jsonioutil.cpp
 $SCRIPTPATH/src/jsonminifier.cpp
 $SCRIPTPATH/src/jsonparser.cpp
+$SCRIPTPATH/src/jsonstream.cpp
 $SCRIPTPATH/src/arm64/bitmask.h
 $SCRIPTPATH/src/haswell/bitmask.h
 $SCRIPTPATH/src/westmere/bitmask.h
@@ -57,6 +58,7 @@ $SCRIPTPATH/include/simdjson/parsedjsoniterator.h
 $SCRIPTPATH/include/simdjson/stage1_find_marks.h
 $SCRIPTPATH/include/simdjson/stage2_build_tape.h
 $SCRIPTPATH/include/simdjson/jsonparser.h
+$SCRIPTPATH/include/simdjson/jsonstream.h
 "
 
 for i in ${ALLCHEADERS} ${ALLCFILES}; do
@@ -134,17 +136,36 @@ cat <<< '
 #include "simdjson.h"
 #include "simdjson.cpp"
 int main(int argc, char *argv[]) {
-  if(argc < 2) {
-    std::cerr << "Please specify a filename " << std::endl;
+  if(argc < 3) {
+    std::cerr << "Please specify filenames " << std::endl;
   }
   const char * filename = argv[1];
   simdjson::padded_string p = simdjson::get_corpus(filename);
   simdjson::ParsedJson pj = simdjson::build_parsed_json(p); // do the parsing
   if( ! pj.is_valid() ) {
-    std::cout << "not valid" << std::endl;
+    std::cout << "build_parsed_json not valid" << std::endl;
   } else {
-    std::cout << "valid" << std::endl;
+    std::cout << "build_parsed_json valid" << std::endl;
   }
+
+  //JsonStream
+  const char * filename2 = argv[2];
+  simdjson::padded_string p2 = simdjson::get_corpus(filename2);
+  simdjson::ParsedJson pj2;
+  simdjson::JsonStream js{p2.data(), p2.size()};
+  int parse_res = simdjson::SUCCESS_AND_HAS_MORE;
+
+  while (parse_res == simdjson::SUCCESS_AND_HAS_MORE) {
+            parse_res = js.json_parse(pj2);
+        }
+
+  if( ! pj2.is_valid()) {
+    std::cout << "JsonStream not valid" << std::endl;
+  } else {
+    std::cout << "JsonStream valid" << std::endl;
+  }
+
+
   return EXIT_SUCCESS;
 }
 ' >>  "${DEMOCPP}"
@@ -160,16 +181,16 @@ echo "Giving final instructions:"
 CPPBIN=${DEMOCPP%%.*}
 
 echo "Try :"
-echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json "
+echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson"
 
 SINGLEHDR=$SCRIPTPATH/singleheader
 echo "Copying files to $SCRIPTPATH/singleheader "
 mkdir -p $SINGLEHDR
-echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json " > $SINGLEHDR/README.md
+echo "c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" > $SINGLEHDR/README.md
 cp ${AMAL_C} ${AMAL_H}  ${DEMOCPP} $SINGLEHDR
 ls $SINGLEHDR
 
-cd $SINGLEHDR && c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json
+cd $SINGLEHDR && c++ -O3 -std=c++17 -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
 
 lowercase(){
     echo "$1" | tr 'A-Z' 'a-z'
 
@@ -6,4 +6,5 @@ target_include_directories(${SIMDJSON_LIB_NAME}
 
 add_cpp_benchmark(parse)
 add_cpp_benchmark(statisticalmodel)
+add_cpp_benchmark(parse_stream)
 add_executable(perfdiff perfdiff.cpp)
@@ -0,0 +1,157 @@
+#include <iostream>
+#include <algorithm>
+#include <chrono>
+#include <vector>
+#include "simdjson/jsonstream.h"
+#include <map>
+#include "simdjson/jsonparser.h"
+#include "simdjson/parsedjson.h"
+
+#define NB_ITERATION 5
+#define MIN_BATCH_SIZE 100000
+#define MAX_BATCH_SIZE 10000000
+
+bool test_baseline = false;
+bool test_per_batch = true;
+bool test_best_batch = true;
+
+bool compare(std::pair<size_t, double> i, std::pair<size_t, double> j){
+    return i.second > j.second;
+}
+
+int main (int argc, char *argv[]){
+
+    if (argc <= 1) {
+        std::cerr << "Usage: " << argv[0] << " <jsonfile>" << std::endl;
+        exit(1);
+    }
+    const char *filename = argv[1];
+    simdjson::padded_string p;
+    try {
+        std::wclog << "loading " << filename << "\n" << std::endl;
+        simdjson::get_corpus(filename).swap(p);
+    } catch (const std::exception &) { // caught by reference to base
+        std::cerr << "Could not load the file " << filename << std::endl;
+        return EXIT_FAILURE;
+    }
+if (test_baseline) {
+    std::wclog << "Baseline: Getline + normal parse... " << std::endl;
+    std::cout << "Gigabytes/second\t" << "Nb of documents parsed" << std::endl;
+    for (auto i = 0; i < 3; i++) {
+        //Actual test
+        simdjson::ParsedJson pj;
+        bool allocok = pj.allocate_capacity(p.size());
+        if (!allocok) {
+            std::cerr << "failed to allocate memory" << std::endl;
+            return EXIT_FAILURE;
+        }
+        std::istringstream ss(std::string(p.data(), p.size()));
+
+        auto start = std::chrono::steady_clock::now();
+        int count = 0;
+        std::string line;
+        int parse_res = simdjson::SUCCESS;
+        while (getline(ss, line)) {
+            parse_res = simdjson::json_parse(line, pj);
+            count++;
+        }
+
+        auto end = std::chrono::steady_clock::now();
+
+        std::chrono::duration<double> secs = end - start;
+        double speedinGBs = (p.size()) / (secs.count() * 1000000000.0);
+        std::cout << speedinGBs << "\t\t\t\t" << count << std::endl;
+
+        if (parse_res != simdjson::SUCCESS) {
+            std::cerr << "Parsing failed" << std::endl;
+            exit(1);
+        }
+    }
+}
+
+std::map<size_t, double> batch_size_res;
+if(test_per_batch) {
+    std::wclog << "Jsonstream: Speed per batch_size... from " << MIN_BATCH_SIZE
+               << " bytes to " << MAX_BATCH_SIZE << " bytes..." << std::endl;
+    std::cout << "Batch Size\t" << "Gigabytes/second\t" << "Nb of documents parsed" << std::endl;
+    for (size_t i = MIN_BATCH_SIZE; i <= MAX_BATCH_SIZE; i += (MAX_BATCH_SIZE - MIN_BATCH_SIZE) / 200) {
+        batch_size_res.insert(std::pair<size_t, double>(i, 0));
+        int count;
+        for (size_t j = 0; j < 5; j++) {
+            //Actual test
+            simdjson::ParsedJson pj;
+            simdjson::JsonStream js{p.data(), p.size(), i};
+            int parse_res = simdjson::SUCCESS_AND_HAS_MORE;
+
+            auto start = std::chrono::steady_clock::now();
+            count = 0;
+            while (parse_res == simdjson::SUCCESS_AND_HAS_MORE) {
+                parse_res = js.json_parse(pj);
+                count++;
+            }
+
+            auto end = std::chrono::steady_clock::now();
+
+            std::chrono::duration<double> secs = end - start;
+            double speedinGBs = (p.size()) / (secs.count() * 1000000000.0);
+            if (speedinGBs > batch_size_res.at(i))
+                batch_size_res[i] = speedinGBs;
+
+            if (parse_res != simdjson::SUCCESS) {
+                std::wcerr << "Parsing failed with: " << simdjson::error_message(parse_res).c_str() << std::endl;
+                exit(1);
+            }
+        }
+        std::cout << i << "\t\t" << batch_size_res.at(i) << "\t\t\t\t" << count << std::endl;
+
+    }
+}
+if(test_best_batch) {
+    size_t optimal_batch_size;
+    if(test_per_batch)
+        optimal_batch_size = (*min_element(batch_size_res.begin(), batch_size_res.end(), compare)).first;
+    else
+        optimal_batch_size = MIN_BATCH_SIZE;
+    std::wclog << "Starting speed test... Best of " << NB_ITERATION << " iterations..." << std::endl;
+    std::wclog << "Seemingly optimal batch_size: " << optimal_batch_size << "..." << std::endl;
+    std::vector<double> res;
+    for (int i = 0; i < NB_ITERATION; i++) {
+
+        //Actual test
+        simdjson::ParsedJson pj;
+        simdjson::JsonStream js{p.data(), p.size(), 4000000};
+        int parse_res = simdjson::SUCCESS_AND_HAS_MORE;
+
+        auto start = std::chrono::steady_clock::now();
+
+        while (parse_res == simdjson::SUCCESS_AND_HAS_MORE) {
+            parse_res = js.json_parse(pj);
+        }
+
+
+        auto end = std::chrono::steady_clock::now();
+
+        std::chrono::duration<double> secs = end - start;
+        res.push_back(secs.count());
+
+        if (parse_res != simdjson::SUCCESS) {
+            std::cerr << "Parsing failed" << std::endl;
+            exit(1);
+        }
+
+    }
+
+    std::min(res.begin(), res.end());
+
+    double min_result = *min_element(res.begin(), res.end());
+    double speedinGBs = (p.size()) / (min_result * 1000000000.0);
+
+
+    std::cout << "Min:  " << min_result << " bytes read: " << p.size()
+              << " Gigabytes/second: " << speedinGBs << std::endl;
+}
+
+
+
+    return 0;
+}
@@ -0,0 +1,43 @@
+#ifndef SIMDJSON_JSONSTREAM_H
+#define SIMDJSON_JSONSTREAM_H
+
+
+#include <algorithm>
+#include "simdjson/stage1_find_marks.h"
+#include "simdjson/stage2_build_tape.h"
+#include "simdjson/simdjson.h"
+
+namespace simdjson {
+
+    class JsonStream {
+    public:
+        JsonStream(const char *buf, size_t len, size_t batch_size = 1000000);
+
+        JsonStream(const std::string &s, size_t batch_size = 1000000) : JsonStream(s.data(), s.size(), batch_size) {};
+
+        int json_parse(ParsedJson &pj);
+
+        void set_new_buffer(const char *buf, size_t len);
+
+        void set_new_buffer(const std::string &s) { set_new_buffer(s.data(), s.size()); }
+
+        size_t get_current_buffer_loc() const;
+
+        size_t get_n_parsed_docs() const;
+
+        size_t get_n_bytes_parsed() const;
+
+    private:
+        const char *_buf;
+        size_t _len;
+        size_t _batch_size;
+        size_t next_json{0};
+        bool error_on_last_attempt{false};
+        bool load_next_batch{true};
+        size_t current_buffer_loc{0};
+        size_t n_parsed_docs{0};
+        size_t n_bytes_parsed{0};
+    };
+
+}
+#endif //SIMDJSON_JSONSTREAM_H
@@ -22,6 +22,7 @@ enum class Architecture {
 
 enum ErrorValues {
   SUCCESS = 0,
+  SUCCESS_AND_HAS_MORE, //No errors and buffer still has more data
   CAPACITY,    // This ParsedJson can't support a document that big
   MEMALLOC,    // Error allocating memory, most likely out of memory
   TAPE_ERROR,  // Something went wrong while writing to the tape (stage 2), this
 
@@ -7,11 +7,21 @@
 namespace simdjson {
 
 template <Architecture T = Architecture::NATIVE>
-int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj);
+int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming);
+
+template <Architecture T = Architecture::NATIVE>
+int find_structural_bits(const char *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) {
+  return find_structural_bits<T>((const uint8_t *)buf, len, pj, streaming);
+}
+
+template <Architecture T = Architecture::NATIVE>
+int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj){
+    return find_structural_bits<T>((const uint8_t *)buf, len, pj, false);
+}
 
 template <Architecture T = Architecture::NATIVE>
 int find_structural_bits(const char *buf, size_t len, simdjson::ParsedJson &pj) {
-  return find_structural_bits((const uint8_t *)buf, len, pj);
+    return find_structural_bits<T>((const uint8_t *)buf, len, pj, false);
 }
 
 }; // namespace simdjson
 
@@ -18,6 +18,19 @@ int unified_machine(const char *buf, size_t len, ParsedJson &pj) {
   return unified_machine<T>(reinterpret_cast<const uint8_t *>(buf), len, pj);
 }
 
+
+
+// Streaming
+template <Architecture T = Architecture::NATIVE>
+WARN_UNUSED int
+unified_machine(const uint8_t *buf, size_t len, ParsedJson &pj, size_t &next_json);
+
+template <Architecture T = Architecture::NATIVE>
+int unified_machine(const char *buf, size_t len, ParsedJson &pj, size_t &next_json) {
+    return unified_machine<T>(reinterpret_cast<const uint8_t *>(buf), len, pj, next_json);
+}
+
+
 } // namespace simdjson
 
 #endif