Make perf validation more stable, check no-AVX as well (simdjson#275)

jkeiser · lemire · commit c8d50a60602f · 2019-08-15T20:43:21.000-04:00
* Compare against v0.2.1, fail only if perf is less 6 times in a row

* Run AVX and no-AVX perf tests in Circle CI

* Set % difference threshold to 0.2%
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -24,6 +24,10 @@ jobs:
           name: Running tests (gcc)
           command: make quiettest amalgamate
 
+      - run:
+          name: Comparing perf against reference (gcc)
+          command: make checkperf
+
       - run:
           name: Building (gcc, cmake, dynamic)
           command: |
@@ -51,6 +55,7 @@ jobs:
           command: |
             cd buildstatic
             make test
+
       - run:
           name: Building (gcc, cmake, sanitize)
           command: |
@@ -88,6 +93,10 @@ jobs:
       - run:
           name: Running tests (gcc)
           command: ARCHFLAGS="-march=nehalem" make quiettest amalgamate
+          
+      - run:
+          name: Comparing perf against reference (gcc)
+          command: ARCHFLAGS="-march=nehalem" make checkperf
 
       - run:
           name: Building (gcc, cmake, dynamic)
@@ -154,6 +163,10 @@ jobs:
           name: Running tests (clang)
           command: make quiettest amalgamate
 
+      - run:
+          name: Comparing perf against reference (clang)
+          command: make checkperf
+
       - run:
           name: Building (clang, cmake, dynamic)
           command: |
@@ -220,6 +233,10 @@ jobs:
           name: Running tests (clang)
           command: ARCHFLAGS="-march=nehalem"  make quiettest amalgamate
 
+      - run:
+          name: Comparing perf against reference (clang)
+          command: ARCHFLAGS="-march=nehalem"  make checkperf
+
       - run:
           name: Building (clang, cmake, dynamic)
           command: |
diff --git a/Makefile b/Makefile
@@ -1,3 +1,4 @@
+REFERENCE_VERSION = v0.2.1
 
 .SUFFIXES:
 #
@@ -127,7 +128,7 @@ perfdiff: benchmark/perfdiff.cpp
 	$(CXX) $(CXXFLAGS) -o perfdiff benchmark/perfdiff.cpp $(LIBFLAGS)
 
 checkperf:
-	bash ./scripts/checkperf.sh
+	bash ./scripts/checkperf.sh $(REFERENCE_VERSION)
 
 statisticalmodel: benchmark/statisticalmodel.cpp $(HEADERS) $(LIBFILES)
 	$(CXX) $(CXXFLAGS) -o statisticalmodel $(LIBFILES) benchmark/statisticalmodel.cpp $(LIBFLAGS)
diff --git a/benchmark/perfdiff.cpp b/benchmark/perfdiff.cpp
@@ -44,30 +44,37 @@ double readThroughput(std::string parseOutput) {
     return result / numResults;
 }
 
-const double ERROR_MARGIN = 10; // 10%
-const double INTERLEAVED_ATTEMPTS = 4;
+const double INTERLEAVED_ATTEMPTS = 6;
+const double PERCENT_DIFFERENCE_THRESHOLD = -0.2;
 
 int main(int argc, char *argv[]) {
     if (argc != 3) {
-        std::cerr << "Usage: " << argv[0] << " <new parse cmd> <reference parse cmd>";
+        std::cerr << "Usage: " << argv[0] << " <new parse cmd> <reference parse cmd>" << std::endl;
         return 1;
     }
-    double newThroughput = 0;
-    double referenceThroughput = 0;
     for (int attempt=0; attempt < INTERLEAVED_ATTEMPTS; attempt++) {
-        newThroughput += readThroughput(exec(argv[1]));
-        referenceThroughput += readThroughput(exec(argv[2]));
-    }
-    newThroughput /= INTERLEAVED_ATTEMPTS;
-    referenceThroughput /= INTERLEAVED_ATTEMPTS;
+        if (attempt > 0) {
+            std::cout << "Running again to check whether it's a fluke ..." << std::endl;
+        }
+        std::cout << "Attempt #" << (attempt+1) << " of up to " << INTERLEAVED_ATTEMPTS << std::endl;
 
-    std::cout << "New throughput: " << newThroughput << std::endl;
-    std::cout << "Ref throughput: " << referenceThroughput << std::endl;
-    double percentDifference = ((newThroughput / referenceThroughput) - 1.0) * 100;
-    std::cout << "Difference: " << percentDifference << "%" << std::endl;
-    if (percentDifference < -ERROR_MARGIN) {
-        std::cerr << "New throughput is more than " << ERROR_MARGIN << "% degraded from reference throughput!" << std::endl;
-        return 1;
+        // Read new throughput
+        double newThroughput = readThroughput(exec(argv[1]));
+        std::cout << "New throughput: " << newThroughput << std::endl;
+
+        // Read reference throughput
+        double referenceThroughput = readThroughput(exec(argv[2]));
+        std::cout << "Ref throughput: " << referenceThroughput << std::endl;
+
+        // Check if % difference > 0
+        double percentDifference = ((newThroughput / referenceThroughput) - 1.0) * 100;
+        std::cout << "Difference: " << percentDifference << "%" << std::endl;
+        if (percentDifference >= PERCENT_DIFFERENCE_THRESHOLD) {
+            std::cout << "New throughput is same or better." << std::endl;
+            return 0;
+        } else {
+            std::cout << "New throughput is lower!";
+        }
     }
-    return 0;
+    return 1;
 }
diff --git a/scripts/checkperf.sh b/scripts/checkperf.sh
@@ -9,6 +9,7 @@ if [ -z "$*" ]; then perftests="jsonexamples/twitter.json"; else perftests=$*; f
 
 # Clone and build the reference branch's parse
 echo "Cloning and build the reference branch ($reference_branch) ..."
+current=$SCRIPTPATH/..
 reference=$current/benchbranch/$reference_branch
 rm -rf $reference
 mkdir -p $reference
@@ -18,7 +19,6 @@ make parse
 
 # Build the current branch's parse
 echo "Building the current branch ..."
-current=$SCRIPTPATH/..
 cd $current
 make clean
 make parse
