Take care of the order of benchmarks and remove outliers

komamitsu · komamitsu · commit 9b2f4e32d9d8 · 2015-12-25T13:16:01.000+09:00
diff --git a/msgpack-jackson/src/test/java/org/msgpack/jackson/dataformat/benchmark/Benchmarker.java b/msgpack-jackson/src/test/java/org/msgpack/jackson/dataformat/benchmark/Benchmarker.java
@@ -19,6 +19,7 @@
 import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 public class Benchmarker
@@ -58,32 +59,34 @@ public void run(int count, int warmupCount)
             throws Exception
     {
         List<Tuple<String, double[]>> benchmarksResults = new ArrayList<Tuple<String, double[]>>(benchmarkableList.size());
-
         for (Benchmarkable benchmark : benchmarkableList) {
-            double[] durations = new double[count];
-
-            for (int i = 0; i < count + warmupCount; i++) {
-                if (i >= warmupCount) {
-                    System.gc();
-                }
+            benchmarksResults.add(new Tuple<String, double[]>(benchmark.label, new double[count]));
+        }
 
+        for (int i = 0; i < count + warmupCount; i++) {
+            for (int bi = 0; bi < benchmarkableList.size(); bi++) {
+                Benchmarkable benchmark = benchmarkableList.get(bi);
                 long currentTimeNanos = System.nanoTime();
                 benchmark.run();
 
                 if (i >= warmupCount) {
-                    durations[i - warmupCount] = (System.nanoTime() - currentTimeNanos) / 1000000.0;
+                    benchmarksResults.get(bi).second[i - warmupCount] = (System.nanoTime() - currentTimeNanos) / 1000000.0;
                 }
             }
-            benchmarksResults.add(new Tuple<String, double[]>(benchmark.label, durations));
         }
 
         for (Tuple<String, double[]> benchmarkResult : benchmarksResults) {
             printStat(benchmarkResult.first, benchmarkResult.second);
         }
     }
 
-    private void printStat(String label, double[] values)
+    private void printStat(String label, double[] origValues)
     {
+        double[] values = origValues;
+        Arrays.sort(origValues);
+        if (origValues.length > 2) {
+            values = Arrays.copyOfRange(origValues, 1, origValues.length - 1);
+        }
         StandardDeviation standardDeviation = new StandardDeviation();
         System.out.println(label + ":");
         System.out.println(String.format("  mean : %8.3f", StatUtils.mean(values)));
