Skip to content

Commit 6192b39

Browse files
sokrasokra
committed
use mathematical correct confidence interval
reset git tree correctly when loading stuff fix error when errors in benchmark occur
1 parent d9ac043 commit 6192b39

File tree

1 file changed

+41
-9
lines changed

1 file changed

+41
-9
lines changed

‎test/BenchmarkTestCases.benchmark.js‎

Lines changed: 41 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,18 @@ describe("BenchmarkTestCases", function() {
4040
try {
4141
fs.mkdirSync(baselinePath);
4242
} catch(e) {}
43-
git(baselinePath).raw(["--git-dir", path.join(rootPath, ".git"), "reset", "--hard", baselineRevision], err => {
43+
const gitIndex = path.resolve(rootPath, ".git/index");
44+
const index = fs.readFileSync(gitIndex);
45+
git(rootPath).raw(["rev-list", "-n", "1", "HEAD"], (err, prevHead) => {
4446
if(err) return callback(err);
45-
doLoadWebpack();
47+
git(baselinePath).raw(["--git-dir", path.join(rootPath, ".git"), "reset", "--hard", baselineRevision], err => {
48+
if(err) return callback(err);
49+
git(rootPath).raw(["reset", "--soft", prevHead.split("\n")[0]], err => {
50+
if(err) return callback(err);
51+
fs.writeFileSync(gitIndex, index);
52+
doLoadWebpack();
53+
});
54+
});
4655
});
4756
}
4857

@@ -100,6 +109,22 @@ describe("BenchmarkTestCases", function() {
100109
});
101110
}
102111

112+
function tDistribution(n) {
113+
// two-sided, 90%
114+
// https://en.wikipedia.org/wiki/Student%27s_t-distribution
115+
if(n <= 30) {
116+
// 1 2 ...
117+
const data = [6.314, 2.920, 2.353, 2.132, 2.015, 1.943, 1.895, 1.860, 1.833, 1.812, 1.796, 1.782, 1.771, 1.761, 1.753, 1.746, 1.740, 1.734, 1.729, 1.725, 1.721, 1.717, 1.714, 1.711, 1.708, 1.706, 1.703, 1.701, 1.699, 1.697];
118+
return data[n - 1];
119+
} else if(n <= 120) {
120+
// 40 50 60 70 80 90 100 110 120
121+
const data = [1.684, 1.676, 1.671, 1.667, 1.664, 1.662, 1.660, 1.659, 1.658]
122+
return data[Math.floor(n / 10) - 4];
123+
} else {
124+
return 1.645;
125+
}
126+
}
127+
103128
function runBenchmark(webpack, config, callback) {
104129
// warmup
105130
const warmupCompiler = webpack(config, (err, stats) => {
@@ -108,11 +133,11 @@ describe("BenchmarkTestCases", function() {
108133
const compiler = webpack(config, (err, stats) => {
109134
compiler.purgeInputFileSystem();
110135
if(err) {
111-
deferred.reject(err);
136+
callback(err);
112137
return;
113138
}
114139
if(stats.hasErrors()) {
115-
deferred.reject(new Error(stats.toJson().errors.join("\n\n")));
140+
callback(new Error(stats.toJson().errors.join("\n\n")));
116141
return;
117142
}
118143
deferred.resolve();
@@ -122,6 +147,13 @@ describe("BenchmarkTestCases", function() {
122147
defer: true,
123148
initCount: 1,
124149
onComplete: function() {
150+
const stats = bench.stats;
151+
const n = stats.sample.length;
152+
const nSqrt = Math.sqrt(n);
153+
const z = tDistribution(n - 1);
154+
stats.minConfidence = stats.mean - z * stats.deviation / nSqrt;
155+
stats.maxConfidence = stats.mean + z * stats.deviation / nSqrt;
156+
stats.text = `${Math.round(stats.mean * 1000)}ms ± ${Math.round(stats.deviation * 1000)}ms [${Math.round(stats.minConfidence * 1000)}ms; ${Math.round(stats.maxConfidence * 1000)}ms]`;
125157
callback(null, bench.stats);
126158
},
127159
onError: callback
@@ -153,7 +185,7 @@ describe("BenchmarkTestCases", function() {
153185
if(!config.output.path) config.output.path = outputDirectory;
154186
runBenchmark(baseline.webpack, config, (err, stats) => {
155187
if(err) return done(err);
156-
console.log(` ${baseline.name} ${Math.round(stats.mean * 1000)}ms ± ${Math.round(stats.deviation * 1000)}ms`);
188+
console.log(` ${baseline.name} ${stats.text}`);
157189
if(baseline.name === "HEAD")
158190
headStats = stats;
159191
else
@@ -164,10 +196,10 @@ describe("BenchmarkTestCases", function() {
164196

165197
if(baseline.name !== "HEAD") {
166198
it(`HEAD should not be slower than ${baseline.name} (${baseline.rev})`, function() {
167-
if(baselineStats.mean + baselineStats.deviation < headStats.mean - headStats.deviation) {
168-
throw new Error(`HEAD (${baselineStats.mean} ± ${baselineStats.deviation}) is slower than ${baseline.name} (${headStats.mean} ± ${headStats.deviation})`);
169-
} else if(baselineStats.mean - baselineStats.deviation > headStats.mean + headStats.deviation) {
170-
console.log(`======> HEAD is ${Math.round(baselineStats.mean / headStats.mean * 100 - 100)}% faster than ${baseline.name}!`);
199+
if(baselineStats.maxConfidence < headStats.minConfidence) {
200+
throw new Error(`HEAD (${headStats.text}) is slower than ${baseline.name} (${baselineStats.text}) (90% confidence)`);
201+
} else if(baselineStats.minConfidence > headStats.maxConfidence) {
202+
console.log(`======> HEAD is ${Math.round(baselineStats.mean / headStats.mean * 100 - 100)}% faster than ${baseline.name} (90% confidence)!`);
171203
}
172204
});
173205
}

0 commit comments

Comments
 (0)