Skip to content

Commit b1bbc68

Browse files
bzozbzoz
committed
benchmark: support for multiple http benchmarkers
This adds support for multiple HTTP benchmarkers. Adds autocannon as the secondary benchmarker. PR-URL: nodejs#8140 Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
1 parent f6e33ef commit b1bbc68

8 files changed

Lines changed: 243 additions & 73 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -627,13 +627,6 @@ ifeq ($(XZ), 0)
627627
ssh $(STAGINGSERVER) "touch nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/node-$(FULLVERSION)-$(OSTYPE)-$(ARCH).tar.xz.done"
628628
endif
629629

630-
haswrk=$(shell which wrk > /dev/null 2>&1; echo $$?)
631-
wrk:
632-
ifneq ($(haswrk), 0)
633-
@echo "please install wrk before proceeding. More information can be found in benchmark/README.md." >&2
634-
@exit 1
635-
endif
636-
637630
bench-net: all
638631
@$(NODE) benchmark/run.js net
639632

@@ -643,7 +636,7 @@ bench-crypto: all
643636
bench-tls: all
644637
@$(NODE) benchmark/run.js tls
645638

646-
bench-http: wrk all
639+
bench-http: all
647640
@$(NODE) benchmark/run.js http
648641

649642
bench-fs: all

‎benchmark/README.md‎

Lines changed: 62 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,25 @@ This folder contains benchmarks to measure the performance of the Node.js APIs.
1414

1515
## Prerequisites
1616

17-
Most of the http benchmarks require [`wrk`][wrk] to be installed. It may be
18-
available through your preferred package manager. If not, `wrk` can be built
19-
[from source][wrk] via `make`.
17+
Most of the HTTP benchmarks require a benchmarker to be installed, this can be
18+
either [`wrk`][wrk] or [`autocannon`][autocannon].
19+
20+
`Autocannon` is a Node script that can be installed using
21+
`npm install -g autocannon`. It will use the Node executable that is in the
22+
path, hence if you want to compare two HTTP benchmark runs make sure that the
23+
Node version in the path is not altered.
24+
25+
`wrk` may be available through your preferred package manger. If not, you can
26+
easily build it [from source][wrk] via `make`.
27+
28+
By default `wrk` will be used as benchmarker. If it is not available
29+
`autocannon` will be used in it its place. When creating a HTTP benchmark you
30+
can specify which benchmarker should be used. You can force a specific
31+
benchmarker to be used by providing it as an argument, e. g.:
32+
33+
`node benchmark/run.js --set benchmarker=autocannon http`
34+
35+
`node benchmark/http/simple.js benchmarker=autocannon`
2036

2137
To analyze the results `R` should be installed. Check you package manager or
2238
download it from https://www.r-project.org/.
@@ -287,5 +303,48 @@ function main(conf) {
287303
}
288304
```
289305

306+
## Creating HTTP benchmark
307+
308+
The `bench` object returned by `createBenchmark` implements
309+
`http(options, callback)` method. It can be used to run external tool to
310+
benchmark HTTP servers.
311+
312+
```js
313+
'use strict';
314+
315+
const common = require('../common.js');
316+
317+
const bench = common.createBenchmark(main, {
318+
kb: [64, 128, 256, 1024],
319+
connections: [100, 500]
320+
});
321+
322+
function main(conf) {
323+
const http = require('http');
324+
const len = conf.kb * 1024;
325+
const chunk = Buffer.alloc(len, 'x');
326+
const server = http.createServer(function(req, res) {
327+
res.end(chunk);
328+
});
329+
330+
server.listen(common.PORT, function() {
331+
bench.http({
332+
connections: conf.connections,
333+
}, function() {
334+
server.close();
335+
});
336+
});
337+
}
338+
```
339+
340+
Supported options keys are:
341+
* `port` - defaults to `common.PORT`
342+
* `path` - defaults to `/`
343+
* `connections` - number of concurrent connections to use, defaults to 100
344+
* `duration` - duration of the benchmark in seconds, defaults to 10
345+
* `benchmarker` - benchmarker to use, defaults to
346+
`common.default_http_benchmarker`
347+
348+
[autocannon]: https://github.com/mcollina/autocannon
290349
[wrk]: https://github.com/wg/wrk
291350
[t-test]: https://en.wikipedia.org/wiki/Student%27s_t-test#Equal_or_unequal_sample_sizes.2C_unequal_variances

‎benchmark/_http-benchmarkers.js‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
'use strict';
2+
3+
const child_process = require('child_process');
4+
5+
// The port used by servers and wrk
6+
exports.PORT = process.env.PORT || 12346;
7+
8+
function AutocannonBenchmarker() {
9+
this.name = 'autocannon';
10+
this.autocannon_exe = process.platform === 'win32'
11+
? 'autocannon.cmd'
12+
: 'autocannon';
13+
const result = child_process.spawnSync(this.autocannon_exe, ['-h']);
14+
this.present = !(result.error && result.error.code === 'ENOENT');
15+
}
16+
17+
AutocannonBenchmarker.prototype.create = function(options) {
18+
const args = ['-d', options.duration, '-c', options.connections, '-j', '-n',
19+
`http://127.0.0.1:${options.port}${options.path}` ];
20+
const child = child_process.spawn(this.autocannon_exe, args);
21+
return child;
22+
};
23+
24+
AutocannonBenchmarker.prototype.processResults = function(output) {
25+
let result;
26+
try {
27+
result = JSON.parse(output);
28+
} catch (err) {
29+
// Do nothing, let next line handle this
30+
}
31+
if (!result || !result.requests || !result.requests.average) {
32+
return undefined;
33+
} else {
34+
return result.requests.average;
35+
}
36+
};
37+
38+
function WrkBenchmarker() {
39+
this.name = 'wrk';
40+
this.regexp = /Requests\/sec:[ \t]+([0-9\.]+)/;
41+
const result = child_process.spawnSync('wrk', ['-h']);
42+
this.present = !(result.error && result.error.code === 'ENOENT');
43+
}
44+
45+
WrkBenchmarker.prototype.create = function(options) {
46+
const args = ['-d', options.duration, '-c', options.connections, '-t', 8,
47+
`http://127.0.0.1:${options.port}${options.path}` ];
48+
const child = child_process.spawn('wrk', args);
49+
return child;
50+
};
51+
52+
WrkBenchmarker.prototype.processResults = function(output) {
53+
const match = output.match(this.regexp);
54+
const result = match && +match[1];
55+
if (!result) {
56+
return undefined;
57+
} else {
58+
return result;
59+
}
60+
};
61+
62+
const http_benchmarkers = [ new WrkBenchmarker(),
63+
new AutocannonBenchmarker() ];
64+
65+
const benchmarkers = {};
66+
67+
http_benchmarkers.forEach((benchmarker) => {
68+
benchmarkers[benchmarker.name] = benchmarker;
69+
if (!exports.default_http_benchmarker && benchmarker.present) {
70+
exports.default_http_benchmarker = benchmarker.name;
71+
}
72+
});
73+
74+
exports.run = function(options, callback) {
75+
options = Object.assign({
76+
port: exports.PORT,
77+
path: '/',
78+
connections: 100,
79+
duration: 10,
80+
benchmarker: exports.default_http_benchmarker
81+
}, options);
82+
if (!options.benchmarker) {
83+
callback(new Error('Could not locate any of the required http ' +
84+
'benchmarkers. Check benchmark/README.md for further ' +
85+
'instructions.'));
86+
return;
87+
}
88+
const benchmarker = benchmarkers[options.benchmarker];
89+
if (!benchmarker) {
90+
callback(new Error(`Requested benchmarker '${options.benchmarker}' is ` +
91+
'not supported'));
92+
return;
93+
}
94+
if (!benchmarker.present) {
95+
callback(new Error(`Requested benchmarker '${options.benchmarker}' is ` +
96+
'not installed'));
97+
return;
98+
}
99+
100+
const benchmarker_start = process.hrtime();
101+
102+
const child = benchmarker.create(options);
103+
104+
child.stderr.pipe(process.stderr);
105+
106+
let stdout = '';
107+
child.stdout.on('data', (chunk) => stdout += chunk.toString());
108+
109+
child.once('close', function(code) {
110+
const elapsed = process.hrtime(benchmarker_start);
111+
if (code) {
112+
let error_message = `${options.benchmarker} failed with ${code}.`;
113+
if (stdout !== '') {
114+
error_message += ` Output: ${stdout}`;
115+
}
116+
callback(new Error(error_message), code);
117+
return;
118+
}
119+
120+
const result = benchmarker.processResults(stdout);
121+
if (!result) {
122+
callback(new Error(`${options.benchmarker} produced strange output: ` +
123+
stdout, code));
124+
return;
125+
}
126+
127+
callback(null, code, options.benchmarker, result, elapsed);
128+
});
129+
130+
};

‎benchmark/common.js‎

Lines changed: 36 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
'use strict';
22

33
const child_process = require('child_process');
4-
5-
// The port used by servers and wrk
6-
exports.PORT = process.env.PORT || 12346;
4+
const http_benchmarkers = require('./_http-benchmarkers.js');
75

86
exports.createBenchmark = function(fn, options) {
97
return new Benchmark(fn, options);
108
};
119

1210
function Benchmark(fn, options) {
1311
this.name = require.main.filename.slice(__dirname.length + 1);
14-
this.options = this._parseArgs(process.argv.slice(2), options);
12+
const parsed_args = this._parseArgs(process.argv.slice(2), options);
13+
this.options = parsed_args.cli;
14+
this.extra_options = parsed_args.extra;
1515
this.queue = this._queue(this.options);
1616
this.config = this.queue[0];
1717

@@ -29,7 +29,7 @@ function Benchmark(fn, options) {
2929

3030
Benchmark.prototype._parseArgs = function(argv, options) {
3131
const cliOptions = Object.assign({}, options);
32-
32+
const extraOptions = {};
3333
// Parse configuration arguments
3434
for (const arg of argv) {
3535
const match = arg.match(/^(.+?)=([\s\S]*)$/);
@@ -38,14 +38,16 @@ Benchmark.prototype._parseArgs = function(argv, options) {
3838
process.exit(1);
3939
}
4040

41-
// Infer the type from the options object and parse accordingly
42-
const isNumber = typeof options[match[1]][0] === 'number';
43-
const value = isNumber ? +match[2] : match[2];
44-
45-
cliOptions[match[1]] = [value];
41+
if (options[match[1]]) {
42+
// Infer the type from the options object and parse accordingly
43+
const isNumber = typeof options[match[1]][0] === 'number';
44+
const value = isNumber ? +match[2] : match[2];
45+
cliOptions[match[1]] = [value];
46+
} else {
47+
extraOptions[match[1]] = match[2];
48+
}
4649
}
47-
48-
return cliOptions;
50+
return { cli: cliOptions, extra: extraOptions };
4951
};
5052

5153
Benchmark.prototype._queue = function(options) {
@@ -88,51 +90,29 @@ Benchmark.prototype._queue = function(options) {
8890
return queue;
8991
};
9092

91-
function hasWrk() {
92-
const result = child_process.spawnSync('wrk', ['-h']);
93-
if (result.error && result.error.code === 'ENOENT') {
94-
console.error('Couldn\'t locate `wrk` which is needed for running ' +
95-
'benchmarks. Check benchmark/README.md for further instructions.');
96-
process.exit(1);
97-
}
98-
}
93+
// Benchmark an http server.
94+
exports.default_http_benchmarker =
95+
http_benchmarkers.default_http_benchmarker;
96+
exports.PORT = http_benchmarkers.PORT;
9997

100-
// benchmark an http server.
101-
const WRK_REGEXP = /Requests\/sec:[ \t]+([0-9\.]+)/;
102-
Benchmark.prototype.http = function(urlPath, args, cb) {
103-
hasWrk();
98+
Benchmark.prototype.http = function(options, cb) {
10499
const self = this;
105-
106-
const urlFull = 'http://127.0.0.1:' + exports.PORT + urlPath;
107-
args = args.concat(urlFull);
108-
109-
const childStart = process.hrtime();
110-
const child = child_process.spawn('wrk', args);
111-
child.stderr.pipe(process.stderr);
112-
113-
// Collect stdout
114-
let stdout = '';
115-
child.stdout.on('data', (chunk) => stdout += chunk.toString());
116-
117-
child.once('close', function(code) {
118-
const elapsed = process.hrtime(childStart);
119-
if (cb) cb(code);
120-
121-
if (code) {
122-
console.error('wrk failed with ' + code);
123-
process.exit(code);
100+
const http_options = Object.assign({ }, options);
101+
http_options.benchmarker = http_options.benchmarker ||
102+
self.config.benchmarker ||
103+
self.extra_options.benchmarker ||
104+
exports.default_http_benchmarker;
105+
http_benchmarkers.run(http_options, function(error, code, used_benchmarker,
106+
result, elapsed) {
107+
if (cb) {
108+
cb(code);
124109
}
125-
126-
// Extract requests pr second and check for odd results
127-
const match = stdout.match(WRK_REGEXP);
128-
if (!match || match.length <= 1) {
129-
console.error('wrk produced strange output:');
130-
console.error(stdout);
131-
process.exit(1);
110+
if (error) {
111+
console.error(error);
112+
process.exit(code || 1);
132113
}
133-
134-
// Report rate
135-
self.report(+match[1], elapsed);
114+
self.config.benchmarker = used_benchmarker;
115+
self.report(result, elapsed);
136116
});
137117
};
138118

@@ -152,6 +132,9 @@ Benchmark.prototype._run = function() {
152132
for (const key of Object.keys(config)) {
153133
childArgs.push(`${key}=${config[key]}`);
154134
}
135+
for (const key of Object.keys(self.extra_options)) {
136+
childArgs.push(`${key}=${self.extra_options[key]}`);
137+
}
155138

156139
const child = child_process.fork(require.main.filename, childArgs, {
157140
env: childEnv

‎benchmark/http/chunked.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ function main(conf) {
2020
const http = require('http');
2121
var chunk = Buffer.alloc(conf.size, '8');
2222

23-
var args = ['-d', '10s', '-t', 8, '-c', conf.c];
24-
2523
var server = http.createServer(function(req, res) {
2624
function send(left) {
2725
if (left === 0) return res.end();
@@ -34,7 +32,9 @@ function main(conf) {
3432
});
3533

3634
server.listen(common.PORT, function() {
37-
bench.http('/', args, function() {
35+
bench.http({
36+
connections: conf.c
37+
}, function() {
3838
server.close();
3939
});
4040
});

0 commit comments

Comments
 (0)