Skip to content

Commit 301bf18

Browse files
alan-agius4dgp1130
alan-agius4
authored and
dgp1130
committed
feat(@angular-devkit/benchmark): add capabilities to benchmark watch processes
1 parent e72f97d commit 301bf18

File tree

15 files changed

+312
-29
lines changed

15 files changed

+312
-29
lines changed

‎etc/api/angular_devkit/benchmark/src/index.d.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ export declare const aggregateMetrics: (m1: Metric | AggregatedMetric, m2: Metri
1919

2020
export declare type BenchmarkReporter = (command: Command, groups: MetricGroup[]) => void;
2121

22-
export declare type Capture = (process: MonitoredProcess) => Observable<MetricGroup>;
22+
export declare type Capture = (stats: Observable<AggregatedProcessStats>) => Observable<MetricGroup>;
2323

2424
export declare class Command {
2525
args: string[];
@@ -40,7 +40,8 @@ export declare class LocalMonitoredProcess implements MonitoredProcess {
4040
stats$: Observable<AggregatedProcessStats>;
4141
stderr$: Observable<Buffer>;
4242
stdout$: Observable<Buffer>;
43-
constructor(command: Command);
43+
constructor(command: Command, useProcessTime?: boolean);
44+
resetElapsedTimer(): void;
4445
run(): Observable<number>;
4546
}
4647

@@ -89,3 +90,11 @@ export interface RunBenchmarkOptions {
8990
reporters: BenchmarkReporter[];
9091
retries?: number;
9192
}
93+
94+
export declare function runBenchmarkWatch({ command, captures, reporters, iterations, retries, logger, watchMatcher, watchTimeout, watchCommand, }: RunBenchmarkWatchOptions): Observable<MetricGroup[]>;
95+
96+
export interface RunBenchmarkWatchOptions extends RunBenchmarkOptions {
97+
watchCommand: Command;
98+
watchMatcher: string;
99+
watchTimeout?: number;
100+
}

‎packages/angular_devkit/benchmark/BUILD‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ ts_library(
4545
"src/test/exit-code-one.js",
4646
"src/test/fibonacci.js",
4747
"src/test/test-script.js",
48+
"src/test/watch-test-cmd.js",
49+
"src/test/watch-test-file.txt",
50+
"src/test/watch-test-script.js",
4851
],
4952
# @external_begin
5053
tsconfig = "//:tsconfig-test.json",

‎packages/angular_devkit/benchmark/README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,21 @@ $ benchmark -- node fibonacci.js 40
7272
[benchmark] Peak Memory usage: 22.34 MB (22.32, 22.34, 22.34, 22.35, 22.35)
7373
```
7474

75+
## Example in watch mode
76+
77+
```
78+
benchmark --verbose --watch-timeout=10000 --watch-matcher="Compiled successfully" --watch-script watch-script.js -- ng serve
79+
[benchmark] Benchmarking process over 5 iterations, with up to 5 retries.
80+
[benchmark] ng serve (at D:\sandbox\latest-project)
81+
[benchmark] Process Stats
82+
[benchmark] Elapsed Time: 470.40 ms (820.00, 557.00, 231.00, 509.00, 235.00)
83+
[benchmark] Average Process usage: 2.00 process(es) (2.00, 2.00, 2.00, 2.00, 2.00)
84+
[benchmark] Peak Process usage: 2.00 process(es) (2.00, 2.00, 2.00, 2.00, 2.00)
85+
[benchmark] Average CPU usage: 33.77 % (31.27, 0.00, 101.70, 35.90, 0.00)
86+
[benchmark] Peak CPU usage: 59.72 % (125.10, 0.00, 101.70, 71.80, 0.00)
87+
[benchmark] Average Memory usage: 665.49 MB (619.84, 657.17, 669.47, 685.19, 695.76)
88+
[benchmark] Peak Memory usage: 672.44 MB (639.87, 661.04, 669.47, 689.66, 702.14)
89+
```
7590

7691
## API Usage
7792

‎packages/angular_devkit/benchmark/src/default-stats-capture.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import { cumulativeMovingAverage, max } from './utils';
1313

1414

1515
export const defaultStatsCapture: Capture = (
16-
process: MonitoredProcess,
16+
stats: Observable<AggregatedProcessStats>,
1717
): Observable<MetricGroup> => {
1818
type Accumulator = {
1919
elapsed: number,
@@ -34,7 +34,7 @@ export const defaultStatsCapture: Capture = (
3434
peakMemory: 0,
3535
};
3636

37-
return process.stats$.pipe(
37+
return stats.pipe(
3838
reduce<AggregatedProcessStats, Accumulator>((acc, val, idx) => ({
3939
elapsed: val.elapsed,
4040
avgProcesses: cumulativeMovingAverage(acc.avgProcesses, val.processes, idx),

‎packages/angular_devkit/benchmark/src/default-stats-capture_spec.ts‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import { Observable } from 'rxjs';
99
import { toArray } from 'rxjs/operators';
1010
import { defaultStatsCapture } from './default-stats-capture';
11-
import { AggregatedProcessStats, MonitoredProcess } from './interfaces';
11+
import { AggregatedProcessStats } from './interfaces';
1212

1313

1414
describe('defaultStatsCapture', () => {
@@ -29,9 +29,8 @@ describe('defaultStatsCapture', () => {
2929
});
3030
obs.complete();
3131
});
32-
const process = { stats$ } as {} as MonitoredProcess;
3332

34-
const res = await defaultStatsCapture(process).pipe(toArray()).toPromise();
33+
const res = await defaultStatsCapture(stats$).pipe(toArray()).toPromise();
3534
expect(res).toEqual([{
3635
name: 'Process Stats',
3736
metrics: [

‎packages/angular_devkit/benchmark/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,6 @@ export * from './default-reporter';
1212
export * from './default-stats-capture';
1313
export * from './monitored-process';
1414
export * from './run-benchmark';
15+
export * from './run-benchmark-watch';
1516
export * from './utils';
1617
export * from './main';

‎packages/angular_devkit/benchmark/src/interfaces.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ export interface MetricGroup {
5151
metrics: (Metric | AggregatedMetric)[];
5252
}
5353

54-
export type Capture = (process: MonitoredProcess) => Observable<MetricGroup>;
54+
export type Capture = (stats: Observable<AggregatedProcessStats>) => Observable<MetricGroup>;
55+
5556

5657
// TODO: might need to allow reporters to say they are finished.
5758
export type BenchmarkReporter = (command: Command, groups: MetricGroup[]) => void;

‎packages/angular_devkit/benchmark/src/main.ts‎

Lines changed: 55 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { Command } from '../src/command';
1616
import { defaultReporter } from '../src/default-reporter';
1717
import { defaultStatsCapture } from '../src/default-stats-capture';
1818
import { runBenchmark } from '../src/run-benchmark';
19+
import { runBenchmarkWatch } from './run-benchmark-watch';
1920

2021

2122
export interface MainOptions {
@@ -47,6 +48,9 @@ export async function main({
4748
--output-file File to output benchmark log to.
4849
--overwrite-output-file If the output file should be overwritten rather than appended to.
4950
--prefix Logging prefix.
51+
--watch-matcher Text to match in stdout to mark an iteration complete.
52+
--watch-timeout The maximum time in 'ms' to wait for the text specified in the matcher to be matched. Default is 10000.
53+
--watch-script Script to run before each watch iteration.
5054
5155
Example:
5256
benchmark --iterations=3 -- node my-script.js
@@ -63,19 +67,27 @@ export async function main({
6367
'output-file': string | null;
6468
cwd: string;
6569
prefix: string;
70+
'watch-timeout': number;
71+
'watch-matcher'?: string;
72+
'watch-script'?: string;
6673
'--': string[] | null;
6774
}
6875

6976
// Parse the command line.
7077
const argv = minimist(args, {
7178
boolean: ['help', 'verbose', 'overwrite-output-file'],
79+
string: [
80+
'watch-matcher',
81+
'watch-script',
82+
],
7283
default: {
7384
'exit-code': 0,
7485
'iterations': 5,
7586
'retries': 5,
7687
'output-file': null,
7788
'cwd': process.cwd(),
7889
'prefix': '[benchmark]',
90+
'watch-timeout': 10000,
7991
},
8092
'--': true,
8193
}) as {} as BenchmarkCliArgv;
@@ -127,6 +139,29 @@ export async function main({
127139

128140
const commandArgv = argv['--'];
129141

142+
const {
143+
'watch-timeout': watchTimeout,
144+
'watch-matcher': watchMatcher,
145+
'watch-script': watchScript,
146+
'exit-code': exitCode,
147+
'output-file': outFile,
148+
iterations,
149+
retries,
150+
} = argv;
151+
152+
// Exit early if we can't find the command to benchmark.
153+
if (watchMatcher && !watchScript) {
154+
logger.fatal(`Cannot use --watch-matcher without specifying --watch-script.`);
155+
156+
return 1;
157+
}
158+
159+
if (!watchMatcher && watchScript) {
160+
logger.fatal(`Cannot use --watch-script without specifying --watch-matcher.`);
161+
162+
return 1;
163+
}
164+
130165
// Exit early if we can't find the command to benchmark.
131166
if (!commandArgv || !Array.isArray(argv['--']) || (argv['--'] as Array<string>).length < 1) {
132167
logger.fatal(`Missing command, see benchmark --help for help.`);
@@ -135,32 +170,42 @@ export async function main({
135170
}
136171

137172
// Setup file logging.
138-
if (argv['output-file'] !== null) {
173+
if (outFile !== null) {
139174
if (argv['overwrite-output-file']) {
140-
writeFileSync(argv['output-file'] as string, '');
175+
writeFileSync(outFile, '');
141176
}
142177
logger.pipe(filter(entry => (entry.level != 'debug' || argv['verbose'])))
143-
.subscribe(entry => appendFileSync(argv['output-file'] as string, `${entry.message}\n`));
178+
.subscribe(entry => appendFileSync(outFile, `${entry.message}\n`));
144179
}
145180

146181
// Run benchmark on given command, capturing stats and reporting them.
147-
const exitCode = argv['exit-code'];
148182
const cmd = commandArgv[0];
149183
const cmdArgs = commandArgv.slice(1);
150184
const command = new Command(cmd, cmdArgs, argv['cwd'], exitCode);
151185
const captures = [defaultStatsCapture];
152186
const reporters = [defaultReporter(logger)];
153-
const iterations = argv['iterations'];
154-
const retries = argv['retries'];
155187

156188
logger.info(`Benchmarking process over ${iterations} iterations, with up to ${retries} retries.`);
157189
logger.info(` ${command.toString()}`);
158190

159-
let res;
160191
try {
161-
res = await runBenchmark(
162-
{ command, captures, reporters, iterations, retries, logger },
163-
).pipe(toArray()).toPromise();
192+
let res$;
193+
if (watchMatcher && watchScript) {
194+
res$ = runBenchmarkWatch({
195+
command, captures, reporters, iterations, retries, logger,
196+
watchCommand: new Command('node', [watchScript]), watchMatcher, watchTimeout,
197+
});
198+
} else {
199+
res$ = runBenchmark(
200+
{ command, captures, reporters, iterations, retries, logger },
201+
);
202+
}
203+
204+
const res = await res$.pipe(toArray()).toPromise();
205+
if (res.length === 0) {
206+
return 1;
207+
}
208+
164209
} catch (error) {
165210
if (error.message) {
166211
logger.fatal(error.message);
@@ -171,10 +216,6 @@ export async function main({
171216
return 1;
172217
}
173218

174-
if (res.length === 0) {
175-
return 1;
176-
}
177-
178219
return 0;
179220
}
180221

‎packages/angular_devkit/benchmark/src/main_spec.ts‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'fs';
99
import { basename, dirname, join } from 'path';
1010
import { main } from './main';
11+
1112
// tslint:disable-next-line:no-implicit-dependencies
1213
const temp = require('temp');
1314

@@ -23,9 +24,11 @@ class MockWriteStream {
2324
}
2425
}
2526

26-
describe('benchmark binary', () => {
27+
fdescribe('benchmark binary', () => {
2728
const benchmarkScript = require.resolve(join(__dirname, './test/fibonacci.js'));
2829
const exitCodeOneScript = require.resolve(join(__dirname, './test/exit-code-one.js'));
30+
const benchmarkWatchScript = require.resolve(join(__dirname, './test/watch-test-cmd.js'));
31+
const watchTriggerScript = require.resolve(join(__dirname, './test/watch-test-script.js'));
2932
const outputFileRoot = temp.mkdirSync('benchmark-binary-spec-');
3033
const outputFile = join(outputFileRoot, 'output.log');
3134
let stdout: MockWriteStream, stderr: MockWriteStream;
@@ -142,4 +145,51 @@ describe('benchmark binary', () => {
142145
stdout.lines.forEach(line => expect(line).toMatch(/^\[abc\]/));
143146
expect(res).toEqual(0);
144147
});
148+
149+
it('uses watch-script and watch-matcher', async () => {
150+
const args = [
151+
'--watch-matcher',
152+
'Complete',
153+
'--watch-script',
154+
watchTriggerScript,
155+
'--',
156+
'node',
157+
benchmarkWatchScript,
158+
];
159+
const res = await main({ args, stdout, stderr });
160+
expect(stdout.lines).toContain('[benchmark] Process Stats\n');
161+
expect(res).toEqual(0);
162+
}, 30000);
163+
164+
it('should not fail with exit code', async () => {
165+
const args = [
166+
'--watch-matcher',
167+
'Complete',
168+
'--watch-script',
169+
watchTriggerScript,
170+
'--',
171+
'node',
172+
exitCodeOneScript,
173+
];
174+
const res = await main({ args, stdout, stderr });
175+
expect(stderr.lines).toContain('[benchmark] Maximum number of retries (5) for command was exceeded.\n');
176+
expect(res).toEqual(1);
177+
});
178+
179+
it('should error when watch-timeout is exceeded', async () => {
180+
const args = [
181+
'--watch-timeout',
182+
'20',
183+
'--watch-matcher',
184+
'Wrong Match',
185+
'--watch-script',
186+
watchTriggerScript,
187+
'--',
188+
'node',
189+
benchmarkWatchScript,
190+
];
191+
const res = await main({ args, stdout, stderr });
192+
expect(stderr.lines).toContain('[benchmark] Timeout has occurred\n');
193+
expect(res).toEqual(1);
194+
});
145195
});

‎packages/angular_devkit/benchmark/src/monitored-process.ts‎

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,14 +31,21 @@ export class LocalMonitoredProcess implements MonitoredProcess {
3131
stats$: Observable<AggregatedProcessStats> = this.stats.asObservable();
3232
stdout$: Observable<Buffer> = this.stdout.asObservable();
3333
stderr$: Observable<Buffer> = this.stderr.asObservable();
34+
private elapsedTimer: number;
3435

35-
constructor(private command: Command) { }
36+
constructor(
37+
private command: Command,
38+
private useProcessTime = true,
39+
) { }
3640

3741
run(): Observable<number> {
3842
return new Observable(obs => {
3943
const { cmd, cwd, args } = this.command;
40-
const spawnOptions: SpawnOptions = { cwd };
44+
const spawnOptions: SpawnOptions = { cwd, shell: true };
4145

46+
if (!this.useProcessTime) {
47+
this.resetElapsedTimer();
48+
}
4249
// Spawn the process.
4350
const childProcess = spawn(cmd, args, spawnOptions);
4451

@@ -63,7 +70,14 @@ export class LocalMonitoredProcess implements MonitoredProcess {
6370
}
6471

6572
return {
66-
processes, cpu, memory, pid, ppid, ctime, elapsed, timestamp,
73+
processes,
74+
cpu,
75+
memory,
76+
pid,
77+
ppid,
78+
ctime,
79+
elapsed: this.useProcessTime ? elapsed : (Date.now() - this.elapsedTimer),
80+
timestamp,
6781
} as AggregatedProcessStats;
6882
}),
6983
tap(stats => this.stats.next(stats)),
@@ -100,7 +114,15 @@ export class LocalMonitoredProcess implements MonitoredProcess {
100114
processExitCb = killChildProcess;
101115

102116
// Cleanup on unsubscription.
103-
return () => childProcess.kill();
117+
return killChildProcess;
104118
});
105119
}
120+
121+
resetElapsedTimer() {
122+
if (this.useProcessTime) {
123+
throw new Error(`Cannot reset elapsed timer when using process time. Set 'useProcessTime' to false.`);
124+
}
125+
126+
this.elapsedTimer = Date.now();
127+
}
106128
}

0 commit comments

Comments
 (0)