Skip to content

Commit b447190

Browse files
swissspidyswissspidy
committed
Build/Test Tools: Compare results in performance measurement workflow.
This extends the performance test workflow added in [55459] to also run tests against the target branch (if running on a pull request) or the previous commit (if running on trunk). Those results are then compared with the ones from the current commit, and the difference is displayed as a GitHub Actions workflow summary for convenience. Props mukesh27, flixos90, desrosj, joemcgill, swissspidy. Fixes #58358, #58359. See #56150. git-svn-id: https://develop.svn.wordpress.org/trunk@56506 602fd350-edb4-49c9-b593-d223f7449a82
1 parent 36fe58a commit b447190

2 files changed

Lines changed: 209 additions & 4 deletions

File tree

‎.github/workflows/performance.yml‎

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ env:
4444
# changed if we want to normalize results against a different baseline.
4545
BASE_TAG: '6.1.1'
4646
LOCAL_DIR: build
47+
TARGET_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.ref || '' }}
48+
TARGET_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
4749

4850
jobs:
4951
# Runs the performance test suite.
@@ -65,9 +67,17 @@ jobs:
6567
# - Install MU plugin.
6668
# - Run performance tests (current commit).
6769
# - Print performance tests results.
70+
# - Check out target commit (target branch or previous commit).
71+
# - Install npm dependencies.
72+
# - Build WordPress.
73+
# - Run performance tests (previous/target commit).
74+
# - Print target performance tests results.
75+
# - Reset to original commit.
6876
# - Set the environment to the baseline version.
6977
# - Run baseline performance tests.
70-
# - Print base line performance tests results.
78+
# - Print baseline performance tests results.
79+
# - Compare results with base.
80+
# - Add workflow summary.
7181
# - Set the base sha.
7282
# - Set commit details.
7383
# - Publish performance results.
@@ -152,7 +162,31 @@ jobs:
152162
run: npm run test:performance
153163

154164
- name: Print performance tests results
155-
run: "node ./tests/performance/results.js"
165+
run: node ./tests/performance/results.js
166+
167+
- name: Check out target commit (target branch or previous commit)
168+
run: |
169+
if [[ -z "$TARGET_REF" ]]; then
170+
git fetch -n origin $TARGET_SHA
171+
else
172+
git fetch -n origin $TARGET_REF
173+
fi
174+
git reset --hard $TARGET_SHA
175+
176+
- name: Install npm dependencies
177+
run: npm ci
178+
179+
- name: Build WordPress
180+
run: npm run build
181+
182+
- name: Run target performance tests (base/previous commit)
183+
run: npm run test:performance -- --prefix=before
184+
185+
- name: Print target performance tests results
186+
run: node ./tests/performance/results.js --prefix=before
187+
188+
- name: Reset to original commit
189+
run: git reset --hard $GITHUB_SHA
156190

157191
- name: Set the environment to the baseline version
158192
run: |
@@ -162,8 +196,14 @@ jobs:
162196
- name: Run baseline performance tests
163197
run: npm run test:performance -- --prefix=base
164198

165-
- name: Print base line performance tests results
166-
run: "node ./tests/performance/results.js --prefix=base"
199+
- name: Print baseline performance tests results
200+
run: node ./tests/performance/results.js --prefix=base
201+
202+
- name: Compare results with base
203+
run: node ./tests/performance/compare-results.js ${{ runner.temp }}/summary.md
204+
205+
- name: Add workflow summary
206+
run: cat ${{ runner.temp }}/summary.md >> $GITHUB_STEP_SUMMARY
167207

168208
- name: Set the base sha
169209
# Only needed when publishing results.

‎tests/performance/compare-results.js‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
#!/usr/bin/env node
2+
3+
/**
4+
* External dependencies.
5+
*/
6+
const fs = require( 'fs' );
7+
const path = require( 'path' );
8+
const { median } = require( './utils' );
9+
10+
/**
11+
* Parse test files into JSON objects.
12+
*
13+
* @param {string} fileName The name of the file.
14+
* @returns An array of parsed objects from each file.
15+
*/
16+
const parseFile = ( fileName ) =>
17+
JSON.parse(
18+
fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' )
19+
);
20+
21+
// The list of test suites to log.
22+
const testSuites = [ 'home-block-theme', 'home-classic-theme' ];
23+
24+
// The current commit's results.
25+
const testResults = Object.fromEntries(
26+
testSuites.map( ( key ) => [
27+
key,
28+
parseFile( `${ key }.test.results.json` ),
29+
] )
30+
);
31+
32+
// The previous commit's results.
33+
const prevResults = Object.fromEntries(
34+
testSuites.map( ( key ) => [
35+
key,
36+
parseFile( `before-${ key }.test.results.json` ),
37+
] )
38+
);
39+
40+
const args = process.argv.slice( 2 );
41+
42+
const summaryFile = args[ 0 ];
43+
44+
/**
45+
* Formats an array of objects as a Markdown table.
46+
*
47+
* For example, this array:
48+
*
49+
* [
50+
* {
51+
* foo: 123,
52+
* bar: 456,
53+
* baz: 'Yes',
54+
* },
55+
* {
56+
* foo: 777,
57+
* bar: 999,
58+
* baz: 'No',
59+
* }
60+
* ]
61+
*
62+
* Will result in the following table:
63+
*
64+
* | foo | bar | baz |
65+
* |-----|-----|-----|
66+
* | 123 | 456 | Yes |
67+
* | 777 | 999 | No |
68+
*
69+
* @param {Array<Object>} rows Table rows.
70+
* @returns {string} Markdown table content.
71+
*/
72+
function formatAsMarkdownTable( rows ) {
73+
let result = '';
74+
const headers = Object.keys( rows[ 0 ] );
75+
for ( const header of headers ) {
76+
result += `| ${ header } `;
77+
}
78+
result += '|\n';
79+
for ( const header of headers ) {
80+
result += '| ------ ';
81+
}
82+
result += '|\n';
83+
84+
for ( const row of rows ) {
85+
for ( const value of Object.values( row ) ) {
86+
result += `| ${ value } `;
87+
}
88+
result += '|\n';
89+
}
90+
91+
return result;
92+
}
93+
94+
/**
95+
* Returns a Markdown link to a Git commit on the current GitHub repository.
96+
*
97+
* For example, turns `a5c3785ed8d6a35868bc169f07e40e889087fd2e`
98+
* into (https://github.com/wordpress/wordpress-develop/commit/36fe58a8c64dcc83fc21bddd5fcf054aef4efb27)[36fe58a].
99+
*
100+
* @param {string} sha Commit SHA.
101+
* @return string Link
102+
*/
103+
function linkToSha(sha) {
104+
const repoName = process.env.GITHUB_REPOSITORY || 'wordpress/wordpress-develop';
105+
106+
return `[${sha.slice(0, 7)}](https://github.com/${repoName}/commit/${sha})`;
107+
}
108+
109+
let summaryMarkdown = `# Performance Test Results\n\n`;
110+
111+
if ( process.env.GITHUB_SHA ) {
112+
summaryMarkdown += `🛎️ Performance test results for ${ linkToSha( process.env.GITHUB_SHA ) } are in!\n\n`;
113+
} else {
114+
summaryMarkdown += `🛎️ Performance test results are in!\n\n`;
115+
}
116+
117+
if ( process.env.TARGET_SHA ) {
118+
summaryMarkdown += `This compares the results from this commit with the ones from ${ linkToSha( process.env.TARGET_SHA ) }.\n\n`;
119+
}
120+
121+
if ( process.env.GITHUB_SHA ) {
122+
summaryMarkdown += `**Note:** Due to the nature of how GitHub Actions work, some variance in the results is expected.\n\n`;
123+
}
124+
125+
console.log( 'Performance Test Results\n' );
126+
127+
console.log( 'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n' );
128+
129+
for ( const key of testSuites ) {
130+
const current = testResults[ key ];
131+
const prev = prevResults[ key ];
132+
133+
const title = ( key.charAt( 0 ).toUpperCase() + key.slice( 1 ) ).replace(
134+
/-+/g,
135+
' '
136+
);
137+
138+
const rows = [];
139+
140+
for ( const [ metric, values ] of Object.entries( current ) ) {
141+
const value = median( values );
142+
const prevValue = median( prev[ metric ] );
143+
144+
const delta = value - prevValue;
145+
const percentage = Math.round( ( delta / value ) * 100 );
146+
rows.push( {
147+
Metric: metric,
148+
Before: `${ prevValue.toFixed( 2 ) } ms`,
149+
After: `${ value.toFixed( 2 ) } ms`,
150+
'Diff abs.': `${ delta.toFixed( 2 ) } ms`,
151+
'Diff %': `${ percentage.toFixed( 2 ) } %`,
152+
} );
153+
}
154+
155+
summaryMarkdown += `## ${ title }\n\n`;
156+
summaryMarkdown += `${ formatAsMarkdownTable( rows ) }\n`;
157+
158+
console.log( title );
159+
console.table( rows );
160+
}
161+
162+
fs.writeFileSync(
163+
summaryFile,
164+
summaryMarkdown
165+
);

0 commit comments

Comments
 (0)