Skip to content

Commit e692c18

Browse files
author
Casey Hillers
authored
[customer_testing] Refactor runner to be testable (flutter#76831)
1 parent ca2bef6 commit e692c18

7 files changed

Lines changed: 411 additions & 229 deletions

File tree

‎dev/customer_testing/lib/customer_test.dart‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:io';
6+
7+
import 'package:meta/meta.dart';
8+
9+
@immutable
10+
class CustomerTest {
11+
factory CustomerTest(File testFile) {
12+
final String errorPrefix = 'Could not parse: ${testFile.path}\n';
13+
final List<String> contacts = <String>[];
14+
final List<String> fetch = <String>[];
15+
final List<Directory> update = <Directory>[];
16+
final List<String> test = <String>[];
17+
bool hasTests = false;
18+
for (final String line in testFile.readAsLinesSync().map((String line) => line.trim())) {
19+
if (line.isEmpty) {
20+
// blank line
21+
} else if (line.startsWith('#')) {
22+
// comment
23+
} else if (line.startsWith('contact=')) {
24+
contacts.add(line.substring(8));
25+
} else if (line.startsWith('fetch=')) {
26+
fetch.add(line.substring(6));
27+
} else if (line.startsWith('update=')) {
28+
update.add(Directory(line.substring(7)));
29+
} else if (line.startsWith('test=')) {
30+
hasTests = true;
31+
test.add(line.substring(5));
32+
} else if (line.startsWith('test.windows=')) {
33+
hasTests = true;
34+
if (Platform.isWindows)
35+
test.add(line.substring(13));
36+
} else if (line.startsWith('test.macos=')) {
37+
hasTests = true;
38+
if (Platform.isMacOS)
39+
test.add(line.substring(11));
40+
} else if (line.startsWith('test.linux=')) {
41+
hasTests = true;
42+
if (Platform.isLinux)
43+
test.add(line.substring(11));
44+
} else if (line.startsWith('test.posix=')) {
45+
hasTests = true;
46+
if (Platform.isLinux || Platform.isMacOS)
47+
test.add(line.substring(11));
48+
} else {
49+
throw FormatException('${errorPrefix}Unexpected directive:\n$line');
50+
}
51+
}
52+
if (contacts.isEmpty)
53+
throw FormatException('${errorPrefix}No contacts specified. At least one contact e-mail address must be specified.');
54+
for (final String email in contacts) {
55+
if (!email.contains(_email) || email.endsWith('@example.com'))
56+
throw FormatException('${errorPrefix}The following e-mail address appears to be an invalid e-mail address: $email');
57+
}
58+
if (fetch.isEmpty)
59+
throw FormatException('${errorPrefix}No "fetch" directives specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
60+
if (fetch.length < 2)
61+
throw FormatException('${errorPrefix}Only one "fetch" directive specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
62+
if (!fetch[0].contains(_fetch1))
63+
throw FormatException('${errorPrefix}First "fetch" directive does not match expected pattern (expected "git clone https://github.com/USERNAME/REPOSITORY.git tests").');
64+
if (!fetch[1].contains(_fetch2))
65+
throw FormatException('${errorPrefix}Second "fetch" directive does not match expected pattern (expected "git -C tests checkout HASH").');
66+
if (update.isEmpty)
67+
throw FormatException('${errorPrefix}No "update" directives specified. At least one directory must be specified. (It can be "." to just upgrade the root of the repository.)');
68+
if (!hasTests)
69+
throw FormatException('${errorPrefix}No "test" directives specified. At least one command must be specified to run tests.');
70+
return CustomerTest._(
71+
List<String>.unmodifiable(contacts),
72+
List<String>.unmodifiable(fetch),
73+
List<Directory>.unmodifiable(update),
74+
List<String>.unmodifiable(test),
75+
);
76+
}
77+
78+
const CustomerTest._(this.contacts, this.fetch, this.update, this.tests);
79+
80+
// (e-mail regexp from HTML standard)
81+
static final RegExp _email = RegExp(r"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$");
82+
static final RegExp _fetch1 = RegExp(r'^git(?: -c core.longPaths=true)? clone https://github.com/[-a-zA-Z0-9]+/[-_a-zA-Z0-9]+.git tests$');
83+
static final RegExp _fetch2 = RegExp(r'^git(?: -c core.longPaths=true)? -C tests checkout [0-9a-f]+$');
84+
85+
final List<String> contacts;
86+
final List<String> fetch;
87+
final List<Directory> update;
88+
final List<String> tests;
89+
}

‎dev/customer_testing/lib/runner.dart‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:convert';
6+
import 'dart:io';
7+
8+
import 'package:path/path.dart' as path;
9+
10+
import 'customer_test.dart';
11+
12+
Future<bool> runTests({
13+
int repeat = 1,
14+
bool skipOnFetchFailure = false,
15+
bool verbose = false,
16+
int numberShards = 1,
17+
int shardIndex = 0,
18+
List<File> files,
19+
}) async {
20+
if (verbose)
21+
print('Starting run_tests.dart...');
22+
23+
// Best attempt at evenly splitting tests among the shards
24+
final List<File> shardedFiles = <File>[];
25+
for (int i = shardIndex; i < files.length; i += numberShards) {
26+
shardedFiles.add(files[i]);
27+
}
28+
29+
int testCount = 0;
30+
int failures = 0;
31+
32+
if (verbose) {
33+
final String s = files.length == 1 ? '' : 's';
34+
final String ss = shardedFiles.length == 1 ? '' : 's';
35+
print('${files.length} file$s specified. ${shardedFiles.length} test$ss in shard #$shardIndex.');
36+
print('');
37+
}
38+
39+
if (verbose) {
40+
print('Tests in this shard:');
41+
for (final File file in shardedFiles)
42+
print(file.path);
43+
}
44+
print('');
45+
46+
for (final File file in shardedFiles) {
47+
if (verbose)
48+
print('Processing ${file.path}...');
49+
CustomerTest instructions;
50+
try {
51+
instructions = CustomerTest(file);
52+
} on FormatException catch (error) {
53+
print('ERROR: ${error.message}');
54+
print('');
55+
failures += 1;
56+
continue;
57+
} on FileSystemException catch (error) {
58+
print('ERROR: ${error.message}');
59+
print(' ${file.path}');
60+
print('');
61+
failures += 1;
62+
continue;
63+
}
64+
65+
final Directory checkout = Directory.systemTemp.createTempSync('flutter_customer_testing.${path.basenameWithoutExtension(file.path)}.');
66+
if (verbose)
67+
print('Created temporary directory: ${checkout.path}');
68+
try {
69+
bool success;
70+
bool showContacts = false;
71+
for (final String fetchCommand in instructions.fetch) {
72+
success = await shell(fetchCommand, checkout, verbose: verbose, silentFailure: skipOnFetchFailure);
73+
if (!success) {
74+
if (skipOnFetchFailure) {
75+
if (verbose) {
76+
print('Skipping (fetch failed).');
77+
} else {
78+
print('Skipping ${file.path} (fetch failed).');
79+
}
80+
} else {
81+
print('ERROR: Failed to fetch repository.');
82+
failures += 1;
83+
showContacts = true;
84+
}
85+
break;
86+
}
87+
}
88+
assert(success != null);
89+
if (success) {
90+
if (verbose)
91+
print('Running tests...');
92+
final Directory tests = Directory(path.join(checkout.path, 'tests'));
93+
// TODO(ianh): Once we have a way to update source code, run that command in each directory of instructions.update
94+
for (int iteration = 0; iteration < repeat; iteration += 1) {
95+
if (verbose && repeat > 1)
96+
print('Round ${iteration + 1} of $repeat.');
97+
for (final String testCommand in instructions.tests) {
98+
testCount += 1;
99+
success = await shell(testCommand, tests, verbose: verbose);
100+
if (!success) {
101+
print('ERROR: One or more tests from ${path.basenameWithoutExtension(file.path)} failed.');
102+
failures += 1;
103+
showContacts = true;
104+
break;
105+
}
106+
}
107+
}
108+
if (verbose && success)
109+
print('Tests finished.');
110+
}
111+
if (showContacts) {
112+
final String s = instructions.contacts.length == 1 ? '' : 's';
113+
print('Contact$s: ${instructions.contacts.join(", ")}');
114+
}
115+
} finally {
116+
if (verbose)
117+
print('Deleting temporary directory...');
118+
try {
119+
checkout.deleteSync(recursive: true);
120+
} on FileSystemException {
121+
print('Failed to delete "${checkout.path}".');
122+
}
123+
}
124+
if (verbose)
125+
print('');
126+
}
127+
if (failures > 0) {
128+
final String s = failures == 1 ? '' : 's';
129+
print('$failures failure$s.');
130+
return false;
131+
}
132+
print('$testCount tests all passed!');
133+
return true;
134+
}
135+
136+
final RegExp _spaces = RegExp(r' +');
137+
138+
Future<bool> shell(String command, Directory directory, { bool verbose = false, bool silentFailure = false }) async {
139+
if (verbose)
140+
print('>> $command');
141+
Process process;
142+
if (Platform.isWindows) {
143+
process = await Process.start('CMD.EXE', <String>['/S', '/C', command], workingDirectory: directory.path);
144+
} else {
145+
final List<String> segments = command.trim().split(_spaces);
146+
process = await Process.start(segments.first, segments.skip(1).toList(), workingDirectory: directory.path);
147+
}
148+
final List<String> output = <String>[];
149+
utf8.decoder.bind(process.stdout).transform(const LineSplitter()).listen(verbose ? printLog : output.add);
150+
utf8.decoder.bind(process.stderr).transform(const LineSplitter()).listen(verbose ? printLog : output.add);
151+
final bool success = await process.exitCode == 0;
152+
if (success || silentFailure)
153+
return success;
154+
if (!verbose) {
155+
print('>> $command');
156+
output.forEach(printLog);
157+
}
158+
return success;
159+
}
160+
161+
void printLog(String line) {
162+
print('| $line'.trimRight());
163+
}

‎dev/customer_testing/pubspec.yaml‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,42 @@ dependencies:
1919
string_scanner: 1.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
2020
term_glyph: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
2121

22-
# PUBSPEC CHECKSUM: 5fe8
22+
dev_dependencies:
23+
test: 1.16.5
24+
25+
_fe_analyzer_shared: 14.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
26+
analyzer: 0.41.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
27+
boolean_selector: 2.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
28+
cli_util: 0.3.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
29+
convert: 3.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
30+
coverage: 0.15.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
31+
crypto: 3.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
32+
http_multi_server: 2.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
33+
http_parser: 3.1.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
34+
io: 1.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
35+
js: 0.6.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
36+
logging: 0.11.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
37+
matcher: 0.12.10 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
38+
mime: 1.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
39+
node_preamble: 1.4.13 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
40+
package_config: 1.9.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
41+
pool: 1.5.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
42+
pub_semver: 1.4.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
43+
shelf: 0.7.9 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
44+
shelf_packages_handler: 2.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
45+
shelf_static: 0.2.9+2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
46+
shelf_web_socket: 0.2.4+1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
47+
source_map_stack_trace: 2.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
48+
source_maps: 0.10.10 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
49+
stack_trace: 1.10.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
50+
stream_channel: 2.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
51+
test_api: 0.2.19 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
52+
test_core: 0.3.15 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
53+
typed_data: 1.3.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
54+
vm_service: 6.0.1-nullsafety.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
55+
watcher: 1.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
56+
web_socket_channel: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
57+
webkit_inspection_protocol: 0.7.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
58+
yaml: 3.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
59+
60+
# PUBSPEC CHECKSUM: 86cb

0 commit comments

Comments
 (0)