1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5import 'dart:convert';
6import 'dart:io';
7
8import 'package:path/path.dart' as path;
9
10import 'customer_test.dart';
11
12Future<bool> runTests({
13 int repeat = 1,
14 bool skipOnFetchFailure = false,
15 bool verbose = false,
16 int numberShards = 1,
17 int shardIndex = 0,
18 required List<File> files,
19}) async {
20 if (verbose) {
21 print('Starting run_tests.dart...');
22 }
23
24 // Best attempt at evenly splitting tests among the shards
25 final List<File> shardedFiles = <File>[];
26 for (int i = shardIndex; i < files.length; i += numberShards) {
27 shardedFiles.add(files[i]);
28 }
29
30 int testCount = 0;
31 int failures = 0;
32
33 if (verbose) {
34 final String s = files.length == 1 ? '' : 's';
35 if (numberShards > 1) {
36 final String ss = shardedFiles.length == 1 ? '' : 's';
37 print(
38 '${files.length} file$s specified. ${shardedFiles.length} test$ss in shard #$shardIndex ($numberShards shards total).',
39 );
40 } else {
41 print('${files.length} file$s specified.');
42 }
43 print('');
44 }
45
46 if (verbose) {
47 if (numberShards > 1) {
48 print('Tests in this shard:');
49 } else {
50 print('Tests:');
51 }
52 for (final File file in shardedFiles) {
53 print(file.path);
54 }
55 }
56 print('');
57
58 for (final File file in shardedFiles) {
59 // Always print name of running task for debugging individual customer test
60 // suites.
61 print('Processing ${file.path}...');
62
63 void failure(String message) {
64 print('ERROR: $message');
65 failures += 1;
66 }
67
68 CustomerTest instructions;
69 try {
70 instructions = CustomerTest(file);
71 } on FormatException catch (error) {
72 failure(error.message);
73 print('');
74 continue;
75 } on FileSystemException catch (error) {
76 failure(error.message);
77 print('');
78 continue;
79 }
80
81 bool success = true;
82
83 final Directory checkout = Directory.systemTemp.createTempSync(
84 'flutter_customer_testing.${path.basenameWithoutExtension(file.path)}.',
85 );
86 if (verbose) {
87 print('Created temporary directory: ${checkout.path}');
88 }
89 try {
90 assert(instructions.fetch.isNotEmpty);
91 for (final String fetchCommand in instructions.fetch) {
92 success = await shell(
93 fetchCommand,
94 checkout,
95 verbose: verbose,
96 silentFailure: skipOnFetchFailure,
97 );
98 if (!success) {
99 if (skipOnFetchFailure) {
100 if (verbose) {
101 print('Skipping (fetch failed).');
102 } else {
103 print('Skipping ${file.path} (fetch failed).');
104 }
105 } else {
106 failure('Failed to fetch repository.');
107 }
108 break;
109 }
110 }
111 if (success) {
112 final Directory customerRepo = Directory(path.join(checkout.path, 'tests'));
113 for (final String setupCommand in instructions.setup) {
114 if (verbose) {
115 print('Running setup command: $setupCommand');
116 }
117 success = await shell(setupCommand, customerRepo, verbose: verbose);
118 if (!success) {
119 failure('Setup command failed: $setupCommand');
120 break;
121 }
122 }
123 for (final Directory updateDirectory in instructions.update) {
124 final Directory resolvedUpdateDirectory = Directory(
125 path.join(customerRepo.path, updateDirectory.path),
126 );
127 if (verbose) {
128 print('Updating code in ${resolvedUpdateDirectory.path}...');
129 }
130 if (!File(path.join(resolvedUpdateDirectory.path, 'pubspec.yaml')).existsSync()) {
131 failure(
132 'The directory ${updateDirectory.path}, which was specified as an update directory, does not contain a "pubspec.yaml" file.',
133 );
134 success = false;
135 break;
136 }
137 success = await shell('flutter packages get', resolvedUpdateDirectory, verbose: verbose);
138 if (!success) {
139 failure(
140 'Could not run "flutter pub get" in ${updateDirectory.path}, which was specified as an update directory.',
141 );
142 break;
143 }
144 success = await shell('dart fix --apply', resolvedUpdateDirectory, verbose: verbose);
145 if (!success) {
146 failure(
147 'Could not run "dart fix" in ${updateDirectory.path}, which was specified as an update directory.',
148 );
149 break;
150 }
151 }
152 if (success) {
153 if (verbose) {
154 print('Running tests...');
155 }
156 if (instructions.iterations != null && instructions.iterations! < repeat) {
157 if (verbose) {
158 final String s = instructions.iterations == 1 ? '' : 's';
159 print(
160 'Limiting to ${instructions.iterations} round$s rather than $repeat rounds because of "iterations" directive.',
161 );
162 }
163 repeat = instructions.iterations!;
164 }
165 final Stopwatch stopwatch = Stopwatch()..start();
166 for (int iteration = 0; iteration < repeat; iteration += 1) {
167 if (verbose && repeat > 1) {
168 print('Round ${iteration + 1} of $repeat.');
169 }
170 for (final String testCommand in instructions.tests) {
171 testCount += 1;
172 success = await shell(testCommand, customerRepo, verbose: verbose);
173 if (!success) {
174 failure(
175 'One or more tests from ${path.basenameWithoutExtension(file.path)} failed.',
176 );
177 break;
178 }
179 }
180 }
181 stopwatch.stop();
182 // Always print test runtime for debugging.
183 print(
184 'Tests finished in ${(stopwatch.elapsed.inSeconds / repeat).toStringAsFixed(2)} seconds per iteration.',
185 );
186 }
187 }
188 } finally {
189 if (verbose) {
190 print('Deleting temporary directory...');
191 }
192 try {
193 checkout.deleteSync(recursive: true);
194 } on FileSystemException {
195 print('Failed to delete "${checkout.path}".');
196 }
197 }
198 if (!success) {
199 final String s = instructions.contacts.length == 1 ? '' : 's';
200 print('Contact$s: ${instructions.contacts.join(", ")}');
201 }
202 if (verbose || !success) {
203 print('');
204 }
205 }
206 if (failures > 0) {
207 final String s = failures == 1 ? '' : 's';
208 print('$failures failure$s.');
209 return false;
210 }
211 print('$testCount tests all passed!');
212 return true;
213}
214
215final RegExp _spaces = RegExp(r' +');
216
217Future<bool> shell(
218 String command,
219 Directory directory, {
220 bool verbose = false,
221 bool silentFailure = false,
222 void Function()? failedCallback,
223}) async {
224 if (verbose) {
225 print('>> $command');
226 }
227 Process process;
228 if (Platform.isWindows) {
229 process = await Process.start('CMD.EXE', <String>[
230 '/S',
231 '/C',
232 command,
233 ], workingDirectory: directory.path);
234 } else {
235 final List<String> segments = command.trim().split(_spaces);
236 process = await Process.start(
237 segments.first,
238 segments.skip(1).toList(),
239 workingDirectory: directory.path,
240 );
241 }
242 final List<String> output = <String>[];
243 utf8.decoder
244 .bind(process.stdout)
245 .transform(const LineSplitter())
246 .listen(verbose ? printLog : output.add);
247 utf8.decoder
248 .bind(process.stderr)
249 .transform(const LineSplitter())
250 .listen(verbose ? printLog : output.add);
251 final bool success = await process.exitCode == 0;
252 if (success || silentFailure) {
253 return success;
254 }
255 if (!verbose) {
256 if (failedCallback != null) {
257 failedCallback();
258 }
259 print('>> $command');
260 output.forEach(printLog);
261 }
262 return success;
263}
264
265void printLog(String line) {
266 print('| $line'.trimRight());
267}
268