1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5import 'dart:convert';
6import 'dart:io' show Directory, File, FileSystemEntity, Platform, Process;
7import 'dart:typed_data';
8
9import 'package:archive/archive.dart';
10import 'package:path/path.dart' as path;
11
12import '../run_command.dart';
13import '../utils.dart';
14import 'run_test_harness_tests.dart';
15
16Future<void> frameworkTestsRunner() async {
17 final List<String> trackWidgetCreationAlternatives = <String>[
18 '--track-widget-creation',
19 '--no-track-widget-creation',
20 ];
21
22 Future<void> runWidgets() async {
23 printProgress('${green}Running packages/flutter tests $reset for ${cyan}test/widgets/$reset');
24 for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
25 await runFlutterTest(
26 path.join(flutterRoot, 'packages', 'flutter'),
27 options: <String>[trackWidgetCreationOption],
28 tests: <String>[path.join('test', 'widgets') + path.separator],
29 );
30 }
31 // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
32 for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
33 await runFlutterTest(
34 path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'),
35 options: <String>[trackWidgetCreationOption],
36 fatalWarnings: false, // until we've migrated video_player
37 );
38 }
39 // Run release mode tests (see packages/flutter/test_release/README.md)
40 await runFlutterTest(
41 path.join(flutterRoot, 'packages', 'flutter'),
42 options: <String>['--dart-define=dart.vm.product=true'],
43 tests: <String>['test_release${path.separator}'],
44 );
45 // Run profile mode tests (see packages/flutter/test_profile/README.md)
46 await runFlutterTest(
47 path.join(flutterRoot, 'packages', 'flutter'),
48 options: <String>[
49 '--dart-define=dart.vm.product=false',
50 '--dart-define=dart.vm.profile=true',
51 ],
52 tests: <String>['test_profile${path.separator}'],
53 );
54 }
55
56 Future<void> runImpeller() async {
57 printProgress('${green}Running packages/flutter tests $reset in Impeller$reset');
58 await runFlutterTest(
59 path.join(flutterRoot, 'packages', 'flutter'),
60 options: <String>['--enable-impeller'],
61 );
62 }
63
64 Future<void> runLibraries() async {
65 final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test'))
66 .listSync(followLinks: false)
67 .whereType<Directory>()
68 .where((Directory dir) => !dir.path.endsWith('widgets'))
69 .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
70 .toList();
71 printProgress(
72 '${green}Running packages/flutter tests$reset for $cyan${tests.join(", ")}$reset',
73 );
74 for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
75 await runFlutterTest(
76 path.join(flutterRoot, 'packages', 'flutter'),
77 options: <String>[trackWidgetCreationOption],
78 tests: tests,
79 );
80 }
81 }
82
83 Future<void> runExampleTests() async {
84 await runCommand(flutter, <String>[
85 'config',
86 '--enable-${Platform.operatingSystem}-desktop',
87 ], workingDirectory: flutterRoot);
88 await runCommand(dart, <String>[
89 path.join(flutterRoot, 'dev', 'tools', 'examples_smoke_test.dart'),
90 ], workingDirectory: path.join(flutterRoot, 'examples', 'api'));
91 for (final FileSystemEntity entity in Directory(
92 path.join(flutterRoot, 'examples'),
93 ).listSync()) {
94 if (entity is! Directory || !Directory(path.join(entity.path, 'test')).existsSync()) {
95 continue;
96 }
97 await runFlutterTest(entity.path);
98 }
99 }
100
101 Future<void> runTracingTests() async {
102 final String tracingDirectory = path.join(flutterRoot, 'dev', 'tracing_tests');
103
104 // run the tests for debug mode
105 await runFlutterTest(tracingDirectory, options: <String>['--enable-vmservice']);
106
107 Future<List<String>> verifyTracingAppBuild({
108 required String modeArgument,
109 required String sourceFile,
110 required Set<String> allowed,
111 required Set<String> disallowed,
112 }) async {
113 try {
114 await runCommand(flutter, <String>[
115 'build',
116 'appbundle',
117 '--$modeArgument',
118 path.join('lib', sourceFile),
119 ], workingDirectory: tracingDirectory);
120 final Archive archive = ZipDecoder().decodeBytes(
121 File(
122 path.join(
123 tracingDirectory,
124 'build',
125 'app',
126 'outputs',
127 'bundle',
128 modeArgument,
129 'app-$modeArgument.aab',
130 ),
131 ).readAsBytesSync(),
132 );
133 final ArchiveFile libapp = archive.findFile('base/lib/arm64-v8a/libapp.so')!;
134 final Uint8List libappBytes = libapp.content as Uint8List; // bytes decompressed here
135 final String libappStrings = utf8.decode(libappBytes, allowMalformed: true);
136 await runCommand(flutter, <String>['clean'], workingDirectory: tracingDirectory);
137 final List<String> results = <String>[];
138 for (final String pattern in allowed) {
139 if (!libappStrings.contains(pattern)) {
140 results.add(
141 'When building with --$modeArgument, expected to find "$pattern" in libapp.so but could not find it.',
142 );
143 }
144 }
145 for (final String pattern in disallowed) {
146 if (libappStrings.contains(pattern)) {
147 results.add(
148 'When building with --$modeArgument, expected to not find "$pattern" in libapp.so but did find it.',
149 );
150 }
151 }
152 return results;
153 } catch (error, stackTrace) {
154 return <String>[error.toString(), ...stackTrace.toString().trimRight().split('\n')];
155 }
156 }
157
158 final List<String> results = <String>[];
159 results.addAll(
160 await verifyTracingAppBuild(
161 modeArgument: 'profile',
162 sourceFile: 'control.dart', // this is the control, the other two below are the actual test
163 allowed: <String>{
164 'TIMELINE ARGUMENTS TEST CONTROL FILE',
165 'toTimelineArguments used in non-debug build', // we call toTimelineArguments directly to check the message does exist
166 },
167 disallowed: <String>{'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE'},
168 ),
169 );
170 results.addAll(
171 await verifyTracingAppBuild(
172 modeArgument: 'profile',
173 sourceFile: 'test.dart',
174 allowed: <String>{
175 'BUILT IN PROFILE MODE', 'RenderTest.performResize called', // controls
176 'BUILD', 'LAYOUT', 'PAINT', // we output these to the timeline in profile builds
177 // (LAYOUT and PAINT also exist because of NEEDS-LAYOUT and NEEDS-PAINT in RenderObject.toStringShort)
178 },
179 disallowed: <String>{
180 'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE',
181 'TestWidget.debugFillProperties called',
182 'RenderTest.debugFillProperties called', // debug only
183 'toTimelineArguments used in non-debug build', // entire function should get dropped by tree shaker
184 },
185 ),
186 );
187 results.addAll(
188 await verifyTracingAppBuild(
189 modeArgument: 'release',
190 sourceFile: 'test.dart',
191 allowed: <String>{
192 'BUILT IN RELEASE MODE', 'RenderTest.performResize called', // controls
193 },
194 disallowed: <String>{
195 'BUILT IN DEBUG MODE', 'BUILT IN PROFILE MODE',
196 'BUILD',
197 'LAYOUT',
198 'PAINT', // these are only used in Timeline.startSync calls that should not appear in release builds
199 'TestWidget.debugFillProperties called',
200 'RenderTest.debugFillProperties called', // debug only
201 'toTimelineArguments used in non-debug build', // not included in release builds
202 },
203 ),
204 );
205 if (results.isNotEmpty) {
206 foundError(results);
207 }
208 }
209
210 Future<void> runFixTests(String package) async {
211 final List<String> args = <String>['fix', '--compare-to-golden'];
212 await runCommand(
213 dart,
214 args,
215 workingDirectory: path.join(flutterRoot, 'packages', package, 'test_fixes'),
216 );
217 }
218
219 Future<void> runPrivateTests() async {
220 final List<String> args = <String>['run', 'bin/test_private.dart'];
221 final Map<String, String> environment = <String, String>{
222 'FLUTTER_ROOT': flutterRoot,
223 if (Directory(pubCache).existsSync()) 'PUB_CACHE': pubCache,
224 };
225 adjustEnvironmentToEnableFlutterAsserts(environment);
226 await runCommand(
227 dart,
228 args,
229 workingDirectory: path.join(flutterRoot, 'packages', 'flutter', 'test_private'),
230 environment: environment,
231 );
232 }
233
234 // Tests that take longer than average to run. This is usually because they
235 // need to compile something large or make use of the analyzer for the test.
236 // These tests need to be platform agnostic as they are only run on a linux
237 // machine to save on execution time and cost.
238 Future<void> runSlow() async {
239 printProgress(
240 '${green}Running slow package tests$reset for directories other than packages/flutter',
241 );
242 await runTracingTests();
243 await runFixTests('flutter');
244 await runFixTests('flutter_test');
245 await runFixTests('integration_test');
246 await runFixTests('flutter_driver');
247 await runPrivateTests();
248
249 // Run java unit tests for integration_test
250 //
251 // Generate Gradle wrapper if it doesn't exist.
252 Process.runSync(
253 flutter,
254 <String>['build', 'apk', '--config-only'],
255 workingDirectory: path.join(
256 flutterRoot,
257 'packages',
258 'integration_test',
259 'example',
260 'android',
261 ),
262 );
263 await runCommand(
264 path.join(flutterRoot, 'packages', 'integration_test', 'example', 'android', 'gradlew$bat'),
265 <String>[
266 ':integration_test:testDebugUnitTest',
267 '--tests',
268 'dev.flutter.plugins.integration_test.FlutterDeviceScreenshotTest',
269 ],
270 workingDirectory: path.join(
271 flutterRoot,
272 'packages',
273 'integration_test',
274 'example',
275 'android',
276 ),
277 );
278 }
279
280 Future<void> runMisc() async {
281 printProgress(
282 '${green}Running package tests$reset for directories other than packages/flutter',
283 );
284 await testHarnessTestsRunner();
285 await runExampleTests();
286 await runFlutterTest(
287 path.join(flutterRoot, 'dev', 'a11y_assessments'),
288 tests: <String>['test'],
289 );
290 await runDartTest(path.join(flutterRoot, 'dev', 'bots'));
291 await runDartTest(
292 path.join(flutterRoot, 'dev', 'devicelab'),
293 ensurePrecompiledTool: false, // See https://github.com/flutter/flutter/issues/86209
294 );
295 await runDartTest(path.join(flutterRoot, 'dev', 'packages_autoroller'));
296 // TODO(gspencergoog): Remove the exception for fatalWarnings once https://github.com/flutter/flutter/issues/113782 has landed.
297 await runFlutterTest(
298 path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'),
299 fatalWarnings: false,
300 );
301 await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'ui'));
302 await runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
303 await runFlutterTest(path.join(flutterRoot, 'dev', 'tools'));
304 await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'));
305 await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_defaults'));
306 await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_keycodes'));
307 await runFlutterTest(path.join(flutterRoot, 'dev', 'benchmarks', 'test_apps', 'stocks'));
308 await runFlutterTest(
309 path.join(flutterRoot, 'packages', 'flutter_driver'),
310 tests: <String>[path.join('test', 'src', 'real_tests')],
311 );
312 await runFlutterTest(
313 path.join(flutterRoot, 'packages', 'integration_test'),
314 options: <String>[
315 '--enable-vmservice',
316 // Web-specific tests depend on Chromium, so they run as part of the web_long_running_tests shard.
317 '--exclude-tags=web',
318 ],
319 );
320 await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens'));
321 await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'));
322 await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
323 await runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'));
324 const String httpClientWarning =
325 'Warning: At least one test in this suite creates an HttpClient. When running a test suite that uses\n'
326 'TestWidgetsFlutterBinding, all HTTP requests will return status code 400, and no network request\n'
327 'will actually be made. Any test expecting a real network connection and status code will fail.\n'
328 'To test code that needs an HttpClient, provide your own HttpClient implementation to the code under\n'
329 'test, so that your test can consistently provide a testable response to the code under test.';
330 await runFlutterTest(
331 path.join(flutterRoot, 'packages', 'flutter_test'),
332 script: path.join('test', 'bindings_test_failure.dart'),
333 expectFailure: true,
334 printOutput: false,
335 outputChecker: (CommandResult result) {
336 final Iterable<Match> matches = httpClientWarning.allMatches(result.flattenedStdout!);
337 if (matches.isEmpty || matches.length > 1) {
338 return 'Failed to print warning about HttpClientUsage, or printed it too many times.\n\n'
339 'stdout:\n${result.flattenedStdout}\n\n'
340 'stderr:\n${result.flattenedStderr}';
341 }
342 return null;
343 },
344 );
345 }
346
347 await selectSubshard(<String, ShardRunner>{
348 'widgets': runWidgets,
349 'libraries': runLibraries,
350 'slow': runSlow,
351 'misc': runMisc,
352 'impeller': runImpeller,
353 });
354}
355