1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5import 'dart:convert';
6import 'dart:io' show stderr;
7import 'dart:typed_data';
8
9import 'package:convert/convert.dart';
10import 'package:crypto/crypto.dart';
11import 'package:file/file.dart';
12import 'package:http/http.dart' as http;
13import 'package:path/path.dart' as path;
14import 'package:platform/platform.dart' show LocalPlatform, Platform;
15import 'package:pool/pool.dart';
16import 'package:process/process.dart';
17
18import 'common.dart';
19import 'process_runner.dart';
20
21typedef HttpReader = Future<Uint8List> Function(Uri url, {Map<String, String> headers});
22
23/// Creates a pre-populated Flutter archive from a git repo.
24class ArchiveCreator {
25 /// [tempDir] is the directory to use for creating the archive. The script
26 /// will place several GiB of data there, so it should have available space.
27 ///
28 /// The processManager argument is used to inject a mock of [ProcessManager] for
29 /// testing purposes.
30 ///
31 /// If subprocessOutput is true, then output from processes invoked during
32 /// archive creation is echoed to stderr and stdout.
33 factory ArchiveCreator(
34 Directory tempDir,
35 Directory outputDir,
36 String revision,
37 Branch branch, {
38 required FileSystem fs,
39 HttpReader? httpReader,
40 Platform platform = const LocalPlatform(),
41 ProcessManager? processManager,
42 bool strict = true,
43 bool subprocessOutput = true,
44 }) {
45 final Directory flutterRoot = fs.directory(path.join(tempDir.path, 'flutter'));
46 final ProcessRunner processRunner = ProcessRunner(
47 processManager: processManager,
48 subprocessOutput: subprocessOutput,
49 platform: platform,
50 )..environment['PUB_CACHE'] = path.join(tempDir.path, '.pub-cache');
51 final String flutterExecutable = path.join(flutterRoot.absolute.path, 'bin', 'flutter');
52 final String dartExecutable = path.join(
53 flutterRoot.absolute.path,
54 'bin',
55 'cache',
56 'dart-sdk',
57 'bin',
58 'dart',
59 );
60
61 return ArchiveCreator._(
62 tempDir: tempDir,
63 platform: platform,
64 flutterRoot: flutterRoot,
65 fs: fs,
66 outputDir: outputDir,
67 revision: revision,
68 branch: branch,
69 strict: strict,
70 processRunner: processRunner,
71 httpReader: httpReader ?? http.readBytes,
72 flutterExecutable: flutterExecutable,
73 dartExecutable: dartExecutable,
74 );
75 }
76
77 ArchiveCreator._({
78 required this.branch,
79 required String dartExecutable,
80 required this.fs,
81 required String flutterExecutable,
82 required this.flutterRoot,
83 required this.httpReader,
84 required this.outputDir,
85 required this.platform,
86 required ProcessRunner processRunner,
87 required this.revision,
88 required this.strict,
89 required this.tempDir,
90 }) : assert(revision.length == 40),
91 _processRunner = processRunner,
92 _flutter = flutterExecutable,
93 _dart = dartExecutable;
94
95 /// The platform to use for the environment and determining which
96 /// platform we're running on.
97 final Platform platform;
98
99 /// The branch to build the archive for. The branch must contain [revision].
100 final Branch branch;
101
102 /// The git revision hash to build the archive for. This revision has
103 /// to be available in the [branch], although it doesn't have to be
104 /// at HEAD, since we clone the branch and then reset to this revision
105 /// to create the archive.
106 final String revision;
107
108 /// The flutter root directory in the [tempDir].
109 final Directory flutterRoot;
110
111 /// The temporary directory used to build the archive in.
112 final Directory tempDir;
113
114 /// The directory to write the output file to.
115 final Directory outputDir;
116
117 final FileSystem fs;
118
119 /// True if the creator should be strict about checking requirements or not.
120 ///
121 /// In strict mode, will insist that the [revision] be a tagged revision.
122 final bool strict;
123
124 final Uri _minGitUri = Uri.parse(mingitForWindowsUrl);
125 final ProcessRunner _processRunner;
126
127 /// Used to tell the [ArchiveCreator] which function to use for reading
128 /// bytes from a URL. Used in tests to inject a fake reader. Defaults to
129 /// [http.readBytes].
130 final HttpReader httpReader;
131
132 final Map<String, String> _version = <String, String>{};
133 late String _flutter;
134 late String _dart;
135
136 late final Future<String> _dartArch = (() async {
137 // Parse 'arch' out of a string like '... "os_arch"\n'.
138 return (await _runDart(<String>[
139 '--version',
140 ])).trim().split(' ').last.replaceAll('"', '').split('_')[1];
141 })();
142
143 /// Returns a default archive name when given a Git revision.
144 /// Used when an output filename is not given.
145 Future<String> get _archiveName async {
146 final String os = platform.operatingSystem.toLowerCase();
147 // Include the intended host architecture in the file name for non-x64.
148 final String arch = await _dartArch == 'x64' ? '' : '${await _dartArch}_';
149 // We don't use .tar.xz on Mac because although it can unpack them
150 // on the command line (with tar), the "Archive Utility" that runs
151 // when you double-click on them just does some crazy behavior (it
152 // converts it to a compressed cpio archive, and when you double
153 // click on that, it converts it back to .tar.xz, without ever
154 // unpacking it!) So, we use .zip for Mac, and the files are about
155 // 220MB larger than they need to be. :-(
156 final String suffix = platform.isLinux ? 'tar.xz' : 'zip';
157 final String package = '${os}_$arch${_version[frameworkVersionTag]}';
158 return 'flutter_$package-${branch.name}.$suffix';
159 }
160
161 /// Checks out the flutter repo and prepares it for other operations.
162 ///
163 /// Returns the version for this release as obtained from the git tags, and
164 /// the dart version as obtained from `flutter --version`.
165 Future<Map<String, String>> initializeRepo() async {
166 await _checkoutFlutter();
167 if (_version.isEmpty) {
168 _version.addAll(await _getVersion());
169 }
170 return _version;
171 }
172
173 /// Performs all of the steps needed to create an archive.
174 Future<File> createArchive() async {
175 assert(_version.isNotEmpty, 'Must run initializeRepo before createArchive');
176 final File outputFile = fs.file(path.join(outputDir.absolute.path, await _archiveName));
177 await _installMinGitIfNeeded();
178 await _populateCaches();
179 await _validate();
180 await _archiveFiles(outputFile);
181 return outputFile;
182 }
183
184 /// Validates the integrity of the release package.
185 ///
186 /// Currently only checks that macOS binaries are codesigned. Will throw a
187 /// [PreparePackageException] if the test fails.
188 Future<void> _validate() async {
189 // Only validate in strict mode, which means `--publish`
190 if (!strict || !platform.isMacOS) {
191 return;
192 }
193 // Validate that the dart binary is codesigned
194 try {
195 await _processRunner.runProcess(<String>[
196 'codesign',
197 '-vvvv',
198 '--check-notarization',
199 _dart,
200 ], workingDirectory: flutterRoot);
201 } on PreparePackageException catch (e) {
202 throw PreparePackageException('The binary $_dart was not codesigned!\n${e.message}');
203 }
204 }
205
206 /// Returns the version map of this release, according the to tags in the
207 /// repo and the output of `flutter --version --machine`.
208 ///
209 /// This looks for the tag attached to [revision] and, if it doesn't find one,
210 /// git will give an error.
211 ///
212 /// If [strict] is true, the exact [revision] must be tagged to return the
213 /// version. If [strict] is not true, will look backwards in time starting at
214 /// [revision] to find the most recent version tag.
215 ///
216 /// The version found as a git tag is added to the information given by
217 /// `flutter --version --machine` with the `frameworkVersionFromGit` tag, and
218 /// returned.
219 Future<Map<String, String>> _getVersion() async {
220 String gitVersion;
221 if (strict) {
222 try {
223 gitVersion = await _runGit(<String>['describe', '--tags', '--exact-match', revision]);
224 } on PreparePackageException catch (exception) {
225 throw PreparePackageException(
226 'Git error when checking for a version tag attached to revision $revision.\n'
227 'Perhaps there is no tag at that revision?:\n'
228 '$exception',
229 );
230 }
231 } else {
232 gitVersion = await _runGit(<String>['describe', '--tags', '--abbrev=0', revision]);
233 }
234 // Run flutter command twice, once to make sure the flutter command is built
235 // and ready (and thus won't output any junk on stdout the second time), and
236 // once to capture theJSON output. The second run should be fast.
237 await _runFlutter(<String>['--version', '--machine']);
238 final String versionJson = await _runFlutter(<String>['--version', '--machine']);
239 final Map<String, String> versionMap = <String, String>{};
240 final Map<String, dynamic> result = json.decode(versionJson) as Map<String, dynamic>;
241 result.forEach((String key, dynamic value) => versionMap[key] = value.toString());
242 versionMap[frameworkVersionTag] = gitVersion;
243 versionMap[dartTargetArchTag] = await _dartArch;
244 return versionMap;
245 }
246
247 /// Clone the Flutter repo and make sure that the git environment is sane
248 /// for when the user will unpack it.
249 Future<void> _checkoutFlutter() async {
250 // We want the user to start out the in the specified branch instead of a
251 // detached head. To do that, we need to make sure the branch points at the
252 // desired revision.
253 await _runGit(<String>['clone', '-b', branch.name, gobMirror], workingDirectory: tempDir);
254 await _runGit(<String>['reset', '--hard', revision]);
255
256 // Make the origin point to github instead of the chromium mirror.
257 await _runGit(<String>['remote', 'set-url', 'origin', githubRepo]);
258
259 // Minify `.git` footprint (saving about ~100 MB as of Oct 2022)
260 await _runGit(<String>['gc', '--prune=now', '--aggressive']);
261 }
262
263 /// Retrieve the MinGit executable from storage and unpack it.
264 Future<void> _installMinGitIfNeeded() async {
265 if (!platform.isWindows) {
266 return;
267 }
268 final Uint8List data = await httpReader(_minGitUri);
269 final File gitFile = fs.file(path.join(tempDir.absolute.path, 'mingit.zip'));
270 await gitFile.writeAsBytes(data, flush: true);
271
272 final Directory minGitPath = fs.directory(
273 path.join(flutterRoot.absolute.path, 'bin', 'mingit'),
274 );
275 await minGitPath.create(recursive: true);
276 await _unzipArchive(gitFile, workingDirectory: minGitPath);
277 }
278
279 /// Downloads an archive of every package that is present in the temporary
280 /// pub-cache from pub.dev. Stores the archives in
281 /// $flutterRoot/.pub-preload-cache.
282 ///
283 /// These archives will be installed in the user-level cache on first
284 /// following flutter command that accesses the cache.
285 ///
286 /// Precondition: all packages currently in the PUB_CACHE of [_processRunner]
287 /// are installed from pub.dev.
288 Future<void> _downloadPubPackageArchives() async {
289 final Pool pool = Pool(10); // Number of simultaneous downloads.
290 final http.Client client = http.Client();
291 final Directory preloadCache = fs.directory(path.join(flutterRoot.path, '.pub-preload-cache'));
292 preloadCache.createSync(recursive: true);
293
294 /// Fetch a single package.
295 Future<void> fetchPackageArchive(String name, String version) async {
296 await pool.withResource(() async {
297 stderr.write('Fetching package archive for $name-$version.\n');
298 int retries = 7;
299 while (true) {
300 retries -= 1;
301 try {
302 final Uri packageListingUrl = Uri.parse('https://pub.dev/api/packages/$name');
303 // Fetch the package listing to obtain the package download url.
304 final http.Response packageListingResponse = await client.get(packageListingUrl);
305 if (packageListingResponse.statusCode != 200) {
306 throw Exception(
307 'Downloading $packageListingUrl failed. Status code ${packageListingResponse.statusCode}.',
308 );
309 }
310 final dynamic decodedPackageListing = json.decode(packageListingResponse.body);
311 if (decodedPackageListing is! Map) {
312 throw const FormatException('Package listing should be a map');
313 }
314 final dynamic versions = decodedPackageListing['versions'];
315 if (versions is! List) {
316 throw const FormatException('.versions should be a list');
317 }
318 final Map<String, dynamic> versionDescription =
319 versions.firstWhere(
320 (dynamic description) {
321 if (description is! Map) {
322 throw const FormatException('.versions elements should be maps');
323 }
324 return description['version'] == version;
325 },
326 orElse: () =>
327 throw FormatException('Could not find $name-$version in package listing'),
328 )
329 as Map<String, dynamic>;
330 final dynamic downloadUrl = versionDescription['archive_url'];
331 if (downloadUrl is! String) {
332 throw const FormatException('archive_url should be a string');
333 }
334 final dynamic archiveSha256 = versionDescription['archive_sha256'];
335 if (archiveSha256 is! String) {
336 throw const FormatException('archive_sha256 should be a string');
337 }
338 final http.Request request = http.Request('get', Uri.parse(downloadUrl));
339 final http.StreamedResponse response = await client.send(request);
340 if (response.statusCode != 200) {
341 throw Exception(
342 'Downloading ${request.url} failed. Status code ${response.statusCode}.',
343 );
344 }
345 final File archiveFile = fs.file(path.join(preloadCache.path, '$name-$version.tar.gz'));
346 await response.stream.pipe(archiveFile.openWrite());
347 final Stream<List<int>> archiveStream = archiveFile.openRead();
348 final Digest r = await sha256.bind(archiveStream).first;
349 if (hex.encode(r.bytes) != archiveSha256) {
350 throw Exception('Hash mismatch of downloaded archive');
351 }
352 } on Exception catch (e) {
353 stderr.write('Failed downloading $name-$version. $e\n');
354 if (retries > 0) {
355 stderr.write('Retrying download of $name-$version...');
356 // Retry.
357 continue;
358 } else {
359 rethrow;
360 }
361 }
362 break;
363 }
364 });
365 }
366
367 final Map<String, dynamic> cacheDescription =
368 json.decode(await _runFlutter(<String>['pub', 'cache', 'list'])) as Map<String, dynamic>;
369 final Map<String, dynamic> packages = cacheDescription['packages'] as Map<String, dynamic>;
370 final List<Future<void>> downloads = <Future<void>>[];
371 for (final MapEntry<String, dynamic> package in packages.entries) {
372 final String name = package.key;
373 final Map<String, dynamic> versions = package.value as Map<String, dynamic>;
374 for (final String version in versions.keys) {
375 downloads.add(fetchPackageArchive(name, version));
376 }
377 }
378 await Future.wait(downloads);
379 client.close();
380 }
381
382 /// Prepare the archive repo so that it has all of the caches warmed up and
383 /// is configured for the user to begin working.
384 Future<void> _populateCaches() async {
385 await _runFlutter(<String>['doctor']);
386 await _runFlutter(<String>['update-packages']);
387 await _runFlutter(<String>['precache']);
388 await _runFlutter(<String>['ide-config']);
389
390 // Create each of the templates, since they will call 'pub get' on
391 // themselves when created, and this will warm the cache with their
392 // dependencies too.
393 for (final String template in <String>['app', 'package', 'plugin']) {
394 final String createName = path.join(tempDir.path, 'create_$template');
395 await _runFlutter(
396 <String>['create', '--template=$template', createName],
397 // Run it outside the cloned Flutter repo to not nest git repos, since
398 // they'll be git repos themselves too.
399 workingDirectory: tempDir,
400 );
401 }
402 await _downloadPubPackageArchives();
403 // Yes, we could just skip all .packages files when constructing
404 // the archive, but some are checked in, and we don't want to skip
405 // those.
406 await _runGit(<String>[
407 'clean',
408 '-f',
409 // Do not -X as it could lead to entire bin/cache getting cleaned
410 '-x',
411 '--',
412 '**/.packages',
413 ]);
414
415 /// Remove package_config files and any contents in .dart_tool
416 await _runGit(<String>['clean', '-f', '-x', '--', '**/.dart_tool/']);
417
418 // Ensure the above commands do not clean out the cache
419 final Directory flutterCache = fs.directory(
420 path.join(flutterRoot.absolute.path, 'bin', 'cache'),
421 );
422 if (!flutterCache.existsSync()) {
423 throw Exception('The flutter cache was not found at ${flutterCache.path}!');
424 }
425
426 /// Remove git subfolder from .pub-cache, this contains the flutter goldens
427 /// and new flutter_gallery.
428 final Directory gitCache = fs.directory(
429 path.join(flutterRoot.absolute.path, '.pub-cache', 'git'),
430 );
431 if (gitCache.existsSync()) {
432 gitCache.deleteSync(recursive: true);
433 }
434 }
435
436 /// Write the archive to the given output file.
437 Future<void> _archiveFiles(File outputFile) async {
438 if (outputFile.path.toLowerCase().endsWith('.zip')) {
439 await _createZipArchive(outputFile, flutterRoot);
440 } else if (outputFile.path.toLowerCase().endsWith('.tar.xz')) {
441 await _createTarArchive(outputFile, flutterRoot);
442 }
443 }
444
445 Future<String> _runDart(List<String> args, {Directory? workingDirectory}) {
446 return _processRunner.runProcess(<String>[
447 _dart,
448 ...args,
449 ], workingDirectory: workingDirectory ?? flutterRoot);
450 }
451
452 Future<String> _runFlutter(List<String> args, {Directory? workingDirectory}) {
453 return _processRunner.runProcess(<String>[
454 _flutter,
455 ...args,
456 ], workingDirectory: workingDirectory ?? flutterRoot);
457 }
458
459 Future<String> _runGit(List<String> args, {Directory? workingDirectory}) {
460 return _processRunner.runProcess(<String>[
461 'git',
462 ...args,
463 ], workingDirectory: workingDirectory ?? flutterRoot);
464 }
465
466 /// Unpacks the given zip file into the currentDirectory (if set), or the
467 /// same directory as the archive.
468 Future<String> _unzipArchive(File archive, {Directory? workingDirectory}) {
469 workingDirectory ??= fs.directory(path.dirname(archive.absolute.path));
470 List<String> commandLine;
471 if (platform.isWindows) {
472 commandLine = <String>['7za', 'x', archive.absolute.path];
473 } else {
474 commandLine = <String>['unzip', archive.absolute.path];
475 }
476 return _processRunner.runProcess(commandLine, workingDirectory: workingDirectory);
477 }
478
479 /// Create a zip archive from the directory source.
480 Future<String> _createZipArchive(File output, Directory source) async {
481 List<String> commandLine;
482 if (platform.isWindows) {
483 // Unhide the .git folder, https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/attrib.
484 await _processRunner.runProcess(<String>[
485 'attrib',
486 '-h',
487 '.git',
488 ], workingDirectory: fs.directory(source.absolute.path));
489 commandLine = <String>[
490 '7za',
491 'a',
492 '-tzip',
493 '-mx=9',
494 output.absolute.path,
495 path.basename(source.path),
496 ];
497 } else {
498 commandLine = <String>[
499 'zip',
500 '-r',
501 '-9',
502 '--symlinks',
503 output.absolute.path,
504 path.basename(source.path),
505 ];
506 }
507 return _processRunner.runProcess(
508 commandLine,
509 workingDirectory: fs.directory(path.dirname(source.absolute.path)),
510 );
511 }
512
513 /// Create a tar archive from the directory source.
514 Future<String> _createTarArchive(File output, Directory source) {
515 return _processRunner.runProcess(<String>[
516 'tar',
517 'cJf',
518 output.absolute.path,
519 // Print out input files as they get added, to debug hangs
520 '--verbose',
521 path.basename(source.absolute.path),
522 ], workingDirectory: fs.directory(path.dirname(source.absolute.path)));
523 }
524}
525