1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5/// This script removes published archives from the cloud storage and the
6/// corresponding JSON metadata file that the website uses to determine what
7/// releases are available.
8///
9/// If asked to remove a release that is currently the release on that channel,
10/// it will replace that release with the next most recent release on that
11/// channel.
12library;
13
14import 'dart:async';
15import 'dart:convert';
16import 'dart:io' hide Platform;
17
18import 'package:args/args.dart';
19import 'package:path/path.dart' as path;
20import 'package:platform/platform.dart' show LocalPlatform, Platform;
21import 'package:process/process.dart';
22
23const String gsBase = 'gs://flutter_infra_release';
24const String releaseFolder = '/releases';
25const String gsReleaseFolder = '$gsBase$releaseFolder';
26const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release';
27
28/// Exception class for when a process fails to run, so we can catch
29/// it and provide something more readable than a stack trace.
30class UnpublishException implements Exception {
31 UnpublishException(this.message, [this.result]);
32
33 final String message;
34 final ProcessResult? result;
35 int get exitCode => result?.exitCode ?? -1;
36
37 @override
38 String toString() {
39 String output = runtimeType.toString();
40 output += ': $message';
41 final String stderr = result?.stderr as String? ?? '';
42 if (stderr.isNotEmpty) {
43 output += ':\n$stderr';
44 }
45 return output;
46 }
47}
48
49enum Channel { dev, beta, stable }
50
51String getChannelName(Channel channel) {
52 return switch (channel) {
53 Channel.beta => 'beta',
54 Channel.dev => 'dev',
55 Channel.stable => 'stable',
56 };
57}
58
59Channel fromChannelName(String? name) {
60 return switch (name) {
61 'beta' => Channel.beta,
62 'dev' => Channel.dev,
63 'stable' => Channel.stable,
64 _ => throw ArgumentError('Invalid channel name.'),
65 };
66}
67
68enum PublishedPlatform { linux, macos, windows }
69
70String getPublishedPlatform(PublishedPlatform platform) {
71 return switch (platform) {
72 PublishedPlatform.linux => 'linux',
73 PublishedPlatform.macos => 'macos',
74 PublishedPlatform.windows => 'windows',
75 };
76}
77
78PublishedPlatform fromPublishedPlatform(String name) {
79 return switch (name) {
80 'linux' => PublishedPlatform.linux,
81 'macos' => PublishedPlatform.macos,
82 'windows' => PublishedPlatform.windows,
83 _ => throw ArgumentError('Invalid published platform name.'),
84 };
85}
86
87/// A helper class for classes that want to run a process, optionally have the
88/// stderr and stdout reported as the process runs, and capture the stdout
89/// properly without dropping any.
90class ProcessRunner {
91 /// Creates a [ProcessRunner].
92 ///
93 /// The [processManager], [subprocessOutput], and [platform] arguments must
94 /// not be null.
95 ProcessRunner({
96 this.processManager = const LocalProcessManager(),
97 this.subprocessOutput = true,
98 this.defaultWorkingDirectory,
99 this.platform = const LocalPlatform(),
100 }) {
101 environment = Map<String, String>.from(platform.environment);
102 }
103
104 /// The platform to use for a starting environment.
105 final Platform platform;
106
107 /// Set [subprocessOutput] to show output as processes run. Stdout from the
108 /// process will be printed to stdout, and stderr printed to stderr.
109 final bool subprocessOutput;
110
111 /// Set the [processManager] in order to inject a test instance to perform
112 /// testing.
113 final ProcessManager processManager;
114
115 /// Sets the default directory used when `workingDirectory` is not specified
116 /// to [runProcess].
117 final Directory? defaultWorkingDirectory;
118
119 /// The environment to run processes with.
120 late Map<String, String> environment;
121
122 /// Run the command and arguments in `commandLine` as a sub-process from
123 /// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses
124 /// [Directory.current] if [defaultWorkingDirectory] is not set.
125 ///
126 /// Set `failOk` if [runProcess] should not throw an exception when the
127 /// command completes with a non-zero exit code.
128 Future<String> runProcess(
129 List<String> commandLine, {
130 Directory? workingDirectory,
131 bool failOk = false,
132 }) async {
133 workingDirectory ??= defaultWorkingDirectory ?? Directory.current;
134 if (subprocessOutput) {
135 stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n');
136 }
137 final List<int> output = <int>[];
138 final Completer<void> stdoutComplete = Completer<void>();
139 final Completer<void> stderrComplete = Completer<void>();
140 late Process process;
141 Future<int> allComplete() async {
142 await stderrComplete.future;
143 await stdoutComplete.future;
144 return process.exitCode;
145 }
146
147 try {
148 process = await processManager.start(
149 commandLine,
150 workingDirectory: workingDirectory.absolute.path,
151 environment: environment,
152 );
153 process.stdout.listen((List<int> event) {
154 output.addAll(event);
155 if (subprocessOutput) {
156 stdout.add(event);
157 }
158 }, onDone: () async => stdoutComplete.complete());
159 if (subprocessOutput) {
160 process.stderr.listen((List<int> event) {
161 stderr.add(event);
162 }, onDone: () async => stderrComplete.complete());
163 } else {
164 stderrComplete.complete();
165 }
166 } on ProcessException catch (e) {
167 final String message =
168 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
169 'failed with:\n$e';
170 throw UnpublishException(message);
171 } on ArgumentError catch (e) {
172 final String message =
173 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
174 'failed with:\n$e';
175 throw UnpublishException(message);
176 }
177
178 final int exitCode = await allComplete();
179 if (exitCode != 0 && !failOk) {
180 final String message =
181 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed';
182 throw UnpublishException(message, ProcessResult(0, exitCode, null, 'returned $exitCode'));
183 }
184 return utf8.decoder.convert(output).trim();
185 }
186}
187
188class ArchiveUnpublisher {
189 ArchiveUnpublisher(
190 this.tempDir,
191 this.revisionsBeingRemoved,
192 this.channels,
193 this.platform, {
194 this.confirmed = false,
195 ProcessManager? processManager,
196 bool subprocessOutput = true,
197 }) : assert(revisionsBeingRemoved.length == 40),
198 metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
199 _processRunner = ProcessRunner(
200 processManager: processManager ?? const LocalProcessManager(),
201 subprocessOutput: subprocessOutput,
202 );
203
204 final PublishedPlatform platform;
205 final String metadataGsPath;
206 final Set<Channel> channels;
207 final Set<String> revisionsBeingRemoved;
208 final bool confirmed;
209 final Directory tempDir;
210 final ProcessRunner _processRunner;
211 static String getMetadataFilename(PublishedPlatform platform) =>
212 'releases_${getPublishedPlatform(platform)}.json';
213
214 /// Remove the archive from Google Storage.
215 Future<void> unpublishArchive() async {
216 final Map<String, dynamic> jsonData = await _loadMetadata();
217 final List<Map<String, String>> releases = (jsonData['releases'] as List<dynamic>)
218 .map<Map<String, String>>((dynamic entry) {
219 final Map<String, dynamic> mapEntry = entry as Map<String, dynamic>;
220 return mapEntry.cast<String, String>();
221 })
222 .toList();
223 final Map<Channel, Map<String, String>> paths = await _getArchivePaths(releases);
224 releases.removeWhere(
225 (Map<String, String> value) =>
226 revisionsBeingRemoved.contains(value['hash']) &&
227 channels.contains(fromChannelName(value['channel'])),
228 );
229 releases.sort((Map<String, String> a, Map<String, String> b) {
230 final DateTime aDate = DateTime.parse(a['release_date']!);
231 final DateTime bDate = DateTime.parse(b['release_date']!);
232 return bDate.compareTo(aDate);
233 });
234 jsonData['releases'] = releases;
235 for (final Channel channel in channels) {
236 if (!revisionsBeingRemoved.contains(
237 (jsonData['current_release'] as Map<String, dynamic>)[getChannelName(channel)],
238 )) {
239 // Don't replace the current release if it's not one of the revisions we're removing.
240 continue;
241 }
242 final Map<String, String> replacementRelease = releases.firstWhere(
243 (Map<String, String> value) => value['channel'] == getChannelName(channel),
244 );
245 (jsonData['current_release'] as Map<String, dynamic>)[getChannelName(channel)] =
246 replacementRelease['hash'];
247 print(
248 '${confirmed ? 'Reverting' : 'Would revert'} current ${getChannelName(channel)} '
249 '${getPublishedPlatform(platform)} release to ${replacementRelease['hash']} (version ${replacementRelease['version']}).',
250 );
251 }
252 await _cloudRemoveArchive(paths);
253 await _updateMetadata(jsonData);
254 }
255
256 Future<Map<Channel, Map<String, String>>> _getArchivePaths(
257 List<Map<String, String>> releases,
258 ) async {
259 final Set<String> hashes = <String>{};
260 final Map<Channel, Map<String, String>> paths = <Channel, Map<String, String>>{};
261 for (final Map<String, String> revision in releases) {
262 final String hash = revision['hash']!;
263 final Channel channel = fromChannelName(revision['channel']);
264 hashes.add(hash);
265 if (revisionsBeingRemoved.contains(hash) && channels.contains(channel)) {
266 paths[channel] ??= <String, String>{};
267 paths[channel]![hash] = revision['archive']!;
268 }
269 }
270 final Set<String> missingRevisions = revisionsBeingRemoved.difference(
271 hashes.intersection(revisionsBeingRemoved),
272 );
273 if (missingRevisions.isNotEmpty) {
274 final bool plural = missingRevisions.length > 1;
275 throw UnpublishException(
276 'Revision${plural ? 's' : ''} $missingRevisions ${plural ? 'are' : 'is'} not present in the server metadata.',
277 );
278 }
279 return paths;
280 }
281
282 Future<Map<String, dynamic>> _loadMetadata() async {
283 final File metadataFile = File(path.join(tempDir.absolute.path, getMetadataFilename(platform)));
284 // Always run this, even in dry runs.
285 await _runGsUtil(<String>['cp', metadataGsPath, metadataFile.absolute.path], confirm: true);
286 final String currentMetadata = metadataFile.readAsStringSync();
287 if (currentMetadata.isEmpty) {
288 throw UnpublishException('Empty metadata received from server');
289 }
290
291 Map<String, dynamic> jsonData;
292 try {
293 jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
294 } on FormatException catch (e) {
295 throw UnpublishException('Unable to parse JSON metadata received from cloud: $e');
296 }
297
298 return jsonData;
299 }
300
301 Future<void> _updateMetadata(Map<String, dynamic> jsonData) async {
302 // We can't just cat the metadata from the server with 'gsutil cat', because
303 // Windows wants to echo the commands that execute in gsutil.bat to the
304 // stdout when we do that. So, we copy the file locally and then read it
305 // back in.
306 final File metadataFile = File(path.join(tempDir.absolute.path, getMetadataFilename(platform)));
307 const JsonEncoder encoder = JsonEncoder.withIndent(' ');
308 metadataFile.writeAsStringSync(encoder.convert(jsonData));
309 print(
310 '${confirmed ? 'Overwriting' : 'Would overwrite'} $metadataGsPath with contents of ${metadataFile.absolute.path}',
311 );
312 await _cloudReplaceDest(metadataFile.absolute.path, metadataGsPath);
313 }
314
315 Future<String> _runGsUtil(
316 List<String> args, {
317 Directory? workingDirectory,
318 bool failOk = false,
319 bool confirm = false,
320 }) async {
321 final List<String> command = <String>['gsutil', '--', ...args];
322 if (confirm) {
323 return _processRunner.runProcess(command, workingDirectory: workingDirectory, failOk: failOk);
324 } else {
325 print('Would run: ${command.join(' ')}');
326 return '';
327 }
328 }
329
330 Future<void> _cloudRemoveArchive(Map<Channel, Map<String, String>> paths) async {
331 final List<String> files = <String>[];
332 print('${confirmed ? 'Removing' : 'Would remove'} the following release archives:');
333 for (final Channel channel in paths.keys) {
334 final Map<String, String> hashes = paths[channel]!;
335 for (final String hash in hashes.keys) {
336 final String file = '$gsReleaseFolder/${hashes[hash]}';
337 files.add(file);
338 print(' $file');
339 }
340 }
341 await _runGsUtil(<String>['rm', ...files], failOk: true, confirm: confirmed);
342 }
343
344 Future<String> _cloudReplaceDest(String src, String dest) async {
345 assert(dest.startsWith('gs:'), '_cloudReplaceDest must have a destination in cloud storage.');
346 assert(!src.startsWith('gs:'), '_cloudReplaceDest must have a local source file.');
347 // We often don't have permission to overwrite, but
348 // we have permission to remove, so that's what we do first.
349 await _runGsUtil(<String>['rm', dest], failOk: true, confirm: confirmed);
350 String? mimeType;
351 if (dest.endsWith('.tar.xz')) {
352 mimeType = 'application/x-gtar';
353 }
354 if (dest.endsWith('.zip')) {
355 mimeType = 'application/zip';
356 }
357 if (dest.endsWith('.json')) {
358 mimeType = 'application/json';
359 }
360 final List<String> args = <String>[
361 // Use our preferred MIME type for the files we care about
362 // and let gsutil figure it out for anything else.
363 if (mimeType != null) ...<String>['-h', 'Content-Type:$mimeType'],
364 ...<String>['cp', src, dest],
365 ];
366 return _runGsUtil(args, confirm: confirmed);
367 }
368}
369
370void _printBanner(String message) {
371 final String banner = '*** $message ***';
372 print('\n');
373 print('*' * banner.length);
374 print(banner);
375 print('*' * banner.length);
376 print('\n');
377}
378
379/// Prepares a flutter git repo to be removed from the published cloud storage.
380Future<void> main(List<String> rawArguments) async {
381 final List<String> allowedChannelValues = Channel.values
382 .map<String>((Channel channel) => getChannelName(channel))
383 .toList();
384 final List<String> allowedPlatformNames = PublishedPlatform.values
385 .map<String>((PublishedPlatform platform) => getPublishedPlatform(platform))
386 .toList();
387 final ArgParser argParser = ArgParser();
388 argParser.addOption(
389 'temp_dir',
390 help:
391 'A location where temporary files may be written. Defaults to a '
392 'directory in the system temp folder. If a temp_dir is not '
393 'specified, then by default a generated temporary directory will be '
394 'created, used, and removed automatically when the script exits.',
395 );
396 argParser.addMultiOption(
397 'revision',
398 help:
399 'The Flutter git repo revisions to remove from the published site. '
400 'Must be full 40-character hashes. More than one may be specified, '
401 'either by giving the option more than once, or by giving a comma '
402 'separated list. Required.',
403 );
404 argParser.addMultiOption(
405 'channel',
406 allowed: allowedChannelValues,
407 help:
408 'The Flutter channels to remove the archives corresponding to the '
409 'revisions given with --revision. More than one may be specified, '
410 'either by giving the option more than once, or by giving a '
411 'comma separated list. If not specified, then the archives from all '
412 'channels that a revision appears in will be removed.',
413 );
414 argParser.addMultiOption(
415 'platform',
416 allowed: allowedPlatformNames,
417 help:
418 'The Flutter platforms to remove the archive from. May specify more '
419 'than one, either by giving the option more than once, or by giving a '
420 'comma separated list. If not specified, then the archives from all '
421 'platforms that a revision appears in will be removed.',
422 );
423 argParser.addFlag(
424 'confirm',
425 help:
426 'If set, will actually remove the archive from Google Cloud Storage '
427 'upon successful execution of this script. Published archives will be '
428 'removed from this directory: $baseUrl$releaseFolder. This option '
429 'must be set to perform any action on the server, otherwise only a dry '
430 'run is performed.',
431 );
432 argParser.addFlag('help', negatable: false, help: 'Print help for this command.');
433
434 final ArgResults parsedArguments = argParser.parse(rawArguments);
435
436 if (parsedArguments['help'] as bool) {
437 print(argParser.usage);
438 exit(0);
439 }
440
441 void errorExit(String message, {int exitCode = -1}) {
442 stderr.write('Error: $message\n\n');
443 stderr.write('${argParser.usage}\n');
444 exit(exitCode);
445 }
446
447 final List<String> revisions = parsedArguments['revision'] as List<String>;
448 if (revisions.isEmpty) {
449 errorExit('Invalid argument: at least one --revision must be specified.');
450 }
451 for (final String revision in revisions) {
452 if (revision.length != 40) {
453 errorExit(
454 'Invalid argument: --revision "$revision" must be the entire hash, not just a prefix.',
455 );
456 }
457 if (revision.contains(RegExp(r'[^a-fA-F0-9]'))) {
458 errorExit('Invalid argument: --revision "$revision" contains non-hex characters.');
459 }
460 }
461
462 final String tempDirArg = parsedArguments['temp_dir'] as String;
463 Directory tempDir;
464 bool removeTempDir = false;
465 if (tempDirArg.isEmpty) {
466 tempDir = Directory.systemTemp.createTempSync('flutter_package.');
467 removeTempDir = true;
468 } else {
469 tempDir = Directory(tempDirArg);
470 if (!tempDir.existsSync()) {
471 errorExit("Temporary directory $tempDirArg doesn't exist.");
472 }
473 }
474
475 if (!(parsedArguments['confirm'] as bool)) {
476 _printBanner(
477 'This will be just a dry run. To actually perform the changes below, re-run with --confirm argument.',
478 );
479 }
480
481 final List<String> channelArg = parsedArguments['channel'] as List<String>;
482 final List<String> channelOptions = channelArg.isNotEmpty ? channelArg : allowedChannelValues;
483 final Set<Channel> channels = channelOptions
484 .map<Channel>((String value) => fromChannelName(value))
485 .toSet();
486 final List<String> platformArg = parsedArguments['platform'] as List<String>;
487 final List<String> platformOptions = platformArg.isNotEmpty ? platformArg : allowedPlatformNames;
488 final List<PublishedPlatform> platforms = platformOptions
489 .map<PublishedPlatform>((String value) => fromPublishedPlatform(value))
490 .toList();
491 int exitCode = 0;
492 late String message;
493 late String stack;
494 try {
495 for (final PublishedPlatform platform in platforms) {
496 final ArchiveUnpublisher publisher = ArchiveUnpublisher(
497 tempDir,
498 revisions.toSet(),
499 channels,
500 platform,
501 confirmed: parsedArguments['confirm'] as bool,
502 );
503 await publisher.unpublishArchive();
504 }
505 } on UnpublishException catch (e, s) {
506 exitCode = e.exitCode;
507 message = e.message;
508 stack = s.toString();
509 } catch (e, s) {
510 exitCode = -1;
511 message = e.toString();
512 stack = s.toString();
513 } finally {
514 if (removeTempDir) {
515 tempDir.deleteSync(recursive: true);
516 }
517 if (exitCode != 0) {
518 errorExit('$message\n$stack', exitCode: exitCode);
519 }
520 if (!(parsedArguments['confirm'] as bool)) {
521 _printBanner(
522 'This was just a dry run. To actually perform the above changes, re-run with --confirm argument.',
523 );
524 }
525 exit(0);
526 }
527}
528