1// Copyright 2014 The Flutter Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5import 'dart:convert';
6
7import 'package:convert/convert.dart';
8import 'package:crypto/crypto.dart';
9import 'package:file/file.dart';
10import 'package:path/path.dart' as path;
11import 'package:platform/platform.dart' show LocalPlatform, Platform;
12import 'package:process/process.dart';
13
14import 'common.dart';
15import 'process_runner.dart';
16
17class ArchivePublisher {
18 ArchivePublisher(
19 this.tempDir,
20 this.revision,
21 this.branch,
22 this.version,
23 this.outputFile,
24 this.dryRun, {
25 ProcessManager? processManager,
26 bool subprocessOutput = true,
27 required this.fs,
28 this.platform = const LocalPlatform(),
29 }) : assert(revision.length == 40),
30 platformName = platform.operatingSystem.toLowerCase(),
31 metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
32 _processRunner = ProcessRunner(
33 processManager: processManager,
34 subprocessOutput: subprocessOutput,
35 );
36
37 final Platform platform;
38 final FileSystem fs;
39 final String platformName;
40 final String metadataGsPath;
41 final Branch branch;
42 final String revision;
43 final Map<String, String> version;
44 final Directory tempDir;
45 final File outputFile;
46 final ProcessRunner _processRunner;
47 final bool dryRun;
48 String get destinationArchivePath =>
49 '${branch.name}/$platformName/${path.basename(outputFile.path)}';
50 static String getMetadataFilename(Platform platform) =>
51 'releases_${platform.operatingSystem.toLowerCase()}.json';
52
53 Future<String> _getChecksum(File archiveFile) async {
54 final AccumulatorSink<Digest> digestSink = AccumulatorSink<Digest>();
55 final ByteConversionSink sink = sha256.startChunkedConversion(digestSink);
56
57 final Stream<List<int>> stream = archiveFile.openRead();
58 await stream.forEach((List<int> chunk) {
59 sink.add(chunk);
60 });
61 sink.close();
62 return digestSink.events.single.toString();
63 }
64
65 /// Publish the archive to Google Storage.
66 ///
67 /// This method will throw if the target archive already exists on cloud
68 /// storage.
69 Future<void> publishArchive([bool forceUpload = false]) async {
70 final String destGsPath = '$gsReleaseFolder/$destinationArchivePath';
71 if (!forceUpload) {
72 if (await _cloudPathExists(destGsPath) && !dryRun) {
73 throw PreparePackageException('File $destGsPath already exists on cloud storage!');
74 }
75 }
76 await _cloudCopy(src: outputFile.absolute.path, dest: destGsPath);
77 assert(tempDir.existsSync());
78 final String gcsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}';
79 await _publishMetadata(gcsPath);
80 }
81
82 /// Downloads and updates the metadata file without publishing it.
83 Future<void> generateLocalMetadata() async {
84 await _updateMetadata('$gsReleaseFolder/${getMetadataFilename(platform)}');
85 }
86
87 Future<Map<String, dynamic>> _addRelease(Map<String, dynamic> jsonData) async {
88 jsonData['base_url'] = '$baseUrl$releaseFolder';
89 if (!jsonData.containsKey('current_release')) {
90 jsonData['current_release'] = <String, String>{};
91 }
92 (jsonData['current_release'] as Map<String, dynamic>)[branch.name] = revision;
93 if (!jsonData.containsKey('releases')) {
94 jsonData['releases'] = <Map<String, dynamic>>[];
95 }
96
97 final Map<String, dynamic> newEntry = <String, dynamic>{};
98 newEntry['hash'] = revision;
99 newEntry['channel'] = branch.name;
100 newEntry['version'] = version[frameworkVersionTag];
101 newEntry['dart_sdk_version'] = version[dartVersionTag];
102 newEntry['dart_sdk_arch'] = version[dartTargetArchTag];
103 newEntry['release_date'] = DateTime.now().toUtc().toIso8601String();
104 newEntry['archive'] = destinationArchivePath;
105 newEntry['sha256'] = await _getChecksum(outputFile);
106
107 // Search for any entries with the same hash and channel and remove them.
108 final List<dynamic> releases = jsonData['releases'] as List<dynamic>;
109 jsonData['releases'] =
110 <Map<String, dynamic>>[
111 for (final Map<String, dynamic> entry in releases.cast<Map<String, dynamic>>())
112 if (entry['hash'] != newEntry['hash'] ||
113 entry['channel'] != newEntry['channel'] ||
114 entry['dart_sdk_arch'] != newEntry['dart_sdk_arch'])
115 entry,
116 newEntry,
117 ]..sort((Map<String, dynamic> a, Map<String, dynamic> b) {
118 final DateTime aDate = DateTime.parse(a['release_date'] as String);
119 final DateTime bDate = DateTime.parse(b['release_date'] as String);
120 return bDate.compareTo(aDate);
121 });
122 return jsonData;
123 }
124
125 Future<void> _updateMetadata(String gsPath) async {
126 // We can't just cat the metadata from the server with 'gsutil cat', because
127 // Windows wants to echo the commands that execute in gsutil.bat to the
128 // stdout when we do that. So, we copy the file locally and then read it
129 // back in.
130 final File metadataFile = fs.file(
131 path.join(tempDir.absolute.path, getMetadataFilename(platform)),
132 );
133 await _runGsUtil(<String>['cp', gsPath, metadataFile.absolute.path]);
134 Map<String, dynamic> jsonData = <String, dynamic>{};
135 if (!dryRun) {
136 final String currentMetadata = metadataFile.readAsStringSync();
137 if (currentMetadata.isEmpty) {
138 throw PreparePackageException('Empty metadata received from server');
139 }
140 try {
141 jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
142 } on FormatException catch (e) {
143 throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e');
144 }
145 }
146 // Run _addRelease, even on a dry run, so we can inspect the metadata on a
147 // dry run. On a dry run, the only thing in the metadata file be the new
148 // release.
149 jsonData = await _addRelease(jsonData);
150
151 const JsonEncoder encoder = JsonEncoder.withIndent(' ');
152 metadataFile.writeAsStringSync(encoder.convert(jsonData));
153 }
154
155 /// Publishes the metadata file to GCS.
156 Future<void> _publishMetadata(String gsPath) async {
157 final File metadataFile = fs.file(
158 path.join(tempDir.absolute.path, getMetadataFilename(platform)),
159 );
160 await _cloudCopy(
161 src: metadataFile.absolute.path,
162 dest: gsPath,
163 // This metadata file is used by the website, so we don't want a long
164 // latency between publishing a release and it being available on the
165 // site.
166 cacheSeconds: shortCacheSeconds,
167 );
168 }
169
170 Future<String> _runGsUtil(
171 List<String> args, {
172 Directory? workingDirectory,
173 bool failOk = false,
174 }) async {
175 if (dryRun) {
176 print('gsutil.py -- $args');
177 return '';
178 }
179 return _processRunner.runProcess(
180 <String>[
181 'python3',
182 path.join(platform.environment['DEPOT_TOOLS']!, 'gsutil.py'),
183 '--',
184 ...args,
185 ],
186 workingDirectory: workingDirectory,
187 failOk: failOk,
188 );
189 }
190
191 /// Determine if a file exists at a given [cloudPath].
192 Future<bool> _cloudPathExists(String cloudPath) async {
193 try {
194 await _runGsUtil(<String>['stat', cloudPath]);
195 } on PreparePackageException {
196 // `gsutil stat gs://path/to/file` will exit with 1 if file does not exist
197 return false;
198 }
199 return true;
200 }
201
202 Future<String> _cloudCopy({required String src, required String dest, int? cacheSeconds}) async {
203 // We often don't have permission to overwrite, but
204 // we have permission to remove, so that's what we do.
205 await _runGsUtil(<String>['rm', dest], failOk: true);
206 String? mimeType;
207 if (dest.endsWith('.tar.xz')) {
208 mimeType = 'application/x-gtar';
209 }
210 if (dest.endsWith('.zip')) {
211 mimeType = 'application/zip';
212 }
213 if (dest.endsWith('.json')) {
214 mimeType = 'application/json';
215 }
216 return _runGsUtil(<String>[
217 // Use our preferred MIME type for the files we care about
218 // and let gsutil figure it out for anything else.
219 if (mimeType != null) ...<String>['-h', 'Content-Type:$mimeType'],
220 if (cacheSeconds != null) ...<String>['-h', 'Cache-Control:max-age=$cacheSeconds'],
221 'cp',
222 src,
223 dest,
224 ]);
225 }
226}
227