forked from flutter/flutter
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmanifest.yaml
More file actions
371 lines (314 loc) · 11.6 KB
/
manifest.yaml
File metadata and controls
371 lines (314 loc) · 11.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
# Describes the tasks we run in the continuous integration (CI) environment.
#
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
# every master commit.
#
# [1] github.com/flutter/cocoon
# CI tasks.
#
# Each key in this dictionary is the unique name of a task, which also
# corresponds to a file in the "bin/" directory that the task runner will run.
#
# Required properties:
#
# * description: free form string
# describes what the task is about
# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
# tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
# icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
# run device tests on red Travis.
# * required_agent_capabilities: a list of strings
# list of capabilities a devicelab agent must have to be able to run this task.
#
# Optional properties:
#
# * flaky: boolean true or false
# whether the task is considered flaky; the result of running a flaky task does not affect
# the overall build status. Tests should be marked flaky when newly added, until they have
# been proved for a few cycles. Tests that are actually flaky but not being actively worked
# on should be hidden because they just cause confusion otherwise.
# * timeout_in_minutes: integer
# a custom task timeout, specified in minutes.
tasks:
# Deviceless tests
# TODO: make these not require "has-android-device"; it is only there to
# ensure we have the Android SDK.
complex_layout__build:
description: >
Collects various performance metrics from AOT builds of the Complex
Layout sample app.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
basic_material_app__size:
description: >
Measures the APK size of a basic material app.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
# Android on-device tests
complex_layout_scroll_perf__timeline_summary:
description: >
Measures the runtime performance of the Complex Layout sample app on
Android.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
flavors_test:
description: >
Checks that flavored builds work on Android.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
flaky: true
channels_integration_test:
description: >
Checks that platform channels work on Android.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
platform_interaction_test:
description: >
Checks platform interaction on Android.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
platform_channel_sample_test:
description: >
Runs a driver test on the Platform Channel sample app on Android.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
complex_layout__start_up:
description: >
Measures the startup time of the Complex Layout sample app on Android.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
hot_mode_dev_cycle__benchmark:
description: >
Measures the performance of Dart VM hot patching feature.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
hot_mode_dev_cycle__preview_dart_2_benchmark:
description: >
Measures the performance of Dart VM hot patching feature under
--preview-dart-2 option, that enables Dart 2.0 frontend.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
flaky: true
complex_layout_scroll_perf__memory:
description: >
Measures memory usage of the scroll performance test.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
hello_world__memory:
description: >
Measures starting memory usage of the simplest Flutter app.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
microbenchmarks:
description: >
Runs benchmarks from dev/benchmarks/microbenchmarks.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
timeout_in_minutes: 30
flutter_view__start_up:
description: >
Verifies that Flutter View can be used from an Android project.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
integration_ui:
description: >
Runs end-to-end Flutter tests on Android.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
commands_test:
description: >
Runs tests of flutter run commands.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
service_extensions_test:
description: >
Validates our service protocol extensions.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
flaky: true
android_sample_catalog_generator:
description: >
Builds sample catalog markdown pages and Android screenshots
stage: devicelab
required_agent_capabilities: ["has-android-device"]
flaky: true
complex_layout_semantics_perf:
description: >
Measures duration of building the initial semantics tree.
stage: devicelab
required_agent_capabilities: ["linux/android"]
routing_test:
description: >
Verifies that `flutter drive --route` still works. No performance numbers.
stage: devicelab
required_agent_capabilities: ["linux/android"]
gradle_plugin_test:
description: >
Verifies that the Flutter Gradle plugin supports standard and custom Android build types.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery_instrumentation_test:
description: >
Same as flutter_gallery__transition_perf but uses Android instrumentation
framework, and therefore does not require a host computer to run. This
test can run on off-the-shelf infrastructures, such as Firebase Test Lab.
stage: devicelab
required_agent_capabilities: ["linux/android"]
# iOS on-device tests
flavors_test_ios:
description: >
Checks that flavored builds work on iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
flaky: true
channels_integration_test_ios:
description: >
Checks that platform channels work on iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
platform_interaction_test_ios:
description: >
Checks platform interaction on iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
platform_channel_sample_test_ios:
description: >
Runs a driver test on the Platform Channel sample app on iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
complex_layout_scroll_perf_ios__timeline_summary:
description: >
Measures the runtime performance of the Complex Layout sample app on
iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
# flutter_gallery_ios__start_up:
# description: >
# Measures the startup time of the Flutter Gallery app on iOS.
# stage: devicelab_ios
# required_agent_capabilities: ["has-ios-device"]
# flaky: true
# complex_layout_ios__start_up:
# description: >
# Measures the startup time of the Complex Layout sample app on iOS.
# stage: devicelab_ios
# required_agent_capabilities: ["has-ios-device"]
# flaky: true
flutter_gallery_ios__transition_perf:
description: >
Measures the performance of screen transitions in Flutter Gallery on
iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
basic_material_app_ios__size:
description: >
Measures the IPA size of a basic material app.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
# microbenchmarks_ios:
# description: >
# Runs benchmarks from dev/benchmarks/microbenchmarks on iOS.
# stage: devicelab_ios
# required_agent_capabilities: ["has-ios-device"]
# timeout_in_minutes: 30
# flaky: true
# flutter_view_ios__start_up:
# description: >
# Verifies that Flutter View can be used from an iOS project.
# stage: devicelab_ios
# required_agent_capabilities: ["has-ios-device"]
# flaky: true
integration_ui_ios:
description: >
Runs end-to-end Flutter tests on iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
ios_sample_catalog_generator:
description: >
Builds sample catalog markdown pages and iOS screenshots
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
# Tests running on Windows host
flavors_test_win:
description: >
Checks that flavored builds work on Windows.
stage: devicelab_win
required_agent_capabilities: ["windows"]
flaky: true
channels_integration_test_win:
description: >
Checks that platform channels work when app is launched from Windows.
stage: devicelab_win
required_agent_capabilities: ["windows"]
flutter_gallery_win__build:
description: >
Collects various performance metrics from AOT builds of the Flutter
Gallery on Windows.
stage: devicelab_win
required_agent_capabilities: ["windows"]
hot_mode_dev_cycle_win__benchmark:
description: >
Measures the performance of Dart VM hot patching feature on Windows.
stage: devicelab_win
required_agent_capabilities: ["windows"]
# Tests running on Linux hosts
hot_mode_dev_cycle_linux__benchmark:
description: >
Measures the performance of Dart VM hot patching feature on a Linux host.
stage: devicelab
required_agent_capabilities: ["linux/android"]
dartdocs:
description: >
Tracks how many members are still lacking documentation.
stage: devicelab
required_agent_capabilities: ["linux/android"]
technical_debt__cost:
description: >
Estimates our technical debt (TODOs, analyzer ignores, etc).
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery__build:
description: >
Collects various performance metrics from AOT builds of the Flutter
Gallery.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery__preview_dart_2_build:
description: >
Collects various performance metrics from AOT builds of the Flutter
Gallery under --preview-dart-2 that enables new Dart 2.0 frontend.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery__start_up:
description: >
Measures the startup time of the Flutter Gallery app on Android.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery__transition_perf:
description: >
Measures the performance of screen transitions in Flutter Gallery on
Android.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery__transition_perf_with_semantics:
description: >
Measures the delta in performance of screen transitions without and
with semantics enabled.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flaky: true
flutter_gallery__memory_nav:
description: >
Measures memory usage after repeated navigation in Gallery.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery__back_button_memory:
description: >
Measures memory usage after Android app suspend and resume.
stage: devicelab
required_agent_capabilities: ["linux/android"]
analyzer_benchmark:
description: >
Measures the speed of Dart analyzer.
stage: devicelab
required_agent_capabilities: ["linux/android"]