Rerun devicelab task from test runner (flutter#86394)

keyonghan · web-flow · commit 4d96a3fd508c · 2021-07-14T15:23:10.000-07:00
diff --git a/dev/devicelab/README.md b/dev/devicelab/README.md
@@ -27,7 +27,9 @@ When a device in the lab is free, it will pickup tasks that need to be completed
 
 1. If the task succeeds, the test runner reports the success and uploads its performance metrics to Flutter's infrastructure. Not
 all tasks record performance metrics.
-2. If the task fails, the test runner reports the failure to Flutter's infrastructure and no performance metrics are collected
+2. If task fails, an auto rerun happens. Whenever the last run succeeds, the task will be reported as a success. For this case,
+a flake will be flagged and populated to the test result.
+3. If the task fails in all reruns, the test runner reports the failure to Flutter's infrastructure and no performance metrics are collected
 
 ## Running tests locally
 
diff --git a/dev/devicelab/lib/framework/cocoon.dart b/dev/devicelab/lib/framework/cocoon.dart
@@ -47,6 +47,9 @@ class Cocoon {
   /// Url used to send results to.
   static const String baseCocoonApiUrl = 'https://flutter-dashboard.appspot.com/api';
 
+  /// Threshold to auto retry a failed test.
+  static const int retryNumber = 2;
+
   /// Underlying [FileSystem] to use.
   final FileSystem fs;
 
diff --git a/dev/devicelab/lib/framework/runner.dart b/dev/devicelab/lib/framework/runner.dart
@@ -4,6 +4,7 @@
 
 import 'dart:async';
 import 'dart:convert';
+// import 'dart:core' as core;
 import 'dart:io';
 
 import 'package:flutter_devicelab/common.dart';
@@ -15,6 +16,16 @@ import 'devices.dart';
 import 'task_result.dart';
 import 'utils.dart';
 
+/// Run a list of tasks.
+///
+/// For each task, an auto rerun will be triggered when task fails.
+///
+/// If the task succeeds the first time, it will be recorded as successful.
+///
+/// If the task fails first, but gets passed in the end, the
+/// test will be recorded as successful but with a flake flag.
+///
+/// If the task fails all reruns, it will be recorded as failed.
 Future<void> runTasks(
   List<String> taskNames, {
   bool exitOnFirstTestFailure = false,
@@ -26,33 +37,45 @@ Future<void> runTasks(
   String? luciBuilder,
   String? resultsPath,
   List<String>? taskArgs,
+  @visibleForTesting Map<String, String>? isolateParams,
+  @visibleForTesting Function(String) print = print,
+  @visibleForTesting List<String>? logs,
 }) async {
   for (final String taskName in taskNames) {
-    section('Running task "$taskName"');
-    final TaskResult result = await runTask(
-      taskName,
-      deviceId: deviceId,
-      localEngine: localEngine,
-      localEngineSrcPath: localEngineSrcPath,
-      silent: silent,
-      taskArgs: taskArgs,
-    );
-
-    print('Task result:');
-    print(const JsonEncoder.withIndent('  ').convert(result));
-    section('Finished task "$taskName"');
-
-    if (resultsPath != null) {
-      final Cocoon cocoon = Cocoon();
-      await cocoon.writeTaskResultToFile(
-        builderName: luciBuilder,
-        gitBranch: gitBranch,
-        result: result,
+    TaskResult result = TaskResult.success(null);
+    int retry = 0;
+    while (retry <= Cocoon.retryNumber) {
+      result = await rerunTask(
+        taskName,
+        deviceId: deviceId,
+        localEngine: localEngine,
+        localEngineSrcPath: localEngineSrcPath,
+        silent: silent,
+        taskArgs: taskArgs,
         resultsPath: resultsPath,
+        gitBranch: gitBranch,
+        luciBuilder: luciBuilder,
+        isolateParams: isolateParams,
       );
+
+      section('Flaky status for "$taskName"');
+      if (!result.succeeded) {
+        retry++;
+      } else {
+        if (retry > 0) {
+          print('Total ${retry+1} executions: $retry failures and 1 success');
+          print('flaky: true');
+        } else {
+          print('Total ${retry+1} executions: 1 success');
+          print('flaky: false');
+        }
+        break;
+      }
     }
 
     if (!result.succeeded) {
+      print('Total $retry executions: 0 success');
+      print('flaky: false');
       exitCode = 1;
       if (exitOnFirstTestFailure) {
         return;
@@ -61,6 +84,48 @@ Future<void> runTasks(
   }
 }
 
+/// A rerun wrapper for `runTask`.
+///
+/// This separates reruns in separate sections.
+Future<TaskResult> rerunTask(
+  String taskName, {
+  String? deviceId,
+  String? localEngine,
+  String? localEngineSrcPath,
+  bool silent = false,
+  List<String>? taskArgs,
+  String? resultsPath,
+  String? gitBranch,
+  String? luciBuilder,
+  @visibleForTesting Map<String, String>? isolateParams,
+}) async {
+  section('Running task "$taskName"');
+  final TaskResult result = await runTask(
+    taskName,
+    deviceId: deviceId,
+    localEngine: localEngine,
+    localEngineSrcPath: localEngineSrcPath,
+    silent: silent,
+    taskArgs: taskArgs,
+    isolateParams: isolateParams,
+  );
+
+  print('Task result:');
+  print(const JsonEncoder.withIndent('  ').convert(result));
+  section('Finished task "$taskName"');
+
+  if (resultsPath != null) {
+    final Cocoon cocoon = Cocoon();
+    await cocoon.writeTaskResultToFile(
+      builderName: luciBuilder,
+      gitBranch: gitBranch,
+      result: result,
+      resultsPath: resultsPath,
+    );
+  }
+  return result;
+}
+
 /// Runs a task in a separate Dart VM and collects the result using the VM
 /// service protocol.
 ///
diff --git a/dev/devicelab/test/runner_test.dart b/dev/devicelab/test/runner_test.dart
@@ -0,0 +1,47 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// @dart = 2.8
+
+import 'package:flutter_devicelab/framework/runner.dart';
+
+import 'common.dart';
+
+void main() {
+  final Map<String, String> isolateParams = <String, String>{
+    'runFlutterConfig': 'false',
+    'runProcessCleanup': 'false',
+    'timeoutInMinutes': '1',
+  };
+  List<String> printLog;
+  void print(String s) => printLog.add(s);
+
+  group('run.dart script', () {
+    test('Reruns - Test passes the first time.', () async {
+      printLog = <String>[];
+      await runTasks(
+        <String>['smoke_test_success'],
+        isolateParams: isolateParams,
+        print: print,
+        logs: printLog,
+      );
+      expect(printLog.length, 2);
+      expect(printLog[0], 'Total 1 executions: 1 success');
+      expect(printLog[1], 'flaky: false');
+    });
+
+    test('Reruns - Test fails all reruns.', () async {
+      printLog = <String>[];
+      await runTasks(
+        <String>['smoke_test_failure'],
+        isolateParams: isolateParams,
+        print: print,
+        logs: printLog,
+      );
+      expect(printLog.length, 2);
+      expect(printLog[0], 'Total 3 executions: 0 success');
+      expect(printLog[1], 'flaky: false');
+    });
+  });
+}
