[cypress] Support Cypress 13.4

ramosbugs · ramosbugs · commit 8cb9af1bb23c · 2023-10-31T00:12:02.000-07:00
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -132,6 +132,7 @@ jobs:
           - "13.1"
           - "13.2"
           - "13.3"
+          - "13.4"
     steps:
       - uses: actions/checkout@v4
 
@@ -242,6 +243,7 @@ jobs:
           - "13.1"
           - "13.2"
           - "13.3"
+          - "13.4"
     steps:
       - uses: actions/checkout@v4
 
diff --git a/packages/cypress-plugin/test/integration/src/hook-failures.test.ts b/packages/cypress-plugin/test/integration/src/hook-failures.test.ts
@@ -1,6 +1,14 @@
 // Copyright (c) 2023 Developer Innovations, LLC
 
 import { integrationTest, integrationTestSuite } from "./test-wrappers";
+import semverLt from "semver/functions/lt";
+import cypressPackage from "cypress/package.json";
+
+// Cypress 13.4 broke the handling of multiple hook failures. Prior to that version, Cypress/Mocha
+// reported both errors as failures of the first test in the suite, and then skipped all remaining
+// tests. Beginning in 13.4, Cypress skips all tests in the suite and never reports either error.
+// This was most likely introduced in https://github.com/cypress-io/cypress/pull/27930.
+const supportMultipleHookErrors = semverLt(cypressPackage.version, "13.4.0");
 
 integrationTestSuite((mockBackend) => {
   it("run should succeed when before() fails and both tests are quarantined", (done) =>
@@ -303,28 +311,31 @@ integrationTestSuite((mockBackend) => {
       done
     ));
 
-  it("multiple before() hook errors", (done) =>
-    integrationTest(
-      {
-        params: {
-          multipleHookErrors: true,
-          specNameStubs: ["hook-fail"],
-        },
-        expectedExitCode: 1,
-        summaryTotals: {
-          icon: "fail",
-          numFailing: 1,
-          numFlaky: 0,
-          numPassing: 0,
-          numPending: 0,
-          numQuarantined: 0,
-          numSkipped: 1,
-          numTests: 2,
+  (supportMultipleHookErrors ? it : it.skip)(
+    "multiple before() hook errors",
+    (done) =>
+      integrationTest(
+        {
+          params: {
+            multipleHookErrors: true,
+            specNameStubs: ["hook-fail"],
+          },
+          expectedExitCode: 1,
+          summaryTotals: {
+            icon: "fail",
+            numFailing: 1,
+            numFlaky: 0,
+            numPassing: 0,
+            numPending: 0,
+            numQuarantined: 0,
+            numSkipped: 1,
+            numTests: 2,
+          },
         },
-      },
-      mockBackend,
-      done
-    ));
+        mockBackend,
+        done
+      )
+  );
 
   it("test and afterEach() hook errors", (done) =>
     integrationTest(
diff --git a/packages/cypress-plugin/test/integration/src/verify-output.ts b/packages/cypress-plugin/test/integration/src/verify-output.ts
@@ -22,11 +22,18 @@ import {
 import { expect as expectExt } from "@jest/globals";
 import escapeStringRegexp from "escape-string-regexp";
 import { TestAttemptResult } from "@unflakable/js-api";
+import semverGte from "semver/functions/gte";
+import cypressPackage from "cypress/package.json";
 
 const THROWN_ERROR = "\x1B[0m\x1B[31m     Error\x1B[0m\x1B[90m";
 const FAIL_SYMBOL = process.platform === "win32" ? "×" : "✖";
 const PASS_SYMBOL = process.platform === "win32" ? "√" : "✓";
 
+// Cypress 13.4 changed the number of retries returned after a retry passes (i.e., when a test is
+// flaky). See:
+// https://github.com/cypress-io/cypress/blob/5a95541c3c4e48bfc67a54642abc949576fa6f05/packages/driver/patches/mocha%2B7.0.1.dev.patch
+const adjustRetriesOnPass = semverGte(cypressPackage.version, "13.4.0");
+
 const verifySpecOutput = (
   params: TestCaseParams,
   specOutputs: SpecOutput[],
@@ -141,7 +148,7 @@ const verifySpecOutputs = (
                           new RegExp(
                             // eslint-disable-next-line no-control-regex
                             `^ {2}\x1B\\[35m {2}${PASS_SYMBOL}\x1B\\[39m\x1B\\[90m mixed: flake should be quarantined\x1B\\[0m\x1B\\[35m \\[flaky, quarantined]\x1B\\[39m\x1B\\[33m \\(attempt 2 of ${
-                              expectedRetries + 1
+                              adjustRetriesOnPass ? 2 : expectedRetries + 1
                             }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$`
                           )
                         ),
@@ -169,7 +176,7 @@ const verifySpecOutputs = (
                   new RegExp(
                     // eslint-disable-next-line no-control-regex
                     `^ {2}\x1B\\[33m {2}${PASS_SYMBOL}\x1B\\[0m\x1B\\[90m mixed: flake should be quarantined\x1B\\[0m\x1B\\[33m \\[flaky]\x1B\\[0m\x1B\\[33m \\(attempt 2 of ${
-                      expectedRetries + 1
+                      adjustRetriesOnPass ? 2 : expectedRetries + 1
                     }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$`
                   )
                 ),
@@ -208,15 +215,15 @@ const verifySpecOutputs = (
                       new RegExp(
                         // eslint-disable-next-line no-control-regex
                         `^ {2}\x1B\\[35m {2}${PASS_SYMBOL}\x1B\\[39m\x1B\\[90m mixed: should be flaky\x1B\\[0m\x1B\\[35m \\[flaky, quarantined]\x1B\\[39m\x1B\\[33m \\(attempt 2 of ${
-                          expectedRetries + 1
+                          adjustRetriesOnPass ? 2 : expectedRetries + 1
                         }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$`
                       )
                     )
                   : expectExt.stringMatching(
                       new RegExp(
                         // eslint-disable-next-line no-control-regex
                         `^ {2}\x1B\\[33m {2}${PASS_SYMBOL}\x1B\\[0m\x1B\\[90m mixed: should be flaky\x1B\\[0m\x1B\\[33m \\[flaky]\x1B\\[0m\x1B\\[33m \\(attempt 2 of ${
-                          expectedRetries + 1
+                          adjustRetriesOnPass ? 2 : expectedRetries + 1
                         }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$`
                       )
                     ),
@@ -708,7 +715,7 @@ const verifySpecOutputs = (
                         `^\x1B\\[35m {2}${PASS_SYMBOL}\x1B\\[39m\x1B\\[90m should be flaky${escapeStringRegexp(
                           expectedFlakeTestNameSuffix
                         )}\x1B\\[0m\x1B\\[35m \\[flaky, quarantined]\x1B\\[39m\x1B\\[33m \\(attempt 2 of ${
-                          expectedRetries + 1
+                          adjustRetriesOnPass ? 2 : expectedRetries + 1
                         }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$`
                       )
                     )
@@ -718,7 +725,7 @@ const verifySpecOutputs = (
                         `^\x1B\\[33m {2}${PASS_SYMBOL}\x1B\\[0m\x1B\\[90m should be flaky${escapeStringRegexp(
                           expectedFlakeTestNameSuffix
                         )}\x1B\\[0m\x1B\\[33m \\[flaky]\x1B\\[0m\x1B\\[33m \\(attempt 2 of ${
-                          expectedRetries + 1
+                          adjustRetriesOnPass ? 2 : expectedRetries + 1
                         }\\)\x1B\\[0m\x1B\\[(?:33|90)m \\([0-9]+.+?\\)\x1B\\[0m$`
                       )
                     ),
