Skip to content
This repository was archived by the owner on Sep 1, 2024. It is now read-only.

Commit 89111d2

Browse files
committed
[jest] Implement experimental test-independence
1 parent 05f810b commit 89111d2

40 files changed

+1714
-339
lines changed

.github/workflows/ci.yaml

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ jobs:
2828
cache: yarn
2929

3030
- id: install
31+
env:
32+
CYPRESS_INSTALL_BINARY: "0"
3133
run: yarn install --immutable
3234

3335
- run: yarn build
@@ -50,6 +52,16 @@ jobs:
5052
name: js-api
5153
path: packages/js-api/package.tgz
5254

55+
- name: Run @unflakable/cypress-plugin unit tests
56+
env:
57+
FORCE_COLOR: "1"
58+
run: yarn workspace @unflakable/cypress-plugin test
59+
60+
- name: Run @unflakable/jest-plugin unit tests
61+
env:
62+
FORCE_COLOR: "1"
63+
run: yarn workspace @unflakable/jest-plugin test
64+
5365
- if: ${{ always() && steps.install.outcome == 'success' }}
5466
run: yarn lint
5567

@@ -189,7 +201,8 @@ jobs:
189201
UNFLAKABLE_API_KEY=${{ secrets.UNFLAKABLE_API_KEY }} \
190202
yarn workspace cypress-integration test \
191203
--reporters @unflakable/jest-plugin/dist/reporter \
192-
--runner @unflakable/jest-plugin/dist/runner
204+
--runner @unflakable/jest-plugin/dist/runner \
205+
--testRunner @unflakable/jest-plugin/dist/test-runner
193206
194207
cypress_windows_integration_tests:
195208
name: "Cypress ${{ matrix.cypress }} Windows Node ${{ matrix.node }} Integration Tests"
@@ -294,7 +307,8 @@ jobs:
294307
run: |
295308
yarn workspace cypress-integration test `
296309
--reporters @unflakable/jest-plugin/dist/reporter `
297-
--runner @unflakable/jest-plugin/dist/runner
310+
--runner @unflakable/jest-plugin/dist/runner `
311+
--testRunner @unflakable/jest-plugin/dist/test-runner
298312
299313
jest_linux_integration_tests:
300314
name: "Jest ${{ matrix.jest }} Linux Node ${{ matrix.node }} Integration Tests"

packages/cypress-plugin/src/main.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,11 @@ const main = async (): Promise<void> => {
265265
? path.resolve(process.cwd(), runOptions.project)
266266
: process.cwd();
267267

268-
const unflakableConfig = await loadConfig(projectRoot, args["test-suite-id"]);
268+
const unflakableConfig = await loadConfig(
269+
projectRoot,
270+
() => [{}, []],
271+
args["test-suite-id"]
272+
);
269273
debug(`Unflakable plugin is ${unflakableConfig.enabled ? "en" : "dis"}abled`);
270274

271275
let configFile: string | undefined = undefined;

packages/cypress-plugin/test/integration/src/run-test-case.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -615,7 +615,7 @@ export const runTestCase = async (
615615
);
616616

617617
const configMockParams: CosmiconfigMockParams = {
618-
searchFrom: path.resolve(projectPath(params)),
618+
expectedSearchFrom: path.resolve(projectPath(params)),
619619
searchResult:
620620
params.config !== null
621621
? {

packages/cypress-plugin/test/integration/tsconfig.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@
77
// that involve lots of Node.JS processes from inside the browser.
88
"types": ["jest", "jest-expect-message", "node"]
99
},
10-
"include": [".eslintrc.js", "jest.config.js", "src"]
10+
"include": [".eslintrc.js", "jest.config.js", "src", "unflakable.js"]
1111
}
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
// Copyright (c) 2023 Developer Innovations, LLC
2+
3+
module.exports = {
4+
__unstableIsFailureTestIndependent: [
5+
// Cypress sometimes hangs waiting for Chrome tabs to close. See:
6+
// https://github.com/cypress-io/cypress/issues/27360
7+
// https://github.com/cypress-io/cypress/blob/fe54cf504aefcfa6b621a90baa57e345cfa09548/packages/server/lib/modes/run.ts#L676-L680
8+
// NB: This requires DEBUG="cypress:server:run" (at a minimum).
9+
/attempting to close the browser tab(?:(?!resetting server state).)*$/s,
10+
/Still waiting to connect to Edge, retrying in 1 second.*(?:Error: Test timed out after|All promises were rejected)/s,
11+
],
12+
};
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// Copyright (c) 2023 Developer Innovations, LLC
2+
3+
// jest-circus doesn't export the types for runner. See:
4+
// https://github.com/jestjs/jest/blob/6d2632adae0f0fa1fe116d3b475fd9783d0de1b5/packages/jest-circus/runner.js#L10-L9
5+
// https://github.com/jestjs/jest/blob/6d2632adae0f0fa1fe116d3b475fd9783d0de1b5/packages/jest-runner/src/types.ts#L34
6+
declare module "jest-circus/runner" {
7+
import { Config } from "@jest/types";
8+
import { JestEnvironment } from "@jest/environment";
9+
import {
10+
TestResult,
11+
AssertionResult,
12+
SerializableError,
13+
Test,
14+
} from "@jest/test-result";
15+
16+
// Exported by newer Jest versions but not older ones prior to 26.2.0.
17+
export declare type TestEvents = {
18+
"test-file-start": [Test];
19+
"test-file-success": [Test, TestResult];
20+
"test-file-failure": [Test, SerializableError];
21+
"test-case-result": [string, AssertionResult];
22+
};
23+
24+
export declare type TestFileEvent<
25+
T extends keyof TestEvents = keyof TestEvents
26+
> = (eventName: T, args: TestEvents[T]) => unknown;
27+
28+
export declare type UnsubscribeFn = () => void;
29+
30+
export declare type TestFramework = (
31+
globalConfig: Config.GlobalConfig,
32+
config: Config.ProjectConfig,
33+
environment: JestEnvironment,
34+
runtime: unknown,
35+
testPath: string,
36+
sendMessageToJest?: TestFileEvent
37+
) => Promise<TestResult>;
38+
39+
const initialize: TestFramework;
40+
export default initialize;
41+
}
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Copyright (c) 2023 Developer Innovations, LLC
2+
3+
/** @type {import('ts-jest').JestConfigWithTsJest} */
4+
module.exports = {
5+
extensionsToTreatAsEsm: [".ts"],
6+
roots: ["src"],
7+
testEnvironment: "node",
8+
testTimeout: 60000,
9+
transform: {
10+
"^.+\\.[tj]s$": [
11+
"ts-jest",
12+
{
13+
tsconfig: "tsconfig.json",
14+
},
15+
],
16+
},
17+
verbose: true,
18+
};

packages/jest-plugin/package.json

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,18 @@
1717
"./dist/runner": {
1818
"types": "./dist/runner.d.ts",
1919
"default": "./dist/runner.js"
20+
},
21+
"./dist/test-runner": {
22+
"types": "./dist/test-runner.d.ts",
23+
"default": "./dist/test-runner.js"
2024
}
2125
},
2226
"files": [
2327
"README.md",
2428
"dist/**/*.js",
2529
"dist/reporter.d.ts",
26-
"dist/runner.d.ts"
30+
"dist/runner.d.ts",
31+
"dist/test-runner.d.ts"
2732
],
2833
"dependencies": {
2934
"@unflakable/js-api": "workspace:^",
@@ -32,6 +37,7 @@
3237
"debug": "^4.3.3",
3338
"deep-equal": "^2.0.5",
3439
"escape-string-regexp": "^4.0.0",
40+
"semver": "^7.5.4",
3541
"simple-git": "^3.16.0"
3642
},
3743
"devDependencies": {
@@ -50,6 +56,9 @@
5056
"@types/jest": "25.1.0 - 29",
5157
"@unflakable/plugins-common": "workspace:^",
5258
"exit": "^0.1.2",
59+
"jest": "25.1.0 - 29",
60+
"jest-circus": "25.1.0 - 29",
61+
"jest-environment-node": "25.1.0 - 29",
5362
"jest-runner": "25.1.0 - 29",
5463
"jest-util": "25.1.0 - 29",
5564
"rimraf": "^5.0.1",
@@ -62,6 +71,7 @@
6271
"scripts": {
6372
"build": "yarn clean && tsc --noEmit && rollup --config",
6473
"build:watch": "rollup --config --watch",
65-
"clean": "rimraf dist/"
74+
"clean": "rimraf dist/",
75+
"test": "jest --useStderr --verbose"
6676
}
6777
}

packages/jest-plugin/rollup.config.mjs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ const isExternal = (id) =>
2929
*/
3030
export default [
3131
{
32-
input: ["src/reporter.ts", "src/runner.ts"],
32+
input: ["src/reporter.ts", "src/runner.ts", "src/test-runner.ts"],
3333
output: {
3434
dir: "dist",
3535
format: "cjs",
@@ -57,6 +57,7 @@ export default [
5757
// NB: This should include every exported .d.ts from package.json.
5858
"dist/reporter.d.ts",
5959
"dist/runner.d.ts",
60+
"dist/test-runner.d.ts",
6061
],
6162
output: {
6263
dir: ".",
Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
// Copyright (c) 2023 Developer Innovations, LLC
2+
3+
import { loadConfig } from "./config";
4+
import { cosmiconfigSync, Options } from "cosmiconfig";
5+
import {
6+
setCosmiconfigSync,
7+
UnflakableConfigFile,
8+
} from "@unflakable/plugins-common";
9+
import { IsFailureTestIndependentFn } from "./types";
10+
11+
const MOCK_SUITE_ID = "MOCK_SUITE_ID";
12+
const SEARCH_FROM = ".";
13+
14+
const throwUnimplemented = (): never => {
15+
throw new Error("unimplemented");
16+
};
17+
18+
const setMockConfig = (
19+
config: Partial<
20+
UnflakableConfigFile & {
21+
__unstableIsFailureTestIndependent:
22+
| string
23+
| RegExp
24+
| (string | RegExp)[]
25+
| IsFailureTestIndependentFn;
26+
}
27+
>
28+
): void => {
29+
setCosmiconfigSync(
30+
(
31+
moduleName: string,
32+
options?: Options
33+
): ReturnType<typeof cosmiconfigSync> => {
34+
expect(moduleName).toBe("unflakable");
35+
expect(options?.searchPlaces).toContain("package.json");
36+
expect(options?.searchPlaces).toContain("unflakable.json");
37+
expect(options?.searchPlaces).toContain("unflakable.js");
38+
expect(options?.searchPlaces).toContain("unflakable.yaml");
39+
expect(options?.searchPlaces).toContain("unflakable.yml");
40+
return {
41+
clearCaches: throwUnimplemented,
42+
clearLoadCache: throwUnimplemented,
43+
clearSearchCache: throwUnimplemented,
44+
load: throwUnimplemented,
45+
search: (
46+
searchFrom?: string
47+
): ReturnType<ReturnType<typeof cosmiconfigSync>["search"]> => {
48+
expect(searchFrom).toBe(SEARCH_FROM);
49+
return {
50+
config,
51+
filepath: "unflakable.js",
52+
isEmpty: false,
53+
};
54+
},
55+
};
56+
}
57+
);
58+
};
59+
60+
describe("__unstableIsFailureTestIndependent", () => {
61+
it("should default to undefined", () => {
62+
setMockConfig({ testSuiteId: MOCK_SUITE_ID });
63+
const config = loadConfig(SEARCH_FROM);
64+
expect(config.testSuiteId).toBe(MOCK_SUITE_ID);
65+
expect(config.isFailureTestIndependent).toBeUndefined();
66+
});
67+
68+
it("should accept a string regex", () => {
69+
setMockConfig({
70+
testSuiteId: MOCK_SUITE_ID,
71+
__unstableIsFailureTestIndependent: ".*",
72+
});
73+
const config = loadConfig(SEARCH_FROM);
74+
expect(config.testSuiteId).toBe(MOCK_SUITE_ID);
75+
expect(Array.isArray(config.isFailureTestIndependent)).toBe(true);
76+
expect(config.isFailureTestIndependent).toHaveLength(1);
77+
expect((config.isFailureTestIndependent as RegExp[])[0]).toBeInstanceOf(
78+
RegExp
79+
);
80+
expect((config.isFailureTestIndependent as RegExp[])[0].source).toBe(".*");
81+
expect((config.isFailureTestIndependent as RegExp[])[0].flags).toBe("");
82+
});
83+
84+
it("should accept a RegExp object", () => {
85+
setMockConfig({
86+
testSuiteId: MOCK_SUITE_ID,
87+
__unstableIsFailureTestIndependent: /.*/gs,
88+
});
89+
const config = loadConfig(SEARCH_FROM);
90+
expect(config.testSuiteId).toBe(MOCK_SUITE_ID);
91+
expect(Array.isArray(config.isFailureTestIndependent)).toBe(true);
92+
expect(config.isFailureTestIndependent).toHaveLength(1);
93+
expect((config.isFailureTestIndependent as RegExp[])[0]).toBeInstanceOf(
94+
RegExp
95+
);
96+
expect((config.isFailureTestIndependent as RegExp[])[0].source).toBe(".*");
97+
expect((config.isFailureTestIndependent as RegExp[])[0].flags).toBe("gs");
98+
});
99+
100+
it("should accept an array of strings/RegExps object", () => {
101+
setMockConfig({
102+
testSuiteId: MOCK_SUITE_ID,
103+
__unstableIsFailureTestIndependent: [/foo/s, /bar/g, "baz", ".*"],
104+
});
105+
const config = loadConfig(SEARCH_FROM);
106+
expect(config.testSuiteId).toBe(MOCK_SUITE_ID);
107+
expect(Array.isArray(config.isFailureTestIndependent)).toBe(true);
108+
expect(config.isFailureTestIndependent).toHaveLength(4);
109+
expect((config.isFailureTestIndependent as RegExp[])[0]).toBeInstanceOf(
110+
RegExp
111+
);
112+
expect((config.isFailureTestIndependent as RegExp[])[0].source).toBe("foo");
113+
expect((config.isFailureTestIndependent as RegExp[])[0].flags).toBe("s");
114+
expect((config.isFailureTestIndependent as RegExp[])[1]).toBeInstanceOf(
115+
RegExp
116+
);
117+
expect((config.isFailureTestIndependent as RegExp[])[1].source).toBe("bar");
118+
expect((config.isFailureTestIndependent as RegExp[])[1].flags).toBe("g");
119+
expect((config.isFailureTestIndependent as RegExp[])[2]).toBeInstanceOf(
120+
RegExp
121+
);
122+
expect((config.isFailureTestIndependent as RegExp[])[2].source).toBe("baz");
123+
expect((config.isFailureTestIndependent as RegExp[])[2].flags).toBe("");
124+
expect((config.isFailureTestIndependent as RegExp[])[3]).toBeInstanceOf(
125+
RegExp
126+
);
127+
expect((config.isFailureTestIndependent as RegExp[])[3].source).toBe(".*");
128+
expect((config.isFailureTestIndependent as RegExp[])[3].flags).toBe("");
129+
});
130+
});

0 commit comments

Comments
 (0)