Skip to content

Commit fffaebc

Browse files
rfelberrfelber
authored
Merge pull request #389 from secureCodeBox/retry-integration-tests
Retry Failing Integration Tests up to 3 times
2 parents 674465b + 5931c69 commit fffaebc

17 files changed

Lines changed: 229 additions & 124 deletions

‎tests/integration/hooks/finding-post-processing.test.js‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
const { scan } = require('../helpers')
1+
const retry = require("../retry");
22

3-
test(
3+
const { scan } = require("../helpers");
4+
5+
retry(
46
"Finding Post Processing after test-scan",
7+
3,
58
async () => {
69
const { severities, count } = await scan(
710
"finding-post-processing",
@@ -11,7 +14,7 @@ test(
1114
);
1215

1316
expect(count).toBe(2);
14-
expect(severities.high).toBe(1)
17+
expect(severities.high).toBe(1);
1518
},
1619
3 * 60 * 1000
1720
);

‎tests/integration/hooks/notification-hook.test.js‎

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,13 @@
1+
const retry = require("../retry");
2+
13
const { scan } = require("../helpers");
2-
const k8s = require('@kubernetes/client-node');
4+
const k8s = require("@kubernetes/client-node");
35

4-
test(
6+
retry(
57
"should trigger notification",
8+
3,
69
async () => {
7-
await scan(
8-
"test-scan-notification-web-hook",
9-
"test-scan",
10-
[],
11-
90
12-
);
10+
await scan("test-scan-notification-web-hook", "test-scan", [], 90);
1311

1412
const WEBHOOK = "http-webhook";
1513
const NAMESPACE = "integration-tests";
@@ -20,12 +18,12 @@ test(
2018
const k8sApi = kc.makeApiClient(k8s.CoreV1Api);
2119

2220
function containsPod(item) {
23-
return item.metadata.name.includes(WEBHOOK)
21+
return item.metadata.name.includes(WEBHOOK);
2422
}
2523

2624
let podName;
27-
await k8sApi.listNamespacedPod(NAMESPACE, 'true').then((res) => {
28-
let podArray = res.body.items.filter((containsPod));
25+
await k8sApi.listNamespacedPod(NAMESPACE, "true").then((res) => {
26+
let podArray = res.body.items.filter(containsPod);
2927
if (podArray.length === 0) {
3028
throw new Error(`Did not find Pod for "${WEBHOOK}" Hook`);
3129
}
@@ -39,26 +37,35 @@ test(
3937
k8sApi,
4038
podName,
4139
namespace: NAMESPACE,
42-
containerName
43-
}
40+
containerName,
41+
};
4442
const result = await delayedRepeat(isHookTriggered, params, 1000, 10);
4543

46-
expect(result).toBe(true)
44+
expect(result).toBe(true);
4745
},
4846
3 * 60 * 1000
4947
);
5048

5149
async function isHookTriggered(params) {
52-
console.log("Fetch Container Logs...")
53-
let containerLog = await params.k8sApi.readNamespacedPodLog(params.podName, params.namespace, params.containerName, false);
50+
console.log("Fetch Container Logs...");
51+
let containerLog = await params.k8sApi.readNamespacedPodLog(
52+
params.podName,
53+
params.namespace,
54+
params.containerName,
55+
false
56+
);
5457
return containerLog.body.includes("/slack-notification");
5558
}
5659

60+
const sleep = (durationInMs) =>
61+
new Promise((resolve) => setTimeout(resolve, durationInMs));
5762

58-
const sleep = durationInMs =>
59-
new Promise(resolve => setTimeout(resolve, durationInMs));
60-
61-
async function delayedRepeat(fun, functionParamObject, intervalInMs, maxRetries,) {
63+
async function delayedRepeat(
64+
fun,
65+
functionParamObject,
66+
intervalInMs,
67+
maxRetries
68+
) {
6269
for (let i = 0; i < maxRetries; i++) {
6370
const condition = await fun(functionParamObject);
6471
if (condition) {
@@ -68,5 +75,5 @@ async function delayedRepeat(fun, functionParamObject, intervalInMs, maxRetries,
6875
await sleep(intervalInMs);
6976
}
7077

71-
throw new Error("Reached max retries")
78+
throw new Error("Reached max retries");
7279
}

‎tests/integration/retry.js‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
// Vendored from https://www.npmjs.com/package/jest-retries MIT License
2+
// Includes adjustments to pass in timeout values to the underlying jest test function
3+
4+
function runTest(handler) {
5+
return new Promise((resolve, reject) => {
6+
const result = handler((err) => (err ? reject(err) : resolve()));
7+
8+
if (result && result.then) {
9+
result.catch(reject).then(resolve);
10+
} else {
11+
resolve();
12+
}
13+
});
14+
}
15+
16+
async function retry(description, retries, handler, ...args) {
17+
if (!description || typeof description !== "string") {
18+
throw new Error("Invalid argument, description must be a string");
19+
}
20+
21+
if (typeof retries === "function" && !handler) {
22+
handler = retries;
23+
retries = 1;
24+
}
25+
26+
if (!retries || typeof retries !== "number" || retries < 1) {
27+
throw new Error("Invalid argument, retries must be a greather than 0");
28+
}
29+
30+
test(
31+
description,
32+
async () => {
33+
let latestError;
34+
for (let tries = 0; tries < retries; tries++) {
35+
try {
36+
await runTest(handler);
37+
return;
38+
} catch (error) {
39+
latestError = error;
40+
}
41+
}
42+
43+
throw latestError;
44+
},
45+
...args
46+
);
47+
}
48+
49+
module.exports = retry;

‎tests/integration/scanner/amass.test.js‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
1-
const {scan} = require('../helpers');
1+
const retry = require("../retry");
22

3-
test(
4-
'amass should find at least 20 subdomains',
3+
const { scan } = require("../helpers");
4+
5+
retry(
6+
"amass should find at least 20 subdomains",
7+
3,
58
async () => {
6-
const {count} = await scan(
7-
'amass-scanner-dummy-scan',
8-
'amass',
9-
['-passive', '-noalts', '-norecursive', '-d', 'owasp.org'],
9+
const { count } = await scan(
10+
"amass-scanner-dummy-scan",
11+
"amass",
12+
["-passive", "-noalts", "-norecursive", "-d", "owasp.org"],
1013
90
1114
);
1215
expect(count).toBeGreaterThanOrEqual(20);

‎tests/integration/scanner/cascade-nmap-ncrack.test.js‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
const { cascadingScan } = require('../helpers')
1+
const retry = require("../retry");
22

3-
test(
3+
const { cascadingScan } = require("../helpers");
4+
5+
retry(
46
"Cascading Scan nmap -> ncrack on dummy-ssh",
7+
3,
58
async () => {
69
const { categories, severities, count } = await cascadingScan(
710
"nmap-dummy-ssh",
@@ -11,23 +14,19 @@ test(
1114
nameCascade: "ncrack-ssh",
1215
matchLabels: {
1316
"securecodebox.io/invasive": "invasive",
14-
"securecodebox.io/intensive": "high"
15-
}
17+
"securecodebox.io/intensive": "high",
18+
},
1619
},
1720
120
1821
);
1922

2023
expect(count).toBe(1);
21-
expect(categories).toEqual(
22-
{
23-
"Discovered Credentials": 1,
24-
}
25-
);
26-
expect(severities).toEqual(
27-
{
28-
"high": 1,
29-
}
30-
);
24+
expect(categories).toEqual({
25+
"Discovered Credentials": 1,
26+
});
27+
expect(severities).toEqual({
28+
high: 1,
29+
});
3130
},
3231
3 * 60 * 1000
3332
);

‎tests/integration/scanner/cascade-nmap-sslyze.test.js‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1-
const { cascadingScan } = require('../helpers')
1+
const retry = require("../retry");
22

3-
test(
3+
const { cascadingScan } = require("../helpers");
4+
5+
retry(
46
"Cascading Scan nmap -> sslyze on unsafe-https",
7+
3,
58
async () => {
69
const { categories, severities, count } = await cascadingScan(
710
"nmap-unsafe-https-sslyze",
@@ -11,8 +14,8 @@ test(
1114
nameCascade: "https-tls-scan",
1215
matchLabels: {
1316
"securecodebox.io/invasive": "non-invasive",
14-
"securecodebox.io/intensive": "light"
15-
}
17+
"securecodebox.io/intensive": "light",
18+
},
1619
},
1720
4 * 60
1821
);

‎tests/integration/scanner/git-repo-scanner.test.js‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,21 @@
1-
const {scan} = require('../helpers');
1+
const retry = require("../retry");
22

3-
test(
4-
'gitleaks should find at least 1 repository in the GitHub secureCodeBox organisation',
3+
const { scan } = require("../helpers");
4+
5+
retry(
6+
"gitleaks should find at least 1 repository in the GitHub secureCodeBox organisation",
7+
3,
58
async () => {
69
// This integration tests runs about 30min because of the GitHub Public API call rate limit.
710
// If you want to speed up you need to add an valid access token like: ['--git-type', 'github', '--organization', 'secureCodeBox', '--access-token', '23476VALID2345TOKEN'],
8-
const {count} = await scan(
9-
'git-repo-scanner-dummy-scan',
10-
'git-repo-scanner',
11-
['--git-type', 'github', '--organization', 'secureCodeBox'],
11+
const { count } = await scan(
12+
"git-repo-scanner-dummy-scan",
13+
"git-repo-scanner",
14+
["--git-type", "github", "--organization", "secureCodeBox"],
1215
90
1316
);
1417
// There must be >= 28 Repositories found in the GitHub secureCodeBox organisation.
1518
expect(count).toBeGreaterThanOrEqual(28);
1619
},
1720
3 * 60 * 1000
1821
);
19-

‎tests/integration/scanner/gitleaks.test.js‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,31 @@
1-
const {scan} = require('../helpers');
1+
const retry = require("../retry");
22

3-
test(
4-
'gitleaks should find 1 credential in the testfiles',
3+
const { scan } = require("../helpers");
4+
5+
retry(
6+
"gitleaks should find 1 credential in the testfiles",
7+
3,
58
async () => {
6-
const {categories, severities, count} = await scan(
7-
'gitleaks-dummy-scan',
8-
'gitleaks',
9-
['-r', 'https://github.com/secureCodeBox/secureCodeBox', '--commit=ec0fe179ccf178b56fcd51d1730448bc64bb9ab5', '--config-path', '/home/config_all.toml'],
9+
const { categories, severities, count } = await scan(
10+
"gitleaks-dummy-scan",
11+
"gitleaks",
12+
[
13+
"-r",
14+
"https://github.com/secureCodeBox/secureCodeBox",
15+
"--commit=ec0fe179ccf178b56fcd51d1730448bc64bb9ab5",
16+
"--config-path",
17+
"/home/config_all.toml",
18+
],
1019
90
1120
);
1221

1322
expect(count).toBe(1);
14-
expect(categories).toEqual(
15-
{
16-
'Potential Secret': 1
17-
}
18-
);
19-
expect(severities).toEqual(
20-
{
21-
'high': 1
22-
}
23-
);
23+
expect(categories).toEqual({
24+
"Potential Secret": 1,
25+
});
26+
expect(severities).toEqual({
27+
high: 1,
28+
});
2429
},
2530
3 * 60 * 1000
2631
);
27-

‎tests/integration/scanner/kube-hunter.test.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1+
const retry = require("../retry");
2+
13
const { scan } = require("../helpers");
24

3-
test(
5+
retry(
46
"kube-hunter should find a fixed number of findings for the kind cluster",
7+
3,
58
async () => {
69
await scan(
710
"kube-hunter-in-cluster",

‎tests/integration/scanner/kubeaudit.test.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
1+
const retry = require("../retry");
2+
13
const { scan } = require("../helpers");
24

3-
test(
5+
retry(
46
"kubeaudit should run and check the jshop in kubeaudit-tests namespace",
7+
3,
58
async () => {
69
const { categories, severities } = await scan(
710
"kubeaudit-tests",

0 commit comments

Comments
 (0)