From d7979582c1bceb8a16a67836dfea5771703ba204 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 17 Jul 2025 13:19:13 -0600
Subject: [PATCH 001/256] abort removal if var set

---
 .openshift-ci/dispatch.sh | 3 +++
 tests/e2e/lib.sh          | 6 ++++++
 2 files changed, 9 insertions(+)

diff --git a/.openshift-ci/dispatch.sh b/.openshift-ci/dispatch.sh
index 968ff14e0c72d..a08e4f8ee9f82 100755
--- a/.openshift-ci/dispatch.sh
+++ b/.openshift-ci/dispatch.sh
@@ -17,6 +17,9 @@ if [[ -f "${SHARED_DIR:-}/shared_env" ]]; then
     source "${SHARED_DIR:-}/shared_env"
 fi
 
+set |grep ROX
+echo "REMOVE_EXISTING_STACKROX_RESOURCES:${REMOVE_EXISTING_STACKROX_RESOURCES:-}"
+
 openshift_ci_mods
 openshift_ci_import_creds
 create_exit_trap
diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index a6620200422ce..739b001756a7f 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -983,6 +983,12 @@ collect_and_check_stackrox_logs() {
 # system tests against the same cluster.
 # shellcheck disable=SC2120
 remove_existing_stackrox_resources() {
+    set | grep ROX
+    if [[ "${REMOVE_EXISTING_STACKROX_RESOURCES:-false}" == 'false' ]]; then
+      info 'Skipped removal of existing stackrox resources [REMOVE_EXISTING_STACKROX_RESOURCES=false].'
+      return
+    fi
+
     info "Will remove any existing stackrox resources"
     local namespaces=( "$@" )
     local psps_supported=false

From e461d6f5392ce72db6f5ac504b8d4d03f3bc7199 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 17 Jul 2025 23:31:28 -0600
Subject: [PATCH 002/256] default remove srox

---
 tests/e2e/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 739b001756a7f..7366138ade39a 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -984,7 +984,7 @@ collect_and_check_stackrox_logs() {
 # shellcheck disable=SC2120
 remove_existing_stackrox_resources() {
     set | grep ROX
-    if [[ "${REMOVE_EXISTING_STACKROX_RESOURCES:-false}" == 'false' ]]; then
+    if [[ "${REMOVE_EXISTING_STACKROX_RESOURCES:-true}" == 'false' ]]; then
       info 'Skipped removal of existing stackrox resources [REMOVE_EXISTING_STACKROX_RESOURCES=false].'
       return
     fi

From 6c11f4cddb1eda4ebee6663e26948c7a52fd5245 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 17 Jul 2025 23:32:19 -0600
Subject: [PATCH 003/256] remove rox check

---
 .openshift-ci/dispatch.sh | 1 -
 tests/e2e/lib.sh          | 1 -
 2 files changed, 2 deletions(-)

diff --git a/.openshift-ci/dispatch.sh b/.openshift-ci/dispatch.sh
index a08e4f8ee9f82..c3bc16e17f1a9 100755
--- a/.openshift-ci/dispatch.sh
+++ b/.openshift-ci/dispatch.sh
@@ -17,7 +17,6 @@ if [[ -f "${SHARED_DIR:-}/shared_env" ]]; then
     source "${SHARED_DIR:-}/shared_env"
 fi
 
-set |grep ROX
 echo "REMOVE_EXISTING_STACKROX_RESOURCES:${REMOVE_EXISTING_STACKROX_RESOURCES:-}"
 
 openshift_ci_mods
diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 7366138ade39a..9ee940d9c445c 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -983,7 +983,6 @@ collect_and_check_stackrox_logs() {
 # system tests against the same cluster.
 # shellcheck disable=SC2120
 remove_existing_stackrox_resources() {
-    set | grep ROX
     if [[ "${REMOVE_EXISTING_STACKROX_RESOURCES:-true}" == 'false' ]]; then
       info 'Skipped removal of existing stackrox resources [REMOVE_EXISTING_STACKROX_RESOURCES=false].'
       return

From 439d715d9a87b03c9b1abc017d210da01a2c40c4 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 18 Jul 2025 16:04:21 -0600
Subject: [PATCH 004/256] skip config

---
 qa-tests-backend/scripts/run-part-1.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/qa-tests-backend/scripts/run-part-1.sh b/qa-tests-backend/scripts/run-part-1.sh
index 88624e64c56b3..f113476aa5c43 100755
--- a/qa-tests-backend/scripts/run-part-1.sh
+++ b/qa-tests-backend/scripts/run-part-1.sh
@@ -25,7 +25,14 @@ set -euo pipefail
 run_part_1() {
     info "Starting test (qa-tests-backend part I)"
 
-    config_part_1
+    if [[ "${REMOVE_EXISTING_STACKROX_RESOURCES:-true}" == 'false' ]]; then
+      info 'Skipping config. Tests will be performed on the existing stackrox resources.'
+      kubectl get ns || true
+      kubectl get pods -n stackrox || true
+      { kubectl get pods -A | grep stackrox; } || true
+    else
+      config_part_1
+    fi
     test_part_1
 }
 

From 1bcfcc3d626793274de9cfa49ef6ab6d1b4e7c3b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 25 Jul 2025 21:25:03 -0600
Subject: [PATCH 005/256] re-use config if using existing stackrox

---
 qa-tests-backend/scripts/run-part-1.sh | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/qa-tests-backend/scripts/run-part-1.sh b/qa-tests-backend/scripts/run-part-1.sh
index 316eb2593f665..96cd642c595a5 100755
--- a/qa-tests-backend/scripts/run-part-1.sh
+++ b/qa-tests-backend/scripts/run-part-1.sh
@@ -74,6 +74,21 @@ reuse_config_part_1() {
 
     export_test_environment
     setup_deployment_env false false
+
+    if [[ "${REMOVE_EXISTING_STACKROX_RESOURCES:-true}" == 'false' ]]; then
+      DEPLOY_DIR="deploy/${ORCHESTRATOR_FLAVOR}"
+
+      # requires "stackrox" namespace
+      deploy_webhook_server "$ROOT/$DEPLOY_DIR/webhook_server_certs"
+
+      get_ECR_docker_pull_password
+
+      wait_for_api "${EXISTING_STACKROX_NAMESPACE:-stackrox}"
+      export_central_basic_auth_creds
+
+      export CLUSTER="${ORCHESTRATOR_FLAVOR^^}"
+      return 0
+    fi
     export_default_TLS_certs "$ROOT/$DEPLOY_DIR/default_TLS_certs"
     export_client_TLS_certs "$ROOT/$DEPLOY_DIR/client_TLS_certs"
 

From bbea8c8dc4ac70752e3977f4d4323f9df6e2d4d5 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 25 Jul 2025 22:32:43 -0600
Subject: [PATCH 006/256] get rox pass from secrets

---
 tests/e2e/lib.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index beb09483fcba7..71237f56cc339 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -514,6 +514,9 @@ export_central_basic_auth_creds() {
     elif [[ -n "${ROX_ADMIN_PASSWORD:-}" ]]; then
         info "Using existing ROX_ADMIN_PASSWORD env"
     else
+        ROX_ADMIN_PASSWORD=$(kubectl -n stackrox get secret central-htpasswd -o go-template='{{index .data "password" | base64decode}}')
+    fi 
+    if [[ -z "${ROX_ADMIN_PASSWORD:-}" ]]; then
         echo "Expected to find file ${DEPLOY_DIR}/central-deploy/password or ROX_ADMIN_PASSWORD env"
         exit 1
     fi

From e224106d2c21747d0f9f98de9d368335c1099917 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 25 Jul 2025 22:41:35 -0600
Subject: [PATCH 007/256] lint

---
 tests/e2e/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 71237f56cc339..5665bfc739efb 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -515,7 +515,7 @@ export_central_basic_auth_creds() {
         info "Using existing ROX_ADMIN_PASSWORD env"
     else
         ROX_ADMIN_PASSWORD=$(kubectl -n stackrox get secret central-htpasswd -o go-template='{{index .data "password" | base64decode}}')
-    fi 
+    fi
     if [[ -z "${ROX_ADMIN_PASSWORD:-}" ]]; then
         echo "Expected to find file ${DEPLOY_DIR}/central-deploy/password or ROX_ADMIN_PASSWORD env"
         exit 1

From e1586931730c08c1cba0dbcf5595a89fe4f843c3 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 30 Jul 2025 16:45:41 -0600
Subject: [PATCH 008/256] find infra cluster

---
 .github/workflows/e2e-tests.yaml | 50 ++++++++++++++++++++++++++++++++
 .openshift-ci/begin.sh           |  2 ++
 scripts/ci/lib.sh                | 48 ++++++++++++++++++++++++++++++
 3 files changed, 100 insertions(+)
 create mode 100644 .github/workflows/e2e-tests.yaml

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
new file mode 100644
index 0000000000000..0fbbaa9bc58fd
--- /dev/null
+++ b/.github/workflows/e2e-tests.yaml
@@ -0,0 +1,50 @@
+name: E2E tests
+on:
+  push:
+  pull_request:
+env:
+  ROX_PRODUCT_BRANDING: stackrox
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+
+  gke-e2e-tests:
+    env:
+      ARTIFACT_DIR: junit-reports/
+    runs-on: ubuntu-latest
+    container:
+      image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.9
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - uses: ./.github/actions/job-preamble
+      with:
+        gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
+
+    - name: Create artifacts dir
+      run: mkdir -p "$ARTIFACT_DIR"
+
+    - name: Cache Go dependencies
+      uses: ./.github/actions/cache-go-dependencies
+
+    - name: Check Generated
+      run: scripts/ci/jobs/check-generated.sh
+
+    - name: Check Go version is in sync in .golangci.yml and go.mod
+      run: |
+        [ "$(awk '/^go /{print $2}' go.mod | cut -d. -f 1-2)" = "$(yq e .run.go .golangci.yml)" ]
+
+    - name: make golangci-lint
+      run: make golangci-lint
+
+    - name: Check Cache golangci-lint
+      run: make golangci-lint-cache-status
diff --git a/.openshift-ci/begin.sh b/.openshift-ci/begin.sh
index dcb3e34f61b65..68006ffe075b7 100755
--- a/.openshift-ci/begin.sh
+++ b/.openshift-ci/begin.sh
@@ -18,6 +18,8 @@ openshift_ci_import_creds
 setup_gcp
 set_ci_shared_export started_at "$(date -u +%s)"
 
+test_on_infra
+
 if [[ -z "${SHARED_DIR:-}" ]]; then
     echo "ERROR: There is no SHARED_DIR for step env sharing"
     exit 0 # not fatal but worth highlighting
diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 491388d97d6b1..88b1e95367a0e 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2406,6 +2406,54 @@ _record_cluster_info() {
     set_ci_shared_export "cut_container_runtime_version" "$containerRuntimeVersion"
 }
 
+get_infra_cluster_files() {
+    local cluster_name="${1}"
+    local data_dir="/tmp/artifacts-${cluster_name}"
+    infractl --version \
+      || { curl --silent -o infractl https://infra.rox.systems/downloads/infractl-linux-amd64;
+           chmod u+x infractl; }
+    infractl whoami
+    for I in {1..5}; do
+        infractl artifacts ${cluster_name} \
+          --download-dir "${data_dir}"
+        if [[ -d /tmp/artifacts-${cluster_name} ]]; then
+          break
+        fi
+        printf "%s - Waiting for infra cluster [${cluster_name}]" "$(date -u)"
+        sleep 60
+    done
+    echo $cluster_name; kubectl get nodes -o wide; done
+    cp "${data_dir}/kubeconfig" "${SHARED_DIR}/kubeconfig" \
+        || cp "${data_dir}/auth/kubeconfig" "${SHARED_DIR}/kubeconfig" || true
+    cp "${data_dir}/dotenv" "${SHARED_DIR}/dotenv" || true
+    ls -la "${SHARED_DIR:-/tmp}"
+    grep NAME "${SHARED_DIR:-/tmp}/shared_env" || true
+    grep CLUSTER "${SHARED_DIR:-/tmp}/shared_env" || true
+}
+
+test_on_infra() {
+    local event_json body
+    echo "PULL_NUMBER=${PULL_NUMBER:-}"
+    REPO_NAME=${REPO_NAME:-stackrox}
+    REPO_OWNER=${REPO_OWNER:-stackrox}
+    event_json=$(set -x; curl --silent \
+      -H "X-GitHub-Api-Version: 2022-11-28" \
+      -H "Accept: application/vnd.github+json" \
+      -o - "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls/${PULL_NUMBER}")
+    body=$(jq -r '.body' <<<"${event_json}" \
+      | tee >(cat >&2) | grep '^/test-on-infra') || return
+    while read -r cmd cluster_name job_name_match; do
+      if [[ "${JOB_NAME:-}" == *"$job_name_match"* ]]; then
+        echo "Matching job_name string to *${job_name_match}* for infra cluster ${cluster_name}."
+        echo "https://infra.rox.systems/cluster/${cluster_name}"
+        echo "Tests will run on this cluster instead of starting a new cluster."
+        get_infra_cluster_files "${cluster_name}"
+        echo "CLUSTER_NAME=${cluster_name}" >> "${SHARED_DIR}/"
+        break
+      fi
+    done <<<"$body"
+}
+
 if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
     if [[ "$#" -lt 1 ]]; then
         die "When invoked at the command line a method is required."

From 70c0c7f77142cefb24d3a150111f91789e770dd8 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 30 Jul 2025 17:23:05 -0600
Subject: [PATCH 009/256] lint

---
 scripts/ci/lib.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 88b1e95367a0e..35cdddf6bf1b6 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2422,7 +2422,8 @@ get_infra_cluster_files() {
         printf "%s - Waiting for infra cluster [${cluster_name}]" "$(date -u)"
         sleep 60
     done
-    echo $cluster_name; kubectl get nodes -o wide; done
+    echo "$cluster_name"
+    kubectl get nodes -o wide
     cp "${data_dir}/kubeconfig" "${SHARED_DIR}/kubeconfig" \
         || cp "${data_dir}/auth/kubeconfig" "${SHARED_DIR}/kubeconfig" || true
     cp "${data_dir}/dotenv" "${SHARED_DIR}/dotenv" || true

From 738a26cd2ca09e8eadb71b0f65c7174c50a3bdb2 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 30 Jul 2025 18:01:41 -0600
Subject: [PATCH 010/256] pwd infractl

---
 scripts/ci/lib.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 35cdddf6bf1b6..732c1e2952d2d 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2412,9 +2412,9 @@ get_infra_cluster_files() {
     infractl --version \
       || { curl --silent -o infractl https://infra.rox.systems/downloads/infractl-linux-amd64;
            chmod u+x infractl; }
-    infractl whoami
+    ./infractl whoami
     for I in {1..5}; do
-        infractl artifacts ${cluster_name} \
+        ./infractl artifacts ${cluster_name} \
           --download-dir "${data_dir}"
         if [[ -d /tmp/artifacts-${cluster_name} ]]; then
           break

From 95fded3778fa063e87393336ced8e2b2a3a00248 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 31 Jul 2025 07:38:57 -0600
Subject: [PATCH 011/256] remove autolinks

---
 scripts/ci/lib.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 732c1e2952d2d..83b5f95a88e68 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2444,6 +2444,8 @@ test_on_infra() {
     body=$(jq -r '.body' <<<"${event_json}" \
       | tee >(cat >&2) | grep '^/test-on-infra') || return
     while read -r cmd cluster_name job_name_match; do
+      job_name_match=${job_name_match%%]*}
+      job_name_match=${job_name_match##*[}
       if [[ "${JOB_NAME:-}" == *"$job_name_match"* ]]; then
         echo "Matching job_name string to *${job_name_match}* for infra cluster ${cluster_name}."
         echo "https://infra.rox.systems/cluster/${cluster_name}"

From 197a06aea1e84afd7443e8cc60d066c4b676d6c6 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 31 Jul 2025 10:29:46 -0600
Subject: [PATCH 012/256] check env

---
 scripts/ci/lib.sh | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 83b5f95a88e68..e83be8f76bba7 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2409,12 +2409,15 @@ _record_cluster_info() {
 get_infra_cluster_files() {
     local cluster_name="${1}"
     local data_dir="/tmp/artifacts-${cluster_name}"
+    set -x
+    ls -la "${SHARED_DIR:-/tmp}" || true
+    grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
     infractl --version \
       || { curl --silent -o infractl https://infra.rox.systems/downloads/infractl-linux-amd64;
-           chmod u+x infractl; }
-    ./infractl whoami
+           install infractl /usr/local/bin/ || install infractl /usr/bin/; }
+    infractl whoami
     for I in {1..5}; do
-        ./infractl artifacts ${cluster_name} \
+        infractl artifacts ${cluster_name} \
           --download-dir "${data_dir}"
         if [[ -d /tmp/artifacts-${cluster_name} ]]; then
           break
@@ -2427,9 +2430,11 @@ get_infra_cluster_files() {
     cp "${data_dir}/kubeconfig" "${SHARED_DIR}/kubeconfig" \
         || cp "${data_dir}/auth/kubeconfig" "${SHARED_DIR}/kubeconfig" || true
     cp "${data_dir}/dotenv" "${SHARED_DIR}/dotenv" || true
-    ls -la "${SHARED_DIR:-/tmp}"
+    ls -la "${SHARED_DIR:-/tmp}" || true
+    grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
     grep NAME "${SHARED_DIR:-/tmp}/shared_env" || true
     grep CLUSTER "${SHARED_DIR:-/tmp}/shared_env" || true
+    set +e
 }
 
 test_on_infra() {
@@ -2446,10 +2451,10 @@ test_on_infra() {
     while read -r cmd cluster_name job_name_match; do
       job_name_match=${job_name_match%%]*}
       job_name_match=${job_name_match##*[}
+      echo "Comparing job_name string '${JOB_NAME:-}' to *${job_name_match}* for infra cluster ${cluster_name}."
       if [[ "${JOB_NAME:-}" == *"$job_name_match"* ]]; then
-        echo "Matching job_name string to *${job_name_match}* for infra cluster ${cluster_name}."
+        echo "Match found. Tests will run on ${cluster_name} instead of starting a new cluster."
         echo "https://infra.rox.systems/cluster/${cluster_name}"
-        echo "Tests will run on this cluster instead of starting a new cluster."
         get_infra_cluster_files "${cluster_name}"
         echo "CLUSTER_NAME=${cluster_name}" >> "${SHARED_DIR}/"
         break

From 5c3970eaabe4d7a69a73e5dbf51f53fa8bda6da7 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 31 Jul 2025 12:54:22 -0600
Subject: [PATCH 013/256] try in gha

---
 .github/workflows/e2e-tests.yaml | 41 ++++++++------------------------
 1 file changed, 10 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 0fbbaa9bc58fd..4657c18250808 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -10,41 +10,20 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  setup:
-    runs-on: ubuntu-latest
-
-  gke-e2e-tests:
-    env:
-      ARTIFACT_DIR: junit-reports/
+  e2e-tests:
     runs-on: ubuntu-latest
     container:
       image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.9
     steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        ref: ${{ github.event.pull_request.head.sha }}
-
-    - uses: ./.github/actions/job-preamble
-      with:
-        gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
-
-    - name: Create artifacts dir
-      run: mkdir -p "$ARTIFACT_DIR"
-
-    - name: Cache Go dependencies
-      uses: ./.github/actions/cache-go-dependencies
-
-    - name: Check Generated
-      run: scripts/ci/jobs/check-generated.sh
+      - uses: actions/checkout@v4
 
-    - name: Check Go version is in sync in .golangci.yml and go.mod
-      run: |
-        [ "$(awk '/^go /{print $2}' go.mod | cut -d. -f 1-2)" = "$(yq e .run.go .golangci.yml)" ]
+      - uses: ./.github/actions/job-preamble
+        with:
+          gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
 
-    - name: make golangci-lint
-      run: make golangci-lint
+      - continue-on-error: true
+        run: |
+          scripts/ci/lib.sh test_on_infra
 
-    - name: Check Cache golangci-lint
-      run: make golangci-lint-cache-status
+      - run: |
+          scripts/ci/jobs/gke_qa_e2e_tests.py

From 9bb80ccaed409f838c50da648340f41632d11419 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 09:52:32 -0600
Subject: [PATCH 014/256] check path

---
 scripts/ci/lib.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index e83be8f76bba7..08c93c2166524 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2412,9 +2412,10 @@ get_infra_cluster_files() {
     set -x
     ls -la "${SHARED_DIR:-/tmp}" || true
     grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
+    echo "$PATH"
     infractl --version \
       || { curl --silent -o infractl https://infra.rox.systems/downloads/infractl-linux-amd64;
-           install infractl /usr/local/bin/ || install infractl /usr/bin/; }
+           install infractl ${GOPATH:-/opt}/bin/infractl || install infractl ~/.local/bin || install infractl /usr/local/bin/ || install infractl /usr/bin/ || { chmod u+x ./infractl; export PATH="$PATH:$PWD"; }
     infractl whoami
     for I in {1..5}; do
         infractl artifacts ${cluster_name} \

From 56f0438ec8fe5d63f636df42672c4d51e6364582 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 12:27:57 -0600
Subject: [PATCH 015/256] end brace

---
 scripts/ci/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 08c93c2166524..b401842da0f45 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2415,7 +2415,7 @@ get_infra_cluster_files() {
     echo "$PATH"
     infractl --version \
       || { curl --silent -o infractl https://infra.rox.systems/downloads/infractl-linux-amd64;
-           install infractl ${GOPATH:-/opt}/bin/infractl || install infractl ~/.local/bin || install infractl /usr/local/bin/ || install infractl /usr/bin/ || { chmod u+x ./infractl; export PATH="$PATH:$PWD"; }
+           install infractl ${GOPATH:-/opt}/bin/infractl || install infractl ~/.local/bin || install infractl /usr/local/bin/ || install infractl /usr/bin/ || { chmod u+x ./infractl; export PATH="$PATH:$PWD"; } }
     infractl whoami
     for I in {1..5}; do
         infractl artifacts ${cluster_name} \

From 999656b78422d307537e3d10e02c590995e3b1c2 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:36:08 -0600
Subject: [PATCH 016/256] only run on prs

---
 .github/workflows/e2e-tests.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 4657c18250808..e1ef8878aae05 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -1,6 +1,5 @@
 name: E2E tests
 on:
-  push:
   pull_request:
 env:
   ROX_PRODUCT_BRANDING: stackrox

From 6b3c941940cf1931a32b3edf9f1d4b26d4beed88 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:40:25 -0600
Subject: [PATCH 017/256] reuse base from .openshift-ci

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index e1ef8878aae05..40d220c07f575 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -25,4 +25,5 @@ jobs:
           scripts/ci/lib.sh test_on_infra
 
       - run: |
+          export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci
           scripts/ci/jobs/gke_qa_e2e_tests.py

From b96492ef64d2644755ef7fe400ad8439c50016bf Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:44:52 -0600
Subject: [PATCH 018/256] endquote

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 40d220c07f575..41f1c9a33fd74 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -25,5 +25,5 @@ jobs:
           scripts/ci/lib.sh test_on_infra
 
       - run: |
-          export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci
+          export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
           scripts/ci/jobs/gke_qa_e2e_tests.py

From 2bed6a5f46cffe2c7d1d0619b5a454b71ac1b366 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:50:58 -0600
Subject: [PATCH 019/256] set pr var if gha

---
 scripts/ci/lib.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index b401842da0f45..a67f7d3933295 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2440,7 +2440,8 @@ get_infra_cluster_files() {
 
 test_on_infra() {
     local event_json body
-    echo "PULL_NUMBER=${PULL_NUMBER:-}"
+    PULL_NUMBER="${PULL_NUMBER:-${GITHUB_REF_NAME:+${GITHUB_REF_NAME%%/merge}}}"
+    echo "PULL_NUMBER=${PULL_NUMBER}"
     REPO_NAME=${REPO_NAME:-stackrox}
     REPO_OWNER=${REPO_OWNER:-stackrox}
     event_json=$(set -x; curl --silent \

From a46a140b853dd6b1db0849ba4d832af996264915 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:57:02 -0600
Subject: [PATCH 020/256] use runner_temp

---
 scripts/ci/lib.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index a67f7d3933295..6469508a71a33 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2440,8 +2440,10 @@ get_infra_cluster_files() {
 
 test_on_infra() {
     local event_json body
-    PULL_NUMBER="${PULL_NUMBER:-${GITHUB_REF_NAME:+${GITHUB_REF_NAME%%/merge}}}"
+    export PULL_NUMBER="${PULL_NUMBER:-${GITHUB_REF_NAME:+${GITHUB_REF_NAME%%/merge}}}"
     echo "PULL_NUMBER=${PULL_NUMBER}"
+    export SHARED_DIR="${SHARED_DIR:-${GITHUB_WORKSPACE:+${RUNNER_TEMP}}}"
+    echo "SHARED_DIR=${SHARED_DIR}"
     REPO_NAME=${REPO_NAME:-stackrox}
     REPO_OWNER=${REPO_OWNER:-stackrox}
     event_json=$(set -x; curl --silent \

From 6aa0cc726d654fbad77ac214a3fbb6864162ef5d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 13:58:54 -0600
Subject: [PATCH 021/256] or use /tmp

---
 scripts/ci/lib.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 6469508a71a33..4209be23ba484 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2443,6 +2443,10 @@ test_on_infra() {
     export PULL_NUMBER="${PULL_NUMBER:-${GITHUB_REF_NAME:+${GITHUB_REF_NAME%%/merge}}}"
     echo "PULL_NUMBER=${PULL_NUMBER}"
     export SHARED_DIR="${SHARED_DIR:-${GITHUB_WORKSPACE:+${RUNNER_TEMP}}}"
+    if ! touch "${SHARED_DIR}/file_write_test"; then
+      export SHARED_DIR="/tmp/"
+      touch "${SHARED_DIR}/file_write_test"
+    fi
     echo "SHARED_DIR=${SHARED_DIR}"
     REPO_NAME=${REPO_NAME:-stackrox}
     REPO_OWNER=${REPO_OWNER:-stackrox}

From cb8dcc422f52c3959cae3fb5e784d12de62fb59e Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:00:09 -0600
Subject: [PATCH 022/256] share with other steps

---
 scripts/ci/lib.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 4209be23ba484..dff0953cce4e5 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2441,13 +2441,13 @@ get_infra_cluster_files() {
 test_on_infra() {
     local event_json body
     export PULL_NUMBER="${PULL_NUMBER:-${GITHUB_REF_NAME:+${GITHUB_REF_NAME%%/merge}}}"
-    echo "PULL_NUMBER=${PULL_NUMBER}"
+    echo "PULL_NUMBER=${PULL_NUMBER}" | tee -a "$GITHUB_ENV"
     export SHARED_DIR="${SHARED_DIR:-${GITHUB_WORKSPACE:+${RUNNER_TEMP}}}"
     if ! touch "${SHARED_DIR}/file_write_test"; then
       export SHARED_DIR="/tmp/"
       touch "${SHARED_DIR}/file_write_test"
     fi
-    echo "SHARED_DIR=${SHARED_DIR}"
+    echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"
     REPO_NAME=${REPO_NAME:-stackrox}
     REPO_OWNER=${REPO_OWNER:-stackrox}
     event_json=$(set -x; curl --silent \

From dabf27928fa9ffe8805a8194e8a46c43ef6ade8a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:03:26 -0600
Subject: [PATCH 023/256] env in step yaml

---
 .github/workflows/e2e-tests.yaml | 2 ++
 scripts/ci/lib.sh                | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 41f1c9a33fd74..26e57f8919773 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -23,6 +23,8 @@ jobs:
       - continue-on-error: true
         run: |
           scripts/ci/lib.sh test_on_infra
+          echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"
+          echo "REMOVE_EXISTING_STACKROX_RESOURCES=false" | tee -a "$GITHUB_ENV"
 
       - run: |
           export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index dff0953cce4e5..4209be23ba484 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2441,13 +2441,13 @@ get_infra_cluster_files() {
 test_on_infra() {
     local event_json body
     export PULL_NUMBER="${PULL_NUMBER:-${GITHUB_REF_NAME:+${GITHUB_REF_NAME%%/merge}}}"
-    echo "PULL_NUMBER=${PULL_NUMBER}" | tee -a "$GITHUB_ENV"
+    echo "PULL_NUMBER=${PULL_NUMBER}"
     export SHARED_DIR="${SHARED_DIR:-${GITHUB_WORKSPACE:+${RUNNER_TEMP}}}"
     if ! touch "${SHARED_DIR}/file_write_test"; then
       export SHARED_DIR="/tmp/"
       touch "${SHARED_DIR}/file_write_test"
     fi
-    echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"
+    echo "SHARED_DIR=${SHARED_DIR}"
     REPO_NAME=${REPO_NAME:-stackrox}
     REPO_OWNER=${REPO_OWNER:-stackrox}
     event_json=$(set -x; curl --silent \

From 697dfc06ed57362b007a9ff78b61aedc65d68758 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:07:24 -0600
Subject: [PATCH 024/256] add job name

---
 .github/workflows/e2e-tests.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 26e57f8919773..c2a782c550029 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -25,6 +25,10 @@ jobs:
           scripts/ci/lib.sh test_on_infra
           echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"
           echo "REMOVE_EXISTING_STACKROX_RESOURCES=false" | tee -a "$GITHUB_ENV"
+          echo "JOB_NAME=ocp-4-19-qa-e2e-tests" | tee -a "$GITHUB_ENV"
+          echo "JOB_NAME_SAFE=qa-e2e-tests" | tee -a "$GITHUB_ENV"
+          echo "CI=true" | tee -a "$GITHUB_ENV"
+          echo "GRADLE_USER_HOME="${HOME}" | tee -a "$GITHUB_ENV"
 
       - run: |
           export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"

From eea52be76fb9f2e12f8a3b0ed0f0d9231d296b4f Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:08:31 -0600
Subject: [PATCH 025/256] in same shell

---
 .github/workflows/e2e-tests.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index c2a782c550029..d669bcdb2447b 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -22,7 +22,8 @@ jobs:
 
       - continue-on-error: true
         run: |
-          scripts/ci/lib.sh test_on_infra
+          source scripts/ci/lib.sh
+          test_on_infra
           echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"
           echo "REMOVE_EXISTING_STACKROX_RESOURCES=false" | tee -a "$GITHUB_ENV"
           echo "JOB_NAME=ocp-4-19-qa-e2e-tests" | tee -a "$GITHUB_ENV"

From b1f018d16b4f5d22f88240f53da93835f594bdd5 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:29:39 -0600
Subject: [PATCH 026/256] use ocp script

---
 .github/workflows/e2e-tests.yaml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index d669bcdb2447b..0770973ecb274 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -29,8 +29,12 @@ jobs:
           echo "JOB_NAME=ocp-4-19-qa-e2e-tests" | tee -a "$GITHUB_ENV"
           echo "JOB_NAME_SAFE=qa-e2e-tests" | tee -a "$GITHUB_ENV"
           echo "CI=true" | tee -a "$GITHUB_ENV"
-          echo "GRADLE_USER_HOME="${HOME}" | tee -a "$GITHUB_ENV"
+          echo "GRADLE_USER_HOME=${HOME}" | tee -a "$GITHUB_ENV"
+          echo "CI_JOB_NAME=${JOB_NAME}" | tee -a "$GITHUB_ENV"
+          ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/tmp/}}"
+          mkdir -p "$ARTIFACT_DIR"
+          echo "ARTIFACT_DIR=${ARTIFACT_DIR}" | tee -a "$GITHUB_ENV"
 
       - run: |
           export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
-          scripts/ci/jobs/gke_qa_e2e_tests.py
+          scripts/ci/jobs/ocp_qa_e2e_tests.py

From 19a2e0bf0130eac5f39f146a67a38c9a143c01e2 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:31:49 -0600
Subject: [PATCH 027/256] just use the dispatch

---
 .github/workflows/e2e-tests.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 0770973ecb274..e4b39192f486f 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -37,4 +37,5 @@ jobs:
 
       - run: |
           export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
-          scripts/ci/jobs/ocp_qa_e2e_tests.py
+          # scripts/ci/jobs/ocp_qa_e2e_tests.py
+          bash -x .openshift-ci/dispatch.sh "$JOB_NAME"

From c29f0af7e6a2272aed00ea05a3ce6ce3c8c8b7d0 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:33:34 -0600
Subject: [PATCH 028/256] fetch all history

---
 .github/workflows/e2e-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index e4b39192f486f..de3c9e810f5c2 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -15,6 +15,8 @@ jobs:
       image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.9
     steps:
       - uses: actions/checkout@v4
+        with:
+          depth: 0
 
       - uses: ./.github/actions/job-preamble
         with:

From 158e02f2857f4cc2e338ef2eab180eae5cf51d31 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:35:21 -0600
Subject: [PATCH 029/256] reuse job name

---
 .github/workflows/e2e-tests.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index de3c9e810f5c2..3a175f204fd23 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -28,7 +28,8 @@ jobs:
           test_on_infra
           echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"
           echo "REMOVE_EXISTING_STACKROX_RESOURCES=false" | tee -a "$GITHUB_ENV"
-          echo "JOB_NAME=ocp-4-19-qa-e2e-tests" | tee -a "$GITHUB_ENV"
+          export "JOB_NAME=${JOB_NAME:-ocp-4-19-qa-e2e-tests}" | tee -a "$GITHUB_ENV"
+          echo "JOB_NAME=${JOB_NAME}" | tee -a "$GITHUB_ENV"
           echo "JOB_NAME_SAFE=qa-e2e-tests" | tee -a "$GITHUB_ENV"
           echo "CI=true" | tee -a "$GITHUB_ENV"
           echo "GRADLE_USER_HOME=${HOME}" | tee -a "$GITHUB_ENV"

From 4a261a56e31ed7d441c714d423aefc86c7471d17 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:36:02 -0600
Subject: [PATCH 030/256] fetch-depth

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 3a175f204fd23..fabfd51afb815 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -16,7 +16,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          depth: 0
+          fetch-depth: 0
 
       - uses: ./.github/actions/job-preamble
         with:

From a867c06727c36e8760e4e3f2741deb04b2f9b394 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:39:08 -0600
Subject: [PATCH 031/256] fix export name

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index fabfd51afb815..4e9e6a83a1551 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -28,7 +28,7 @@ jobs:
           test_on_infra
           echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"
           echo "REMOVE_EXISTING_STACKROX_RESOURCES=false" | tee -a "$GITHUB_ENV"
-          export "JOB_NAME=${JOB_NAME:-ocp-4-19-qa-e2e-tests}" | tee -a "$GITHUB_ENV"
+          export JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}" | tee -a "$GITHUB_ENV"
           echo "JOB_NAME=${JOB_NAME}" | tee -a "$GITHUB_ENV"
           echo "JOB_NAME_SAFE=qa-e2e-tests" | tee -a "$GITHUB_ENV"
           echo "CI=true" | tee -a "$GITHUB_ENV"

From 4f77ad9de5cf32a537b81606cee30904b845411e Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:42:33 -0600
Subject: [PATCH 032/256] not tee with export

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 4e9e6a83a1551..5dc3d20328c23 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -28,7 +28,7 @@ jobs:
           test_on_infra
           echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"
           echo "REMOVE_EXISTING_STACKROX_RESOURCES=false" | tee -a "$GITHUB_ENV"
-          export JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}" | tee -a "$GITHUB_ENV"
+          export JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
           echo "JOB_NAME=${JOB_NAME}" | tee -a "$GITHUB_ENV"
           echo "JOB_NAME_SAFE=qa-e2e-tests" | tee -a "$GITHUB_ENV"
           echo "CI=true" | tee -a "$GITHUB_ENV"

From 59a827cf795ce8a45001c09c95f27d5a9710dfb3 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:43:54 -0600
Subject: [PATCH 033/256] direct python script run

---
 .github/workflows/e2e-tests.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 5dc3d20328c23..1f80b11c40064 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -40,5 +40,6 @@ jobs:
 
       - run: |
           export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
-          # scripts/ci/jobs/ocp_qa_e2e_tests.py
-          bash -x .openshift-ci/dispatch.sh "$JOB_NAME"
+          cat "$GITHUB_ENV" || true
+          scripts/ci/jobs/ocp_qa_e2e_tests.py \
+            || bash -x .openshift-ci/dispatch.sh "$JOB_NAME"

From 4c1d7edb6826e8db83cdaeee17683c5d32045673 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:47:02 -0600
Subject: [PATCH 034/256] set job name before infra check

---
 .github/workflows/e2e-tests.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 1f80b11c40064..d8a846948baa5 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -24,8 +24,6 @@ jobs:
 
       - continue-on-error: true
         run: |
-          source scripts/ci/lib.sh
-          test_on_infra
           echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"
           echo "REMOVE_EXISTING_STACKROX_RESOURCES=false" | tee -a "$GITHUB_ENV"
           export JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
@@ -38,6 +36,9 @@ jobs:
           mkdir -p "$ARTIFACT_DIR"
           echo "ARTIFACT_DIR=${ARTIFACT_DIR}" | tee -a "$GITHUB_ENV"
 
+          source scripts/ci/lib.sh
+          test_on_infra
+
       - run: |
           export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
           cat "$GITHUB_ENV" || true

From d83b41adcbc3a04e54d4e1938732233bf6790bfc Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:48:41 -0600
Subject: [PATCH 035/256] list here

---
 scripts/ci/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 4209be23ba484..7dde5d8a4e519 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2414,7 +2414,7 @@ get_infra_cluster_files() {
     grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
     echo "$PATH"
     infractl --version \
-      || { curl --silent -o infractl https://infra.rox.systems/downloads/infractl-linux-amd64;
+      || { curl --silent -o infractl https://infra.rox.systems/downloads/infractl-linux-amd64; file ./infractl || true; ls -latr ./;
            install infractl ${GOPATH:-/opt}/bin/infractl || install infractl ~/.local/bin || install infractl /usr/local/bin/ || install infractl /usr/bin/ || { chmod u+x ./infractl; export PATH="$PATH:$PWD"; } }
     infractl whoami
     for I in {1..5}; do

From 622a8358e13ba86b1bbfd46d31a489c02ad35838 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 14:56:44 -0600
Subject: [PATCH 036/256] use gha infractl

---
 .github/workflows/e2e-tests.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index d8a846948baa5..24516eb6c822b 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -22,6 +22,10 @@ jobs:
         with:
           gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
 
+      - uses: stackrox/actions/infra/install-infractl@main
+
+      - run: infractl version >> "$GITHUB_STEP_SUMMARY"
+
       - continue-on-error: true
         run: |
           echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"

From 76786deb879b955741bf7828174a5ab14c41ebc3 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 15:00:37 -0600
Subject: [PATCH 037/256] use public path

---
 .github/workflows/e2e-tests.yaml | 4 ----
 scripts/ci/lib.sh                | 2 +-
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 24516eb6c822b..d8a846948baa5 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -22,10 +22,6 @@ jobs:
         with:
           gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
 
-      - uses: stackrox/actions/infra/install-infractl@main
-
-      - run: infractl version >> "$GITHUB_STEP_SUMMARY"
-
       - continue-on-error: true
         run: |
           echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"
diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 7dde5d8a4e519..24b73bf04b3a1 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2414,7 +2414,7 @@ get_infra_cluster_files() {
     grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
     echo "$PATH"
     infractl --version \
-      || { curl --silent -o infractl https://infra.rox.systems/downloads/infractl-linux-amd64; file ./infractl || true; ls -latr ./;
+      || { curl -L --silent -o infractl https://infra.rox.systems/v1/cli/linux/amd64/upgrade ; file ./infractl || true; ls -latr ./;
            install infractl ${GOPATH:-/opt}/bin/infractl || install infractl ~/.local/bin || install infractl /usr/local/bin/ || install infractl /usr/bin/ || { chmod u+x ./infractl; export PATH="$PATH:$PWD"; } }
     infractl whoami
     for I in {1..5}; do

From 28398fa339baa49999d32d24223213c836a71334 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 15:13:51 -0600
Subject: [PATCH 038/256] decode from base64

---
 .github/workflows/e2e-tests.yaml | 4 +++-
 scripts/ci/lib.sh                | 7 +++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index d8a846948baa5..3a03f55a7d216 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -39,8 +39,10 @@ jobs:
           source scripts/ci/lib.sh
           test_on_infra
 
+          echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
+
       - run: |
           export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
           cat "$GITHUB_ENV" || true
           scripts/ci/jobs/ocp_qa_e2e_tests.py \
-            || bash -x .openshift-ci/dispatch.sh "$JOB_NAME"
+            echo end  #|| bash -x .openshift-ci/dispatch.sh "$JOB_NAME"
diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 24b73bf04b3a1..ff1677c7d8a82 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2414,8 +2414,11 @@ get_infra_cluster_files() {
     grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
     echo "$PATH"
     infractl --version \
-      || { curl -L --silent -o infractl https://infra.rox.systems/v1/cli/linux/amd64/upgrade ; file ./infractl || true; ls -latr ./;
-           install infractl ${GOPATH:-/opt}/bin/infractl || install infractl ~/.local/bin || install infractl /usr/local/bin/ || install infractl /usr/bin/ || { chmod u+x ./infractl; export PATH="$PATH:$PWD"; } }
+      || { curl --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \
+        | jq -r ".result.fileChunk" \
+        | base64 -d > ./infractl;
+          file ./infractl || true; ls -latr ./;
+          install infractl ${GOPATH:-/opt}/bin/infractl || install infractl ~/.local/bin || install infractl /usr/local/bin/ || install infractl /usr/bin/ || { chmod u+x ./infractl; export PATH="$PATH:$PWD"; } }
     infractl whoami
     for I in {1..5}; do
         infractl artifacts ${cluster_name} \

From 1d6dac0ebea40dfc77a7eccd48e621435d81cf00 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 15:31:59 -0600
Subject: [PATCH 039/256] retry curl

---
 scripts/ci/lib.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index ff1677c7d8a82..91da4b1b94600 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2453,10 +2453,19 @@ test_on_infra() {
     echo "SHARED_DIR=${SHARED_DIR}"
     REPO_NAME=${REPO_NAME:-stackrox}
     REPO_OWNER=${REPO_OWNER:-stackrox}
+    local tries=4
+    while [[ -z "$event_json" ]]; do
     event_json=$(set -x; curl --silent \
       -H "X-GitHub-Api-Version: 2022-11-28" \
       -H "Accept: application/vnd.github+json" \
       -o - "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls/${PULL_NUMBER}")
+      if [[ $tries -gt 0 && -z "$event_json" ]]; then
+        sleep 30
+        echo $(( tries-- ))
+      else
+        break
+      fi
+    done
     body=$(jq -r '.body' <<<"${event_json}" \
       | tee >(cat >&2) | grep '^/test-on-infra') || return
     while read -r cmd cluster_name job_name_match; do

From 43f8757957cf02f58fe0539735951e8255099a2f Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 1 Aug 2025 15:56:46 -0600
Subject: [PATCH 040/256] set empty

---
 scripts/ci/lib.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 91da4b1b94600..cffb994df6579 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2454,6 +2454,7 @@ test_on_infra() {
     REPO_NAME=${REPO_NAME:-stackrox}
     REPO_OWNER=${REPO_OWNER:-stackrox}
     local tries=4
+    event_json=''
     while [[ -z "$event_json" ]]; do
     event_json=$(set -x; curl --silent \
       -H "X-GitHub-Api-Version: 2022-11-28" \

From bb1d8e6df9ac712450db949a33aacec08bf2367e Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 10:54:32 -0600
Subject: [PATCH 041/256] use infra token secret

---
 .github/workflows/e2e-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 3a03f55a7d216..40001bf5e4444 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -13,6 +13,8 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.9
+    env:
+      INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
     steps:
       - uses: actions/checkout@v4
         with:

From 2500e2133a6adc4375c2f540ed866754eb0dd2b3 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 11:08:34 -0600
Subject: [PATCH 042/256] export kubeconfig

---
 scripts/ci/lib.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index cffb994df6579..f28e9b46432fe 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2430,15 +2430,17 @@ get_infra_cluster_files() {
         sleep 60
     done
     echo "$cluster_name"
-    kubectl get nodes -o wide
     cp "${data_dir}/kubeconfig" "${SHARED_DIR}/kubeconfig" \
         || cp "${data_dir}/auth/kubeconfig" "${SHARED_DIR}/kubeconfig" || true
+
     cp "${data_dir}/dotenv" "${SHARED_DIR}/dotenv" || true
     ls -la "${SHARED_DIR:-/tmp}" || true
     grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
     grep NAME "${SHARED_DIR:-/tmp}/shared_env" || true
     grep CLUSTER "${SHARED_DIR:-/tmp}/shared_env" || true
-    set +e
+
+    export KUBECONFIG="${SHARED_DIR}/kubeconfig"
+    kubectl get nodes -o wide
 }
 
 test_on_infra() {

From bdda74cfb7db03851e05d30d4813b07a95ea5eee Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 11:15:17 -0600
Subject: [PATCH 043/256] add to shared_env

---
 scripts/ci/lib.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index f28e9b46432fe..270cf65b6c02f 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2479,7 +2479,8 @@ test_on_infra() {
         echo "Match found. Tests will run on ${cluster_name} instead of starting a new cluster."
         echo "https://infra.rox.systems/cluster/${cluster_name}"
         get_infra_cluster_files "${cluster_name}"
-        echo "CLUSTER_NAME=${cluster_name}" >> "${SHARED_DIR}/"
+        echo "CLUSTER_NAME=${cluster_name}" >> "${SHARED_DIR}/shared_env" || true
+        echo "KUBECONFIG=${KUBECONFIG}" >> "${SHARED_DIR}/shared_env" || true
         break
       fi
     done <<<"$body"

From a0a2646505f5989c55fefe617618b30b88366969 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 11:29:37 -0600
Subject: [PATCH 044/256] quay auth

---
 .github/workflows/e2e-tests.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 40001bf5e4444..e319ce5ba70aa 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -15,6 +15,11 @@ jobs:
       image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.9
     env:
       INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
+      POD_SECURITY_POLICIES: "false"
+      ROX_VULN_MGMT_UNIFIED_CVE_DEFERRAL: "true"
+      ROX_PROCESSES_LISTENING_ON_PORT: "false"
+      REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
+      REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
     steps:
       - uses: actions/checkout@v4
         with:

From 5533999dcf77f08a5c2e40ac360e5cc2fa38d72d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 11:35:02 -0600
Subject: [PATCH 045/256] add image-prefetcher quay auth

---
 .github/workflows/e2e-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index e319ce5ba70aa..26bd148589205 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -20,6 +20,8 @@ jobs:
       ROX_PROCESSES_LISTENING_ON_PORT: "false"
       REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
       REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
+      QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
+      QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
     steps:
       - uses: actions/checkout@v4
         with:

From 0fb344fd4619ab3ef67b7a5e95cecdcebd0bc958 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 11:42:04 -0600
Subject: [PATCH 046/256] add token for image-prefetcher

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 26bd148589205..dec74f6244ee1 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -22,6 +22,7 @@ jobs:
       REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
       QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
       QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
+      QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }}
     steps:
       - uses: actions/checkout@v4
         with:

From b4d1ff10395f56553aca8d915347ff85757ab8f8 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 11:42:36 -0600
Subject: [PATCH 047/256] try without container

---
 .github/workflows/e2e-tests.yaml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index dec74f6244ee1..1626394f8e9dc 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -4,15 +4,9 @@ on:
 env:
   ROX_PRODUCT_BRANDING: stackrox
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
-  cancel-in-progress: true
-
 jobs:
   e2e-tests:
     runs-on: ubuntu-latest
-    container:
-      image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.4.9
     env:
       INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
       POD_SECURITY_POLICIES: "false"

From aa11a47bf0072a3aa7c0ed5865b77166949855b5 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 13:03:05 -0600
Subject: [PATCH 048/256] use pr head sha

---
 .github/workflows/e2e-tests.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 1626394f8e9dc..f252f7870c3c1 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -1,6 +1,13 @@
 name: E2E tests
 on:
   pull_request:
+  workflow_dispatch:
+    inputs:
+      commit:
+        description: 'Commit SHA to test'
+        required: true
+        type: string
+  workflow_call:
 env:
   ROX_PRODUCT_BRANDING: stackrox
 
@@ -21,6 +28,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
       - uses: ./.github/actions/job-preamble
         with:

From 66a0ce7932e90b621a424788e562bac74a19899c Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 13:39:31 -0600
Subject: [PATCH 049/256] try test against target

---
 .github/workflows/e2e-tests.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index f252f7870c3c1..15bb3ab72aa80 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -4,6 +4,10 @@ on:
   workflow_dispatch:
     inputs:
       commit:
+        description: 'Commit SHA of tests'
+        required: true
+        type: string
+      tested_commit:
         description: 'Commit SHA to test'
         required: true
         type: string
@@ -24,6 +28,8 @@ jobs:
       QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
       QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
       QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }}
+      MAIN_IMAGE_TAG: ${{ inputs.tested_commit || github.base_ref }}
+      TAG: ${{ inputs.tested_commit || github.base_ref }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -54,6 +60,8 @@ jobs:
           echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
 
       - run: |
+          set -x
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
           export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
           cat "$GITHUB_ENV" || true
           scripts/ci/jobs/ocp_qa_e2e_tests.py \

From 2e4a53a99470bcb6c415d99f82826972c68089c4 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 13:41:37 -0600
Subject: [PATCH 050/256] export tag

---
 .github/workflows/e2e-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 15bb3ab72aa80..b70d80862af15 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -61,6 +61,8 @@ jobs:
 
       - run: |
           set -x
+          export MAIN_IMAGE_TAG
+          export TAG
           git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
           export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
           cat "$GITHUB_ENV" || true

From a66f6c371aa41355f47bd540a7707ee0788ca281 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 13:49:06 -0600
Subject: [PATCH 051/256] override prefetcher tag

---
 scripts/ci/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 270cf65b6c02f..1edd7bef63272 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -820,7 +820,7 @@ populate_stackrox_image_list() {
     local image_list="$1"
 
     local tag
-    tag="$(make --quiet --no-print-directory tag)"
+    tag="${MAIN_IMAGE_TAG:-$(make --quiet --no-print-directory tag)}"
     local operator_metadata_tag
     operator_metadata_tag="$(echo "v${tag}" | sed 's,x,0,')"
     local operator_controller_tag="${tag//x/0}"

From 123f789120105bc78566870afbee2ab630929100 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 13:51:58 -0600
Subject: [PATCH 052/256] faster clone

---
 .github/workflows/e2e-tests.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index b70d80862af15..1efa0b446fba0 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -33,7 +33,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0
+          fetch-depth: ${{ github.event.pull_request.commits }}
+          fetch-tags: false
+          show-progress: false
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
       - uses: ./.github/actions/job-preamble

From 4f39a0d6620ed48bbbe288bc151f984db7f6cf61 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 13:58:53 -0600
Subject: [PATCH 053/256] use base sha

---
 .github/workflows/e2e-tests.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 1efa0b446fba0..91e0466c9dfb0 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -28,8 +28,8 @@ jobs:
       QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
       QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
       QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }}
-      MAIN_IMAGE_TAG: ${{ inputs.tested_commit || github.base_ref }}
-      TAG: ${{ inputs.tested_commit || github.base_ref }}
+      MAIN_IMAGE_TAG: ${{ inputs.tested_commit || github.event.pull_request.base.sha }}
+      TAG: ${{ inputs.tested_commit || github.event.pull_request.base.sha }}
     steps:
       - uses: actions/checkout@v4
         with:

From 1dbaca272d7e1f4d2cd81a29d0aca941f4566248 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 14:03:04 -0600
Subject: [PATCH 054/256] print tag

---
 .github/workflows/e2e-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 91e0466c9dfb0..61f8d8f10d69d 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -63,6 +63,8 @@ jobs:
 
       - run: |
           set -x
+          echo "MAIN_IMAGE_TAG:${MAIN_IMAGE_TAG:-}"
+          echo "TAG:${TAG:-}"
           export MAIN_IMAGE_TAG
           export TAG
           git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true

From 7bdff215fde9c9c38295013a9d10015a9de2671a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 14:03:56 -0600
Subject: [PATCH 055/256] remove end bracket

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 61f8d8f10d69d..1242d359640ba 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -52,7 +52,7 @@ jobs:
           echo "CI=true" | tee -a "$GITHUB_ENV"
           echo "GRADLE_USER_HOME=${HOME}" | tee -a "$GITHUB_ENV"
           echo "CI_JOB_NAME=${JOB_NAME}" | tee -a "$GITHUB_ENV"
-          ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/tmp/}}"
+          ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/tmp/}"
           mkdir -p "$ARTIFACT_DIR"
           echo "ARTIFACT_DIR=${ARTIFACT_DIR}" | tee -a "$GITHUB_ENV"
 

From 9867327a525e4e39d33bcbc64cb633b40e081cfb Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 14:06:30 -0600
Subject: [PATCH 056/256] remove dir listing

---
 scripts/ci/lib.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 1edd7bef63272..9105e19d7b486 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2417,8 +2417,8 @@ get_infra_cluster_files() {
       || { curl --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \
         | jq -r ".result.fileChunk" \
         | base64 -d > ./infractl;
-          file ./infractl || true; ls -latr ./;
-          install infractl ${GOPATH:-/opt}/bin/infractl || install infractl ~/.local/bin || install infractl /usr/local/bin/ || install infractl /usr/bin/ || { chmod u+x ./infractl; export PATH="$PATH:$PWD"; } }
+          file ./infractl || true;
+          install infractl ${GOPATH:-/opt}/bin/infractl || install infractl /usr/local/bin/; }
     infractl whoami
     for I in {1..5}; do
         infractl artifacts ${cluster_name} \

From 5d69e944dbed6df5fc607d66fdd364b1145a9a5b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 14:16:21 -0600
Subject: [PATCH 057/256] create tag formatted

---
 .github/workflows/e2e-tests.yaml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 1242d359640ba..fb0a39b7f99c8 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -33,11 +33,18 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: ${{ github.event.pull_request.commits }}
+          fetch-depth: ${{ github.event.pull_request.commits + 100 }}
+          # TODO: clone both test and tested as repo's?
           fetch-tags: false
           show-progress: false
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
+      - run: |
+          # example: 4.9.x-457-g9867327a52
+          MAIN_IMAGE_TAG=$(git describe --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG}")
+          echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" >> "$GITHUB_ENV"
+          echo "TAG=${MAIN_IMAGE_TAG}" >> "$GITHUB_ENV"
+
       - uses: ./.github/actions/job-preamble
         with:
           gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}

From a7d5f21306587bf3360a513368e8efd63eb49537 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 14:17:15 -0600
Subject: [PATCH 058/256] fetch all

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index fb0a39b7f99c8..22d001a937e4f 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -33,7 +33,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: ${{ github.event.pull_request.commits + 100 }}
+          fetch-depth: 0 #${{ github.event.pull_request.commits + 100 }}
           # TODO: clone both test and tested as repo's?
           fetch-tags: false
           show-progress: false

From 62aa9f436501a194a6ddbeea1e1f00286d28062b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 14:24:53 -0600
Subject: [PATCH 059/256] install oc

---
 .github/workflows/e2e-tests.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 22d001a937e4f..3ef30f6bae276 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -49,6 +49,13 @@ jobs:
         with:
           gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
 
+      - run: |
+          curl --retry 5 --retry-all-errors -L -o /tmp/openshift-client-linux.tar.gz \
+            https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz
+          tar -C /tmp -xzf /tmp/openshift-client-linux.tar.gz
+          install --mode 755 /tmp/oc /usr/local/bin/
+          install --mode 755 /tmp/kubectl /usr/local/bin/
+
       - continue-on-error: true
         run: |
           echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"

From 5899edf07c977259e8919426eeb72d155cf57b21 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 14:34:17 -0600
Subject: [PATCH 060/256] add scanner

---
 .github/workflows/e2e-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 3ef30f6bae276..5201bd545305f 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -75,6 +75,8 @@ jobs:
 
           echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
 
+      - run: go get github.com/stackrox/scanner
+
       - run: |
           set -x
           echo "MAIN_IMAGE_TAG:${MAIN_IMAGE_TAG:-}"

From a77e8909a4be53a1213403c9967246165ab2f370 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 14:57:33 -0600
Subject: [PATCH 061/256] do not gather

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 5201bd545305f..ea437910104b3 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -30,6 +30,7 @@ jobs:
       QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }}
       MAIN_IMAGE_TAG: ${{ inputs.tested_commit || github.event.pull_request.base.sha }}
       TAG: ${{ inputs.tested_commit || github.event.pull_request.base.sha }}
+      GATHER_QA_TEST_DEBUG_LOGS: "false"
     steps:
       - uses: actions/checkout@v4
         with:

From 421821adf615846332e2bd21591729c41087d749 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 15:14:34 -0600
Subject: [PATCH 062/256] try passwd lookup

---
 .github/workflows/e2e-tests.yaml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index ea437910104b3..f6b980a9ac0ed 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -76,6 +76,16 @@ jobs:
 
           echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
 
+      - run: |
+          set +e
+          set -x
+          if [[ -e "${KUBECONFIG:-}" && -z "${ROX_ADMIN_PASSWORD:-}" ]]; then
+            password="${ROX_ADMIN_PASSWORD:-$(oc -n stackrox get secret admin-pass -o json | jq -r '.data.password' | base64 -d)}"
+            password="${password:-$(oc -n stackrox get secret central-htpasswd -o go-template='{{index .data "password" | base64decode}}')}"
+            echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
+            echo "ROX_ADMIN_PASSWORD=${password}" | tee -a "$GITHUB_ENV"
+          fi
+
       - run: go get github.com/stackrox/scanner
 
       - run: |
@@ -87,5 +97,7 @@ jobs:
           git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
           export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
           cat "$GITHUB_ENV" || true
+          echo "$GATHER_QA_TEST_DEBUG_LOGS"
+          export GATHER_QA_TEST_DEBUG_LOGS
           scripts/ci/jobs/ocp_qa_e2e_tests.py \
             echo end  #|| bash -x .openshift-ci/dispatch.sh "$JOB_NAME"

From 509387178c5d12b51bc86f262c8ccfa2fdf535cb Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 15:15:45 -0600
Subject: [PATCH 063/256] set empty debug collect dir

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index f6b980a9ac0ed..8f15ca8abcc24 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -31,6 +31,7 @@ jobs:
       MAIN_IMAGE_TAG: ${{ inputs.tested_commit || github.event.pull_request.base.sha }}
       TAG: ${{ inputs.tested_commit || github.event.pull_request.base.sha }}
       GATHER_QA_TEST_DEBUG_LOGS: "false"
+      QA_TEST_DEBUG_LOGS: ""
     steps:
       - uses: actions/checkout@v4
         with:

From 59582e34cc1108046e93d2da913dadb51bf91f81 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 15:17:19 -0600
Subject: [PATCH 064/256] hide passwd

---
 .github/workflows/e2e-tests.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 8f15ca8abcc24..b8cd6ab9820a6 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -80,11 +80,11 @@ jobs:
       - run: |
           set +e
           set -x
-          if [[ -e "${KUBECONFIG:-}" && -z "${ROX_ADMIN_PASSWORD:-}" ]]; then
+          if [[ -e "${KUBECONFIG:-}" && -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
             password="${ROX_ADMIN_PASSWORD:-$(oc -n stackrox get secret admin-pass -o json | jq -r '.data.password' | base64 -d)}"
             password="${password:-$(oc -n stackrox get secret central-htpasswd -o go-template='{{index .data "password" | base64decode}}')}"
             echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
-            echo "ROX_ADMIN_PASSWORD=${password}" | tee -a "$GITHUB_ENV"
+            echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
           fi
 
       - run: go get github.com/stackrox/scanner

From 7532e6e2a55cf0ecbee28c9a5d72e1277fc78417 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 15:23:10 -0600
Subject: [PATCH 065/256] add scanner gcr cred

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index b8cd6ab9820a6..1db69f4a04b30 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -25,6 +25,7 @@ jobs:
       ROX_PROCESSES_LISTENING_ON_PORT: "false"
       REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
       REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
+      GOOGLE_CREDENTIALS_GCR_SCANNER_V2: ${{ secrets.GOOGLE_CREDENTIALS_GCR_SCANNER_V2 }}
       QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
       QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
       QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }}

From ee3c542e7dc5512beeaa55a9b548db13a13e1472 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 15:45:19 -0600
Subject: [PATCH 066/256] export empty debug log var

---
 .github/workflows/e2e-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 1db69f4a04b30..7216c2173d98e 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -77,6 +77,7 @@ jobs:
           test_on_infra
 
           echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
+          # API_HOSTNAME
 
       - run: |
           set +e
@@ -101,5 +102,6 @@ jobs:
           cat "$GITHUB_ENV" || true
           echo "$GATHER_QA_TEST_DEBUG_LOGS"
           export GATHER_QA_TEST_DEBUG_LOGS
+          export QA_TEST_DEBUG_LOGS
           scripts/ci/jobs/ocp_qa_e2e_tests.py \
             echo end  #|| bash -x .openshift-ci/dispatch.sh "$JOB_NAME"

From 7526906ff792a2e41ec9df4090ab8405572b47ec Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 15:55:35 -0600
Subject: [PATCH 067/256] post_collect condition

---
 .openshift-ci/base_qa_e2e_test.py   | 16 ++++++++--------
 scripts/ci/jobs/ocp_qa_e2e_tests.py | 13 ++++++++++++-
 2 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/.openshift-ci/base_qa_e2e_test.py b/.openshift-ci/base_qa_e2e_test.py
index 2dccc81553bb7..8bf2c138ea1fa 100755
--- a/.openshift-ci/base_qa_e2e_test.py
+++ b/.openshift-ci/base_qa_e2e_test.py
@@ -10,11 +10,11 @@
     QaE2eDBBackupRestoreTest,
     CustomSetTest,
 )
-from post_tests import PostClusterTest, CheckStackroxLogs, FinalPost
+from post_tests import NullPostTest, PostClusterTest, CheckStackroxLogs, FinalPost
 from runners import ClusterTestSetsRunner
 
 
-def make_qa_e2e_test_runner(cluster):
+def make_qa_e2e_test_runner(cluster, post_collect=True):
     return ClusterTestSetsRunner(
         cluster=cluster,
         initial_pre_test=PreSystemTests(),
@@ -22,27 +22,27 @@ def make_qa_e2e_test_runner(cluster):
             {
                 "name": "QA tests part I",
                 "test": QaE2eTestPart1(),
-                "post_test": PostClusterTest(
+                "post_test": post_collect ? PostClusterTest(
                     check_stackrox_logs=True,
                     artifact_destination_prefix="part-1",
-                ),
+                ) : NullPostTest(),
             },
             {
                 "name": "QA tests part II",
                 "test": QaE2eTestPart2(),
-                "post_test": PostClusterTest(
+                "post_test": post_collect ? PostClusterTest(
                     check_stackrox_logs=True,
                     artifact_destination_prefix="part-2",
-                ),
+                ) : NullPostTest(),
                 "always_run": False,
             },
             {
                 "name": "DB backup and restore",
                 "test": QaE2eDBBackupRestoreTest(),
-                "post_test": CheckStackroxLogs(
+                "post_test": post_collect ? CheckStackroxLogs(
                     check_for_errors_in_stackrox_logs=True,
                     artifact_destination_prefix="db-test",
-                ),
+                ) : NullPostTest(),
                 "always_run": False,
             },
         ],
diff --git a/scripts/ci/jobs/ocp_qa_e2e_tests.py b/scripts/ci/jobs/ocp_qa_e2e_tests.py
index 933e618d8d019..d00e95a310af1 100755
--- a/scripts/ci/jobs/ocp_qa_e2e_tests.py
+++ b/scripts/ci/jobs/ocp_qa_e2e_tests.py
@@ -16,4 +16,15 @@
 os.environ["ROX_RISK_REPROCESSING_INTERVAL"] = "15s"
 os.environ["ROX_SENSOR_CONNECTION_RETRY_MAX_INTERVAL"] = "30s"
 
-make_qa_e2e_test_runner(cluster=AutomationFlavorsCluster()).run()
+make_qa_e2e_test_runner(cluster=AutomationFlavorsCluster(), post_collect=False).run()
+#ClusterTestRunner(
+#    cluster=GKECluster("nongroovy-test"),
+#    pre_test=PreSystemTests(),
+#    test=NonGroovyE2e(),
+#    post_test=PostClusterTest(
+#        check_stackrox_logs=False,
+#    ),
+#    final_post=FinalPost(
+#        store_qa_tests_data=False,
+#    ),
+#).run()

From dffe4191da89368ca34d2a6b77fb77b0367c77e7 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 15:59:51 -0600
Subject: [PATCH 068/256] pythonize

---
 .openshift-ci/base_qa_e2e_test.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.openshift-ci/base_qa_e2e_test.py b/.openshift-ci/base_qa_e2e_test.py
index 8bf2c138ea1fa..163d66e340a88 100755
--- a/.openshift-ci/base_qa_e2e_test.py
+++ b/.openshift-ci/base_qa_e2e_test.py
@@ -22,27 +22,27 @@ def make_qa_e2e_test_runner(cluster, post_collect=True):
             {
                 "name": "QA tests part I",
                 "test": QaE2eTestPart1(),
-                "post_test": post_collect ? PostClusterTest(
+                "post_test": PostClusterTest(
                     check_stackrox_logs=True,
                     artifact_destination_prefix="part-1",
-                ) : NullPostTest(),
+                ) if post_collect else NullPostTest(),
             },
             {
                 "name": "QA tests part II",
                 "test": QaE2eTestPart2(),
-                "post_test": post_collect ? PostClusterTest(
+                "post_test": PostClusterTest(
                     check_stackrox_logs=True,
                     artifact_destination_prefix="part-2",
-                ) : NullPostTest(),
+                ) if post_collect else NullPostTest(),
                 "always_run": False,
             },
             {
                 "name": "DB backup and restore",
                 "test": QaE2eDBBackupRestoreTest(),
-                "post_test": post_collect ? CheckStackroxLogs(
+                "post_test": CheckStackroxLogs(
                     check_for_errors_in_stackrox_logs=True,
                     artifact_destination_prefix="db-test",
-                ) : NullPostTest(),
+                ) if post_collect else NullPostTest(),
                 "always_run": False,
             },
         ],

From dc918308a77ea530c77c3fe518f0053d255af8ec Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 16:10:27 -0600
Subject: [PATCH 069/256] log vars in debug check

---
 .github/workflows/e2e-tests.yaml                     | 3 ++-
 qa-tests-backend/src/main/groovy/util/Helpers.groovy | 4 ++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 7216c2173d98e..52985cd3050ee 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -100,8 +100,9 @@ jobs:
           git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
           export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
           cat "$GITHUB_ENV" || true
-          echo "$GATHER_QA_TEST_DEBUG_LOGS"
+          echo "GATHER_QA_TEST_DEBUG_LOGS:${GATHER_QA_TEST_DEBUG_LOGS:-}"
           export GATHER_QA_TEST_DEBUG_LOGS
+          export "QA_TEST_DEBUG_LOGS:${QA_TEST_DEBUG_LOGS:-}"
           export QA_TEST_DEBUG_LOGS
           scripts/ci/jobs/ocp_qa_e2e_tests.py \
             echo end  #|| bash -x .openshift-ci/dispatch.sh "$JOB_NAME"
diff --git a/qa-tests-backend/src/main/groovy/util/Helpers.groovy b/qa-tests-backend/src/main/groovy/util/Helpers.groovy
index 7a72a27b544df..21de597b66ebf 100644
--- a/qa-tests-backend/src/main/groovy/util/Helpers.groovy
+++ b/qa-tests-backend/src/main/groovy/util/Helpers.groovy
@@ -182,6 +182,10 @@ class Helpers {
     }
 
     private static boolean collectDebug() {
+        log.warn("Debug collection skip check: "+
+                 "(CI: ${Env.IN_CI} ||"+
+                 " GATHER_QA_TEST_DEBUG_LOGS: ${Env.GATHER_QA_TEST_DEBUG_LOGS}) && "+
+                 " QA_TEST_DEBUG_LOGS: ${Env.QA_TEST_DEBUG_LOGS} != ''")
         if ((Env.IN_CI || Env.GATHER_QA_TEST_DEBUG_LOGS) && (Env.QA_TEST_DEBUG_LOGS != "")) {
             return true
         }

From 3ef76f77dfbf3716f37a5a6dbacc010aa7384455 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 16:13:11 -0600
Subject: [PATCH 070/256] echo dir

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 52985cd3050ee..c6b5a086adef0 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -102,7 +102,7 @@ jobs:
           cat "$GITHUB_ENV" || true
           echo "GATHER_QA_TEST_DEBUG_LOGS:${GATHER_QA_TEST_DEBUG_LOGS:-}"
           export GATHER_QA_TEST_DEBUG_LOGS
-          export "QA_TEST_DEBUG_LOGS:${QA_TEST_DEBUG_LOGS:-}"
+          echo "QA_TEST_DEBUG_LOGS:${QA_TEST_DEBUG_LOGS:-}"
           export QA_TEST_DEBUG_LOGS
           scripts/ci/jobs/ocp_qa_e2e_tests.py \
             echo end  #|| bash -x .openshift-ci/dispatch.sh "$JOB_NAME"

From 797d01cededc67246f9b4fa403b372fa62b60f32 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 21:29:49 -0600
Subject: [PATCH 071/256] always log for collect

---
 qa-tests-backend/src/main/groovy/util/Helpers.groovy | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa-tests-backend/src/main/groovy/util/Helpers.groovy b/qa-tests-backend/src/main/groovy/util/Helpers.groovy
index 21de597b66ebf..79a0fd9869890 100644
--- a/qa-tests-backend/src/main/groovy/util/Helpers.groovy
+++ b/qa-tests-backend/src/main/groovy/util/Helpers.groovy
@@ -88,6 +88,7 @@ class Helpers {
 
     @CompileDynamic
     static void collectDebugForFailure(Throwable exception) {
+        log.info("collectDebugForFailure" + exception)
         if (!collectDebug()) {
             return
         }

From 410cc7c83498798fdf5635d83bdde91da053a1cf Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 21:32:54 -0600
Subject: [PATCH 072/256] keep logs dir

---
 tests/e2e/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 5665bfc739efb..39b15768a29f2 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -13,7 +13,7 @@ source "$TEST_ROOT/scripts/ci/lib.sh"
 # shellcheck source=../../scripts/ci/test_state.sh
 source "$TEST_ROOT/scripts/ci/test_state.sh"
 
-export QA_TEST_DEBUG_LOGS="/tmp/qa-tests-backend-logs"
+export QA_TEST_DEBUG_LOGS="${QA_TEST_DEBUG_LOGS:-/tmp/qa-tests-backend-logs}"
 export QA_DEPLOY_WAIT_INFO="/tmp/wait-for-kubectl-object"
 
 # If `envsubst` is contained in a non-standard directory `env -i` won't be able to

From 60e7e9443752cfb47cf156c48bb4439fe5936f89 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 22:02:48 -0600
Subject: [PATCH 073/256] try finding port also

---
 .github/workflows/e2e-tests.yaml |   31 +-
 3                                | 2494 ++++++++++++++++++++++++++++++
 scripts/ci/lib.sh                |    2 -
 3 files changed, 2520 insertions(+), 7 deletions(-)
 create mode 100644 3

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index c6b5a086adef0..e16d5be21d61e 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -82,11 +82,29 @@ jobs:
       - run: |
           set +e
           set -x
-          if [[ -e "${KUBECONFIG:-}" && -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
-            password="${ROX_ADMIN_PASSWORD:-$(oc -n stackrox get secret admin-pass -o json | jq -r '.data.password' | base64 -d)}"
-            password="${password:-$(oc -n stackrox get secret central-htpasswd -o go-template='{{index .data "password" | base64decode}}')}"
-            echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
-            echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
+          if [[ -e "${KUBECONFIG:-}" ]]; then
+            if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
+              password="${ROX_ADMIN_PASSWORD:-$(oc -n stackrox get secret admin-pass -o json | jq -r '.data.password' | base64 -d)}"
+              password="${password:-$(oc -n stackrox get secret central-htpasswd -o go-template='{{index .data "password" | base64decode}}')}"
+              echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
+              echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
+            fi
+            if [[ -z "${API_HOSTNAME:-}" ]]; then
+              echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
+              API_HOSTNAME=$(kubectl -n stackrox get route central -o jsonpath='{.status.ingress[0].host}')
+              API_HOSTNAME=${API_HOSTNAME:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].ip}')}
+              API_HOSTNAME=${API_HOSTNAME:-localhost}
+              echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
+              API_PORT=${API_PORT:-$(kubectl -n stackrox get route central -o jsonpath='{.spec.port.targetPort}')}
+              API_PORT=${API_PORT:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].port}')}
+              API_PORT=${API_PORT//https/443}
+              if [[ "$API_HOSTNAME" == 'localhost' ]]; then
+                kubectl -n stackrox port-forward svc/central 8000:443 &
+                API_PORT=8000
+              fi
+              echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
+              echo "API_PORT=${API_PORT}" | tee -a "$GITHUB_ENV"
+            fi
           fi
 
       - run: go get github.com/stackrox/scanner
@@ -104,5 +122,8 @@ jobs:
           export GATHER_QA_TEST_DEBUG_LOGS
           echo "QA_TEST_DEBUG_LOGS:${QA_TEST_DEBUG_LOGS:-}"
           export QA_TEST_DEBUG_LOGS
+          API_HOSTNAME
+          API_PORT
+          export CLUSTER='OPENSHIFT'
           scripts/ci/jobs/ocp_qa_e2e_tests.py \
             echo end  #|| bash -x .openshift-ci/dispatch.sh "$JOB_NAME"
diff --git a/3 b/3
new file mode 100644
index 0000000000000..a613f12b66f67
--- /dev/null
+++ b/3
@@ -0,0 +1,2494 @@
+#!/usr/bin/env bash
+
+# A library of CI related reusable bash functions
+
+SCRIPTS_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)"
+# shellcheck source=../../scripts/lib.sh
+source "$SCRIPTS_ROOT/scripts/lib.sh"
+# shellcheck source=../../scripts/ci/metrics.sh
+source "$SCRIPTS_ROOT/scripts/ci/metrics.sh"
+# shellcheck source=../../scripts/ci/test_state.sh
+source "$SCRIPTS_ROOT/scripts/ci/test_state.sh"
+# shellcheck source=../../scripts/ci/gcp.sh
+source "$SCRIPTS_ROOT/scripts/ci/gcp.sh"
+
+set -euo pipefail
+
+ensure_CI() {
+    if ! is_CI; then
+        die "A CI environment is required."
+    fi
+}
+
+ci_export() {
+    if [[ "$#" -ne 2 ]]; then
+        die "missing args. usage: ci_export <env-name> <env-value>"
+    fi
+
+    local env_name="$1"
+    local env_value="$2"
+
+    if command -v cci-export >/dev/null; then
+        cci-export "$env_name" "$env_value"
+    else
+        export "$env_name"="$env_value"
+    fi
+}
+
+# set_ci_shared_export() - for openshift-ci and GHA this is state shared between steps.
+set_ci_shared_export() {
+    if [[ "$#" -ne 2 ]]; then
+        die "missing args. usage: set_ci_shared_export <env-name> <env-value>"
+    fi
+
+    ci_export "$@"
+
+    local env_name="$1"
+    local env_value="$2"
+
+    printf 'export %s=%q\n' "${env_name}" "${env_value}" >> "${SHARED_DIR:-/tmp}/shared_env"
+    printf '%s=%q\n' "${env_name}" "${env_value}" >> "${GITHUB_ENV:-/dev/null}"
+}
+
+ci_exit_trap() {
+    local exit_code="$?"
+    info "Executing a general purpose exit trap for CI"
+    echo "Exit code is: ${exit_code}"
+
+    if [[ "${exit_code}" == "0" ]]; then
+        set_ci_shared_export JOB_DISPATCH_OUTCOME "${OUTCOME_PASSED}"
+    elif [[ "${exit_code}" == "130" ]]; then
+        set_ci_shared_export JOB_DISPATCH_OUTCOME "${OUTCOME_CANCELED}"
+    else
+        set_ci_shared_export JOB_DISPATCH_OUTCOME "${OUTCOME_FAILED}"
+    fi
+
+    post_process_test_results "${JOB_SLACK_FAILURE_ATTACHMENTS}" "${JOB_JUNIT2JIRA_SUMMARY_FILE}"
+
+    while [[ -e /tmp/hold ]]; do
+        info "Holding this job for debug"
+        sleep 60
+    done
+
+    handle_dangling_processes
+
+    gate_flaky_tests "${exit_code}"
+}
+
+# handle_dangling_processes() - The OpenShift CI ci-operator will not complete a
+# test job if there are processes remaining that were started by the job. While
+# processes _should_ be cleaned up by their creators it is common that some are
+# not, so this exists as a fail safe.
+handle_dangling_processes() {
+    info "Process state at exit:"
+    ps -e -O ppid
+
+    local psline this_pid pid
+    ps -e -O ppid | while read -r psline; do
+        # trim leading whitespace
+        psline="$(echo "$psline" | xargs)"
+        if [[ "$psline" =~ ^PID ]]; then
+            # Ignoring header
+            continue
+        fi
+        this_pid="$$"
+        if [[ "$psline" =~ ^$this_pid ]]; then
+            echo "Ignoring self: $psline"
+            continue
+        fi
+        # shellcheck disable=SC1087
+        if [[ "$psline" =~ [[:space:]]$this_pid[[:space:]] ]]; then
+            echo "Ignoring child: $psline"
+            continue
+        fi
+        if [[ "$psline" =~ entrypoint|defunct ]]; then
+            echo "Ignoring ci-operator entrypoint or defunct process: $psline"
+            continue
+        fi
+        echo "A candidate to kill: $psline"
+        pid="$(echo "$psline" | cut -d' ' -f1)"
+        echo "Will kill $pid"
+        kill "$pid" || {
+            echo "Error killing $pid"
+        }
+    done
+}
+
+create_exit_trap() {
+    trap ci_exit_trap EXIT
+}
+
+setup_deployment_env() {
+    info "Setting up the deployment environment"
+
+    if [[ "$#" -ne 2 ]]; then
+        die "missing args. usage: setup_deployment_env <docker-login> <use-websocket>"
+    fi
+
+    local docker_login="$1"
+    local use_websocket="$2"
+
+    if [[ "$docker_login" == "true" ]]; then
+        registry_ro_login "quay.io/rhacs-eng"
+    fi
+
+    if [[ "$use_websocket" == "true" ]]; then
+        ci_export CLUSTER_API_ENDPOINT "wss://central.stackrox:443"
+    fi
+
+    ci_export REGISTRY_USERNAME "$QUAY_RHACS_ENG_RO_USERNAME"
+    ci_export REGISTRY_PASSWORD "$QUAY_RHACS_ENG_RO_PASSWORD"
+    if [[ -z "${MAIN_IMAGE_TAG:-}" ]]; then
+        ci_export MAIN_IMAGE_TAG "$(make --quiet --no-print-directory tag)"
+    fi
+
+    ci_export ROX_PRODUCT_BRANDING "RHACS_BRANDING"
+}
+
+get_central_debug_dump() {
+    info "Getting a central debug dump"
+
+    if [[ "$#" -ne 1 ]]; then
+        die "missing arg. usage: get_central_debug_dump <output_dir>"
+    fi
+
+    local output_dir="$1"
+
+    require_environment "API_ENDPOINT"
+    require_environment "ROX_ADMIN_PASSWORD"
+    roxctl -e "${API_ENDPOINT}" --ca "" --insecure-skip-tls-verify \
+        central debug dump --output-dir "${output_dir}"
+    ls -l "${output_dir}"
+}
+
+process_central_metrics() {
+    info "Processing metrics from central debug dump"
+
+    if [[ "$#" -ne 1 ]]; then
+        die "missing arg. usage: process_central_metrics <debug_dump_dir>"
+    fi
+
+    local output_dir="$1"
+
+    local metrics_output
+    local csv_output
+    local debug_dump_zip
+    metrics_output="$(mktemp --suffix=.prom)"
+    csv_output="$(mktemp --suffix=.csv)"
+    # shellcheck disable=SC2012
+    debug_dump_zip="$(ls -t "${output_dir}"/*.zip | head -1)"
+    unzip -p "${debug_dump_zip}" metrics-1 > "${metrics_output}"
+
+    get_prometheus_metrics_parser
+
+    # We need a link to repository. In case it's not part of job spec (e.g., periodic`s)
+    # we will fallback to short commit
+    base_link="$(echo "$JOB_SPEC" | jq ".refs.base_link | select( . != null )" -r)"
+    calculated_base_link="https://github.com/stackrox/stackrox/commit/$(make --quiet --no-print-directory shortcommit)"
+
+    local metadata="build_tag=${STACKROX_BUILD_TAG:-none},build_id=${BUILD_ID:-none},orchestrator_flavor=${ORCHESTRATOR_FLAVOR:-PROW},job_name=${JOB_NAME:-missing},base_link=${base_link:-$calculated_base_link}"
+    prometheus-metric-parser single \
+        --format csv \
+        --file "${metrics_output}" \
+        --labels "${metadata}" \
+        > "${csv_output}"
+
+    setup_gcp
+    save_central_metrics "${csv_output}"
+}
+
+get_prometheus_metrics_parser() {
+    local parserBin
+    local parserDir
+    parserBin=$(make prometheus-metric-parser -C "$ROOT" --silent | tail -1)
+    parserDir=$(dirname "${parserBin}")
+    export PATH="$parserDir":$PATH
+    prometheus-metric-parser help
+}
+
+get_central_diagnostics() {
+    info "Getting central diagnostics"
+
+    if [[ "$#" -ne 1 ]]; then
+        die "missing arg. usage: get_central_diagnostics <output_dir>"
+    fi
+
+    local output_dir="$1"
+
+    require_environment "API_ENDPOINT"
+    require_environment "ROX_ADMIN_PASSWORD"
+    roxctl -e "${API_ENDPOINT}" --ca "" --insecure-skip-tls-verify \
+        central debug download-diagnostics --output-dir "${output_dir}"
+    ls -l "${output_dir}"
+}
+
+push_image_manifest_lists() {
+    info "Pushing main, roxctl and central-db images as manifest lists"
+
+    if [[ "$#" -ne 3 ]]; then
+        die "missing arg. usage: push_image_manifest_lists <push_context> <brand> <architectures (CSV)>"
+    fi
+
+    local push_context="$1"
+    local brand="$2"
+    local architectures="$3"
+
+    local main_image_set=("main" "roxctl" "central-db")
+
+    local registry
+    registry="$(registry_from_branding "$brand")"
+
+    local tag
+    tag="$(make --quiet --no-print-directory tag)"
+
+    registry_rw_login "$registry"
+    for image in "${main_image_set[@]}"; do
+        retry 5 true \
+          "$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:${tag}" "$architectures" | cat
+        if [[ "$push_context" == "merge-to-master" ]]; then
+            retry 5 true \
+              "$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:latest" "$architectures" | cat
+        fi
+    done
+
+    # Push manifest lists for scanner and collector for amd64 only
+    local amd64_image_set=("scanner" "scanner-db" "scanner-slim" "scanner-db-slim" "collector")
+    for image in "${amd64_image_set[@]}"; do
+        retry 5 true \
+          "$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:${tag}" "amd64" | cat
+    done
+}
+
+registry_from_branding() {
+    local branding="$1"
+    if [[ "$branding" == "STACKROX_BRANDING" ]]; then
+        registry="quay.io/stackrox-io"
+    elif [[ "$branding" == "RHACS_BRANDING" ]]; then
+        registry="quay.io/rhacs-eng"
+    else
+        die "$branding is not a supported branding"
+    fi
+    echo "$registry"
+}
+
+push_main_image_set() {
+    info "Pushing main, roxctl and central-db images"
+
+    if [[ "$#" -ne 3 ]]; then
+        die "missing arg. usage: push_main_image_set <push_context> <brand> <arch>"
+    fi
+
+    local push_context="$1"
+    local brand="$2"
+    local arch="$3"
+
+    local main_image_set=("main" "roxctl" "central-db")
+
+    _push_main_image_set() {
+        local registry="$1"
+        local tag="$2"
+
+        for image in "${main_image_set[@]}"; do
+            retry 5 true \
+              docker push "${registry}/${image}:${tag}" | cat
+        done
+    }
+
+    _tag_main_image_set() {
+        local local_tag="$1"
+        local registry="$2"
+        local remote_tag="$3"
+
+        for image in "${main_image_set[@]}"; do
+            docker tag "stackrox/${image}:${local_tag}" "${registry}/${image}:${remote_tag}"
+        done
+    }
+
+    local registry
+    registry="$(registry_from_branding "$brand")"
+
+    local tag
+    tag="$(make --quiet --no-print-directory tag)"
+
+    registry_rw_login "$registry"
+
+    _tag_main_image_set "$tag" "$registry" "$tag-$arch"
+    _push_main_image_set "$registry" "$tag-$arch"
+
+    if [[ "$push_context" == "merge-to-master" ]]; then
+        _tag_main_image_set "$tag" "$registry" "latest-${arch}"
+        _push_main_image_set "$registry" "latest-${arch}"
+    fi
+}
+
+push_scanner_image_manifest_lists() {
+    info "Pushing scanner-v4 and scanner-v4-db images as manifest lists"
+
+    if [[ "$#" -ne 2 ]]; then
+        die "missing arg. usage: push_scanner_image_manifest_lists <registry> <architectures (CSV)>"
+    fi
+
+    local registry="$1"
+    local architectures="$2"
+    local scanner_image_set=("scanner-v4" "scanner-v4-db")
+
+    local tag
+    tag="$(make --quiet --no-print-directory -C scanner tag)"
+    registry_rw_login "$registry"
+    for image in "${scanner_image_set[@]}"; do
+        retry 5 true \
+          "$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:${tag}" "$architectures" | cat
+    done
+}
+
+push_scanner_image_set() {
+    info "Pushing scanner-v4 and scanner-v4-db images"
+
+    if [[ "$#" -ne 2 ]]; then
+        die "missing arg. usage: push_scanner_image_set <registry> <arch>"
+    fi
+
+    local registry="$1"
+    local arch="$2"
+
+    local scanner_image_set=("scanner-v4" "scanner-v4-db")
+
+    _push_scanner_image_set() {
+        local registry="$1"
+        local tag="$2"
+
+        for image in "${scanner_image_set[@]}"; do
+            retry 5 true \
+              docker push "${registry}/${image}:${tag}" | cat
+        done
+    }
+
+    _tag_scanner_image_set() {
+        local local_tag="$1"
+        local registry="$2"
+        local remote_tag="$3"
+
+        for image in "${scanner_image_set[@]}"; do
+            docker tag "stackrox/${image}:${local_tag}" "${registry}/${image}:${remote_tag}"
+        done
+    }
+
+    local tag
+    tag="$(make --quiet --no-print-directory -C scanner tag)"
+
+    registry_rw_login "$registry"
+
+    _tag_scanner_image_set "$tag" "$registry" "$tag-$arch"
+    _push_scanner_image_set "$registry" "$tag-$arch"
+}
+
+registry_rw_login() {
+    if [[ "$#" -ne 1 ]]; then
+        die "missing arg. usage: registry_rw_login <registry>"
+    fi
+
+    local registry="$1"
+
+    case "$registry" in
+        quay.io/rhacs-eng)
+            _login() {
+                # shellcheck disable=SC2317
+                docker login -u "$QUAY_RHACS_ENG_RW_USERNAME" --password-stdin <<<"$QUAY_RHACS_ENG_RW_PASSWORD" quay.io
+            }
+            retry 5 true _login
+            ;;
+        quay.io/stackrox-io)
+            _login() {
+                # shellcheck disable=SC2317
+                docker login -u "$QUAY_STACKROX_IO_RW_USERNAME" --password-stdin <<<"$QUAY_STACKROX_IO_RW_PASSWORD" quay.io
+            }
+            retry 5 true _login
+            ;;
+        *)
+            die "Unsupported registry login: $registry"
+    esac
+}
+
+registry_ro_login() {
+    if [[ "$#" -ne 1 ]]; then
+        die "missing arg. usage: registry_ro_login <registry>"
+    fi
+
+    local registry="$1"
+
+    case "$registry" in
+        quay.io/rhacs-eng)
+            _login() {
+                # shellcheck disable=SC2317
+                docker login -u "$QUAY_RHACS_ENG_RO_USERNAME" --password-stdin <<<"$QUAY_RHACS_ENG_RO_PASSWORD" quay.io
+            }
+            retry 5 true _login
+            ;;
+        *)
+            die "Unsupported registry login: $registry"
+    esac
+}
+
+push_matching_collector_scanner_images() {
+    info "Pushing collector & scanner images tagged with main-version to quay.io/rhacs-eng"
+
+    if [[ "$#" -ne 2 ]]; then
+        die "missing arg. usage: push_matching_collector_scanner_images <brand> <arch>"
+    fi
+
+    local brand="$1"
+    local arch="$2"
+
+    local registry
+    registry="$(registry_from_branding "$brand")"
+
+    _retag() {
+        retry 5 true "$SCRIPTS_ROOT/scripts/ci/pull-retag-push.sh" "$1" "$2"
+    }
+
+    if [[ "$arch" != "amd64" ]]; then
+        echo "Skipping rebundling for non-amd64 arch"
+        exit 0
+    fi
+
+    local main_tag
+    main_tag="$(make --quiet --no-print-directory tag)"
+    local scanner_version
+    scanner_version="$(make --quiet --no-print-directory scanner-tag)"
+    local collector_version
+    collector_version="$(make --quiet --no-print-directory collector-tag)"
+
+    registry_rw_login "${registry}"
+
+    _retag "${registry}/scanner:${scanner_version}"    "${registry}/scanner:${main_tag}-${arch}"
+    _retag "${registry}/scanner-db:${scanner_version}" "${registry}/scanner-db:${main_tag}-${arch}"
+    _retag "${registry}/scanner-slim:${scanner_version}"    "${registry}/scanner-slim:${main_tag}-${arch}"
+    _retag "${registry}/scanner-db-slim:${scanner_version}" "${registry}/scanner-db-slim:${main_tag}-${arch}"
+
+    _retag "${registry}/collector:${collector_version}"      "${registry}/collector:${main_tag}-${arch}"
+}
+
+poll_for_system_test_images() {
+    info "Polling for images required for system tests"
+
+    if [[ "$#" -ne 1 ]]; then
+        die "missing arg. usage: poll_for_system_test_images <seconds to wait>"
+    fi
+
+    local time_limit="$1"
+
+    require_environment "QUAY_RHACS_ENG_BEARER_TOKEN"
+
+    local image_list
+    image_list="$(mktemp)"
+    populate_stackrox_image_list "${image_list}"
+    info "Will poll for: $(awk '{print $1}' "${image_list}")"
+
+    local start_time
+    start_time="$(date '+%s')"
+
+    local tag
+    local image
+    while read -r image tag
+    do
+        while ! check_rhacs_eng_image_exists "$image" "$tag"
+        do
+            info "$image does not exist"
+            if (( $(date '+%s') - start_time > time_limit )); then
+                check_build_workflows "$(get_commit_sha)"
+                die "ERROR: Timed out waiting for images after ${time_limit} seconds"
+            fi
+            sleep 60
+        done
+    done < "$image_list"
+
+    info "All images exist."
+    touch "${STATE_IMAGES_AVAILABLE}"
+}
+
+# Image prefetch is broken into two sets:
+# - prebuilt: test fixture images that already exist prior to CI job execution.
+#   Prefetch for these images can start as soon as the cluster is ready.
+# - system: the product images under test 'stackrox-images'. Prefetch for these
+#   should not start until the images are built to avoid backoff noise in the
+#   prefetcher logs.
+
+image_prefetcher_await_msg_prefix="Waiting for pre-fetcher"
+
+image_prefetcher_prebuilt_start() {
+    info "Starting image pre-fetcher of pre-built images (NOT built for current commit)..."
+    junit_wrap image-prefetcher-prebuilt-start \
+               "Start image pre-fetcher for pre-built images (NOT built for current commit)." \
+               "See log for error details." \
+               _image_prefetcher_prebuilt_start
+}
+
+image_prefetcher_system_start() {
+    info "Starting image pre-fetcher of system images (built for current commit)..."
+    junit_wrap image-prefetcher-system-start \
+               "Start image pre-fetcher for system images (built for current commit)." \
+               "See log for error details." \
+               _image_prefetcher_system_start
+}
+
+_image_prefetcher_prebuilt_start() {
+    # NOTE: when changing this function, make corresponding changes to
+    # _image_prefetcher_prebuilt_await
+
+    case "$CI_JOB_NAME" in
+    *qa-e2e-tests)
+        image_prefetcher_start_set qa-e2e
+        # Override the default image pull policy for containers with quay.io
+        # images to rely on prefetched images. This helps ensure that the static
+        # prefect list stays up to date with additions.
+        ci_export "IMAGE_PULL_POLICY_FOR_QUAY_IO" "Never"
+        ;;
+    *-operator-e2e-tests)
+        image_prefetcher_start_set operator-e2e
+        # TODO(ROX-20508): pre-fetch images of the release from which operator upgrade test starts as well.
+        ;;
+    *)
+        info "No pre-built image prefetching is currently performed for: ${CI_JOB_NAME}."
+        ;;
+    esac
+}
+
+_image_prefetcher_system_start() {
+    # NOTE: when changing this function, make corresponding changes to
+    # _image_prefetcher_system_await
+
+    case "$CI_JOB_NAME" in
+    # ROX-24818: GKE is excluded from system image prefetch as it causes
+    # flakes in test.
+    *-operator-e2e-tests|*ocp*qa-e2e-tests)
+        image_prefetcher_start_set stackrox-images
+        ;;
+    # Enabling scanner V4 installation tests as well, even though they also run on GKE,
+    # for gathering some more data points for CI reliability.
+    *-scanner-v4-install-tests)
+        image_prefetcher_start_set stackrox-images
+        ;;
+    *)
+        info "No system image prefetching is performed for: ${CI_JOB_NAME}."
+        ;;
+    esac
+}
+
+image_prefetcher_start_set() {
+    local ns="prefetch-images"
+    local name="$1"
+
+    make image-prefetcher-deploy-bin
+    local image_prefetcher_deploy_bin
+    image_prefetcher_deploy_bin="$(make print-image-prefetcher-deploy-bin)"
+    local image_prefetcher_version
+    image_prefetcher_version="$(go -C tools/test list -m -f '{{.Version}}' github.com/stackrox/image-prefetcher/deploy)"
+
+    info "Using ${image_prefetcher_deploy_bin} ${image_prefetcher_version} for image prefetch deployment"
+
+    local manifest
+    manifest=$(mktemp)
+
+    case "${ORCHESTRATOR_FLAVOR}" in
+    k8s)
+        flavor=vanilla
+        ;;
+    openshift)
+        flavor=ocp
+        ;;
+    *)
+        die "unsupported ORCHESTRATOR: ${ORCHESTRATOR_FLAVOR}"
+        ;;
+    esac
+
+    # daemonset, etc
+    ${image_prefetcher_deploy_bin} \
+        --version="${image_prefetcher_version}" \
+        --k8s-flavor="$flavor" \
+        --secret=stackrox \
+        --collect-metrics \
+        "$name" > "$manifest"
+
+    # image list
+    local image_list
+    image_list=$(mktemp)
+    populate_prefetcher_image_list "$name" "${image_list}"
+    echo "---" >> "$manifest"
+    kubectl create --dry-run=client -o yaml --namespace=$ns configmap "$name" --from-file="images.txt=$image_list" >> "$manifest"
+
+    # pull secret
+    REGISTRY_PASSWORD="${QUAY_RHACS_ENG_RO_PASSWORD}" \
+    REGISTRY_USERNAME="${QUAY_RHACS_ENG_RO_USERNAME}" \
+    NAMESPACE=$ns \
+      make -C operator stackrox-image-pull-secret
+
+    # apply configmap, daemonset etc
+    retry 5 true kubectl apply --namespace=$ns -f "$manifest"
+    info "Image pre-fetcher is now running in the background. Its status will be checked later (look for message starting with ${image_prefetcher_await_msg_prefix}). Proceeding with other tasks in the meantime."
+    rm -f "$image_list" "$manifest"
+}
+
+image_prefetcher_prebuilt_await() {
+    if [[ "${IMAGE_PREFETCH_DISABLED:-false}" == "true" ]]; then
+        return
+    fi
+
+    info "${image_prefetcher_await_msg_prefix} of pre-built images to complete..."
+    junit_wrap image-prefetcher-prebuilt-await \
+               "Waiting for pre-fetcher of pre-built images to complete." \
+               "See log for error details." \
+               _image_prefetcher_prebuilt_await
+}
+
+image_prefetcher_system_await() {
+    if [[ "${IMAGE_PREFETCH_DISABLED:-false}" == "true" ]]; then
+        return
+    fi
+
+    info "${image_prefetcher_await_msg_prefix} of system images to complete..."
+    junit_wrap image-prefetcher-system-await \
+               "Waiting for pre-fetcher of system images to complete." \
+               "See log for error details." \
+               _image_prefetcher_system_await
+}
+
+_image_prefetcher_prebuilt_await() {
+    case "$CI_JOB_NAME" in
+    *qa-e2e-tests)
+        image_prefetcher_await_set qa-e2e
+        ;;
+    *-operator-e2e-tests)
+        image_prefetcher_await_set operator-e2e
+        # TODO(ROX-20508): pre-fetch images of the release from which operator upgrade test starts as well.
+        ;;
+    *)
+        info "No pre-built image prefetching is currently performed for: ${CI_JOB_NAME}. Nothing to wait for."
+        ;;
+    esac
+}
+
+_image_prefetcher_system_await() {
+    case "$CI_JOB_NAME" in
+    # ROX-24818: GKE is excluded from system image prefetch as it causes
+    # flakes in test.
+    *-operator-e2e-tests|*ocp*qa-e2e-tests)
+        image_prefetcher_await_set stackrox-images
+        ;;
+    # Enabling scanner V4 installation tests as well, even though they also run on GKE,
+    # for gathering some more data points for CI reliability.
+    *-scanner-v4-install-tests)
+        image_prefetcher_await_set stackrox-images
+        ;;
+    *)
+        info "No system image prefetching is performed for: ${CI_JOB_NAME}. Nothing to wait for."
+        ;;
+    esac
+}
+
+image_prefetcher_await_set() {
+    local ns="prefetch-images"
+    local name="$1"
+    local extra_fields='{"build_id": "'"${BUILD_ID:-}"'", "job_name": "'"${JOB_NAME:-}"'", "orchestrator": "'"${ORCHESTRATOR_FLAVOR:-}"'", "build_tag": "'"${STACKROX_BUILD_TAG:-}"'"}'
+
+    info "Waiting for image prefetcher set ${name} to complete..."
+    if kubectl rollout status daemonset "$name" -n "$ns" --timeout 15m; then
+        info "All images in the set are now pre-fetched."
+    else
+        info "WARNING: Pre-fetching failed to complete in time."
+        info "To investigate closer, go to https://console.cloud.google.com/bigquery and run a query such as:"
+        local query
+        query=$(mktemp)
+        cat > "${query}" <<- EOM
+
+            SELECT started_at, duration_ms, image, error
+            FROM \`acs-san-stackroxci.ci_metrics.stackrox_image_prefetches\`
+            WHERE error IS NOT NULL AND
+            $(echo "${extra_fields}" | jq -r '[to_entries | .[] | select(.value != "") | (.key + "=\"" + .value + "\"")] | join(" AND ")')
+            ORDER BY started_at DESC LIMIT 1000
+
+EOM
+        cat "${query}"
+        info "Note: The data is imported into the table periodically: https://github.com/stackrox/stackrox/actions/workflows/batch-load-test-metrics.yml"
+
+        if [[ -n ${ARTIFACT_DIR:-} ]]; then
+            local prefetcher_help="$ARTIFACT_DIR/image-pre-fetcher-${name}-failure-summary.html"
+            cat > "${prefetcher_help}" <<- EOM
+                <html>
+                <head>
+                <title>Image pre-fetcher ${name} failure</title>
+                <style>
+                  body { color: #e8e8e8; background-color: #424242; font-family: "Roboto", "Helvetica", "Arial", sans-serif }
+                  a { color: #ff8caa }
+                  a:visited { color: #ff8caa }
+                </style>
+                </head>
+                <body>
+
+                Waiting for image prefetcher set ${name} to complete timed out.<br>
+                To investigate closer, go to <a target="_blank" href="https://console.cloud.google.com/bigquery">BigQuery</a> and run a query such as the following:
+                <br>
+                <pre>
+EOM
+            cat >> "${prefetcher_help}" "${query}"
+            cat >> "${prefetcher_help}" <<- EOM
+                </pre>
+                Note: The data is imported into the table <a target="_blank" href="https://github.com/stackrox/stackrox/actions/workflows/batch-load-test-metrics.yml">periodically</a>.
+                <br><br>
+                </body>
+                </html>
+EOM
+        fi
+        rm -f "${query}"
+    fi
+    info "Now retrieving prefetcher metrics..."
+    local attempt=0
+    local service="service/${name}-metrics"
+    while [[ -z $(kubectl -n "${ns}" get "${service}" -o jsonpath="{.status.loadBalancer.ingress}" 2>/dev/null) ]]; do
+        if [ "$attempt" -lt "60" ]; then
+            info "Waiting for ${service} to obtain endpoint ..."
+            ((attempt++))
+            sleep 10
+        else
+            info "Something is wrong with the ${service} service. See the following 'describe' output."
+            kubectl -n "${ns}" describe "${service}" || true
+            die "Timeout waiting for ${service} to obtain endpoint!"
+        fi
+    done
+    local endpoint
+    endpoint="$(kubectl -n "${ns}" get "${service}" -o json | service_get_endpoint)"
+    local fetcher_metrics
+    fetcher_metrics="$(mktemp --suffix=.csv)"
+    local fetcher_metrics_json
+    fetcher_metrics_json="$(mktemp --suffix=.json)"
+    local metrics_url="http://${endpoint}:8080/metrics"
+    if ! curl --silent --show-error --fail --retry 3 --retry-connrefused "${metrics_url}" > "${fetcher_metrics_json}"; then
+        die "Failed to fetch prefetcher metrics from ${metrics_url}"
+    fi
+    # See the stackrox_image_prefetches table definition in https://github.com/stackrox/automation-iac/blob/main/resources/testing/stackrox-ci/metrics.tf
+    # for the order of columns.
+    if ! jq --raw-output \
+      --argjson cols '["attempt_id", "started_at", "image", "duration_ms", "node", "size_bytes", "error", "build_id", "job_name", "orchestrator", "build_tag"]' \
+      --argjson extra "${extra_fields}" \
+      'map(.started_at = (.started_at | todate) | ($extra+.) as $row | $cols | map($row[.])) as $rows | $cols, $rows[] | @csv' \
+      "${fetcher_metrics_json}" > "${fetcher_metrics}"; then
+        info "WARNING: Failed to convert image prefetcher metrics to CSV with extra fields ${extra_fields}"
+        info "Dumping the input JSON file:"
+        jq . < "${fetcher_metrics_json}"
+        die "Failed to convert image prefetcher metrics to CSV, aborting."
+    fi
+    rm -f "${fetcher_metrics_json}"
+
+    if save_image_prefetches_metrics "${fetcher_metrics}"; then
+        info "Image pre-fetcher metrics retrieved and saved."
+    else
+        info "WARNING: failed to save image pre-fetcher metrics."
+    fi
+    rm -f "${fetcher_metrics}"
+}
+
+service_get_endpoint() {
+    jq -r '.status.loadBalancer.ingress // error("List of ingress points of LB " + .metadata.name + " is empty.") | .[0] | .hostname // .ip'
+}
+
+populate_prefetcher_image_list() {
+    local name="$1"
+    local image_list="$2"
+
+    case "$name" in
+    stackrox-images)
+        local image_tag_list
+        image_tag_list=$(mktemp)
+        populate_stackrox_image_list "${image_tag_list}"
+        # convert format from "basename tag" to "quay.io/.../basename:tag" expected by pre-fetcher
+        awk '{print "quay.io/rhacs-eng/" $1 ":" $2}' "$image_tag_list" > "$image_list"
+        rm -f "$image_tag_list"
+        ;;
+    operator-e2e)
+        # shellcheck disable=SC2046
+        grep PREFETCH-THIS-IMAGE -h -A 1 $(find operator/tests/ -type f) | awk '$1 == "image:" {print $2}' | sort -u > "$image_list"
+        ;;
+    qa-e2e)
+        cp "$SCRIPTS_ROOT/qa-tests-backend/scripts/images-to-prefetch.txt" "$image_list"
+        ;;
+    *)
+        die "ERROR: An unsupported image prefetcher target was requested: $name"
+        ;;
+    esac
+}
+
+populate_stackrox_image_list() {
+    local image_list="$1"
+
+    local tag
+    tag="${MAIN_IMAGE_TAG:-$(make --quiet --no-print-directory tag)}"
+    local operator_metadata_tag
+    operator_metadata_tag="$(echo "v${tag}" | sed 's,x,0,')"
+    local operator_controller_tag="${tag//x/0}"
+
+    # Require images based on the job
+    case "$CI_JOB_NAME" in
+        *-operator-e2e-tests)
+            cat >> "${image_list}" << END
+stackrox-operator ${operator_controller_tag}
+stackrox-operator-bundle ${operator_metadata_tag}
+stackrox-operator-index ${operator_metadata_tag}
+main ${tag}
+central-db ${tag}
+collector ${tag}
+scanner ${tag}
+scanner-db ${tag}
+scanner-v4 ${tag}
+scanner-v4-db ${tag}
+END
+            ;;
+        *-race-condition-qa-e2e-tests)
+            cat >> "${image_list}" << END
+central-db ${tag}
+main ${tag}-rcd
+roxctl ${tag}
+END
+            if is_in_PR_context && ! pr_has_label "ci-build-race-condition-debug"; then
+                echo "ERROR: Your PR is missing the \"ci-build-race-condition-debug\" label."
+                echo "ERROR: This label is required to build the images for $CI_JOB_NAME."
+                # Quietly continue to allow labels added after tests start.
+                # Otherwise this message will surface in the Prow log when
+                # images timeout out below.
+            fi
+            ;;
+        *-scanner-v4-install-tests)
+            cat >> "${image_list}" << END
+stackrox-operator ${operator_controller_tag}
+stackrox-operator-bundle ${operator_metadata_tag}
+stackrox-operator-index ${operator_metadata_tag}
+main ${tag}
+central-db ${tag}
+collector ${tag}
+scanner ${tag}
+scanner-db ${tag}
+scanner-v4 ${tag}
+scanner-v4-db ${tag}
+roxctl ${tag}
+END
+            ;;
+        *)
+            cat >> "${image_list}" << END
+central-db ${tag}
+main ${tag}
+roxctl ${tag}
+END
+            ;;
+    esac
+
+    if [[ "${DEPLOY_STACKROX_VIA_OPERATOR:-}" == "true" ]]; then
+            cat >> "${image_list}" << END
+stackrox-operator ${operator_controller_tag}
+stackrox-operator-bundle ${operator_metadata_tag}
+stackrox-operator-index ${operator_metadata_tag}
+END
+    fi
+
+    # Remove duplicates.
+    unique="$(mktemp)"
+    sort -u "${image_list}" > "${unique}"
+    cat "${unique}" > "${image_list}"
+    rm -f "${unique}"
+}
+
+check_rhacs_eng_image_exists() {
+    local name="$1"
+    local tag="$2"
+
+    local url="https://quay.io/api/v1/repository/rhacs-eng/$name/tag?specificTag=$tag"
+    info "Checking for $name using $url"
+    local check
+    local extra_args=()
+    local public_images=("stackrox-operator-index")
+    if [[ -n "${QUAY_RHACS_ENG_BEARER_TOKEN:-}" ]]; then
+        extra_args+=("-H" "Authorization: Bearer ${QUAY_RHACS_ENG_BEARER_TOKEN}")
+    else
+        # shellcheck disable=SC2076
+        if [[ ! " ${public_images[*]} " =~ " ${name} " ]]; then
+            info "Warning: Checking for image existence without QUAY_RHACS_ENG_BEARER_TOKEN is not supported for image ${name}:${tag}"
+            info "Warning: It is only supported for the following public image repositories: ${public_images[*]}"
+        fi
+    fi
+    check=$(curl --location -sS "${extra_args[@]}" "$url")
+    echo "$check"
+    [[ "$(jq -r '.tags | first | .name' <<<"$check")" == "$tag" ]]
+}
+
+check_build_workflows() {
+    local commit_sha="$1"
+
+    {
+        echo
+        info "GitHub Actions workflow status for build.yaml:"
+        check-workflow-run \
+            --workflow=build.yaml \
+            --head-SHA="${commit_sha}"
+
+        echo
+        info "GitHub Actions workflow status for scanner-build.yaml:"
+        check-workflow-run \
+            --workflow=scanner-build.yaml \
+            --head-SHA="${commit_sha}"
+    } | tee "${STATE_BUILD_RESULTS}" || true
+}
+
+check_scanner_version() {
+    if ! is_release_version "$(make --quiet --no-print-directory scanner-tag)"; then
+        echo "::error::Scanner tag does not look like a release tag. Please update SCANNER_VERSION file before releasing."
+        exit 1
+    fi
+}
+
+check_collector_version() {
+    if ! is_release_version "$(make --quiet --no-print-directory collector-tag)"; then
+        echo "::error::Collector tag does not look like a release tag. Please update COLLECTOR_VERSION file before releasing."
+        exit 1
+    fi
+}
+
+publish_roxctl() {
+    if [[ "$#" -ne 1 ]]; then
+        die "missing arg. usage: publish_roxctl <tag>"
+    fi
+
+    local tag="$1"
+
+    echo "Push roxctl to gs://sr-roxc & gs://rhacs-openshift-mirror-src/assets" >> "${GITHUB_STEP_SUMMARY}"
+
+    local temp_dir
+    temp_dir="$(mktemp -d)"
+    "${SCRIPTS_ROOT}/scripts/ci/artifacts-publish/prepare-roxctl.sh" . "${temp_dir}"
+    "${SCRIPTS_ROOT}/scripts/ci/artifacts-publish/publish.sh" "${temp_dir}" "${tag}" "gs://sr-roxc"
+    "${SCRIPTS_ROOT}/scripts/ci/artifacts-publish/publish.sh" "${temp_dir}" "${tag}" "gs://rhacs-openshift-mirror-src/assets"
+}
+
+publish_openapispec() {
+    if [[ "$#" -ne 1 ]]; then
+        die "missing arg. usage: publish_openapispec <tag>"
+    fi
+
+    local tag="$1"
+
+    echo "Push OpenAPI spec to gs://rhacs-openshift-mirror-src/assets" >> "${GITHUB_STEP_SUMMARY}"
+
+    local temp_dir
+    temp_dir="$(mktemp -d)"
+    "${SCRIPTS_ROOT}/scripts/ci/artifacts-publish/prepare-openapispec.sh" "${temp_dir}" "${tag}"
+    "${SCRIPTS_ROOT}/scripts/ci/artifacts-publish/publish.sh" "${temp_dir}" "${tag}" "gs://rhacs-openshift-mirror-src/openapi-spec"
+}
+
+push_helm_charts() {
+    if [[ "$#" -ne 1 ]]; then
+        die "missing arg. usage: push_helm_charts <tag>"
+    fi
+
+    local tag="$1"
+
+    echo "Publish Helm charts to github repository stackrox/release-artifacts and create a PR" >> "${GITHUB_STEP_SUMMARY}"
+
+    local central_services_chart_dir
+    local secured_cluster_services_chart_dir
+    central_services_chart_dir="$(mktemp -d)"
+    secured_cluster_services_chart_dir="$(mktemp -d)"
+    roxctl helm output central-services --image-defaults=rhacs --output-dir "${central_services_chart_dir}/rhacs"
+    roxctl helm output central-services --image-defaults=opensource --output-dir "${central_services_chart_dir}/opensource"
+    roxctl helm output secured-cluster-services --image-defaults=rhacs --output-dir "${secured_cluster_services_chart_dir}/rhacs"
+    roxctl helm output secured-cluster-services --image-defaults=opensource --output-dir "${secured_cluster_services_chart_dir}/opensource"
+    "${SCRIPTS_ROOT}/scripts/ci/publish-helm-charts.sh" "${tag}" "${central_services_chart_dir}" "${secured_cluster_services_chart_dir}"
+}
+
+gitbot() {
+    git -c "user.name=${GITHUB_USERNAME}" \
+        -c "user.email=${GITHUB_EMAIL}" \
+        -c "url.https://${GITHUB_TOKEN}:x-oauth-basic@github.com/.insteadOf=https://github.com/" \
+        "${@}"
+}
+
+is_tagged() {
+    local tags
+    tags="$(git tag --contains)"
+    [[ -n "$tags" ]]
+}
+
+is_nightly_run() {
+    [[ "${BUILD_TAG:-}" =~ -nightly- ]] || [[ "${GITHUB_REF:-}" =~ nightly- ]]
+}
+
+is_in_PR_context() {
+    if is_GITHUB_ACTIONS && [[ -n "${GITHUB_BASE_REF:-}" ]]; then
+        return 0
+    elif is_OPENSHIFT_CI && [[ -n "${PULL_NUMBER:-}" ]]; then
+        return 0
+    elif is_OPENSHIFT_CI && [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
+        # bin, test-bin, images
+        local pull_request
+        pull_request=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].pulls[0].number' 2>&1) || return 1
+        [[ "$pull_request" =~ ^[0-9]+$ ]] && return 0
+    fi
+
+    return 1
+}
+
+get_PR_number() {
+    if is_OPENSHIFT_CI && [[ -n "${PULL_NUMBER:-}" ]]; then
+        echo "${PULL_NUMBER}"
+        return 0
+    elif is_OPENSHIFT_CI && [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
+        # bin, test-bin, images
+        local pull_request
+        pull_request=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].pulls[0].number' 2>&1) || {
+            echo 2>&1 "ERROR: Could not determine a PR number"
+            return 1
+        }
+        if [[ "$pull_request" =~ ^[0-9]+$ ]]; then
+            echo "$pull_request"
+            return 0
+        fi
+    elif is_GITHUB_ACTIONS; then
+        local pull_request
+        pull_request=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") || {
+            echo 2>&1 "ERROR: Could not determine a PR number"
+            return 1
+        }
+        if [[ "$pull_request" =~ ^[0-9]+$ ]]; then
+            echo "$pull_request"
+            return 0
+        fi
+    fi
+
+    echo 2>&1 "ERROR: Could not determine a PR number"
+
+    return 1
+}
+
+is_openshift_CI_rehearse_PR() {
+    [[ "$(get_repo_full_name)" == "openshift/release" ]]
+}
+
+get_branch_name() {
+    # Returns the PR branch name (sometimes that's called PR source branch), e.g. 'johndoe/ROX-23456-fix-branch-name'.
+    # For non-PRs, returns branch name where the commit happened, e.g. 'master'.
+    if is_OPENSHIFT_CI; then
+        # Prow variables doc: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables
+        if [[ -n "${PULL_HEAD_REF:-}" ]]; then
+            # presubmit runs
+            echo "${PULL_HEAD_REF}"
+        elif [[ -n "${PULL_BASE_REF:-}" ]]; then
+            # postsubmit and batch runs
+            echo "${PULL_BASE_REF}"
+        elif [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
+            # periodics - CLONEREFS_OPTIONS exists in binary_build_commands and images.
+            local base_ref
+            base_ref="$(jq -r <<<"${CLONEREFS_OPTIONS}" '.refs[0].base_ref')" || die "invalid CLONEREFS_OPTIONS yaml"
+            if [[ "$base_ref" == "null" ]]; then
+                die "expect: base_ref in CLONEREFS_OPTIONS.refs[0]"
+            fi
+            echo "${base_ref}"
+        else
+            die "Expected PULL_HEAD_REF or PULL_BASE_REF or CLONEREFS_OPTIONS"
+        fi
+    elif is_GITHUB_ACTIONS; then
+        # GHA doc: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#default-environment-variables
+        local ref="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-}}"
+        if [[ -z "${ref}" ]]; then
+            die "Expected GITHUB_HEAD_REF or GITHUB_REF_NAME"
+        fi
+        echo "${ref}"
+    else
+        die "unsupported"
+    fi
+}
+
+get_repo_full_name() {
+    if is_GITHUB_ACTIONS; then
+        [[ -n "${GITHUB_REPOSITORY:-}" ]] || die "expect: GITHUB_REPOSITORY"
+        echo "${GITHUB_REPOSITORY}"
+    elif is_OPENSHIFT_CI; then
+        if [[ -n "${REPO_OWNER:-}" ]]; then
+            # presubmit, postsubmit and batch runs
+            # (ref: https://github.com/kubernetes/test-infra/blob/master/prow/jobs.md#job-environment-variables)
+            [[ -n "${REPO_NAME:-}" ]] || die "expect: REPO_NAME"
+            echo "${REPO_OWNER}/${REPO_NAME}"
+        elif [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
+            # periodics - CLONEREFS_OPTIONS exists in binary_build_commands and images.
+            local org
+            local repo
+            org="$(jq -r <<<"${CLONEREFS_OPTIONS}" '.refs[0].org')" || die "invalid CLONEREFS_OPTIONS yaml"
+            repo="$(jq -r <<<"${CLONEREFS_OPTIONS}" '.refs[0].repo')" || die "invalid CLONEREFS_OPTIONS yaml"
+            if [[ "$org" == "null" ]] || [[ "$repo" == "null" ]]; then
+                die "expect: org and repo in CLONEREFS_OPTIONS.refs[0]"
+            fi
+            echo "${org}/${repo}"
+        else
+            die "Expect REPO_OWNER/NAME or CLONEREFS_OPTIONS"
+        fi
+    else
+        die "unsupported"
+    fi
+}
+
+get_commit_sha() {
+    if is_OPENSHIFT_CI; then
+        echo "${PULL_PULL_SHA:-${PULL_BASE_SHA}}"
+    elif is_GITHUB_ACTIONS; then
+        git rev-parse --verify HEAD
+    else
+        die "unsupported"
+    fi
+}
+
+pr_has_label() {
+    if [[ -z "${1:-}" ]]; then
+        die "usage: pr_has_label <expected label> [<pr details>]"
+    fi
+
+    local expected_label="$1"
+    local pr_details
+    local exitstatus=0
+    pr_details="${2:-$(get_pr_details)}" || exitstatus="$?"
+    if [[ "$exitstatus" != "0" ]]; then
+        info "Warning: checking for a label in a non PR context: details: ${pr_details}, exitstatus: ${exitstatus}"
+        return 1
+    fi
+
+    if is_openshift_CI_rehearse_PR; then
+        pr_has_label_in_body "${expected_label}" "$pr_details"
+    else
+        jq '([.labels | .[].name]  // []) | .[]' -r <<<"$pr_details" | grep -qx "${expected_label}"
+    fi
+}
+
+pr_has_label_in_body() {
+    if [[ "$#" -ne 2 ]]; then
+        die "usage: pr_has_label_in_body <expected label> <pr details>"
+    fi
+
+    local expected_label="$1"
+    local pr_details="$2"
+
+    [[ "$(jq -r '.body' <<<"$pr_details")" =~ \/label:[[:space:]]*$expected_label ]]
+}
+
+# pr_has_pragma() - returns true if a pragma exists. A pragma is a key with
+# value in the description body of a PR that influences how CI behaves.
+# e.g. /pragma gk_release_channel:rapid.
+pr_has_pragma() {
+    if [[ "$#" -ne 1 ]]; then
+        die "usage: pr_has_pragma <key>"
+    fi
+
+    local pr_details
+    if ! pr_details="$(get_pr_details)"; then
+        info "Warning: checking for a pragma in a non PR context"
+        return 0
+    fi
+
+    local key_to_check="$1"
+    [[ "$(jq -r '.body' <<<"$pr_details")" =~ \/pragma:[[:space:]]*$key_to_check: ]]
+}
+
+# pr_get_pragma() - outputs the pragma key value if it exists.
+pr_get_pragma() {
+    if [[ "$#" -ne 1 ]]; then
+        die "usage: pr_get_pragma <key>"
+    fi
+
+    local pr_details
+    if ! pr_details="$(get_pr_details)"; then
+        echo ''
+        return 0
+    fi
+
+    local key_to_check="$1"
+    while IFS= read -r line; do
+        if [[ "$line" =~ \/pragma:[[:space:]]*$key_to_check:[[:space:]]*(.+) ]]; then
+            # shellcheck disable=SC2001
+            echo "${BASH_REMATCH[1]}" | sed -e 's/[[:space:]]*$//'
+        fi
+    done <<< "$(jq -r '.body' <<<"$pr_details")"
+}
+
+# get_pr_details() from GitHub and display the result. Exits 1 if not run in CI in a PR context.
+_PR_DETAILS=""
+_PR_DETAILS_CACHE_FILE="/tmp/PR_DETAILS_CACHE.json"
+get_pr_details() {
+    local pull_request
+    local org
+    local repo
+
+    if [[ -n "${_PR_DETAILS}" ]]; then
+        echo "${_PR_DETAILS}"
+        return 0
+    fi
+    if [[ -e "${_PR_DETAILS_CACHE_FILE}" ]]; then
+        _PR_DETAILS="$(cat "${_PR_DETAILS_CACHE_FILE}")"
+        echo "${_PR_DETAILS}"
+        return 0
+    fi
+
+    _not_a_PR() {
+        echo "This does not appear to be a PR context" >&2
+        echo '{ "msg": "this is not a PR" }'
+        exit 1
+    }
+
+    if is_OPENSHIFT_CI; then
+        if [[ -n "${JOB_SPEC:-}" ]]; then
+            pull_request=$(jq -r <<<"$JOB_SPEC" '.refs.pulls[0].number')
+            org=$(jq -r <<<"$JOB_SPEC" '.refs.org')
+            repo=$(jq -r <<<"$JOB_SPEC" '.refs.repo')
+        elif [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
+            pull_request=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].pulls[0].number')
+            org=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].org')
+            repo=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].repo')
+        else
+            echo "Expect a JOB_SPEC or CLONEREFS_OPTIONS" >&2
+            exit 2
+        fi
+        [[ "${pull_request}" == "null" ]] && _not_a_PR
+    elif is_GITHUB_ACTIONS; then
+        pull_request="$(jq -r .pull_request.number "${GITHUB_EVENT_PATH}")" || _not_a_PR
+        [[ "${pull_request}" == "null" ]] && _not_a_PR
+        org="${GITHUB_REPOSITORY_OWNER}"
+        repo="${GITHUB_REPOSITORY#*/}"
+    else
+        echo "Unsupported CI" >&2
+        exit 2
+    fi
+
+    local headers url pr_details
+
+    headers=()
+    if [[ -n "${GITHUB_TOKEN:-}" ]]; then
+        headers+=(-H "Authorization: token ${GITHUB_TOKEN}")
+    fi
+
+    url="https://api.github.com/repos/${org}/${repo}/pulls/${pull_request}"
+
+    if ! pr_details=$(curl --retry 5 --retry-connrefused -sS "${headers[@]}" "${url}"); then
+        echo "Github API error: $pr_details, exit code: $?" >&2
+        exit 2
+    fi
+
+    if [[ "$(jq .id <<<"$pr_details")" == "null" ]]; then
+        # A valid PR response is expected at this point
+        echo "Invalid response from GitHub: $pr_details" >&2
+        exit 2
+    fi
+    _PR_DETAILS="$pr_details"
+    echo "$pr_details" | tee "${_PR_DETAILS_CACHE_FILE}"
+}
+
+openshift_ci_mods() {
+    info "BEGIN OpenShift CI mods"
+
+    openshift_ci_debug
+
+    info "Current Status:"
+    "$ROOT/status.sh" || true
+
+    # For ci_export(), override BASH_ENV from stackrox-test with something that is writable.
+    BASH_ENV=$(mktemp)
+    export BASH_ENV
+
+    # These are not set in the binary_build_commands or image build envs.
+    export CI=true
+    export OPENSHIFT_CI=true
+
+    if is_in_PR_context && ! is_openshift_CI_rehearse_PR; then
+        local sha
+        if [[ -n "${PULL_PULL_SHA:-}" ]]; then
+            sha="${PULL_PULL_SHA}"
+        else
+            sha=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].pulls[0].sha') || echo "WARNING: Cannot find pull sha"
+        fi
+        if [[ -n "${sha:-}" ]] && [[ "$sha" != "null" ]]; then
+            info "Will checkout SHA to match PR: $sha"
+            git checkout "$sha"
+            git submodule update
+        else
+            echo "WARNING: Could not determine a SHA for this PR, ${sha:-}"
+        fi
+    fi
+
+    # Target a tag if HEAD is tagged.
+    BUILD_TAG="$(git describe --exact-match --tags HEAD)" || echo "Warning: Cannot get tag"
+    export BUILD_TAG
+
+    # For gradle
+    export GRADLE_USER_HOME="${HOME}"
+
+    handle_nightly_runs
+
+    info "Status after mods:"
+    "$ROOT/status.sh" || true
+
+    STACKROX_BUILD_TAG=$(make --quiet --no-print-directory tag)
+    export STACKROX_BUILD_TAG
+
+    info "END OpenShift CI mods"
+}
+
+# openshift_ci_debug() - store useful state (env & git) to help debug CI.
+# NOTE: only run before any creds are imported to the environment.
+openshift_ci_debug() {
+    local debug="${ARTIFACT_DIR:-/tmp}/debug.txt"
+
+    echo "Env A-Z dump:" > "${debug}"
+    env | sort | grep -E '^[A-Z]' >> "${debug}" || true
+
+    ensure_writable_home_dir
+
+    # Prevent fatal error "detected dubious ownership in repository" from recent git.
+    git config --global --add safe.directory "$(pwd)"
+
+    echo "Git log:" >> "${debug}"
+    git log --oneline --decorate -n 20 >> "${debug}" || true
+
+    echo "Recent git refs:" >> "${debug}"
+    git for-each-ref --format='%(creatordate) %(refname)' --sort=creatordate | tail -20 >> "${debug}"
+}
+
+ensure_writable_home_dir() {
+    # Single step test jobs do not have HOME
+    if [[ -z "${HOME:-}" ]] || ! touch "${HOME}/openshift-ci-write-test"; then
+        info "HOME (${HOME:-unset}) is not set or not writeable, using mktemp dir"
+        HOME=$( mktemp -d )
+        export HOME
+        info "HOME is now $HOME"
+    fi
+}
+
+openshift_ci_import_creds() {
+    shopt -s nullglob
+    for cred in /tmp/vault/**/[A-Z]*; do
+        export "$(basename "$cred")"="$(cat "$cred")"
+    done
+}
+
+unset_namespace_env_var() {
+    # NAMESPACE is injected by OpenShift CI for the cluster that is running the
+    # tests but this can have side effects for stackrox tests due to its use as
+    # the default namespace e.g. with helm.
+    if [[ -n "${NAMESPACE:-}" ]]; then
+        export OPENSHIFT_CI_NAMESPACE="$NAMESPACE"
+        unset NAMESPACE
+    fi
+}
+
+openshift_ci_e2e_mods() {
+    unset_namespace_env_var
+
+    # The incoming KUBECONFIG is for the openshift/release cluster and not the
+    # e2e test cluster.
+    if [[ -n "${KUBECONFIG:-}" ]]; then
+        info "There is an incoming KUBECONFIG in ${KUBECONFIG}"
+        export OPENSHIFT_CI_KUBECONFIG="$KUBECONFIG"
+    fi
+    KUBECONFIG="$(mktemp)"
+    info "KUBECONFIG set: ${KUBECONFIG}"
+    export KUBECONFIG
+
+    # KUBERNETES_{PORT,SERVICE} env values interact with commandline kubectl tests
+    if env | grep -e ^KUBERNETES_; then
+        local envfile
+        envfile="$(mktemp)"
+        info "Will clear ^KUBERNETES_ env"
+        env | grep -e ^KUBERNETES_ | cut -d= -f1 | awk '{ print "unset", $1 }' > "$envfile"
+        # shellcheck disable=SC1090
+        source "$envfile"
+    fi
+
+    if [[ "${JOB_NAME:-}" =~ -eks- ]]; then
+        # Explicitly set AWS creds from the stackrox-stackrox-e2e-tests vault to
+        # override any from other vaults e.g. automation-flavors.
+        AWS_ACCESS_KEY_ID="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/AWS_ACCESS_KEY_ID)"
+        AWS_SECRET_ACCESS_KEY="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/AWS_SECRET_ACCESS_KEY)"
+        export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+        aws sts get-caller-identity | jq -r '.Arn'
+    fi
+}
+
+operator_e2e_test_setup() {
+    # TODO(ROX-11901): pass the brand explicitly from the CI config file rather than hardcode here
+    registry_ro_login "quay.io/rhacs-eng"
+    export ROX_PRODUCT_BRANDING="RHACS_BRANDING"
+
+    # $NAMESPACE is set by OpenShift CI, but confuses `operator-sdk scorecard` which runs against
+    # a completely different cluster, where this namespace does not even exist.
+    # Note that even though unsetting the variable turns out not to be sufficient for `operator-sdk scorecard`
+    # (still gets the namespace from *somewhere*), we're keeping this here as it might affect other tools.
+    unset_namespace_env_var
+}
+
+handle_nightly_runs() {
+    if ! is_OPENSHIFT_CI; then
+        die "Only for OpenShift CI"
+    fi
+
+    local nightly_tag_prefix
+    nightly_tag_prefix="$(git describe --tags --abbrev=0 --exclude '*-nightly-*')-nightly-"
+    if ! is_in_PR_context && [[ "${JOB_NAME_SAFE:-}" =~ ^nightly- ]]; then
+        ci_export BUILD_TAG "${nightly_tag_prefix}$(date '+%Y%m%d')"
+    fi
+}
+
+handle_nightly_binary_version_mismatch() {
+    if ! is_OPENSHIFT_CI; then
+        die "Only for OpenShift CI"
+    fi
+
+    if is_in_PR_context || ! [[ "${JOB_NAME_SAFE:-}" =~ ^nightly- ]]; then
+        return 0
+    fi
+
+    # JOB_NAME_SAFE is not set in test_binary_build_commands context for
+    # periodics, so the roxctl produced in that step will cause deploy.sh to
+    # fail.
+
+    if ! is_in_PR_context; then
+        info "Debug:"
+        echo "JOB_NAME: ${JOB_NAME:-}"
+        echo "JOB_NAME_SAFE: ${JOB_NAME_SAFE:-}"
+    fi
+
+    info "Correcting binary versions for nightly e2e tests"
+    echo "Current roxctl is: $(command -v roxctl || true), version: $(roxctl version || true)"
+
+    if ! [[ "$(roxctl version || true)" =~ nightly-$(date '+%Y%m%d') ]]; then
+        make cli_host-arch upgrader
+        make cli-install
+        echo "Replacement roxctl is: $(command -v roxctl || true), version: $(roxctl version || true)"
+    fi
+}
+
+store_qa_test_results() {
+    if ! is_OPENSHIFT_CI; then
+        return
+    fi
+
+    local to="${1:-qa-tests}"
+
+    info "Copying qa-tests-backend results to $to"
+
+    for test_results in qa-tests-backend/build/test-results/*; do
+        store_test_results "$test_results" "$to"
+    done
+}
+
+remove_qa_test_results() {
+    rm -rf qa-tests-backend/build/test-results
+}
+
+stored_test_results() {
+    echo "${ARTIFACT_DIR}/junit-$1"
+}
+
+store_test_results() {
+    if [[ "$#" -ne 2 ]]; then
+        die "missing args. usage: store_test_results <from> <to>"
+    fi
+
+    if ! is_OPENSHIFT_CI; then
+        return
+    fi
+
+    local from="$1"
+    local to="$2"
+
+    info "Copying test results from $from to $to"
+
+    local dest
+    dest="$(stored_test_results "$to")"
+
+    mkdir -p "$dest"
+    cp -a "$from" "$dest" || true # (best effort)
+}
+
+post_process_test_results() {
+    if [[ "$#" -ne 2 ]]; then
+        die "missing args. usage: post_process_test_results <slack attachments file.json> <summary output.json>"
+    fi
+
+    if ! is_OPENSHIFT_CI; then
+        return 0
+    fi
+
+    if [[ -z "${ARTIFACT_DIR:-}" ]]; then
+        info "ERROR: ARTIFACT_DIR is not set which is expected in openshift CI"
+        return 0
+    fi
+
+    local slack_attachments_file="$1"
+    local summary_file="$2"
+    local csv_output
+    local extra_args=()
+    local base_link
+    local calculated_base_link
+    local create_jiras
+    local jira_project="ROX"
+    local prow_job_link
+
+    set +u
+    {
+        info "Post processing junit records to JIRA issues, BigQuery metrics and Slack attachments as appropriate"
+
+        prow_job_link="$(make_prow_job_link)"
+
+        if is_in_PR_context; then
+            if pr_has_label "ci-test-junit-processing"; then
+                create_jiras="true"
+            else
+                create_jiras="false"
+            fi
+            jira_project="RS"
+        else
+            if [[ "${PULL_BASE_REF:-unknown}" =~ ^release ]]; then
+                create_jiras="false"
+            elif [[ "${JOB_NAME:-unknown}" =~ interop ]]; then
+                create_jiras="false"
+            else
+                create_jiras="true"
+            fi
+        fi
+
+        if [[ "${create_jiras}" == "false" ]]; then
+            extra_args=(--dry-run)
+            info "Will use junit2jira to create CSV for BigQuery input"
+        else
+            info "Will create JIRA issues for junit failures found in ${ARTIFACT_DIR}"
+        fi
+
+        csv_output="$(mktemp --suffix=.csv)"
+        # We need a link to repository. In case it's not part of job spec (e.g., periodic`s)
+        # we will fallback to short commit
+        base_link="$(echo "$JOB_SPEC" | jq ".refs.base_link | select( . != null )" -r)"
+        calculated_base_link="https://github.com/stackrox/stackrox/commit/$(make --quiet --no-print-directory shortcommit)"
+        curl --retry 5 --retry-connrefused -SsfL https://github.com/stackrox/junit2jira/releases/download/v0.0.24/junit2jira -o junit2jira && \
+        chmod +x junit2jira && \
+        ./junit2jira \
+            -base-link "${base_link:-$calculated_base_link}" \
+            -build-id "${BUILD_ID}" \
+            -build-link "${prow_job_link}" \
+            -build-tag "${STACKROX_BUILD_TAG}" \
+            -csv-output "${csv_output}" \
+            -jira-project "${jira_project}" \
+            -job-name "${JOB_NAME}" \
+            -junit-reports-dir "${ARTIFACT_DIR}" \
+            -orchestrator "${ORCHESTRATOR_FLAVOR:-PROW}" \
+            -threshold 10 \
+            -html-output "$ARTIFACT_DIR/junit2jira-summary.html" \
+            -slack-output "${slack_attachments_file}" \
+            -summary-output "${summary_file}" \
+            "${extra_args[@]}"
+
+        save_test_metrics "${csv_output}"
+    } || true
+    set -u
+}
+
+gate_flaky_tests() {
+    local exit_code="$1"
+
+    if [[ "${exit_code}" == "0" ]]; then
+        exit "${exit_code}"
+    fi
+
+    # Gating flaky tests is enabled only on PRs.
+    if ! is_in_PR_context; then
+        exit "${exit_code}"
+    fi
+
+    # Prepare flakechecker
+    curl --retry 5 --retry-connrefused -SsfL https://github.com/stackrox/junit2jira/releases/download/v0.0.24/flakechecker -o /tmp/flakechecker || exit "${exit_code}"
+    chmod +x /tmp/flakechecker
+    setup_gcp || echo "setup_gcp called"
+
+    # Run flakechecker for failed test
+    local config_file="${SCRIPTS_ROOT}/scripts/ci/flakechecker/flake-config.yml"
+    if /tmp/flakechecker -config-file "${config_file}" -job-name "${JOB_NAME}" -junit-reports-dir "${ARTIFACT_DIR}"; then
+      # Flakechecker exits successfully IF AND ONLY IF it finds a NON-EMPTY set of test failures in the
+      # JUnit report, AND ALL these test failures are found to be known flaky tests defined in flake-config.yml file.
+      # And the recent failure ratio for found failed tests is below threshold defined in flake-config.yml.
+      #
+      # In this case, we change the overall exit code of the job to success, hoping that the failure was only
+      # due to the tests flaky behavior (and not some other issue in the test job).
+      exit 0
+    else
+      # Flakechecker fails in case it identified test failures which do not qualify for suppression, OR
+      # in case there were no test failures at all in the JUnit report (a sign of problems elsewhere in the test job).
+      #
+      # In this case we keep the exit code as it was.
+      exit "${exit_code}"
+    fi
+}
+
+make_prow_job_link() {
+    local prow_job_link="https://prow.ci.openshift.org/view/gs/origin-ci-test/"
+    if is_in_PR_context; then
+        prow_job_link+="pr-logs/pull/stackrox_stackrox/${PULL_NUMBER}/"
+    else
+        prow_job_link+="logs/"
+    fi
+    prow_job_link+="$JOB_NAME/$BUILD_ID"
+    echo "${prow_job_link}"
+}
+
+# There are currently two openshift-ci steps where junit failures are summarized for slack.
+JOB_SLACK_FAILURE_ATTACHMENTS="${SHARED_DIR:-/tmp}/job-slack-failure-attachments.json"
+END_SLACK_FAILURE_ATTACHMENTS="/tmp/end-slack-failure-attachments.json"
+JOB_JUNIT2JIRA_SUMMARY_FILE="${SHARED_DIR:-/tmp}/job-junit2jira-summary.json"
+END_JUNIT2JIRA_SUMMARY_FILE="/tmp/end-junit2jira-summary.json"
+
+send_slack_failure_summary() {
+    if ! is_OPENSHIFT_CI || is_nightly_run; then
+        return 0
+    fi
+
+    if [[ "${PULL_BASE_REF:-unknown}" =~ ^release ]]; then
+        info "Skipping slack message for release branches"
+        return 0
+    fi
+
+    if [[ "${JOB_TYPE:-unknown}" == "periodic" ]]; then
+        info "Skipping slack message for periodics (scheduled prow jobs)"
+        return 0
+    fi
+
+    if is_system_test_without_images; then
+        # Avoid multiple slack messages from the e2e tests waiting for images.
+        info "Skipping slack message for a system test failure when images were not found"
+        return 0
+    fi
+
+    # Check env required for the job link and slack an error message when
+    # undefined.
+    _slack_check_env "BUILD_ID"
+    _slack_check_env "JOB_NAME"
+    local prow_job_link
+    prow_job_link="$(make_prow_job_link)"
+
+    local webhook_url="${TEST_FAILURES_NOTIFY_WEBHOOK}"
+
+    _slack_check_env "PULL_BASE_SHA"
+    local commit_sha="${PULL_BASE_SHA}"
+
+    if is_in_PR_context; then
+        if pr_has_label "ci-test-junit-processing"; then
+            # Send to #acs-slack-ci-integration-testing when testing the
+            # JUNIT -> Jira, BigQuery, Slack pipeline.
+            webhook_url="${SLACK_CI_INTEGRATION_TESTING_WEBHOOK}"
+            commit_sha="${PULL_PULL_SHA}"
+        else
+            info "Skipping slack message for PRs"
+            return 0
+        fi
+    fi
+
+    local org repo
+
+    if [[ -n "${JOB_SPEC:-}" ]]; then
+        org=$(jq -r <<<"$JOB_SPEC" '.refs.org')
+        repo=$(jq -r <<<"$JOB_SPEC" '.refs.repo')
+    elif [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
+        org=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].org')
+        repo=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].repo')
+    else
+        _send_slack_error "Expect a JOB_SPEC or CLONEREFS_OPTIONS"
+        return 1
+    fi
+
+    if [[ "$org" == "null" ]] || [[ "$repo" == "null" ]]; then
+        _send_slack_error "Could not determine org and/or repo"
+        return 1
+    fi
+
+    local commit_details_url="https://api.github.com/repos/${org}/${repo}/commits/${commit_sha}"
+    local exitstatus=0
+    local commit_details
+    commit_details=$(curl --retry 5 --retry-connrefused -sS "${commit_details_url}") || exitstatus="$?"
+    if [[ "$exitstatus" != "0" ]]; then
+        _send_slack_error "Cannot get commit details: ${commit_details}"
+        return 1
+    fi
+
+    _slack_check_env "JOB_NAME_SAFE"
+    local job_name="${JOB_NAME_SAFE#merge-}"
+
+    local commit_msg
+    commit_msg=$(jq -r <<<"$commit_details" '.commit.message') || exitstatus="$?"
+    commit_msg="${commit_msg%%$'\n'*}" # use first line of commit msg
+    local commit_url
+    commit_url=$(jq -r <<<"$commit_details" '.html_url') || exitstatus="$?"
+    local author_name
+    author_name=$(jq -r <<<"$commit_details" '.commit.author.name') || exitstatus="$?"
+    local author_login
+    author_login=$(jq -r <<<"$commit_details" '.author.login') || exitstatus="$?"
+    if [[ "$exitstatus" != "0" ]]; then
+        _send_slack_error "Error parsing the commit details: ${commit_details}"
+        return 1
+    fi
+
+    local mention_author=""
+    _set_mention_author
+
+    local slack_attachments=""
+    _make_slack_failure_attachments
+
+    local slack_mention=""
+    _make_slack_mention
+
+    # shellcheck disable=SC2016
+    local body='
+{
+    "text": "Prow job failure: \($job_name)",
+    "blocks": [
+        {
+            "type": "header",
+            "text": {
+                "type": "plain_text",
+                "text": "Prow job failure: \($job_name)"
+            }
+        },
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "*Commit:* <\($commit_url)|\($commit_msg)>\n*Repo:* \($repo)\n*Author:* \($author_name)\($slack_mention)\n*Log:* \($prow_job_link)"
+            }
+        },
+        {
+            "type": "context",
+            "elements": [
+                {
+                    "type": "mrkdwn",
+                    "text": "You got tagged but have no idea why or what to do? Check <https://docs.google.com/document/d/1d5ga073jkv4CO1kAJqp8MPGpC6E1bwyrCGZ7S5wKg3w/edit#heading=h.li2pdsxtk1hu|this document>."
+                }
+            ]
+        },
+        {
+            "type": "divider"
+        }
+    ],
+    "attachments": $slack_attachments
+}
+'
+
+    local payload
+    if ! payload="$(jq --null-input \
+                       --arg job_name "$job_name" \
+                       --arg commit_url "$commit_url" \
+                       --arg commit_msg "$commit_msg" \
+                       --arg repo "$repo" \
+                       --arg author_name "$author_name" \
+                       --arg slack_mention "$slack_mention" \
+                       --arg prow_job_link "$prow_job_link" \
+                       --argjson slack_attachments "$slack_attachments" \
+                       "$body")"; then
+        _send_slack_error "Error formatting slack message: [${slack_attachments}/${payload}/$?]"
+        return 1
+    fi
+
+    echo -e "About to post:\n$payload"
+
+    local post_output
+    if ! post_output="$(echo "$payload" | \
+                        curl --location --silent --show-error --fail \
+                             --data @- --header 'Content-Type: application/json' \
+                             "$webhook_url")"; then
+        _send_slack_error "Error posting to Slack: [${post_output}/$?]"
+        return 1
+    fi
+}
+
+_set_mention_author() {
+    mention_author="false"
+
+    # Mention the commit author if new JIRA issues were created
+    if [[ -f "${JOB_JUNIT2JIRA_SUMMARY_FILE}" && \
+        "$(jq -r '.newJIRAs' "${JOB_JUNIT2JIRA_SUMMARY_FILE}")" != "0" ]]; then
+        mention_author="true"
+    fi
+    if [[ -f "${END_JUNIT2JIRA_SUMMARY_FILE}" && \
+        "$(jq -r '.newJIRAs' "${END_JUNIT2JIRA_SUMMARY_FILE}")" != "0" ]]; then
+        mention_author="true"
+    fi
+}
+
+_make_slack_mention() {
+    if [[ "${mention_author}" == "true" && "${author_login}" != "dependabot[bot]" ]]; then
+        slack_mention="$("$SCRIPTS_ROOT"/scripts/ci/get-slack-user-id.sh "$author_login")"
+        if [[ -n "$slack_mention" ]]; then
+            slack_mention=", <@${slack_mention}>"
+        else
+            slack_mention=", _unable to resolve Slack user for GitHub login ${author_login}_"
+        fi
+    fi
+}
+
+_make_slack_failure_attachments() {
+    info "Converting junit failures to slack attachments"
+
+    slack_attachments=""
+    if [[ ! -f "${JOB_SLACK_FAILURE_ATTACHMENTS}" ]]; then
+        if [[ "${CREATE_CLUSTER_OUTCOME:-}" == "${OUTCOME_PASSED}" ]]; then
+            slack_attachments+="$(_make_slack_failure_plain_text_block \
+                "Could not parse junit in main test step. Check build logs for more information.")"
+        fi
+    else
+        slack_attachments+="$(cat "${JOB_SLACK_FAILURE_ATTACHMENTS}")"
+    fi
+    if [[ ! -f "${END_SLACK_FAILURE_ATTACHMENTS}" ]]; then
+        slack_attachments+="$(_make_slack_failure_plain_text_block \
+            "Could not parse junit in final test step. Check build logs for more information.")"
+    else
+        slack_attachments+="$(cat "${END_SLACK_FAILURE_ATTACHMENTS}")"
+    fi
+
+    slack_attachments="$(echo "${slack_attachments}" | jq '.[]' | jq -s '.')"
+
+    if [[ "$(echo "${slack_attachments}" | jq 'length')" == "0" ]]; then
+        msg="No junit records were found for this failure. Check build logs \
+and artifacts for more information. Consider adding an \
+issue to improve CI to detect this failure pattern. (Add a CI_Fail_Better label)."
+        slack_attachments="$(_make_slack_failure_plain_text_block "${msg}")"
+
+        # Mention the commit author when the job failed with no JUNIT records
+        mention_author="true"
+    fi
+}
+
+_make_slack_failure_block() {
+    # shellcheck disable=SC2016
+    local body='
+[
+  {
+    "color": "#bb2124",
+    "blocks": [
+      {
+        "type": "section",
+        "text": {
+          "type": "\($section_type)",
+          "text": "\($content)"
+        }
+      }
+    ]
+  }
+]
+'
+    jq --null-input \
+       --arg section_type "$1" \
+       --arg content "$2" \
+       "$body"
+}
+
+_make_slack_failure_plain_text_block() {
+    _make_slack_failure_block "plain_text" "$1"
+}
+
+_make_slack_failure_markdown_block() {
+    _make_slack_failure_block "mrkdwn" "$1"
+}
+
+_send_slack_error() {
+    echo "ERROR: $1"
+    curl -XPOST -d @- -H 'Content-Type: application/json' "${webhook_url}" << __EOM__
+{ "text": "*An error occurred dealing with a job failure:*\n\t- Job: ${prow_job_link}.\n\t- $1." }
+__EOM__
+}
+
+_slack_check_env() {
+    (
+        set +u
+        if [[ -z "$(eval echo "\$$1")" ]]; then
+            _send_slack_error "An expected environment variable is unset/empty: $1"
+            return 1
+        fi
+    )
+}
+
+slack_workflow_failure() {
+    # shellcheck disable=SC2153
+    local github_context="${GITHUB_CONTEXT}"
+    local webhook_url="${TEST_FAILURES_NOTIFY_WEBHOOK}"
+
+    if is_in_PR_context; then
+        if pr_has_label "ci-test-github-action-slack-messages"; then
+            # Send to #acs-slack-ci-integration-testing when testing.
+            webhook_url="${SLACK_CI_INTEGRATION_TESTING_WEBHOOK}"
+        else
+            info "Skipping slack message for PRs"
+            return 0
+        fi
+    fi
+
+    local workflow_name commit_msg commit_url repo author_name author_login repo_url run_id
+    workflow_name=$(jq -r <<<"${github_context}" '.workflow')
+    event_name=$(jq -r <<<"${github_context}" '.event_name')
+    if [[ "${event_name}" == "push" ]]; then
+        commit_msg=$(jq -r <<<"${github_context}" '.event.head_commit.message')
+        commit_msg="${commit_msg%%$'\n'*}" # use first line of commit msg
+        commit_url=$(jq -r <<<"${github_context}" '.event.head_commit.url')
+        author_name=$(jq -r <<<"${github_context}" '.event.head_commit.author.name')
+        author_login=$(jq -r <<<"${github_context}" '.event.head_commit.author.username')
+        repo_url=$(jq -r <<<"${github_context}" '.event.repository.url')
+    else
+        commit_msg="This is a test slack message"
+        commit_url=$(jq -r <<<"${github_context}" '.event.pull_request.diff_url')
+        author_name=$(jq -r <<<"${github_context}" '.actor')
+        author_login=$(jq -r <<<"${github_context}" '.actor')
+        repo_url=$(jq -r <<<"${github_context}" '.event.pull_request.base.repo.html_url')
+    fi
+    repo=$(jq -r <<<"${github_context}" '.repository')
+    run_id=$(jq -r <<<"${github_context}" '.run_id')
+
+    # If global "mention_author" is set use that value.
+    local mention_author="${mention_author:-true}"
+    local slack_mention=""
+    _make_slack_mention
+
+    local attachments=""
+    local job_name job_url
+    IFS=$'\n'
+    for job in $(gh run view --jq '.jobs[] | select(.conclusion == "failure")' --json 'jobs' -R "${repo}" "${run_id}" | jq -sc '.[]')
+    do
+        job_name=$(jq -r <<<"${job}" '.name')
+        job_url=$(jq -r <<<"${job}" '.url')
+        attachments+="$(_make_slack_failure_markdown_block "Job: <${job_url}|${job_name}>")"
+    done
+    attachments="$(echo "${attachments}" | jq '.[]' | jq -s '.')"
+
+    # shellcheck disable=SC2016
+    local body='
+{
+    "text": "\($workflow_name) failed.
+Commit: \($commit_msg).
+Author: \($author_name)\($slack_mention).",
+    "blocks": [
+        {
+            "type": "header",
+            "text": {
+                "type": "plain_text",
+                "text": "\($workflow_name) failed."
+            }
+        },
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": "
+*Commit:* <\($commit_url)|\($commit_msg)>.
+*Repo:* \($repo).
+*Author:* \($author_name)\($slack_mention).
+*Workflow:* \($repo_url)/actions/runs/\($run_id)"
+            }
+        },
+        {
+            "type": "divider"
+        }
+    ],
+    "attachments": $attachments
+}
+'
+
+    local payload
+    payload="$(jq --null-input \
+        --arg workflow_name "${workflow_name}" \
+        --arg commit_url "$commit_url" \
+        --arg commit_msg "$commit_msg" \
+        --arg repo "$repo" \
+        --arg author_name "$author_name" \
+        --arg slack_mention "$slack_mention" \
+        --arg repo_url "$repo_url" \
+        --arg run_id "$run_id" \
+        --argjson attachments "$attachments" \
+       "$body")"
+
+    echo -e "About to post:\n$payload"
+
+    echo "$payload" | curl --location --silent --show-error --fail \
+         --data @- --header 'Content-Type: application/json' \
+         "${webhook_url}"
+}
+
+# junit_wrap() - runs a command and creates a JUNIT record if the command
+# succeeds or fails. Some output of the command is included in the failure
+# JUNIT.
+#
+# WARNING: If this is used to wrap a bash function and not a separate binary
+# or script file there are two side effects that may not be expected:
+#
+# 1. errexit is not propagated to the function context and the function will
+#    continue on error. This is contrary to the typical approach from `set -e`
+#    used throughout this repo.
+# 2. exports are not propagated back to the calling context because the command
+#    runs in a subshell.
+
+junit_wrap() {
+    if [[ "$#" -lt 4 ]]; then
+        die "missing args. usage: junit_wrap <class> <description> <failure_message> <command> [ args ]"
+    fi
+
+    local class="$1"; shift
+    local description="$1"; shift
+    local failure_message="$1"; shift
+    local command_output_file
+    command_output_file="$(mktemp)"
+
+    if "$@" 2>&1 | tee "${command_output_file}"; then
+        save_junit_success "${class}" "${description}"
+        rm -f "${command_output_file}"
+    else
+        local ret_code="$?"
+        local failure_body=""
+        if [[ -n "$failure_message" ]]; then
+            failure_body="${failure_message}
+"
+        fi
+        failure_body="${failure_body}Command output: $(tail --bytes=512 "${command_output_file}")"
+
+        save_junit_failure "${class}" "${description}" "${failure_body}"
+        rm -f "${command_output_file}"
+
+        return ${ret_code}
+    fi
+}
+
+junit_contains_failure() {
+    local dir="$1"
+    if [[ ! -d $dir ]]; then
+        return 1
+    fi
+    # There should be few files in such dir, and they should have well-behaved names,
+    # and "return" does not mix with piping to "while read", so we use a "for" over find.
+    # shellcheck disable=SC2044
+    for f in $(find "$dir" -type f -iname '*.xml'); do
+        if grep -q '<failure ' "$f"; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+get_junit_misc_dir() {
+    echo "${ARTIFACT_DIR}/junit-misc"
+}
+
+_JUNIT_RESULT_SUCCESS="SUCCESS"
+_JUNIT_RESULT_FAILURE="FAILURE"
+_JUNIT_RESULT_SKIPPED="SKIPPED"
+
+save_junit_success() {
+    if [[ "$#" -ne 2 ]]; then
+        die "missing args. usage: save_junit_success <class> <description>"
+    fi
+
+    _save_junit_record "${_JUNIT_RESULT_SUCCESS}" "$@"
+}
+
+save_junit_failure() {
+    if [[ "$#" -ne 3 ]]; then
+        die "missing args. usage: save_junit_failure <class> <description> <details>"
+    fi
+
+    _save_junit_record "${_JUNIT_RESULT_FAILURE}" "$@"
+}
+
+save_junit_skipped() {
+    if [[ "$#" -ne 2 ]]; then
+        die "missing args. usage: save_junit_skipped <class> <description>"
+    fi
+
+    _save_junit_record "${_JUNIT_RESULT_SKIPPED}" "$@"
+}
+
+remove_junit_record() {
+    local class="$1"
+    local junit_dir
+    junit_dir="$(get_junit_misc_dir)"
+    local junit_file="${junit_dir}/junit-${class}.xml"
+    rm -f "${junit_file}"
+}
+
+_save_junit_record() {
+    local disposition="$1"
+    local class="$2"
+    local description="$3"
+    local details="${4:-}"
+
+    if [[ -z "${ARTIFACT_DIR:-}" ]]; then
+        info "Warning: save_junit_success() requires the \$ARTIFACT_DIR variable to be set"
+        return
+    fi
+
+    local junit_dir
+    junit_dir="$(get_junit_misc_dir)"
+    mkdir -p "${junit_dir}/db"
+
+    # base64 encode failure details to condense multilines
+    if [[ $details != "SUCCESS" ]]; then
+        details="$(base64 -w0 <<< "$details")"
+    fi
+
+    # record this instance
+    local record_length=3
+    local record="${junit_dir}/db/${class}.txt"
+    {
+        echo "${description}"
+        echo "${disposition}"
+        echo "${details}"
+     } >> "${record}"
+
+    local tests
+    tests=$(( "$(wc -l < "${record}")" / "${record_length}" ))
+
+    local failures=0
+    local skipped=0
+    local lines
+    readarray -t lines < "${record}"
+    while (( ${#lines[@]} ))
+    do
+        local result="${lines[1]}"
+        if [[ "${result}" == "${_JUNIT_RESULT_FAILURE}" ]]; then
+            failures=$(( failures+1 ))
+        fi
+        if [[ "${result}" == "${_JUNIT_RESULT_SKIPPED}" ]]; then
+            skipped=$(( skipped+1 ))
+        fi
+        lines=( "${lines[@]:${record_length}}" )
+    done
+
+    local junit_file="${junit_dir}/junit-${class}.xml"
+
+    cat << _EO_SUITE_HEADER_ > "${junit_file}"
+<testsuite name="${class}" tests="${tests}" skipped="${skipped}" failures="${failures}" errors="0">
+_EO_SUITE_HEADER_
+
+    readarray -t lines < "${record}"
+    while (( ${#lines[@]} ))
+    do
+        local description="${lines[0]}"
+        local result="${lines[1]}"
+        local details="${lines[2]}"
+
+        # XML escape description
+        description="${description//&/&amp;}"
+        description="${description//\"/&quot;}"
+        description="${description//\'/&#39;}"
+        description="${description//</&lt;}"
+        description="${description//>/&gt;}"
+
+        cat << _EO_CASE_HEADER_ >> "${junit_file}"
+        <testcase name="${description}" classname="${class}">
+_EO_CASE_HEADER_
+
+        if [[ "$result" == "${_JUNIT_RESULT_FAILURE}" ]]; then
+            details="$(base64 --decode <<< "$details")"
+        cat << _EO_FAILURE_ >> "${junit_file}"
+            <failure><![CDATA[${details}]]></failure>
+_EO_FAILURE_
+        fi
+        if [[ "$result" == "${_JUNIT_RESULT_SKIPPED}" ]]; then
+        cat << _EO_SKIPPED_ >> "${junit_file}"
+            <skipped/>
+_EO_SKIPPED_
+        fi
+
+        echo "        </testcase>" >> "${junit_file}"
+
+        lines=( "${lines[@]:3}" )
+    done
+
+    echo "</testsuite>" >> "${junit_file}"
+}
+
+add_build_comment_to_pr() {
+    info "Adding a comment with the build tag to the PR"
+
+    local pr_details
+    local exitstatus=0
+    pr_details="$(get_pr_details)" || exitstatus="$?"
+    if [[ "$exitstatus" != "0" ]]; then
+        echo "DEBUG: Unable to get the PR details from GitHub: $exitstatus"
+        echo "DEBUG: PR details: ${pr_details}"
+        info "Will continue without commenting on the PR"
+        return
+    fi
+
+    # hub-comment is tied to Circle CI env
+    local url
+    url=$(jq -r '.html_url' <<<"$pr_details")
+    export CIRCLE_PULL_REQUEST="$url"
+
+    local sha
+    sha=$(jq -r '.head.sha' <<<"$pr_details")
+    sha=${sha:0:7}
+    export _SHA="$sha"
+
+    local tag
+    tag=$(make tag)
+    export _TAG="$tag"
+
+    local tmpfile
+    tmpfile=$(mktemp)
+    cat > "$tmpfile" <<- EOT
+Images are ready for the commit at {{.Env._SHA}}.
+
+To use with deploy scripts, first \`export MAIN_IMAGE_TAG={{.Env._TAG}}\`.
+EOT
+
+    hub-comment -type build -template-file "$tmpfile"
+}
+
+is_system_test_without_images() {
+    case "${JOB_NAME:-missing}" in
+        *-e2e-tests|*-upgrade-tests|*-version-compatibility-tests)
+            [[ ! -f "${STATE_IMAGES_AVAILABLE}" ]]
+            ;;
+        *)
+            false
+            ;;
+    esac
+}
+
+handle_gha_tagged_build() {
+    if [[ -z "${GITHUB_REF:-}" ]]; then
+        echo "No GITHUB_REF in env"
+        exit 0
+    fi
+    echo "GITHUB_REF: ${GITHUB_REF}"
+    if [[ "${GITHUB_REF:-}" =~ ^refs/tags/ ]]; then
+        tag="${GITHUB_REF#refs/tags/*}"
+        echo "This is a tagged build: $tag"
+        echo "BUILD_TAG=$tag" >> "$GITHUB_ENV"
+    else
+        echo "This is not a tagged build"
+    fi
+}
+
+slack_prow_notice() {
+    info "Slack a notice that prow tests have started"
+
+    if [[ "$#" -lt 1 ]]; then
+        die "missing arg. usage: slack_prow_notice <tag>"
+    fi
+
+    local tag="$1"
+
+    [[ "$tag" =~ $RELEASE_RC_TAG_BASH_REGEX ]] || is_nightly_run || {
+        info "Skipping step as this is not a release, RC or nightly build"
+        return 0
+    }
+
+    local build_url
+    local webhook_url
+    if [[ "$tag" =~ $RELEASE_RC_TAG_BASH_REGEX ]]; then
+        local release
+        release="$(get_release_stream "$tag")"
+        build_url="https://prow.ci.openshift.org/?repo=stackrox%2Fstackrox&job=*release-$release*"
+        if is_release_test_stream "$tag"; then
+            # send to #acs-slack-integration-testing when testing the release process
+            webhook_url="${SLACK_MAIN_WEBHOOK}"
+        else
+            # send to #acs-release-notifications
+            webhook_url="${RELEASE_WORKFLOW_NOTIFY_WEBHOOK}"
+        fi
+    else
+        die "unexpected"
+    fi
+
+    local github_url="https://github.com/stackrox/stackrox/releases/tag/$tag"
+
+    jq -n \
+    --arg build_url "$build_url" \
+    --arg tag "$tag" \
+    --arg github_url "$github_url" \
+    '{"text": ":prow: Prow CI for tag <\($github_url)|\($tag)> started! Check the status of the tests under the following URL: \($build_url)"}' \
+| curl -XPOST -d @- -H 'Content-Type: application/json' "$webhook_url"
+}
+
+gather_debug_for_cluster_under_test() {
+    highlight_cluster_versions
+    record_cluster_info
+}
+
+highlight_cluster_versions() {
+    if [[ -z "${ARTIFACT_DIR:-}" ]]; then
+        info "No place for artifacts, skipping cluster version dump"
+        return
+    fi
+
+    artifact_file="$ARTIFACT_DIR/cluster-version.html"
+
+    cat > "$artifact_file" <<- HEAD
+<html>
+    <head>
+        <title>Cluster Versions</title>
+        <style>
+          body { color: #e8e8e8; background-color: #424242; font-family: "Roboto", "Helvetica", "Arial", sans-serif }
+          a { color: #ff8caa }
+          a:visited { color: #ff8caa }
+        </style>
+    </head>
+    <body>
+HEAD
+
+    local nodes
+    nodes="$(kubectl get nodes -o wide 2>&1 || true)"
+    local kubectl_version
+    kubectl_version="$(kubectl version -o json 2>&1 || true)"
+    local oc_version
+    oc_version="$(oc version -o json 2>&1 || true)"
+
+    cat >> "$artifact_file" << DETAILS
+      <h3>Nodes:</h3>
+      kubectl get nodes -o wide
+      <pre>$nodes</pre>
+      <h3>Versions:</h3>
+      kubectl version -o json
+      <pre>$kubectl_version</pre>
+      oc version -o json
+      <pre>$oc_version</pre>
+DETAILS
+
+    cat >> "$artifact_file" <<- FOOT
+    <br />
+    <br />
+  </body>
+</html>
+FOOT
+}
+
+record_cluster_info() {
+    _record_cluster_info || {
+        # Failure to gather metrics is not a test failure
+        info "WARNING: Recording cluster info failed"
+    }
+}
+
+_record_cluster_info() {
+    info "Record some cluster info"
+
+    # Assumes (a) there is a single cluster under test (cut_*) and (b) all nodes
+    # in the cluster are homogeneous.
+
+    # Product version. Currently used for OpenShift version. Could cover cloud
+    # provider versions for example.
+    local oc_version
+    oc_version="$(oc version -o json 2>&1 || true)"
+    local openshiftVersion
+    openshiftVersion=$(jq -r <<<"$oc_version" '.openshiftVersion')
+    set_ci_shared_export "cut_product_version" "$openshiftVersion"
+
+    # K8s version.
+    local kubectl_version
+    kubectl_version="$(kubectl version -o json 2>&1 || true)"
+    local serverGitVersion
+    serverGitVersion=$(jq -r <<<"$kubectl_version" '.serverVersion.gitVersion')
+    set_ci_shared_export "cut_k8s_version" "$serverGitVersion"
+
+    # Node info: OS, Kernel & Container Runtime.
+    local nodes
+    nodes="$(kubectl get nodes -o json 2>&1 || true)"
+    local osImage
+    osImage=$(jq -r <<<"$nodes" '.items[0].status.nodeInfo.osImage')
+    set_ci_shared_export "cut_os_image" "$osImage"
+
+    local kernelVersion
+    kernelVersion=$(jq -r <<<"$nodes" '.items[0].status.nodeInfo.kernelVersion')
+    set_ci_shared_export "cut_kernel_version" "$kernelVersion"
+
+    local containerRuntimeVersion
+    containerRuntimeVersion=$(jq -r <<<"$nodes" '.items[0].status.nodeInfo.containerRuntimeVersion')
+    set_ci_shared_export "cut_container_runtime_version" "$containerRuntimeVersion"
+}
+
+get_infra_cluster_files() {
+    local cluster_name="${1}"
+    local data_dir="/tmp/artifacts-${cluster_name}"
+    set -x
+    ls -la "${SHARED_DIR:-/tmp}" || true
+    grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
+    echo "$PATH"
+    infractl --version \
+      || { curl --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \
+        | jq -r ".result.fileChunk" \
+        | base64 -d > ./infractl;
+          file ./infractl || true;
+          install infractl ${GOPATH:-/opt}/bin/infractl || install infractl /usr/local/bin/; }
+    infractl whoami
+    for I in {1..5}; do
+        infractl artifacts ${cluster_name} \
+          --download-dir "${data_dir}"
+        if [[ -d /tmp/artifacts-${cluster_name} ]]; then
+          break
+        fi
+        printf "%s - Waiting for infra cluster [${cluster_name}]" "$(date -u)"
+        sleep 60
+    done
+    echo "$cluster_name"
+    cp "${data_dir}/kubeconfig" "${SHARED_DIR}/kubeconfig" \
+        || cp "${data_dir}/auth/kubeconfig" "${SHARED_DIR}/kubeconfig" || true
+
+    cp "${data_dir}/dotenv" "${SHARED_DIR}/dotenv" || true
+    ls -la "${SHARED_DIR:-/tmp}" || true
+    grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
+
+    export KUBECONFIG="${SHARED_DIR}/kubeconfig"
+    kubectl get nodes -o wide
+}
+
+test_on_infra() {
+    local event_json body
+    export PULL_NUMBER="${PULL_NUMBER:-${GITHUB_REF_NAME:+${GITHUB_REF_NAME%%/merge}}}"
+    echo "PULL_NUMBER=${PULL_NUMBER}"
+    export SHARED_DIR="${SHARED_DIR:-${GITHUB_WORKSPACE:+${RUNNER_TEMP}}}"
+    if ! touch "${SHARED_DIR}/file_write_test"; then
+      export SHARED_DIR="/tmp/"
+      touch "${SHARED_DIR}/file_write_test"
+    fi
+    echo "SHARED_DIR=${SHARED_DIR}"
+    REPO_NAME=${REPO_NAME:-stackrox}
+    REPO_OWNER=${REPO_OWNER:-stackrox}
+    local tries=4
+    event_json=''
+    while [[ -z "$event_json" ]]; do
+    event_json=$(set -x; curl --silent \
+      -H "X-GitHub-Api-Version: 2022-11-28" \
+      -H "Accept: application/vnd.github+json" \
+      -o - "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls/${PULL_NUMBER}")
+      if [[ $tries -gt 0 && -z "$event_json" ]]; then
+        sleep 30
+        echo $(( tries-- ))
+      else
+        break
+      fi
+    done
+    body=$(jq -r '.body' <<<"${event_json}" \
+      | tee >(cat >&2) | grep '^/test-on-infra') || return
+    while read -r cmd cluster_name job_name_match; do
+      job_name_match=${job_name_match%%]*}
+      job_name_match=${job_name_match##*[}
+      echo "Comparing job_name string '${JOB_NAME:-}' to *${job_name_match}* for infra cluster ${cluster_name}."
+      if [[ "${JOB_NAME:-}" == *"$job_name_match"* ]]; then
+        echo "Match found. Tests will run on ${cluster_name} instead of starting a new cluster."
+        echo "https://infra.rox.systems/cluster/${cluster_name}"
+        get_infra_cluster_files "${cluster_name}"
+        echo "CLUSTER_NAME=${cluster_name}" >> "${SHARED_DIR}/shared_env" || true
+        echo "KUBECONFIG=${KUBECONFIG}" >> "${SHARED_DIR}/shared_env" || true
+        break
+      fi
+    done <<<"$body"
+}
+
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+    if [[ "$#" -lt 1 ]]; then
+        die "When invoked at the command line a method is required."
+    fi
+    fn="$1"
+    shift
+    "$fn" "$@"
+fi
diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 9105e19d7b486..a613f12b66f67 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2436,8 +2436,6 @@ get_infra_cluster_files() {
     cp "${data_dir}/dotenv" "${SHARED_DIR}/dotenv" || true
     ls -la "${SHARED_DIR:-/tmp}" || true
     grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
-    grep NAME "${SHARED_DIR:-/tmp}/shared_env" || true
-    grep CLUSTER "${SHARED_DIR:-/tmp}/shared_env" || true
 
     export KUBECONFIG="${SHARED_DIR}/kubeconfig"
     kubectl get nodes -o wide

From 4d107a51a7c5ad52ace2fd239251be7207d7f8a6 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 4 Aug 2025 22:05:13 -0600
Subject: [PATCH 074/256] list api hn and port

---
 .github/workflows/e2e-tests.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index e16d5be21d61e..2b6d30c822b2c 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -77,7 +77,6 @@ jobs:
           test_on_infra
 
           echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
-          # API_HOSTNAME
 
       - run: |
           set +e
@@ -122,8 +121,8 @@ jobs:
           export GATHER_QA_TEST_DEBUG_LOGS
           echo "QA_TEST_DEBUG_LOGS:${QA_TEST_DEBUG_LOGS:-}"
           export QA_TEST_DEBUG_LOGS
-          API_HOSTNAME
-          API_PORT
+          echo "$API_HOSTNAME"
+          echo "$API_PORT"
           export CLUSTER='OPENSHIFT'
           scripts/ci/jobs/ocp_qa_e2e_tests.py \
             echo end  #|| bash -x .openshift-ci/dispatch.sh "$JOB_NAME"

From 78ca81ae7f73fa732497b1eef3ed0fd27ff8cd7d Mon Sep 17 00:00:00 2001
From: David Vail <dcvail17@gmail.com>
Date: Tue, 29 Jul 2025 10:22:47 -0400
Subject: [PATCH 075/256] Add Cypress configuration to run tests against OCP
 console

---
 .../cypress/integration-ocp/smoke.test.ts     |  9 ++++
 ui/apps/platform/package.json                 |  1 +
 ui/apps/platform/scripts/cypress-ocp.sh       | 52 +++++++++++++++++++
 3 files changed, 62 insertions(+)
 create mode 100644 ui/apps/platform/cypress/integration-ocp/smoke.test.ts
 create mode 100755 ui/apps/platform/scripts/cypress-ocp.sh

diff --git a/ui/apps/platform/cypress/integration-ocp/smoke.test.ts b/ui/apps/platform/cypress/integration-ocp/smoke.test.ts
new file mode 100644
index 0000000000000..17dcaffd3ff13
--- /dev/null
+++ b/ui/apps/platform/cypress/integration-ocp/smoke.test.ts
@@ -0,0 +1,9 @@
+import withAuth from '../helpers/basicAuth';
+
+describe('Basic connectivity to the OCP plugin', () => {
+    it('should open the OCP web console', () => {
+        cy.visit('/');
+        // TODO Handle auth/skip auth in dev
+        // TODO Handle OCP welcome modal
+    });
+});
diff --git a/ui/apps/platform/package.json b/ui/apps/platform/package.json
index e773df94bffc7..597ee304220b0 100644
--- a/ui/apps/platform/package.json
+++ b/ui/apps/platform/package.json
@@ -106,6 +106,7 @@
         "lint:fix": "eslint --fix --quiet",
         "tsc": "tsc",
         "cypress-open": "./scripts/cypress.sh open --e2e --config defaultBrowser=chrome",
+        "cypress-open:ocp": "./scripts/cypress-ocp.sh open --e2e --config defaultBrowser=chrome",
         "cypress-spec": "./scripts/cypress.sh run --spec",
         "cypress-component": "CYPRESS_COMPONENT_TEST=true ./scripts/cypress-component.sh open --component",
         "test-e2e": "TZ=UTC ./scripts/cypress.sh run --reporter mocha-multi-reporters --reporter-options configFile=cypress/mocha.config.js",
diff --git a/ui/apps/platform/scripts/cypress-ocp.sh b/ui/apps/platform/scripts/cypress-ocp.sh
new file mode 100755
index 0000000000000..4b10f5406c83b
--- /dev/null
+++ b/ui/apps/platform/scripts/cypress-ocp.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+# Opens cypress with environment variables for feature flags and auth
+CLUSTER_API_ENDPOINT="${CLUSTER_API_ENDPOINT:-http://localhost:9000}"
+API_PROXY_BASE_URL="${CLUSTER_API_ENDPOINT}/api/proxy/plugin/advanced-cluster-security/api-service"
+
+if [[ -z "$CLUSTER_USERNAME" || -z "$CLUSTER_PASSWORD" ]]; then
+    echo "CLUSTER_USERNAME and CLUSTER_PASSWORD must be set"
+    exit 1
+fi
+
+curl_cfg() { # Use built-in echo to not expose $2 in the process list.
+  echo -n "$1 = \"${2//[\"\\]/\\&}\""
+}
+
+if [[ -n "$CLUSTER_PASSWORD" ]]; then
+  readarray -t arr < <(curl -sk --config <(curl_cfg user "$CLUSTER_USERNAME:$CLUSTER_PASSWORD") "${API_PROXY_BASE_URL}"/v1/featureflags | jq -cr '.featureFlags[] | {name: .envVar, enabled: .enabled}')
+  for i in "${arr[@]}"; do
+    name=$(echo "$i" | jq -rc .name)
+    val=$(echo "$i" | jq -rc .enabled)
+    export CYPRESS_"${name}"="${val}"
+  done
+fi
+
+# eventually it should be in cypress.config.js: https://github.com/cypress-io/cypress/issues/5218
+artifacts_dir="${TEST_RESULTS_OUTPUT_DIR:-cypress/test-results}/ocp-artifacts"
+export CYPRESS_VIDEOS_FOLDER="${artifacts_dir}/videos"
+export CYPRESS_SCREENSHOTS_FOLDER="${artifacts_dir}/screenshots"
+if [[ -n "${CLUSTER_API_ENDPOINT}" ]]; then
+  export CYPRESS_BASE_URL="${CLUSTER_API_ENDPOINT}"
+fi
+
+export CYPRESS_SPEC_PATTERN='cypress/integration-ocp/**/*.test.{js,ts}'
+
+# be able to skip tests that are not relevant, for example: openshift
+export CYPRESS_ORCHESTRATOR_FLAVOR="${ORCHESTRATOR_FLAVOR}"
+
+# exit if ORCHESTRATOR_FLAVOR is not 'openshift'
+if [ "${ORCHESTRATOR_FLAVOR}" != "openshift" ]; then
+    echo "ORCHESTRATOR_FLAVOR is not 'openshift', skipping cypress-ocp"
+    exit 0
+fi
+
+if [ "$2" == "--spec" ]; then
+    if [ $# -ne 3 ]; then
+        echo "usage: npm run cypress-spec <spec-file>"
+        exit 1
+    fi
+    cypress run --spec "cypress/integration-ocp/$3"
+else
+    DEBUG="cypress*" NO_COLOR=1 cypress "$@" 2> /dev/null
+fi

From 99ffa401dadd52f03434d205a2752b9c7263b609 Mon Sep 17 00:00:00 2001
From: David Vail <dcvail17@gmail.com>
Date: Tue, 29 Jul 2025 15:05:23 -0400
Subject: [PATCH 076/256] (WIP) debug in ci

---
 ui/apps/platform/cypress/helpers/ocpAuth.ts         | 13 +++++++++++++
 .../platform/cypress/integration-ocp/smoke.test.ts  |  4 +++-
 ui/apps/platform/package.json                       |  2 +-
 ui/apps/platform/scripts/cypress-ocp.sh             |  4 ++++
 4 files changed, 21 insertions(+), 2 deletions(-)
 create mode 100644 ui/apps/platform/cypress/helpers/ocpAuth.ts

diff --git a/ui/apps/platform/cypress/helpers/ocpAuth.ts b/ui/apps/platform/cypress/helpers/ocpAuth.ts
new file mode 100644
index 0000000000000..79d1536a339fd
--- /dev/null
+++ b/ui/apps/platform/cypress/helpers/ocpAuth.ts
@@ -0,0 +1,13 @@
+export function withOcpAuth() {
+    if (Cypress.env('OCP_BRIDGE_AUTH_DISABLED')) {
+        return;
+    }
+
+    cy.session('ocp-session-auth', () => {
+        cy.visit('/');
+        cy.get('input[name="username"]').type(Cypress.env('CLUSTER_USERNAME'));
+        cy.get('input[name="password"]').type(Cypress.env('CLUSTER_PASSWORD'));
+        cy.get('button[type="submit"]').click();
+        // TODO Handle OCP welcome modal
+    });
+}
diff --git a/ui/apps/platform/cypress/integration-ocp/smoke.test.ts b/ui/apps/platform/cypress/integration-ocp/smoke.test.ts
index 17dcaffd3ff13..485390e181c8d 100644
--- a/ui/apps/platform/cypress/integration-ocp/smoke.test.ts
+++ b/ui/apps/platform/cypress/integration-ocp/smoke.test.ts
@@ -1,7 +1,9 @@
-import withAuth from '../helpers/basicAuth';
+import { withOcpAuth } from '../helpers/ocpAuth';
 
 describe('Basic connectivity to the OCP plugin', () => {
     it('should open the OCP web console', () => {
+        // withOcpAuth();
+
         cy.visit('/');
         // TODO Handle auth/skip auth in dev
         // TODO Handle OCP welcome modal
diff --git a/ui/apps/platform/package.json b/ui/apps/platform/package.json
index 597ee304220b0..862c9f3fa2834 100644
--- a/ui/apps/platform/package.json
+++ b/ui/apps/platform/package.json
@@ -109,7 +109,7 @@
         "cypress-open:ocp": "./scripts/cypress-ocp.sh open --e2e --config defaultBrowser=chrome",
         "cypress-spec": "./scripts/cypress.sh run --spec",
         "cypress-component": "CYPRESS_COMPONENT_TEST=true ./scripts/cypress-component.sh open --component",
-        "test-e2e": "TZ=UTC ./scripts/cypress.sh run --reporter mocha-multi-reporters --reporter-options configFile=cypress/mocha.config.js",
+        "test-e2e": "TZ=UTC ./scripts/cypress-ocp.sh run --reporter mocha-multi-reporters --reporter-options configFile=cypress/mocha.config.js",
         "test-e2e-local": "TZ=UTC ./scripts/cypress.sh run",
         "test-component": "TZ=UTC CYPRESS_COMPONENT_TEST=true ./scripts/cypress-component.sh run --reporter mocha-multi-reporters --reporter-options configFile=cypress/mocha.config.js --component",
         "generate-graphql-possible-types": "ROX_AUTH_TOKEN=$(./scripts/get-auth-token.sh) node scripts/generate-graphql-possible-types.js",
diff --git a/ui/apps/platform/scripts/cypress-ocp.sh b/ui/apps/platform/scripts/cypress-ocp.sh
index 4b10f5406c83b..105531147a1ff 100755
--- a/ui/apps/platform/scripts/cypress-ocp.sh
+++ b/ui/apps/platform/scripts/cypress-ocp.sh
@@ -35,6 +35,10 @@ export CYPRESS_SPEC_PATTERN='cypress/integration-ocp/**/*.test.{js,ts}'
 # be able to skip tests that are not relevant, for example: openshift
 export CYPRESS_ORCHESTRATOR_FLAVOR="${ORCHESTRATOR_FLAVOR}"
 
+export CYPRESS_OCP_BRIDGE_AUTH_DISABLED="${OCP_BRIDGE_AUTH_DISABLED}"
+export CYPRESS_CLUSTER_USERNAME="${CLUSTER_USERNAME}"
+export CYPRESS_CLUSTER_PASSWORD="${CLUSTER_PASSWORD}"
+
 # exit if ORCHESTRATOR_FLAVOR is not 'openshift'
 if [ "${ORCHESTRATOR_FLAVOR}" != "openshift" ]; then
     echo "ORCHESTRATOR_FLAVOR is not 'openshift', skipping cypress-ocp"

From 0954b4fe17a8ada6129c38aa901e55a1d4258743 Mon Sep 17 00:00:00 2001
From: David Vail <dcvail17@gmail.com>
Date: Tue, 29 Jul 2025 17:23:01 -0400
Subject: [PATCH 077/256] Test ci_export

---
 tests/e2e/lib.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 9673467510667..33cdea32e1865 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -167,6 +167,11 @@ export_test_environment() {
     ci_export REMOTE_CLUSTER_ARCH "${REMOTE_CLUSTER_ARCH:-x86_64}"
     ci_export SETUP_WORKLOAD_IDENTITIES "${SETUP_WORKLOAD_IDENTITIES:-false}"
 
+    # Expose OCP cluster information for UI e2e tests
+    ci_export CLUSTER_API_ENDPOINT "${CLUSTER_API_ENDPOINT:-none}"
+    ci_export CLUSTER_USERNAME "${CLUSTER_USERNAME:-none}"
+    ci_export CLUSTER_PASSWORD "${CLUSTER_PASSWORD:-none}"
+
     ci_export ROX_BASELINE_GENERATION_DURATION "${ROX_BASELINE_GENERATION_DURATION:-1m}"
     ci_export ROX_NETWORK_BASELINE_OBSERVATION_PERIOD "${ROX_NETWORK_BASELINE_OBSERVATION_PERIOD:-2m}"
     ci_export ROX_VULN_MGMT_UNIFIED_CVE_DEFERRAL "${ROX_VULN_MGMT_UNIFIED_CVE_DEFERRAL:-true}"

From d106d5ebddb5ddf83742cad0c335368bd2768df7 Mon Sep 17 00:00:00 2001
From: David Vail <dcvail17@gmail.com>
Date: Wed, 30 Jul 2025 07:33:00 -0400
Subject: [PATCH 078/256] Export when the vars is in scope, perhaps

---
 tests/e2e/lib.sh                        | 11 ++++++-----
 ui/apps/platform/scripts/cypress-ocp.sh |  2 ++
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 33cdea32e1865..5d9992160d416 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -167,11 +167,6 @@ export_test_environment() {
     ci_export REMOTE_CLUSTER_ARCH "${REMOTE_CLUSTER_ARCH:-x86_64}"
     ci_export SETUP_WORKLOAD_IDENTITIES "${SETUP_WORKLOAD_IDENTITIES:-false}"
 
-    # Expose OCP cluster information for UI e2e tests
-    ci_export CLUSTER_API_ENDPOINT "${CLUSTER_API_ENDPOINT:-none}"
-    ci_export CLUSTER_USERNAME "${CLUSTER_USERNAME:-none}"
-    ci_export CLUSTER_PASSWORD "${CLUSTER_PASSWORD:-none}"
-
     ci_export ROX_BASELINE_GENERATION_DURATION "${ROX_BASELINE_GENERATION_DURATION:-1m}"
     ci_export ROX_NETWORK_BASELINE_OBSERVATION_PERIOD "${ROX_NETWORK_BASELINE_OBSERVATION_PERIOD:-2m}"
     ci_export ROX_VULN_MGMT_UNIFIED_CVE_DEFERRAL "${ROX_VULN_MGMT_UNIFIED_CVE_DEFERRAL:-true}"
@@ -1491,6 +1486,12 @@ setup_automation_flavor_e2e_cluster() {
     if [[ "$ci_job" =~ ^osd ]]; then
         info "Logging in to an OSD cluster"
         source "${SHARED_DIR}/dotenv"
+
+        # Expose OCP cluster information for UI e2e tests
+        ci_export CLUSTER_API_ENDPOINT "$CLUSTER_API_ENDPOINT"
+        ci_export CLUSTER_USERNAME "$CLUSTER_USERNAME"
+        ci_export CLUSTER_PASSWORD "$CLUSTER_PASSWORD"
+
         oc login "$CLUSTER_API_ENDPOINT" \
                 --username "$CLUSTER_USERNAME" \
                 --password "$CLUSTER_PASSWORD" \
diff --git a/ui/apps/platform/scripts/cypress-ocp.sh b/ui/apps/platform/scripts/cypress-ocp.sh
index 105531147a1ff..ba25febafeb83 100755
--- a/ui/apps/platform/scripts/cypress-ocp.sh
+++ b/ui/apps/platform/scripts/cypress-ocp.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+env | sort
+
 # Opens cypress with environment variables for feature flags and auth
 CLUSTER_API_ENDPOINT="${CLUSTER_API_ENDPOINT:-http://localhost:9000}"
 API_PROXY_BASE_URL="${CLUSTER_API_ENDPOINT}/api/proxy/plugin/advanced-cluster-security/api-service"

From b8f1dac3111915ac1fefc0d414f27e060b2a7bf2 Mon Sep 17 00:00:00 2001
From: David Vail <dcvail17@gmail.com>
Date: Wed, 30 Jul 2025 09:40:53 -0400
Subject: [PATCH 079/256] Lets try this again, but for jobs that do not start
 with "osd"

---
 tests/e2e/lib.sh | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 5d9992160d416..afeb9003c3d9c 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -202,6 +202,17 @@ export_test_environment() {
         # GKE uses this network for services. Consider it as a private subnet.
         ci_export ROX_NON_AGGREGATED_NETWORKS "${ROX_NON_AGGREGATED_NETWORKS:-34.118.224.0/20}"
     fi
+
+
+    if [[ "${CI_JOB_NAME:-}" =~ ocp-4-[0-9]+-ui-e2e-tests ]]; then
+        info "Exposing OCP cluster information for UI e2e tests"
+        # Expose OCP cluster information for UI e2e tests
+        source "${SHARED_DIR}/dotenv"
+
+        ci_export CLUSTER_API_ENDPOINT "$CLUSTER_API_ENDPOINT"
+        ci_export CLUSTER_USERNAME "$CLUSTER_USERNAME"
+        ci_export CLUSTER_PASSWORD "$CLUSTER_PASSWORD"
+    fi
 }
 
 deploy_stackrox_operator() {
@@ -1487,11 +1498,6 @@ setup_automation_flavor_e2e_cluster() {
         info "Logging in to an OSD cluster"
         source "${SHARED_DIR}/dotenv"
 
-        # Expose OCP cluster information for UI e2e tests
-        ci_export CLUSTER_API_ENDPOINT "$CLUSTER_API_ENDPOINT"
-        ci_export CLUSTER_USERNAME "$CLUSTER_USERNAME"
-        ci_export CLUSTER_PASSWORD "$CLUSTER_PASSWORD"
-
         oc login "$CLUSTER_API_ENDPOINT" \
                 --username "$CLUSTER_USERNAME" \
                 --password "$CLUSTER_PASSWORD" \

From c9a2033dbf5d8b6f63486afd1b04685352bcce87 Mon Sep 17 00:00:00 2001
From: David Vail <dcvail17@gmail.com>
Date: Wed, 30 Jul 2025 11:37:16 -0400
Subject: [PATCH 080/256] One more time with feeling

---
 tests/e2e/lib.sh | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index afeb9003c3d9c..8d48e01f6224a 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -203,15 +203,18 @@ export_test_environment() {
         ci_export ROX_NON_AGGREGATED_NETWORKS "${ROX_NON_AGGREGATED_NETWORKS:-34.118.224.0/20}"
     fi
 
+    info "CI_JOB_NAME: ${CI_JOB_NAME}"
+    source "${SHARED_DIR}/dotenv"
 
+    ci_export CLUSTER_API_ENDPOINT "$CLUSTER_API_ENDPOINT"
+    ci_export CLUSTER_USERNAME "$CLUSTER_USERNAME"
+    ci_export CLUSTER_PASSWORD "$CLUSTER_PASSWORD"
+    
     if [[ "${CI_JOB_NAME:-}" =~ ocp-4-[0-9]+-ui-e2e-tests ]]; then
         info "Exposing OCP cluster information for UI e2e tests"
         # Expose OCP cluster information for UI e2e tests
-        source "${SHARED_DIR}/dotenv"
-
-        ci_export CLUSTER_API_ENDPOINT "$CLUSTER_API_ENDPOINT"
-        ci_export CLUSTER_USERNAME "$CLUSTER_USERNAME"
-        ci_export CLUSTER_PASSWORD "$CLUSTER_PASSWORD"
+     else
+        info "Not exposing OCP cluster information for UI e2e tests"
     fi
 }
 

From d9807e11208c4dc9ba53960c62937e41d5ff6975 Mon Sep 17 00:00:00 2001
From: David Vail <dcvail17@gmail.com>
Date: Mon, 4 Aug 2025 15:57:07 -0400
Subject: [PATCH 081/256] More debug

---
 .openshift-ci/dispatch.sh | 5 +++++
 tests/e2e/lib.sh          | 3 ---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.openshift-ci/dispatch.sh b/.openshift-ci/dispatch.sh
index 968ff14e0c72d..186fdefb8bea7 100755
--- a/.openshift-ci/dispatch.sh
+++ b/.openshift-ci/dispatch.sh
@@ -12,8 +12,13 @@ source "$ROOT/tests/e2e/lib.sh"
 
 set -euo pipefail
 
+echo "SHARED_DIR: ${SHARED_DIR}"
+ls -l "${SHARED_DIR}"
+cat "${SHARED_DIR:-}/*"
+
 if [[ -f "${SHARED_DIR:-}/shared_env" ]]; then
     # shellcheck disable=SC1091
+    cat "${SHARED_DIR:-}/shared_env"
     source "${SHARED_DIR:-}/shared_env"
 fi
 
diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 8d48e01f6224a..51bb24753aa95 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -203,9 +203,6 @@ export_test_environment() {
         ci_export ROX_NON_AGGREGATED_NETWORKS "${ROX_NON_AGGREGATED_NETWORKS:-34.118.224.0/20}"
     fi
 
-    info "CI_JOB_NAME: ${CI_JOB_NAME}"
-    source "${SHARED_DIR}/dotenv"
-
     ci_export CLUSTER_API_ENDPOINT "$CLUSTER_API_ENDPOINT"
     ci_export CLUSTER_USERNAME "$CLUSTER_USERNAME"
     ci_export CLUSTER_PASSWORD "$CLUSTER_PASSWORD"

From ab62f56554a4f96338f10d2a481441843a022fa9 Mon Sep 17 00:00:00 2001
From: David Vail <dcvail17@gmail.com>
Date: Tue, 5 Aug 2025 07:23:32 -0400
Subject: [PATCH 082/256] More debug

---
 .openshift-ci/dispatch.sh | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/.openshift-ci/dispatch.sh b/.openshift-ci/dispatch.sh
index 186fdefb8bea7..087fbec46908f 100755
--- a/.openshift-ci/dispatch.sh
+++ b/.openshift-ci/dispatch.sh
@@ -12,14 +12,18 @@ source "$ROOT/tests/e2e/lib.sh"
 
 set -euo pipefail
 
-echo "SHARED_DIR: ${SHARED_DIR}"
-ls -l "${SHARED_DIR}"
-cat "${SHARED_DIR:-}/*"
-
-if [[ -f "${SHARED_DIR:-}/shared_env" ]]; then
-    # shellcheck disable=SC1091
-    cat "${SHARED_DIR:-}/shared_env"
-    source "${SHARED_DIR:-}/shared_env"
+if [[ -n "${SHARED_DIR:-}" ]]; then
+    echo "SHARED_DIR: ${SHARED_DIR}"
+    ls -l "${SHARED_DIR}"
+    cat "${SHARED_DIR:-}/*"
+    if [[ -f "${SHARED_DIR}/shared_env" ]]; then
+        # shellcheck disable=SC1091
+        source "${SHARED_DIR}/shared_env"
+    else
+        echo "shared_env file does not exist in ${SHARED_DIR}"
+    fi
+else
+    echo "SHARED_DIR is not set"
 fi
 
 openshift_ci_mods

From 92d096f55a742ec5b0d553c280536f3342522ef4 Mon Sep 17 00:00:00 2001
From: David Vail <dcvail17@gmail.com>
Date: Tue, 5 Aug 2025 08:46:03 -0400
Subject: [PATCH 083/256] Update dispatch.sh

---
 .openshift-ci/dispatch.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.openshift-ci/dispatch.sh b/.openshift-ci/dispatch.sh
index 087fbec46908f..4c939d8602bcb 100755
--- a/.openshift-ci/dispatch.sh
+++ b/.openshift-ci/dispatch.sh
@@ -15,8 +15,8 @@ set -euo pipefail
 if [[ -n "${SHARED_DIR:-}" ]]; then
     echo "SHARED_DIR: ${SHARED_DIR}"
     ls -l "${SHARED_DIR}"
-    cat "${SHARED_DIR:-}/*"
     if [[ -f "${SHARED_DIR}/shared_env" ]]; then
+        cat "${SHARED_DIR:-}/shared_env"
         # shellcheck disable=SC1091
         source "${SHARED_DIR}/shared_env"
     else

From 719f480fb45cca8d95b4a22a7576543e86aa29cf Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 5 Aug 2025 12:33:36 -0600
Subject: [PATCH 084/256] use kubectl

---
 .github/workflows/e2e-tests.yaml | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 2b6d30c822b2c..19becd5af273f 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -52,12 +52,13 @@ jobs:
         with:
           gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
 
-      - run: |
-          curl --retry 5 --retry-all-errors -L -o /tmp/openshift-client-linux.tar.gz \
-            https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz
-          tar -C /tmp -xzf /tmp/openshift-client-linux.tar.gz
-          install --mode 755 /tmp/oc /usr/local/bin/
-          install --mode 755 /tmp/kubectl /usr/local/bin/
+      #- if: ${{ !env.ACT }}
+      #  run: |
+      #    curl --retry 5 --retry-all-errors -L -o /tmp/openshift-client-linux.tar.gz \
+      #      https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz
+      #    tar -C /tmp -xzf /tmp/openshift-client-linux.tar.gz
+      #    install --mode 755 /tmp/oc /usr/local/bin/
+      #    install --mode 755 /tmp/kubectl /usr/local/bin/
 
       - continue-on-error: true
         run: |
@@ -83,8 +84,8 @@ jobs:
           set -x
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
-              password="${ROX_ADMIN_PASSWORD:-$(oc -n stackrox get secret admin-pass -o json | jq -r '.data.password' | base64 -d)}"
-              password="${password:-$(oc -n stackrox get secret central-htpasswd -o go-template='{{index .data "password" | base64decode}}')}"
+              password="${ROX_ADMIN_PASSWORD:-$(kubectl -n stackrox get secret admin-pass -o json | jq -r '.data.password' | base64 -d)}"
+              password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o go-template='{{index .data "password" | base64decode}}')}"
               echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
               echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
             fi

From bac17a45a50357e696b39739bf33a448085b1cbf Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 5 Aug 2025 13:14:28 -0600
Subject: [PATCH 085/256] --amend

---
 .github/workflows/e2e-tests.yaml | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 19becd5af273f..6809eca5aabcd 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -36,7 +36,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0 #${{ github.event.pull_request.commits + 100 }}
+          fetch-depth: 0  # ${{ github.event.pull_request.commits }}
           # TODO: clone both test and tested as repo's?
           fetch-tags: false
           show-progress: false
@@ -49,17 +49,10 @@ jobs:
           echo "TAG=${MAIN_IMAGE_TAG}" >> "$GITHUB_ENV"
 
       - uses: ./.github/actions/job-preamble
+        if: ${{ !env.ACT }}
         with:
           gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
 
-      #- if: ${{ !env.ACT }}
-      #  run: |
-      #    curl --retry 5 --retry-all-errors -L -o /tmp/openshift-client-linux.tar.gz \
-      #      https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz
-      #    tar -C /tmp -xzf /tmp/openshift-client-linux.tar.gz
-      #    install --mode 755 /tmp/oc /usr/local/bin/
-      #    install --mode 755 /tmp/kubectl /usr/local/bin/
-
       - continue-on-error: true
         run: |
           echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"

From f317e014e1bb6338cbc449f1c91f0d3f5defd0bc Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 5 Aug 2025 16:14:18 -0600
Subject: [PATCH 086/256] debug ci set

---
 .github/workflows/e2e-tests.yaml | 5 +++--
 deploy/common/k8sbased.sh        | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 6809eca5aabcd..41b3fcefa640b 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -38,13 +38,14 @@ jobs:
         with:
           fetch-depth: 0  # ${{ github.event.pull_request.commits }}
           # TODO: clone both test and tested as repo's?
-          fetch-tags: false
+          #fetch-tags: false
           show-progress: false
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
       - run: |
           # example: 4.9.x-457-g9867327a52
-          MAIN_IMAGE_TAG=$(git describe --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG}")
+          set -x
+          MAIN_IMAGE_TAG=$(git describe --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG:-master}")
           echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" >> "$GITHUB_ENV"
           echo "TAG=${MAIN_IMAGE_TAG}" >> "$GITHUB_ENV"
 
diff --git a/deploy/common/k8sbased.sh b/deploy/common/k8sbased.sh
index 9484f42938405..4277eee9be2b4 100644
--- a/deploy/common/k8sbased.sh
+++ b/deploy/common/k8sbased.sh
@@ -147,6 +147,7 @@ function launch_central {
     local common_dir="${k8s_dir}/../common"
 
     verify_orch
+    echo "CI:$CI"
     if [[ -z "$CI" ]]; then
         prompt_if_central_exists "${central_namespace}"
     fi

From 9a553c007697121bf5662bd5a32f0649d84f38a1 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 5 Aug 2025 17:50:08 -0600
Subject: [PATCH 087/256] restore oc install

---
 .github/workflows/e2e-tests.yaml | 43 +++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 41b3fcefa640b..96eccdc54bbd5 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -26,6 +26,7 @@ jobs:
       REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
       REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
       GOOGLE_CREDENTIALS_GCR_SCANNER_V2: ${{ secrets.GOOGLE_CREDENTIALS_GCR_SCANNER_V2 }}
+      GCP_SERVICE_ACCOUNT_STACKROX_CI: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
       QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
       QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
       QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }}
@@ -33,6 +34,7 @@ jobs:
       TAG: ${{ inputs.tested_commit || github.event.pull_request.base.sha }}
       GATHER_QA_TEST_DEBUG_LOGS: "false"
       QA_TEST_DEBUG_LOGS: ""
+      SHARED_DIR: "/tmp"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -56,7 +58,7 @@ jobs:
 
       - continue-on-error: true
         run: |
-          echo "SHARED_DIR=${SHARED_DIR}" | tee -a "$GITHUB_ENV"
+          echo "SHARED_DIR=${SHARED_DIR:-/tmp}" | tee -a "$GITHUB_ENV"
           echo "REMOVE_EXISTING_STACKROX_RESOURCES=false" | tee -a "$GITHUB_ENV"
           export JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
           echo "JOB_NAME=${JOB_NAME}" | tee -a "$GITHUB_ENV"
@@ -68,9 +70,12 @@ jobs:
           mkdir -p "$ARTIFACT_DIR"
           echo "ARTIFACT_DIR=${ARTIFACT_DIR}" | tee -a "$GITHUB_ENV"
 
+      - run: |
           source scripts/ci/lib.sh
           test_on_infra
+          echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
 
+      - run: |
           echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
 
       - run: |
@@ -83,7 +88,7 @@ jobs:
               echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
               echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
             fi
-            if [[ -z "${API_HOSTNAME:-}" ]]; then
+            if [[ -z "${API_HOSTNAME:-}" || ( $API_HOSTNAME == 'localhost' && -z "$(pgrep -F /tmp/forward.pid)" ) ]]; then
               echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
               API_HOSTNAME=$(kubectl -n stackrox get route central -o jsonpath='{.status.ingress[0].host}')
               API_HOSTNAME=${API_HOSTNAME:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].ip}')}
@@ -93,16 +98,32 @@ jobs:
               API_PORT=${API_PORT:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].port}')}
               API_PORT=${API_PORT//https/443}
               if [[ "$API_HOSTNAME" == 'localhost' ]]; then
-                kubectl -n stackrox port-forward svc/central 8000:443 &
-                API_PORT=8000
+                nohup kubectl -n stackrox port-forward svc/central 8000:443 &
+                echo $! | tee /tmp/forward.pid
+                API_PORT=8001
               fi
               echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
               echo "API_PORT=${API_PORT}" | tee -a "$GITHUB_ENV"
             fi
           fi
 
+      - run: |
+          set +e
+          set -x
+          cat /tmp/forward.pid
+          ps -ef|grep kube
+          ps -ef|grep "$(cat /tmp/forward.pid)"
+
       - run: go get github.com/stackrox/scanner
 
+      - name: install oc cli
+        run: |
+          curl --retry 5 --retry-all-errors -L -o /tmp/openshift-client-linux.tar.gz \
+            https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz
+          tar -C /tmp -xzf /tmp/openshift-client-linux.tar.gz
+          install --mode 755 /tmp/oc /usr/local/bin/
+          install --mode 755 /tmp/kubectl /usr/local/bin/
+
       - run: |
           set -x
           echo "MAIN_IMAGE_TAG:${MAIN_IMAGE_TAG:-}"
@@ -116,8 +137,22 @@ jobs:
           export GATHER_QA_TEST_DEBUG_LOGS
           echo "QA_TEST_DEBUG_LOGS:${QA_TEST_DEBUG_LOGS:-}"
           export QA_TEST_DEBUG_LOGS
+          echo "$ROX_USERNAME"
+          echo "$ROX_ADMIN_PASSWORD"
           echo "$API_HOSTNAME"
           echo "$API_PORT"
           export CLUSTER='OPENSHIFT'
+          export SENSOR_HELM_DEPLOY="true"
+          export ROX_SCANNER_V4="false"
+          export LOCAL_DEPLOYMENT="true"
+          USE_LOCAL_ROXCTL=true
+          export CI=''
+          #unset CI && ./deploy/k8s/deploy-local.sh
+          #make -C qa-tests-backend bat-test
+          #
+          pushd qa-tests-backend
+          make compile
+          TEST_TARGET=bat-test ./gradlew testBegin testParallel testRest --fail-fast
+          exit 0
           scripts/ci/jobs/ocp_qa_e2e_tests.py \
             echo end  #|| bash -x .openshift-ci/dispatch.sh "$JOB_NAME"

From 546945b29bb75f37e827300fed154347db0d118f Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 5 Aug 2025 21:40:39 -0600
Subject: [PATCH 088/256] set timeouts

---
 .github/workflows/e2e-tests.yaml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 96eccdc54bbd5..65a195c4e11e1 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -70,7 +70,10 @@ jobs:
           mkdir -p "$ARTIFACT_DIR"
           echo "ARTIFACT_DIR=${ARTIFACT_DIR}" | tee -a "$GITHUB_ENV"
 
-      - run: |
+      - name: Infra-check
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
           source scripts/ci/lib.sh
           test_on_infra
           echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
@@ -78,7 +81,9 @@ jobs:
       - run: |
           echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
 
-      - run: |
+      - name: test
+        timeout-minutes: 35
+        run: |
           set +e
           set -x
           if [[ -e "${KUBECONFIG:-}" ]]; then

From 20ef1a2232c029b644e18bbceaff724b16bc5326 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 5 Aug 2025 21:47:58 -0600
Subject: [PATCH 089/256] only env files

---
 .github/workflows/e2e-tests.yaml | 1 -
 scripts/ci/lib.sh                | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 65a195c4e11e1..054c9b457ab08 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -117,7 +117,6 @@ jobs:
           set -x
           cat /tmp/forward.pid
           ps -ef|grep kube
-          ps -ef|grep "$(cat /tmp/forward.pid)"
 
       - run: go get github.com/stackrox/scanner
 
diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index a613f12b66f67..da43ad6c79bae 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2411,7 +2411,7 @@ get_infra_cluster_files() {
     local data_dir="/tmp/artifacts-${cluster_name}"
     set -x
     ls -la "${SHARED_DIR:-/tmp}" || true
-    grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
+    grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"*env || true
     echo "$PATH"
     infractl --version \
       || { curl --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \

From f32ed903dd892975ea9eeac12f770f4df0d0d0b1 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 5 Aug 2025 21:56:52 -0600
Subject: [PATCH 090/256] only *env files

---
 scripts/ci/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index da43ad6c79bae..f7c4f24ddaad1 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2435,7 +2435,7 @@ get_infra_cluster_files() {
 
     cp "${data_dir}/dotenv" "${SHARED_DIR}/dotenv" || true
     ls -la "${SHARED_DIR:-/tmp}" || true
-    grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
+    grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"*env || true
 
     export KUBECONFIG="${SHARED_DIR}/kubeconfig"
     kubectl get nodes -o wide

From 7624476134f7f94a56ee893d4f8f7dc608af0b3b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 5 Aug 2025 21:59:11 -0600
Subject: [PATCH 091/256] re-set CI

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 054c9b457ab08..d7982ca689f7f 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -150,7 +150,7 @@ jobs:
           export ROX_SCANNER_V4="false"
           export LOCAL_DEPLOYMENT="true"
           USE_LOCAL_ROXCTL=true
-          export CI=''
+          #export CI=''
           #unset CI && ./deploy/k8s/deploy-local.sh
           #make -C qa-tests-backend bat-test
           #

From 771305bd44f545103c06b15d702a902cf2efb773 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 12:07:54 -0600
Subject: [PATCH 092/256] install rox if not found

---
 .github/workflows/e2e-tests.yaml | 82 ++++++++++++++++++++------------
 1 file changed, 52 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index d7982ca689f7f..77ab98a3f1ebd 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -40,7 +40,7 @@ jobs:
         with:
           fetch-depth: 0  # ${{ github.event.pull_request.commits }}
           # TODO: clone both test and tested as repo's?
-          #fetch-tags: false
+          fetch-tags: false
           show-progress: false
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
@@ -58,14 +58,18 @@ jobs:
 
       - continue-on-error: true
         run: |
+          JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
+          #echo "REMOVE_EXISTING_STACKROX_RESOURCES=false" | tee -a "$GITHUB_ENV"
+
+          echo "CI=true" | tee -a "$GITHUB_ENV"
           echo "SHARED_DIR=${SHARED_DIR:-/tmp}" | tee -a "$GITHUB_ENV"
-          echo "REMOVE_EXISTING_STACKROX_RESOURCES=false" | tee -a "$GITHUB_ENV"
-          export JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
+
+          echo "CI_JOB_NAME=${JOB_NAME}" | tee -a "$GITHUB_ENV"
           echo "JOB_NAME=${JOB_NAME}" | tee -a "$GITHUB_ENV"
           echo "JOB_NAME_SAFE=qa-e2e-tests" | tee -a "$GITHUB_ENV"
-          echo "CI=true" | tee -a "$GITHUB_ENV"
+
           echo "GRADLE_USER_HOME=${HOME}" | tee -a "$GITHUB_ENV"
-          echo "CI_JOB_NAME=${JOB_NAME}" | tee -a "$GITHUB_ENV"
+
           ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/tmp/}"
           mkdir -p "$ARTIFACT_DIR"
           echo "ARTIFACT_DIR=${ARTIFACT_DIR}" | tee -a "$GITHUB_ENV"
@@ -81,10 +85,22 @@ jobs:
       - run: |
           echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
 
-      - name: test
+      - run: |
+          kubectl get namespace stackrox || exit 0
+          export DEPLOY_STACKROX_VIA_OPERATOR="false"
+          export ORCHESTRATOR_FLAVOR="openshift"
+          #export LOAD_BALANCER="lb"
+          #export SCANNER_SUPPORT="true"
+          export MONITORING_SUPPORT="false"
+          export LOCAL_DEPLOYMENT="true"
+          export ROX_BASELINE_GENERATION_DURATION="1m"
+          export ROX_NETWORK_BASELINE_OBSERVATION_PERIOD="2m"
+
+          ./deploy/k8s/deploy.sh
+
+      - name: Stackrox API access
         timeout-minutes: 35
         run: |
-          set +e
           set -x
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
@@ -105,7 +121,7 @@ jobs:
               if [[ "$API_HOSTNAME" == 'localhost' ]]; then
                 nohup kubectl -n stackrox port-forward svc/central 8000:443 &
                 echo $! | tee /tmp/forward.pid
-                API_PORT=8001
+                API_PORT=8000
               fi
               echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
               echo "API_PORT=${API_PORT}" | tee -a "$GITHUB_ENV"
@@ -129,34 +145,40 @@ jobs:
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
       - run: |
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
+
+      - timeout-minutes: 35
+        run: |
           set -x
-          echo "MAIN_IMAGE_TAG:${MAIN_IMAGE_TAG:-}"
-          echo "TAG:${TAG:-}"
+          cat "$GITHUB_ENV" || true
+
+          echo "MAIN_IMAGE_TAG: ${MAIN_IMAGE_TAG:-}"
+          echo "TAG: ${TAG:-}"
           export MAIN_IMAGE_TAG
           export TAG
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
-          export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
-          cat "$GITHUB_ENV" || true
-          echo "GATHER_QA_TEST_DEBUG_LOGS:${GATHER_QA_TEST_DEBUG_LOGS:-}"
+
+          export API_ENDPOINT="${API_HOSTNAME:-localhost}:${API_PORT:-8000}"
+          echo "API_ENDPOINT: ${API_ENDPOINT}"
+
+          #export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
+
+          echo "GATHER_QA_TEST_DEBUG_LOGS: ${GATHER_QA_TEST_DEBUG_LOGS:-}"
+          echo "QA_TEST_DEBUG_LOGS: ${QA_TEST_DEBUG_LOGS:-}"
           export GATHER_QA_TEST_DEBUG_LOGS
-          echo "QA_TEST_DEBUG_LOGS:${QA_TEST_DEBUG_LOGS:-}"
           export QA_TEST_DEBUG_LOGS
-          echo "$ROX_USERNAME"
-          echo "$ROX_ADMIN_PASSWORD"
-          echo "$API_HOSTNAME"
-          echo "$API_PORT"
+
+          echo "ROX_USERNAME: ${ROX_USERNAME:-}"
+          echo "ROX_ADMIN_PASSWORD: ${ROX_ADMIN_PASSWORD+***}"
+          echo "API_HOSTNAME: ${API_HOSTNAME:-}"
+          echo "API_PORT: ${API_PORT:-}"
+
           export CLUSTER='OPENSHIFT'
-          export SENSOR_HELM_DEPLOY="true"
+          export SENSOR_HELM_DEPLOY="false"
           export ROX_SCANNER_V4="false"
           export LOCAL_DEPLOYMENT="true"
-          USE_LOCAL_ROXCTL=true
-          #export CI=''
-          #unset CI && ./deploy/k8s/deploy-local.sh
+          export ORCHESTRATOR_FLAVOR="k8s"
+
+          make -C qa-tests-backend compile
           #make -C qa-tests-backend bat-test
-          #
-          pushd qa-tests-backend
-          make compile
-          TEST_TARGET=bat-test ./gradlew testBegin testParallel testRest --fail-fast
-          exit 0
-          scripts/ci/jobs/ocp_qa_e2e_tests.py \
-            echo end  #|| bash -x .openshift-ci/dispatch.sh "$JOB_NAME"
+
+          # TEST_TARGET=bat-test ./gradlew testBegin testParallel testRest --fail-fast

From 6209dc4997e39451e359fb2669a87d6c8ba6178b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 12:32:46 -0600
Subject: [PATCH 093/256] add curl api check

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 77ab98a3f1ebd..e8e27f5f39913 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -159,6 +159,7 @@ jobs:
 
           export API_ENDPOINT="${API_HOSTNAME:-localhost}:${API_PORT:-8000}"
           echo "API_ENDPOINT: ${API_ENDPOINT}"
+          curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.' || true
 
           #export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
 

From cd7782c5d4efa7629e4e34c8bbe700f70a2974a2 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 15:08:56 -0600
Subject: [PATCH 094/256] vars to stdout

---
 .github/workflows/e2e-tests.yaml              | 132 ++++++++++--------
 .../groovy/services/ClusterService.groovy     |   4 +-
 .../src/main/groovy/util/Env.groovy           |   2 +
 scripts/ci/lib.sh                             |  24 ++--
 4 files changed, 87 insertions(+), 75 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index e8e27f5f39913..705ce42681e33 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -35,6 +35,10 @@ jobs:
       GATHER_QA_TEST_DEBUG_LOGS: "false"
       QA_TEST_DEBUG_LOGS: ""
       SHARED_DIR: "/tmp"
+      CLUSTER: 'OPENSHIFT'
+      SENSOR_HELM_DEPLOY: "false"
+      ROX_SCANNER_V4: "false"
+      ORCHESTRATOR_FLAVOR: "openshift"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -44,62 +48,72 @@ jobs:
           show-progress: false
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
-      - run: |
-          # example: 4.9.x-457-g9867327a52
-          set -x
-          MAIN_IMAGE_TAG=$(git describe --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG:-master}")
-          echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" >> "$GITHUB_ENV"
-          echo "TAG=${MAIN_IMAGE_TAG}" >> "$GITHUB_ENV"
-
       - uses: ./.github/actions/job-preamble
         if: ${{ !env.ACT }}
         with:
           gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
 
-      - continue-on-error: true
+      - name: Check tag lookup
         run: |
-          JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
-          #echo "REMOVE_EXISTING_STACKROX_RESOURCES=false" | tee -a "$GITHUB_ENV"
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
 
-          echo "CI=true" | tee -a "$GITHUB_ENV"
-          echo "SHARED_DIR=${SHARED_DIR:-/tmp}" | tee -a "$GITHUB_ENV"
+      - name: Env variables
+        run: |
+          set -x
 
-          echo "CI_JOB_NAME=${JOB_NAME}" | tee -a "$GITHUB_ENV"
-          echo "JOB_NAME=${JOB_NAME}" | tee -a "$GITHUB_ENV"
-          echo "JOB_NAME_SAFE=qa-e2e-tests" | tee -a "$GITHUB_ENV"
+          # example: 4.9.x-457-g9867327a52
+          MAIN_IMAGE_TAG=$(git describe --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG:-master}")
 
-          echo "GRADLE_USER_HOME=${HOME}" | tee -a "$GITHUB_ENV"
+          JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
 
           ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/tmp/}"
           mkdir -p "$ARTIFACT_DIR"
-          echo "ARTIFACT_DIR=${ARTIFACT_DIR}" | tee -a "$GITHUB_ENV"
 
-      - name: Infra-check
+          tee -a "$GITHUB_ENV" <<EOF
+          MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}
+          TAG=${MAIN_IMAGE_TAG}
+
+          CI=true
+          SHARED_DIR=${SHARED_DIR:-/tmp}
+
+          CI_JOB_NAME=${JOB_NAME}
+          JOB_NAME=${JOB_NAME}
+          JOB_NAME_SAFE=qa-e2e-tests
+          GRADLE_USER_HOME=${HOME}
+
+          ARTIFACT_DIR=${ARTIFACT_DIR}
+          EOF
+
+      - name: Check if testing on Infra cluster
         timeout-minutes: 5
         continue-on-error: true
         run: |
           source scripts/ci/lib.sh
-          test_on_infra
-          echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
-
-      - run: |
+          echo "Cluster vars?"
+          test_on_infra | sed -e 's/$/# sending to env/' | tee -a "$GITHUB_ENV"
+          echo "end cluster vars?"
           echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
+          kubectl get nodes
+          # groovy tests require CLUSTER='OPENSHIFT' for ocp or ='K8S' for GKE.
+          echo "CLUSTER=$(infractl get $CLUSTER_NAME --json | jq -r '.Flavor | match("openshift")')" | tee -a "$GITHUB_ENV"
+
+      - name: Deploy Stackrox
+        if: ${{ steps.cluster-exists.outcome != 'failure' }}
+        env:
+          DEPLOY_STACKROX_VIA_OPERATOR: "false"
+          # LOAD_BALANCER: "lb"
+          # SCANNER_SUPPORT: "true"
+          MONITORING_SUPPORT: "false"
+          LOCAL_DEPLOYMENT: "true"
+          ROX_BASELINE_GENERATION_DURATION: "1m"
+          ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
+        run: |
+          kubectl get namespace stackrox \
+            || ./deploy/k8s/deploy.sh
 
-      - run: |
-          kubectl get namespace stackrox || exit 0
-          export DEPLOY_STACKROX_VIA_OPERATOR="false"
-          export ORCHESTRATOR_FLAVOR="openshift"
-          #export LOAD_BALANCER="lb"
-          #export SCANNER_SUPPORT="true"
-          export MONITORING_SUPPORT="false"
-          export LOCAL_DEPLOYMENT="true"
-          export ROX_BASELINE_GENERATION_DURATION="1m"
-          export ROX_NETWORK_BASELINE_OBSERVATION_PERIOD="2m"
-
-          ./deploy/k8s/deploy.sh
-
-      - name: Stackrox API access
-        timeout-minutes: 35
+      - name: Stackrox service and auth lookup
+        timeout-minutes: 5
+        continue-on-error: true
         run: |
           set -x
           if [[ -e "${KUBECONFIG:-}" ]]; then
@@ -127,8 +141,11 @@ jobs:
               echo "API_PORT=${API_PORT}" | tee -a "$GITHUB_ENV"
             fi
           fi
+          echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
 
-      - run: |
+      - name: Stackrox API test
+        if: ${{ steps.cluster-exists.outcome != 'failure' }}
+        run: |
           set +e
           set -x
           cat /tmp/forward.pid
@@ -138,15 +155,13 @@ jobs:
 
       - name: install oc cli
         run: |
+          # Some tests require `oc`.
           curl --retry 5 --retry-all-errors -L -o /tmp/openshift-client-linux.tar.gz \
             https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz
           tar -C /tmp -xzf /tmp/openshift-client-linux.tar.gz
           install --mode 755 /tmp/oc /usr/local/bin/
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
-      - run: |
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
-
       - timeout-minutes: 35
         run: |
           set -x
@@ -157,29 +172,22 @@ jobs:
           export MAIN_IMAGE_TAG
           export TAG
 
-          export API_ENDPOINT="${API_HOSTNAME:-localhost}:${API_PORT:-8000}"
-          echo "API_ENDPOINT: ${API_ENDPOINT}"
           curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.' || true
 
-          #export PYTHONPATH="${PYTHONPATH:-}:.openshift-ci"
-
-          echo "GATHER_QA_TEST_DEBUG_LOGS: ${GATHER_QA_TEST_DEBUG_LOGS:-}"
-          echo "QA_TEST_DEBUG_LOGS: ${QA_TEST_DEBUG_LOGS:-}"
-          export GATHER_QA_TEST_DEBUG_LOGS
-          export QA_TEST_DEBUG_LOGS
-
-          echo "ROX_USERNAME: ${ROX_USERNAME:-}"
-          echo "ROX_ADMIN_PASSWORD: ${ROX_ADMIN_PASSWORD+***}"
-          echo "API_HOSTNAME: ${API_HOSTNAME:-}"
-          echo "API_PORT: ${API_PORT:-}"
+          make -C qa-tests-backend compile
 
-          export CLUSTER='OPENSHIFT'
-          export SENSOR_HELM_DEPLOY="false"
-          export ROX_SCANNER_V4="false"
-          export LOCAL_DEPLOYMENT="true"
-          export ORCHESTRATOR_FLAVOR="k8s"
+      - timeout-minutes: 35
+        continue-on-error: true
+        run: |
+          make -C qa-tests-backend smoke-test
 
-          make -C qa-tests-backend compile
-          #make -C qa-tests-backend bat-test
+      - timeout-minutes: 35
+        continue-on-error: true
+        run: |
+          make -C qa-tests-backend bat-test
+          # TEST_TARGET=bat-test ./gradlew testBegin testParallelBAT testBAT --fail-fast
 
-          # TEST_TARGET=bat-test ./gradlew testBegin testParallel testRest --fail-fast
+      - timeout-minutes: 35
+        continue-on-error: true
+        run: |
+          make -C qa-tests-backend test
diff --git a/qa-tests-backend/src/main/groovy/services/ClusterService.groovy b/qa-tests-backend/src/main/groovy/services/ClusterService.groovy
index 62ca18ec32064..35e646338f1cd 100644
--- a/qa-tests-backend/src/main/groovy/services/ClusterService.groovy
+++ b/qa-tests-backend/src/main/groovy/services/ClusterService.groovy
@@ -12,6 +12,8 @@ import io.stackrox.proto.storage.ClusterOuterClass.Cluster
 import io.stackrox.proto.storage.ClusterOuterClass.ClusterMetadata.Type
 import io.stackrox.proto.storage.ClusterOuterClass.DynamicClusterConfig
 
+import util.Env
+
 @CompileStatic
 @Slf4j
 class ClusterService extends BaseService {
@@ -30,7 +32,7 @@ class ClusterService extends BaseService {
         return getClusterServiceClient().getCluster(Common.ResourceByID.newBuilder().setId(clusterId).build()).cluster
     }
 
-    static String getClusterId(String name = DEFAULT_CLUSTER_NAME) {
+    static String getClusterId(String name = Env.get("REMOTE_CLUSTER_NAME", DEFAULT_CLUSTER_NAME)) {
         return getClusterServiceClient().getClusters(
                 GetClustersRequest.newBuilder().setQuery("Cluster:${name}").build()
         ).clustersList.find { it.name == name }?.id
diff --git a/qa-tests-backend/src/main/groovy/util/Env.groovy b/qa-tests-backend/src/main/groovy/util/Env.groovy
index 58cde2dd3078a..7859d9643fc55 100644
--- a/qa-tests-backend/src/main/groovy/util/Env.groovy
+++ b/qa-tests-backend/src/main/groovy/util/Env.groovy
@@ -35,6 +35,8 @@ class Env {
     // the remote cluster arch can be ppc64le or s390x, default is x86_64
     static final String REMOTE_CLUSTER_ARCH = System.getenv("REMOTE_CLUSTER_ARCH") ?: "x86_64"
 
+    static final String REMOTE_CLUSTER_NAME = System.getenv("REMOTE_CLUSTER_NAME")
+
     // ONLY_SECURED_CLUSTER specifies that the remote cluster being used to execute tests
     // only has secured-cluster deployed and connects to a remote central
     static final String ONLY_SECURED_CLUSTER = System.getenv("ONLY_SECURED_CLUSTER") ?: "false"
diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index f7c4f24ddaad1..ac12610690b9b 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2450,19 +2450,19 @@ test_on_infra() {
       export SHARED_DIR="/tmp/"
       touch "${SHARED_DIR}/file_write_test"
     fi
-    echo "SHARED_DIR=${SHARED_DIR}"
+    echo "SHARED_DIR=${SHARED_DIR}" >&2
     REPO_NAME=${REPO_NAME:-stackrox}
     REPO_OWNER=${REPO_OWNER:-stackrox}
     local tries=4
     event_json=''
     while [[ -z "$event_json" ]]; do
-    event_json=$(set -x; curl --silent \
-      -H "X-GitHub-Api-Version: 2022-11-28" \
-      -H "Accept: application/vnd.github+json" \
-      -o - "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls/${PULL_NUMBER}")
+      event_json=$(set -x; curl --silent \
+        -H "X-GitHub-Api-Version: 2022-11-28" \
+        -H "Accept: application/vnd.github+json" \
+        -o - "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls/${PULL_NUMBER}")
       if [[ $tries -gt 0 && -z "$event_json" ]]; then
         sleep 30
-        echo $(( tries-- ))
+        echo $(( tries-- )) >&2
       else
         break
       fi
@@ -2472,13 +2472,13 @@ test_on_infra() {
     while read -r cmd cluster_name job_name_match; do
       job_name_match=${job_name_match%%]*}
       job_name_match=${job_name_match##*[}
-      echo "Comparing job_name string '${JOB_NAME:-}' to *${job_name_match}* for infra cluster ${cluster_name}."
+      echo "Comparing job_name string '${JOB_NAME:-}' to *${job_name_match}* for infra cluster ${cluster_name}." >&2
       if [[ "${JOB_NAME:-}" == *"$job_name_match"* ]]; then
-        echo "Match found. Tests will run on ${cluster_name} instead of starting a new cluster."
-        echo "https://infra.rox.systems/cluster/${cluster_name}"
-        get_infra_cluster_files "${cluster_name}"
-        echo "CLUSTER_NAME=${cluster_name}" >> "${SHARED_DIR}/shared_env" || true
-        echo "KUBECONFIG=${KUBECONFIG}" >> "${SHARED_DIR}/shared_env" || true
+        echo "Match found. Tests will run on ${cluster_name} instead of starting a new cluster." >&2
+        echo "https://infra.rox.systems/cluster/${cluster_name}" >&2
+        get_infra_cluster_files "${cluster_name}" >&2
+        echo "CLUSTER_NAME=${cluster_name}" | tee -a "${SHARED_DIR}/shared_env" || true
+        echo "KUBECONFIG=${KUBECONFIG}" | tee -a "${SHARED_DIR}/shared_env" || true
         break
       fi
     done <<<"$body"

From ba6e94f4934fb946f5189ae3d86a962471076b39 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 15:10:47 -0600
Subject: [PATCH 095/256] match in bash

---
 .github/workflows/e2e-tests.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 705ce42681e33..2dd60e82129d5 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -95,7 +95,10 @@ jobs:
           echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
           kubectl get nodes
           # groovy tests require CLUSTER='OPENSHIFT' for ocp or ='K8S' for GKE.
-          echo "CLUSTER=$(infractl get $CLUSTER_NAME --json | jq -r '.Flavor | match("openshift")')" | tee -a "$GITHUB_ENV"
+          if [[ "$(infractl get $CLUSTER_NAME --json | jq -r '.Flavor')" == *"openshift"* ]]; then
+            echo "CLUSTER=OPENSHIFT" | tee -a "$GITHUB_ENV"
+            # default is "K8S"
+          fi
 
       - name: Deploy Stackrox
         if: ${{ steps.cluster-exists.outcome != 'failure' }}

From 3bcfa8b1b41fc151572a13407777eb7826c8caff Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 15:15:18 -0600
Subject: [PATCH 096/256] reduce

---
 .github/workflows/e2e-tests.yaml | 9 +++------
 scripts/ci/lib.sh                | 1 -
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 2dd60e82129d5..ae1d5d23dc931 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -89,13 +89,10 @@ jobs:
         continue-on-error: true
         run: |
           source scripts/ci/lib.sh
-          echo "Cluster vars?"
-          test_on_infra | sed -e 's/$/# sending to env/' | tee -a "$GITHUB_ENV"
-          echo "end cluster vars?"
-          echo "KUBECONFIG=${KUBECONFIG:-${SHARED_DIR}/kubeconfig}" | tee -a "$GITHUB_ENV"
-          kubectl get nodes
+          test_on_infra | sed -e 's/$/  # sending to env/' | tee -a "$GITHUB_ENV"
+          set -x
           # groovy tests require CLUSTER='OPENSHIFT' for ocp or ='K8S' for GKE.
-          if [[ "$(infractl get $CLUSTER_NAME --json | jq -r '.Flavor')" == *"openshift"* ]]; then
+          if [[ "$(infractl get "$CLUSTER_NAME" --json | jq -r '.Flavor')" == *"openshift"* ]]; then
             echo "CLUSTER=OPENSHIFT" | tee -a "$GITHUB_ENV"
             # default is "K8S"
           fi
diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index ac12610690b9b..34c88ccbdcb70 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2409,7 +2409,6 @@ _record_cluster_info() {
 get_infra_cluster_files() {
     local cluster_name="${1}"
     local data_dir="/tmp/artifacts-${cluster_name}"
-    set -x
     ls -la "${SHARED_DIR:-/tmp}" || true
     grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"*env || true
     echo "$PATH"

From ab97ae9ec51922a336319a3c00ba89a0bf4b6e8e Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 15:16:30 -0600
Subject: [PATCH 097/256] allow err

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index ae1d5d23dc931..ac1786bd47046 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -115,6 +115,7 @@ jobs:
         timeout-minutes: 5
         continue-on-error: true
         run: |
+          set +e
           set -x
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then

From ceb02784921a37bef119ceb22dd21c388d96149a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 15:19:17 -0600
Subject: [PATCH 098/256] api test

---
 .github/workflows/e2e-tests.yaml | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index ac1786bd47046..5fb7bb29756b7 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -112,6 +112,7 @@ jobs:
             || ./deploy/k8s/deploy.sh
 
       - name: Stackrox service and auth lookup
+        id: stackrox-api
         timeout-minutes: 5
         continue-on-error: true
         run: |
@@ -125,7 +126,6 @@ jobs:
               echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
             fi
             if [[ -z "${API_HOSTNAME:-}" || ( $API_HOSTNAME == 'localhost' && -z "$(pgrep -F /tmp/forward.pid)" ) ]]; then
-              echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
               API_HOSTNAME=$(kubectl -n stackrox get route central -o jsonpath='{.status.ingress[0].host}')
               API_HOSTNAME=${API_HOSTNAME:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].ip}')}
               API_HOSTNAME=${API_HOSTNAME:-localhost}
@@ -145,12 +145,14 @@ jobs:
           echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
 
       - name: Stackrox API test
-        if: ${{ steps.cluster-exists.outcome != 'failure' }}
+        if: ${{ steps.stackrox-api.outcome != 'failure' }}
         run: |
-          set +e
-          set -x
-          cat /tmp/forward.pid
-          ps -ef|grep kube
+          if [[ "${API_HOSTNAME:-}" == 'localhost' && -e /tmp/forward.pid ]]; then 
+            cat /tmp/forward.pid
+            ps -ef | grep kube || true
+          fi
+
+          curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.' || true
 
       - run: go get github.com/stackrox/scanner
 
@@ -163,18 +165,9 @@ jobs:
           install --mode 755 /tmp/oc /usr/local/bin/
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
-      - timeout-minutes: 35
+      - timeout-minutes: 10
+        continue-on-error: true
         run: |
-          set -x
-          cat "$GITHUB_ENV" || true
-
-          echo "MAIN_IMAGE_TAG: ${MAIN_IMAGE_TAG:-}"
-          echo "TAG: ${TAG:-}"
-          export MAIN_IMAGE_TAG
-          export TAG
-
-          curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.' || true
-
           make -C qa-tests-backend compile
 
       - timeout-minutes: 35

From 595fd3737461e97b285f2342750044284c988b6c Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 15:44:18 -0600
Subject: [PATCH 099/256] toString cluster constant

---
 .github/workflows/e2e-tests.yaml                          | 8 ++++++++
 .../src/main/groovy/services/ClusterService.groovy        | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 5fb7bb29756b7..d8bca99c33764 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -154,6 +154,14 @@ jobs:
 
           curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.' || true
 
+      - name: Get secured cluster name
+        if: ${{ steps.stackrox-api.outcome != 'failure' }}
+        run: |
+          echo "Groovy tests expect a secured cluster. Default name is 'remote' but the Infra demo clusters set names like 'production'. Look up what secured clusters exist:"
+          set -x
+          CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
+          echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
+
       - run: go get github.com/stackrox/scanner
 
       - name: install oc cli
diff --git a/qa-tests-backend/src/main/groovy/services/ClusterService.groovy b/qa-tests-backend/src/main/groovy/services/ClusterService.groovy
index 35e646338f1cd..1bcb4910e29af 100644
--- a/qa-tests-backend/src/main/groovy/services/ClusterService.groovy
+++ b/qa-tests-backend/src/main/groovy/services/ClusterService.groovy
@@ -32,7 +32,7 @@ class ClusterService extends BaseService {
         return getClusterServiceClient().getCluster(Common.ResourceByID.newBuilder().setId(clusterId).build()).cluster
     }
 
-    static String getClusterId(String name = Env.get("REMOTE_CLUSTER_NAME", DEFAULT_CLUSTER_NAME)) {
+    static String getClusterId(String name = Env.get("REMOTE_CLUSTER_NAME", DEFAULT_CLUSTER_NAME.toString())) {
         return getClusterServiceClient().getClusters(
                 GetClustersRequest.newBuilder().setQuery("Cluster:${name}").build()
         ).clustersList.find { it.name == name }?.id

From fafb1cbcf508e8d7d478667fb3069028fb22fc5d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 15:48:40 -0600
Subject: [PATCH 100/256] separate flavor check

---
 .github/workflows/e2e-tests.yaml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index d8bca99c33764..598791a1a293e 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -85,11 +85,18 @@ jobs:
           EOF
 
       - name: Check if testing on Infra cluster
+        id: cluster-exists
         timeout-minutes: 5
         continue-on-error: true
         run: |
           source scripts/ci/lib.sh
           test_on_infra | sed -e 's/$/  # sending to env/' | tee -a "$GITHUB_ENV"
+
+      - name: Determine cluster flavor
+        if: ${{ steps.cluster-exists.outcome != 'failure' }}
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
           set -x
           # groovy tests require CLUSTER='OPENSHIFT' for ocp or ='K8S' for GKE.
           if [[ "$(infractl get "$CLUSTER_NAME" --json | jq -r '.Flavor')" == *"openshift"* ]]; then
@@ -97,7 +104,7 @@ jobs:
             # default is "K8S"
           fi
 
-      - name: Deploy Stackrox
+      - name: Find or Deploy Stackrox
         if: ${{ steps.cluster-exists.outcome != 'failure' }}
         env:
           DEPLOY_STACKROX_VIA_OPERATOR: "false"

From fbda917313c863428b205f8751c2e5489f843a01 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 15:53:57 -0600
Subject: [PATCH 101/256] remove suffix

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 598791a1a293e..252a9370a47d2 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -90,7 +90,7 @@ jobs:
         continue-on-error: true
         run: |
           source scripts/ci/lib.sh
-          test_on_infra | sed -e 's/$/  # sending to env/' | tee -a "$GITHUB_ENV"
+          test_on_infra | tee -a "$GITHUB_ENV"
 
       - name: Determine cluster flavor
         if: ${{ steps.cluster-exists.outcome != 'failure' }}

From 775752ccb9ac74017cc2486b12f4960c274dbc9f Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 15:56:44 -0600
Subject: [PATCH 102/256] install oc only for openshift cluster

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 252a9370a47d2..612e86100b46a 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -172,6 +172,7 @@ jobs:
       - run: go get github.com/stackrox/scanner
 
       - name: install oc cli
+        if: ${{ env.CLUSTER == 'OPENSHIFT' }}
         run: |
           # Some tests require `oc`.
           curl --retry 5 --retry-all-errors -L -o /tmp/openshift-client-linux.tar.gz \

From c1b677b4bc2d800b516fd1b74e587111888210f9 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 16:05:32 -0600
Subject: [PATCH 103/256] xtrace infractl install

---
 scripts/ci/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 34c88ccbdcb70..074586e00a3f2 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2413,7 +2413,7 @@ get_infra_cluster_files() {
     grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"*env || true
     echo "$PATH"
     infractl --version \
-      || { curl --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \
+      || { set -x; curl --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \
         | jq -r ".result.fileChunk" \
         | base64 -d > ./infractl;
           file ./infractl || true;

From e6b0eccf2576d03a5d86fb9e0c42490e1e49d295 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 16:19:45 -0600
Subject: [PATCH 104/256] use jq

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 612e86100b46a..e6ff37d1b3f27 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -128,7 +128,7 @@ jobs:
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
               password="${ROX_ADMIN_PASSWORD:-$(kubectl -n stackrox get secret admin-pass -o json | jq -r '.data.password' | base64 -d)}"
-              password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o go-template='{{index .data "password" | base64decode}}')}"
+              password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -r '.data.password' | base64 -d)}"
               echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
               echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
             fi

From f00ed3bc9e3bcfcb8627013b045ac127a1d66f76 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 16:34:39 -0600
Subject: [PATCH 105/256] use kubeconfig path

---
 .github/workflows/e2e-tests.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index e6ff37d1b3f27..2b9f4ea98761f 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -127,8 +127,9 @@ jobs:
           set -x
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
+              password="${ROX_ADMIN_PASSWORD:-$(find "${KUBECONFIG:%%kubeconfig}" -name admin-pass -exec cat)}"
               password="${ROX_ADMIN_PASSWORD:-$(kubectl -n stackrox get secret admin-pass -o json | jq -r '.data.password' | base64 -d)}"
-              password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -r '.data.password' | base64 -d)}"
+              password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password' | base64 -d)}"
               echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
               echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
             fi

From 681d469c6f8a5c1fc046cd3360e7b7f81e4b1ca3 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 16:35:58 -0600
Subject: [PATCH 106/256] fix operand

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 2b9f4ea98761f..f8d443e18712e 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -127,7 +127,7 @@ jobs:
           set -x
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
-              password="${ROX_ADMIN_PASSWORD:-$(find "${KUBECONFIG:%%kubeconfig}" -name admin-pass -exec cat)}"
+              password="${ROX_ADMIN_PASSWORD:-$(find "${KUBECONFIG%%kubeconfig}" -name admin-pass -exec cat)}"
               password="${ROX_ADMIN_PASSWORD:-$(kubectl -n stackrox get secret admin-pass -o json | jq -r '.data.password' | base64 -d)}"
               password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password' | base64 -d)}"
               echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"

From 9a74b1d9c39dac20603605e2d5d2a1ac3ebd7cfb Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 16:38:42 -0600
Subject: [PATCH 107/256] use shared_dir

---
 scripts/ci/lib.sh | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 074586e00a3f2..4fb8ce2def41e 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2408,10 +2408,8 @@ _record_cluster_info() {
 
 get_infra_cluster_files() {
     local cluster_name="${1}"
-    local data_dir="/tmp/artifacts-${cluster_name}"
-    ls -la "${SHARED_DIR:-/tmp}" || true
-    grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"*env || true
-    echo "$PATH"
+    local data_dir="${SHARED_DIR:-/tmp}"
+    ls -la "${data_dir}" || true
     infractl --version \
       || { set -x; curl --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \
         | jq -r ".result.fileChunk" \
@@ -2422,17 +2420,15 @@ get_infra_cluster_files() {
     for I in {1..5}; do
         infractl artifacts ${cluster_name} \
           --download-dir "${data_dir}"
-        if [[ -d /tmp/artifacts-${cluster_name} ]]; then
+        if [[ -d "${data_dir}" ]]; then
           break
         fi
         printf "%s - Waiting for infra cluster [${cluster_name}]" "$(date -u)"
         sleep 60
     done
     echo "$cluster_name"
-    cp "${data_dir}/kubeconfig" "${SHARED_DIR}/kubeconfig" \
-        || cp "${data_dir}/auth/kubeconfig" "${SHARED_DIR}/kubeconfig" || true
+    #|| cp "${data_dir}/auth/kubeconfig" "${SHARED_DIR}/kubeconfig" || true
 
-    cp "${data_dir}/dotenv" "${SHARED_DIR}/dotenv" || true
     ls -la "${SHARED_DIR:-/tmp}" || true
     grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"*env || true
 

From 63b1abc55f23711ddeecbc0d3825170cf784eb1b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 21:35:16 -0600
Subject: [PATCH 108/256] admin-password full name

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index f8d443e18712e..7fc39520156e6 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -127,7 +127,7 @@ jobs:
           set -x
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
-              password="${ROX_ADMIN_PASSWORD:-$(find "${KUBECONFIG%%kubeconfig}" -name admin-pass -exec cat)}"
+              password="${ROX_ADMIN_PASSWORD:-$(find "${KUBECONFIG%%kubeconfig}" -name admin-password -exec cat)}"
               password="${ROX_ADMIN_PASSWORD:-$(kubectl -n stackrox get secret admin-pass -o json | jq -r '.data.password' | base64 -d)}"
               password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password' | base64 -d)}"
               echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"

From 26abbde21c4e87d4d9a45cf5d9ba7c8d56160f66 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 21:53:39 -0600
Subject: [PATCH 109/256] exec syntax

---
 .github/workflows/e2e-tests.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 7fc39520156e6..7159bb418f5a9 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -105,6 +105,7 @@ jobs:
           fi
 
       - name: Find or Deploy Stackrox
+        id: rox-installed
         if: ${{ steps.cluster-exists.outcome != 'failure' }}
         env:
           DEPLOY_STACKROX_VIA_OPERATOR: "false"
@@ -120,6 +121,7 @@ jobs:
 
       - name: Stackrox service and auth lookup
         id: stackrox-api
+        if: ${{ steps.rox-installed.outcome != 'failure' }}
         timeout-minutes: 5
         continue-on-error: true
         run: |
@@ -127,9 +129,9 @@ jobs:
           set -x
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
-              password="${ROX_ADMIN_PASSWORD:-$(find "${KUBECONFIG%%kubeconfig}" -name admin-password -exec cat)}"
-              password="${ROX_ADMIN_PASSWORD:-$(kubectl -n stackrox get secret admin-pass -o json | jq -r '.data.password' | base64 -d)}"
-              password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password' | base64 -d)}"
+              password="${ROX_ADMIN_PASSWORD:-$(find "${KUBECONFIG%%kubeconfig}" -name admin-password -exec cat {} \;)}"
+              password="${ROX_ADMIN_PASSWORD:-$(kubectl -n stackrox get secret admin-pass -o json | jq -er '.data.password // ""' | base64 -d)}"
+              password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password // ""' | base64 -d)}"
               echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
               echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
             fi

From 90cccde3341b0fb052c093ef80d4c55d33a6910a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 21:57:00 -0600
Subject: [PATCH 110/256] list tmp files

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 7159bb418f5a9..409bf98984e6f 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -127,6 +127,7 @@ jobs:
         run: |
           set +e
           set -x
+          ls -latr /tmp/
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
               password="${ROX_ADMIN_PASSWORD:-$(find "${KUBECONFIG%%kubeconfig}" -name admin-password -exec cat {} \;)}"

From b467f2bedf4f39f80adb7e6edbb365dba56d679f Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 22:06:48 -0600
Subject: [PATCH 111/256] bg git fetch

---
 .github/workflows/e2e-tests.yaml | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 409bf98984e6f..f9189b8630250 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -42,12 +42,18 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # ${{ github.event.pull_request.commits }}
+          #fetch-depth: 0  # ${{ github.event.pull_request.commits }}
           # TODO: clone both test and tested as repo's?
-          fetch-tags: false
+          #fetch-tags: false
           show-progress: false
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
+      - run: |
+          nohup git fetch --quiet --depth=5000 origin +refs/tags/*:refs/tags/* >>/tmp/git.fetching.log 2>&1 &
+          echo $! | tee /tmp/git.fetching.pid
+
+      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+
       - uses: ./.github/actions/job-preamble
         if: ${{ !env.ACT }}
         with:
@@ -84,6 +90,8 @@ jobs:
           ARTIFACT_DIR=${ARTIFACT_DIR}
           EOF
 
+      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+
       - name: Check if testing on Infra cluster
         id: cluster-exists
         timeout-minutes: 5
@@ -92,6 +100,8 @@ jobs:
           source scripts/ci/lib.sh
           test_on_infra | tee -a "$GITHUB_ENV"
 
+      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+
       - name: Determine cluster flavor
         if: ${{ steps.cluster-exists.outcome != 'failure' }}
         timeout-minutes: 5
@@ -119,6 +129,8 @@ jobs:
           kubectl get namespace stackrox \
             || ./deploy/k8s/deploy.sh
 
+      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+
       - name: Stackrox service and auth lookup
         id: stackrox-api
         if: ${{ steps.rox-installed.outcome != 'failure' }}
@@ -155,6 +167,8 @@ jobs:
           fi
           echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
 
+      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+
       - name: Stackrox API test
         if: ${{ steps.stackrox-api.outcome != 'failure' }}
         run: |
@@ -165,6 +179,8 @@ jobs:
 
           curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.' || true
 
+      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+
       - name: Get secured cluster name
         if: ${{ steps.stackrox-api.outcome != 'failure' }}
         run: |

From a8e9c8a77074eaf8718381918c10bcb3aa09db4c Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 22:11:27 -0600
Subject: [PATCH 112/256] check tag lookup

---
 .github/workflows/e2e-tests.yaml | 35 ++++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index f9189b8630250..a53873b1a48b6 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -42,17 +42,20 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          #fetch-depth: 0  # ${{ github.event.pull_request.commits }}
+          fetch-depth: 1  # ${{ github.event.pull_request.commits }}
           # TODO: clone both test and tested as repo's?
-          #fetch-tags: false
+          fetch-tags: false
           show-progress: false
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
+      - name: Fetch tags
+        run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
+
       - run: |
           nohup git fetch --quiet --depth=5000 origin +refs/tags/*:refs/tags/* >>/tmp/git.fetching.log 2>&1 &
           echo $! | tee /tmp/git.fetching.pid
 
-      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
 
       - uses: ./.github/actions/job-preamble
         if: ${{ !env.ACT }}
@@ -60,6 +63,7 @@ jobs:
           gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
 
       - name: Check tag lookup
+        continue-on-error: true
         run: |
           git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
 
@@ -90,7 +94,12 @@ jobs:
           ARTIFACT_DIR=${ARTIFACT_DIR}
           EOF
 
-      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+
+      - name: Check tag lookup
+        continue-on-error: true
+        run: |
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
 
       - name: Check if testing on Infra cluster
         id: cluster-exists
@@ -100,7 +109,12 @@ jobs:
           source scripts/ci/lib.sh
           test_on_infra | tee -a "$GITHUB_ENV"
 
-      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+
+      - name: Check tag lookup
+        continue-on-error: true
+        run: |
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
 
       - name: Determine cluster flavor
         if: ${{ steps.cluster-exists.outcome != 'failure' }}
@@ -129,7 +143,12 @@ jobs:
           kubectl get namespace stackrox \
             || ./deploy/k8s/deploy.sh
 
-      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+
+      - name: Check tag lookup
+        continue-on-error: true
+        run: |
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
 
       - name: Stackrox service and auth lookup
         id: stackrox-api
@@ -167,7 +186,7 @@ jobs:
           fi
           echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
 
-      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
 
       - name: Stackrox API test
         if: ${{ steps.stackrox-api.outcome != 'failure' }}
@@ -179,7 +198,7 @@ jobs:
 
           curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.' || true
 
-      - run: ps -ef -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
 
       - name: Get secured cluster name
         if: ${{ steps.stackrox-api.outcome != 'failure' }}

From b90939cf2dc883e90e78d2914eda09225fdf4ea7 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 22:15:47 -0600
Subject: [PATCH 113/256] try 4. prefix

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index a53873b1a48b6..493f821d84ae7 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -49,7 +49,7 @@ jobs:
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
       - name: Fetch tags
-        run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
+        run: git fetch origin 'refs/tags/4.*:refs/tags/4.*'
 
       - run: |
           nohup git fetch --quiet --depth=5000 origin +refs/tags/*:refs/tags/* >>/tmp/git.fetching.log 2>&1 &

From 8997e14882bcb25b69d16010e1e6f6b8e6043d96 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 22:30:44 -0600
Subject: [PATCH 114/256] only .x

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 493f821d84ae7..2213440e28677 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -49,7 +49,7 @@ jobs:
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
       - name: Fetch tags
-        run: git fetch origin 'refs/tags/4.*:refs/tags/4.*'
+        run: git fetch origin 'refs/tags/*.x:refs/tags/*.x'
 
       - run: |
           nohup git fetch --quiet --depth=5000 origin +refs/tags/*:refs/tags/* >>/tmp/git.fetching.log 2>&1 &

From 3ebbad35dc9b75cdfd52975ff76c0483bfc664d0 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 22:31:06 -0600
Subject: [PATCH 115/256] try .x ending

---
 .github/workflows/e2e-tests.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 2213440e28677..d62b8a68a1b60 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -51,6 +51,11 @@ jobs:
       - name: Fetch tags
         run: git fetch origin 'refs/tags/*.x:refs/tags/*.x'
 
+      - name: Check tag lookup
+        continue-on-error: true
+        run: |
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
+
       - run: |
           nohup git fetch --quiet --depth=5000 origin +refs/tags/*:refs/tags/* >>/tmp/git.fetching.log 2>&1 &
           echo $! | tee /tmp/git.fetching.pid

From 0fdc65034e0693f99b73d63752afc95c21300edd Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:10:38 -0600
Subject: [PATCH 116/256] try shallow-since

---
 .github/workflows/e2e-tests.yaml |    4 +-
 3                                | 2494 ------------------------------
 2 files changed, 3 insertions(+), 2495 deletions(-)
 delete mode 100644 3

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index d62b8a68a1b60..03598f9c4d23b 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -49,7 +49,9 @@ jobs:
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
       - name: Fetch tags
-        run: git fetch origin 'refs/tags/*.x:refs/tags/*.x'
+        run: |
+          git fetch origin --jobs=2 --shallow-since=2025-01-01 'refs/tags/*.x:refs/tags/*.x' '^refs/tags/3.*'
+          git version
 
       - name: Check tag lookup
         continue-on-error: true
diff --git a/3 b/3
deleted file mode 100644
index a613f12b66f67..0000000000000
--- a/3
+++ /dev/null
@@ -1,2494 +0,0 @@
-#!/usr/bin/env bash
-
-# A library of CI related reusable bash functions
-
-SCRIPTS_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)"
-# shellcheck source=../../scripts/lib.sh
-source "$SCRIPTS_ROOT/scripts/lib.sh"
-# shellcheck source=../../scripts/ci/metrics.sh
-source "$SCRIPTS_ROOT/scripts/ci/metrics.sh"
-# shellcheck source=../../scripts/ci/test_state.sh
-source "$SCRIPTS_ROOT/scripts/ci/test_state.sh"
-# shellcheck source=../../scripts/ci/gcp.sh
-source "$SCRIPTS_ROOT/scripts/ci/gcp.sh"
-
-set -euo pipefail
-
-ensure_CI() {
-    if ! is_CI; then
-        die "A CI environment is required."
-    fi
-}
-
-ci_export() {
-    if [[ "$#" -ne 2 ]]; then
-        die "missing args. usage: ci_export <env-name> <env-value>"
-    fi
-
-    local env_name="$1"
-    local env_value="$2"
-
-    if command -v cci-export >/dev/null; then
-        cci-export "$env_name" "$env_value"
-    else
-        export "$env_name"="$env_value"
-    fi
-}
-
-# set_ci_shared_export() - for openshift-ci and GHA this is state shared between steps.
-set_ci_shared_export() {
-    if [[ "$#" -ne 2 ]]; then
-        die "missing args. usage: set_ci_shared_export <env-name> <env-value>"
-    fi
-
-    ci_export "$@"
-
-    local env_name="$1"
-    local env_value="$2"
-
-    printf 'export %s=%q\n' "${env_name}" "${env_value}" >> "${SHARED_DIR:-/tmp}/shared_env"
-    printf '%s=%q\n' "${env_name}" "${env_value}" >> "${GITHUB_ENV:-/dev/null}"
-}
-
-ci_exit_trap() {
-    local exit_code="$?"
-    info "Executing a general purpose exit trap for CI"
-    echo "Exit code is: ${exit_code}"
-
-    if [[ "${exit_code}" == "0" ]]; then
-        set_ci_shared_export JOB_DISPATCH_OUTCOME "${OUTCOME_PASSED}"
-    elif [[ "${exit_code}" == "130" ]]; then
-        set_ci_shared_export JOB_DISPATCH_OUTCOME "${OUTCOME_CANCELED}"
-    else
-        set_ci_shared_export JOB_DISPATCH_OUTCOME "${OUTCOME_FAILED}"
-    fi
-
-    post_process_test_results "${JOB_SLACK_FAILURE_ATTACHMENTS}" "${JOB_JUNIT2JIRA_SUMMARY_FILE}"
-
-    while [[ -e /tmp/hold ]]; do
-        info "Holding this job for debug"
-        sleep 60
-    done
-
-    handle_dangling_processes
-
-    gate_flaky_tests "${exit_code}"
-}
-
-# handle_dangling_processes() - The OpenShift CI ci-operator will not complete a
-# test job if there are processes remaining that were started by the job. While
-# processes _should_ be cleaned up by their creators it is common that some are
-# not, so this exists as a fail safe.
-handle_dangling_processes() {
-    info "Process state at exit:"
-    ps -e -O ppid
-
-    local psline this_pid pid
-    ps -e -O ppid | while read -r psline; do
-        # trim leading whitespace
-        psline="$(echo "$psline" | xargs)"
-        if [[ "$psline" =~ ^PID ]]; then
-            # Ignoring header
-            continue
-        fi
-        this_pid="$$"
-        if [[ "$psline" =~ ^$this_pid ]]; then
-            echo "Ignoring self: $psline"
-            continue
-        fi
-        # shellcheck disable=SC1087
-        if [[ "$psline" =~ [[:space:]]$this_pid[[:space:]] ]]; then
-            echo "Ignoring child: $psline"
-            continue
-        fi
-        if [[ "$psline" =~ entrypoint|defunct ]]; then
-            echo "Ignoring ci-operator entrypoint or defunct process: $psline"
-            continue
-        fi
-        echo "A candidate to kill: $psline"
-        pid="$(echo "$psline" | cut -d' ' -f1)"
-        echo "Will kill $pid"
-        kill "$pid" || {
-            echo "Error killing $pid"
-        }
-    done
-}
-
-create_exit_trap() {
-    trap ci_exit_trap EXIT
-}
-
-setup_deployment_env() {
-    info "Setting up the deployment environment"
-
-    if [[ "$#" -ne 2 ]]; then
-        die "missing args. usage: setup_deployment_env <docker-login> <use-websocket>"
-    fi
-
-    local docker_login="$1"
-    local use_websocket="$2"
-
-    if [[ "$docker_login" == "true" ]]; then
-        registry_ro_login "quay.io/rhacs-eng"
-    fi
-
-    if [[ "$use_websocket" == "true" ]]; then
-        ci_export CLUSTER_API_ENDPOINT "wss://central.stackrox:443"
-    fi
-
-    ci_export REGISTRY_USERNAME "$QUAY_RHACS_ENG_RO_USERNAME"
-    ci_export REGISTRY_PASSWORD "$QUAY_RHACS_ENG_RO_PASSWORD"
-    if [[ -z "${MAIN_IMAGE_TAG:-}" ]]; then
-        ci_export MAIN_IMAGE_TAG "$(make --quiet --no-print-directory tag)"
-    fi
-
-    ci_export ROX_PRODUCT_BRANDING "RHACS_BRANDING"
-}
-
-get_central_debug_dump() {
-    info "Getting a central debug dump"
-
-    if [[ "$#" -ne 1 ]]; then
-        die "missing arg. usage: get_central_debug_dump <output_dir>"
-    fi
-
-    local output_dir="$1"
-
-    require_environment "API_ENDPOINT"
-    require_environment "ROX_ADMIN_PASSWORD"
-    roxctl -e "${API_ENDPOINT}" --ca "" --insecure-skip-tls-verify \
-        central debug dump --output-dir "${output_dir}"
-    ls -l "${output_dir}"
-}
-
-process_central_metrics() {
-    info "Processing metrics from central debug dump"
-
-    if [[ "$#" -ne 1 ]]; then
-        die "missing arg. usage: process_central_metrics <debug_dump_dir>"
-    fi
-
-    local output_dir="$1"
-
-    local metrics_output
-    local csv_output
-    local debug_dump_zip
-    metrics_output="$(mktemp --suffix=.prom)"
-    csv_output="$(mktemp --suffix=.csv)"
-    # shellcheck disable=SC2012
-    debug_dump_zip="$(ls -t "${output_dir}"/*.zip | head -1)"
-    unzip -p "${debug_dump_zip}" metrics-1 > "${metrics_output}"
-
-    get_prometheus_metrics_parser
-
-    # We need a link to repository. In case it's not part of job spec (e.g., periodic`s)
-    # we will fallback to short commit
-    base_link="$(echo "$JOB_SPEC" | jq ".refs.base_link | select( . != null )" -r)"
-    calculated_base_link="https://github.com/stackrox/stackrox/commit/$(make --quiet --no-print-directory shortcommit)"
-
-    local metadata="build_tag=${STACKROX_BUILD_TAG:-none},build_id=${BUILD_ID:-none},orchestrator_flavor=${ORCHESTRATOR_FLAVOR:-PROW},job_name=${JOB_NAME:-missing},base_link=${base_link:-$calculated_base_link}"
-    prometheus-metric-parser single \
-        --format csv \
-        --file "${metrics_output}" \
-        --labels "${metadata}" \
-        > "${csv_output}"
-
-    setup_gcp
-    save_central_metrics "${csv_output}"
-}
-
-get_prometheus_metrics_parser() {
-    local parserBin
-    local parserDir
-    parserBin=$(make prometheus-metric-parser -C "$ROOT" --silent | tail -1)
-    parserDir=$(dirname "${parserBin}")
-    export PATH="$parserDir":$PATH
-    prometheus-metric-parser help
-}
-
-get_central_diagnostics() {
-    info "Getting central diagnostics"
-
-    if [[ "$#" -ne 1 ]]; then
-        die "missing arg. usage: get_central_diagnostics <output_dir>"
-    fi
-
-    local output_dir="$1"
-
-    require_environment "API_ENDPOINT"
-    require_environment "ROX_ADMIN_PASSWORD"
-    roxctl -e "${API_ENDPOINT}" --ca "" --insecure-skip-tls-verify \
-        central debug download-diagnostics --output-dir "${output_dir}"
-    ls -l "${output_dir}"
-}
-
-push_image_manifest_lists() {
-    info "Pushing main, roxctl and central-db images as manifest lists"
-
-    if [[ "$#" -ne 3 ]]; then
-        die "missing arg. usage: push_image_manifest_lists <push_context> <brand> <architectures (CSV)>"
-    fi
-
-    local push_context="$1"
-    local brand="$2"
-    local architectures="$3"
-
-    local main_image_set=("main" "roxctl" "central-db")
-
-    local registry
-    registry="$(registry_from_branding "$brand")"
-
-    local tag
-    tag="$(make --quiet --no-print-directory tag)"
-
-    registry_rw_login "$registry"
-    for image in "${main_image_set[@]}"; do
-        retry 5 true \
-          "$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:${tag}" "$architectures" | cat
-        if [[ "$push_context" == "merge-to-master" ]]; then
-            retry 5 true \
-              "$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:latest" "$architectures" | cat
-        fi
-    done
-
-    # Push manifest lists for scanner and collector for amd64 only
-    local amd64_image_set=("scanner" "scanner-db" "scanner-slim" "scanner-db-slim" "collector")
-    for image in "${amd64_image_set[@]}"; do
-        retry 5 true \
-          "$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:${tag}" "amd64" | cat
-    done
-}
-
-registry_from_branding() {
-    local branding="$1"
-    if [[ "$branding" == "STACKROX_BRANDING" ]]; then
-        registry="quay.io/stackrox-io"
-    elif [[ "$branding" == "RHACS_BRANDING" ]]; then
-        registry="quay.io/rhacs-eng"
-    else
-        die "$branding is not a supported branding"
-    fi
-    echo "$registry"
-}
-
-push_main_image_set() {
-    info "Pushing main, roxctl and central-db images"
-
-    if [[ "$#" -ne 3 ]]; then
-        die "missing arg. usage: push_main_image_set <push_context> <brand> <arch>"
-    fi
-
-    local push_context="$1"
-    local brand="$2"
-    local arch="$3"
-
-    local main_image_set=("main" "roxctl" "central-db")
-
-    _push_main_image_set() {
-        local registry="$1"
-        local tag="$2"
-
-        for image in "${main_image_set[@]}"; do
-            retry 5 true \
-              docker push "${registry}/${image}:${tag}" | cat
-        done
-    }
-
-    _tag_main_image_set() {
-        local local_tag="$1"
-        local registry="$2"
-        local remote_tag="$3"
-
-        for image in "${main_image_set[@]}"; do
-            docker tag "stackrox/${image}:${local_tag}" "${registry}/${image}:${remote_tag}"
-        done
-    }
-
-    local registry
-    registry="$(registry_from_branding "$brand")"
-
-    local tag
-    tag="$(make --quiet --no-print-directory tag)"
-
-    registry_rw_login "$registry"
-
-    _tag_main_image_set "$tag" "$registry" "$tag-$arch"
-    _push_main_image_set "$registry" "$tag-$arch"
-
-    if [[ "$push_context" == "merge-to-master" ]]; then
-        _tag_main_image_set "$tag" "$registry" "latest-${arch}"
-        _push_main_image_set "$registry" "latest-${arch}"
-    fi
-}
-
-push_scanner_image_manifest_lists() {
-    info "Pushing scanner-v4 and scanner-v4-db images as manifest lists"
-
-    if [[ "$#" -ne 2 ]]; then
-        die "missing arg. usage: push_scanner_image_manifest_lists <registry> <architectures (CSV)>"
-    fi
-
-    local registry="$1"
-    local architectures="$2"
-    local scanner_image_set=("scanner-v4" "scanner-v4-db")
-
-    local tag
-    tag="$(make --quiet --no-print-directory -C scanner tag)"
-    registry_rw_login "$registry"
-    for image in "${scanner_image_set[@]}"; do
-        retry 5 true \
-          "$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:${tag}" "$architectures" | cat
-    done
-}
-
-push_scanner_image_set() {
-    info "Pushing scanner-v4 and scanner-v4-db images"
-
-    if [[ "$#" -ne 2 ]]; then
-        die "missing arg. usage: push_scanner_image_set <registry> <arch>"
-    fi
-
-    local registry="$1"
-    local arch="$2"
-
-    local scanner_image_set=("scanner-v4" "scanner-v4-db")
-
-    _push_scanner_image_set() {
-        local registry="$1"
-        local tag="$2"
-
-        for image in "${scanner_image_set[@]}"; do
-            retry 5 true \
-              docker push "${registry}/${image}:${tag}" | cat
-        done
-    }
-
-    _tag_scanner_image_set() {
-        local local_tag="$1"
-        local registry="$2"
-        local remote_tag="$3"
-
-        for image in "${scanner_image_set[@]}"; do
-            docker tag "stackrox/${image}:${local_tag}" "${registry}/${image}:${remote_tag}"
-        done
-    }
-
-    local tag
-    tag="$(make --quiet --no-print-directory -C scanner tag)"
-
-    registry_rw_login "$registry"
-
-    _tag_scanner_image_set "$tag" "$registry" "$tag-$arch"
-    _push_scanner_image_set "$registry" "$tag-$arch"
-}
-
-registry_rw_login() {
-    if [[ "$#" -ne 1 ]]; then
-        die "missing arg. usage: registry_rw_login <registry>"
-    fi
-
-    local registry="$1"
-
-    case "$registry" in
-        quay.io/rhacs-eng)
-            _login() {
-                # shellcheck disable=SC2317
-                docker login -u "$QUAY_RHACS_ENG_RW_USERNAME" --password-stdin <<<"$QUAY_RHACS_ENG_RW_PASSWORD" quay.io
-            }
-            retry 5 true _login
-            ;;
-        quay.io/stackrox-io)
-            _login() {
-                # shellcheck disable=SC2317
-                docker login -u "$QUAY_STACKROX_IO_RW_USERNAME" --password-stdin <<<"$QUAY_STACKROX_IO_RW_PASSWORD" quay.io
-            }
-            retry 5 true _login
-            ;;
-        *)
-            die "Unsupported registry login: $registry"
-    esac
-}
-
-registry_ro_login() {
-    if [[ "$#" -ne 1 ]]; then
-        die "missing arg. usage: registry_ro_login <registry>"
-    fi
-
-    local registry="$1"
-
-    case "$registry" in
-        quay.io/rhacs-eng)
-            _login() {
-                # shellcheck disable=SC2317
-                docker login -u "$QUAY_RHACS_ENG_RO_USERNAME" --password-stdin <<<"$QUAY_RHACS_ENG_RO_PASSWORD" quay.io
-            }
-            retry 5 true _login
-            ;;
-        *)
-            die "Unsupported registry login: $registry"
-    esac
-}
-
-push_matching_collector_scanner_images() {
-    info "Pushing collector & scanner images tagged with main-version to quay.io/rhacs-eng"
-
-    if [[ "$#" -ne 2 ]]; then
-        die "missing arg. usage: push_matching_collector_scanner_images <brand> <arch>"
-    fi
-
-    local brand="$1"
-    local arch="$2"
-
-    local registry
-    registry="$(registry_from_branding "$brand")"
-
-    _retag() {
-        retry 5 true "$SCRIPTS_ROOT/scripts/ci/pull-retag-push.sh" "$1" "$2"
-    }
-
-    if [[ "$arch" != "amd64" ]]; then
-        echo "Skipping rebundling for non-amd64 arch"
-        exit 0
-    fi
-
-    local main_tag
-    main_tag="$(make --quiet --no-print-directory tag)"
-    local scanner_version
-    scanner_version="$(make --quiet --no-print-directory scanner-tag)"
-    local collector_version
-    collector_version="$(make --quiet --no-print-directory collector-tag)"
-
-    registry_rw_login "${registry}"
-
-    _retag "${registry}/scanner:${scanner_version}"    "${registry}/scanner:${main_tag}-${arch}"
-    _retag "${registry}/scanner-db:${scanner_version}" "${registry}/scanner-db:${main_tag}-${arch}"
-    _retag "${registry}/scanner-slim:${scanner_version}"    "${registry}/scanner-slim:${main_tag}-${arch}"
-    _retag "${registry}/scanner-db-slim:${scanner_version}" "${registry}/scanner-db-slim:${main_tag}-${arch}"
-
-    _retag "${registry}/collector:${collector_version}"      "${registry}/collector:${main_tag}-${arch}"
-}
-
-poll_for_system_test_images() {
-    info "Polling for images required for system tests"
-
-    if [[ "$#" -ne 1 ]]; then
-        die "missing arg. usage: poll_for_system_test_images <seconds to wait>"
-    fi
-
-    local time_limit="$1"
-
-    require_environment "QUAY_RHACS_ENG_BEARER_TOKEN"
-
-    local image_list
-    image_list="$(mktemp)"
-    populate_stackrox_image_list "${image_list}"
-    info "Will poll for: $(awk '{print $1}' "${image_list}")"
-
-    local start_time
-    start_time="$(date '+%s')"
-
-    local tag
-    local image
-    while read -r image tag
-    do
-        while ! check_rhacs_eng_image_exists "$image" "$tag"
-        do
-            info "$image does not exist"
-            if (( $(date '+%s') - start_time > time_limit )); then
-                check_build_workflows "$(get_commit_sha)"
-                die "ERROR: Timed out waiting for images after ${time_limit} seconds"
-            fi
-            sleep 60
-        done
-    done < "$image_list"
-
-    info "All images exist."
-    touch "${STATE_IMAGES_AVAILABLE}"
-}
-
-# Image prefetch is broken into two sets:
-# - prebuilt: test fixture images that already exist prior to CI job execution.
-#   Prefetch for these images can start as soon as the cluster is ready.
-# - system: the product images under test 'stackrox-images'. Prefetch for these
-#   should not start until the images are built to avoid backoff noise in the
-#   prefetcher logs.
-
-image_prefetcher_await_msg_prefix="Waiting for pre-fetcher"
-
-image_prefetcher_prebuilt_start() {
-    info "Starting image pre-fetcher of pre-built images (NOT built for current commit)..."
-    junit_wrap image-prefetcher-prebuilt-start \
-               "Start image pre-fetcher for pre-built images (NOT built for current commit)." \
-               "See log for error details." \
-               _image_prefetcher_prebuilt_start
-}
-
-image_prefetcher_system_start() {
-    info "Starting image pre-fetcher of system images (built for current commit)..."
-    junit_wrap image-prefetcher-system-start \
-               "Start image pre-fetcher for system images (built for current commit)." \
-               "See log for error details." \
-               _image_prefetcher_system_start
-}
-
-_image_prefetcher_prebuilt_start() {
-    # NOTE: when changing this function, make corresponding changes to
-    # _image_prefetcher_prebuilt_await
-
-    case "$CI_JOB_NAME" in
-    *qa-e2e-tests)
-        image_prefetcher_start_set qa-e2e
-        # Override the default image pull policy for containers with quay.io
-        # images to rely on prefetched images. This helps ensure that the static
-        # prefect list stays up to date with additions.
-        ci_export "IMAGE_PULL_POLICY_FOR_QUAY_IO" "Never"
-        ;;
-    *-operator-e2e-tests)
-        image_prefetcher_start_set operator-e2e
-        # TODO(ROX-20508): pre-fetch images of the release from which operator upgrade test starts as well.
-        ;;
-    *)
-        info "No pre-built image prefetching is currently performed for: ${CI_JOB_NAME}."
-        ;;
-    esac
-}
-
-_image_prefetcher_system_start() {
-    # NOTE: when changing this function, make corresponding changes to
-    # _image_prefetcher_system_await
-
-    case "$CI_JOB_NAME" in
-    # ROX-24818: GKE is excluded from system image prefetch as it causes
-    # flakes in test.
-    *-operator-e2e-tests|*ocp*qa-e2e-tests)
-        image_prefetcher_start_set stackrox-images
-        ;;
-    # Enabling scanner V4 installation tests as well, even though they also run on GKE,
-    # for gathering some more data points for CI reliability.
-    *-scanner-v4-install-tests)
-        image_prefetcher_start_set stackrox-images
-        ;;
-    *)
-        info "No system image prefetching is performed for: ${CI_JOB_NAME}."
-        ;;
-    esac
-}
-
-image_prefetcher_start_set() {
-    local ns="prefetch-images"
-    local name="$1"
-
-    make image-prefetcher-deploy-bin
-    local image_prefetcher_deploy_bin
-    image_prefetcher_deploy_bin="$(make print-image-prefetcher-deploy-bin)"
-    local image_prefetcher_version
-    image_prefetcher_version="$(go -C tools/test list -m -f '{{.Version}}' github.com/stackrox/image-prefetcher/deploy)"
-
-    info "Using ${image_prefetcher_deploy_bin} ${image_prefetcher_version} for image prefetch deployment"
-
-    local manifest
-    manifest=$(mktemp)
-
-    case "${ORCHESTRATOR_FLAVOR}" in
-    k8s)
-        flavor=vanilla
-        ;;
-    openshift)
-        flavor=ocp
-        ;;
-    *)
-        die "unsupported ORCHESTRATOR: ${ORCHESTRATOR_FLAVOR}"
-        ;;
-    esac
-
-    # daemonset, etc
-    ${image_prefetcher_deploy_bin} \
-        --version="${image_prefetcher_version}" \
-        --k8s-flavor="$flavor" \
-        --secret=stackrox \
-        --collect-metrics \
-        "$name" > "$manifest"
-
-    # image list
-    local image_list
-    image_list=$(mktemp)
-    populate_prefetcher_image_list "$name" "${image_list}"
-    echo "---" >> "$manifest"
-    kubectl create --dry-run=client -o yaml --namespace=$ns configmap "$name" --from-file="images.txt=$image_list" >> "$manifest"
-
-    # pull secret
-    REGISTRY_PASSWORD="${QUAY_RHACS_ENG_RO_PASSWORD}" \
-    REGISTRY_USERNAME="${QUAY_RHACS_ENG_RO_USERNAME}" \
-    NAMESPACE=$ns \
-      make -C operator stackrox-image-pull-secret
-
-    # apply configmap, daemonset etc
-    retry 5 true kubectl apply --namespace=$ns -f "$manifest"
-    info "Image pre-fetcher is now running in the background. Its status will be checked later (look for message starting with ${image_prefetcher_await_msg_prefix}). Proceeding with other tasks in the meantime."
-    rm -f "$image_list" "$manifest"
-}
-
-image_prefetcher_prebuilt_await() {
-    if [[ "${IMAGE_PREFETCH_DISABLED:-false}" == "true" ]]; then
-        return
-    fi
-
-    info "${image_prefetcher_await_msg_prefix} of pre-built images to complete..."
-    junit_wrap image-prefetcher-prebuilt-await \
-               "Waiting for pre-fetcher of pre-built images to complete." \
-               "See log for error details." \
-               _image_prefetcher_prebuilt_await
-}
-
-image_prefetcher_system_await() {
-    if [[ "${IMAGE_PREFETCH_DISABLED:-false}" == "true" ]]; then
-        return
-    fi
-
-    info "${image_prefetcher_await_msg_prefix} of system images to complete..."
-    junit_wrap image-prefetcher-system-await \
-               "Waiting for pre-fetcher of system images to complete." \
-               "See log for error details." \
-               _image_prefetcher_system_await
-}
-
-_image_prefetcher_prebuilt_await() {
-    case "$CI_JOB_NAME" in
-    *qa-e2e-tests)
-        image_prefetcher_await_set qa-e2e
-        ;;
-    *-operator-e2e-tests)
-        image_prefetcher_await_set operator-e2e
-        # TODO(ROX-20508): pre-fetch images of the release from which operator upgrade test starts as well.
-        ;;
-    *)
-        info "No pre-built image prefetching is currently performed for: ${CI_JOB_NAME}. Nothing to wait for."
-        ;;
-    esac
-}
-
-_image_prefetcher_system_await() {
-    case "$CI_JOB_NAME" in
-    # ROX-24818: GKE is excluded from system image prefetch as it causes
-    # flakes in test.
-    *-operator-e2e-tests|*ocp*qa-e2e-tests)
-        image_prefetcher_await_set stackrox-images
-        ;;
-    # Enabling scanner V4 installation tests as well, even though they also run on GKE,
-    # for gathering some more data points for CI reliability.
-    *-scanner-v4-install-tests)
-        image_prefetcher_await_set stackrox-images
-        ;;
-    *)
-        info "No system image prefetching is performed for: ${CI_JOB_NAME}. Nothing to wait for."
-        ;;
-    esac
-}
-
-image_prefetcher_await_set() {
-    local ns="prefetch-images"
-    local name="$1"
-    local extra_fields='{"build_id": "'"${BUILD_ID:-}"'", "job_name": "'"${JOB_NAME:-}"'", "orchestrator": "'"${ORCHESTRATOR_FLAVOR:-}"'", "build_tag": "'"${STACKROX_BUILD_TAG:-}"'"}'
-
-    info "Waiting for image prefetcher set ${name} to complete..."
-    if kubectl rollout status daemonset "$name" -n "$ns" --timeout 15m; then
-        info "All images in the set are now pre-fetched."
-    else
-        info "WARNING: Pre-fetching failed to complete in time."
-        info "To investigate closer, go to https://console.cloud.google.com/bigquery and run a query such as:"
-        local query
-        query=$(mktemp)
-        cat > "${query}" <<- EOM
-
-            SELECT started_at, duration_ms, image, error
-            FROM \`acs-san-stackroxci.ci_metrics.stackrox_image_prefetches\`
-            WHERE error IS NOT NULL AND
-            $(echo "${extra_fields}" | jq -r '[to_entries | .[] | select(.value != "") | (.key + "=\"" + .value + "\"")] | join(" AND ")')
-            ORDER BY started_at DESC LIMIT 1000
-
-EOM
-        cat "${query}"
-        info "Note: The data is imported into the table periodically: https://github.com/stackrox/stackrox/actions/workflows/batch-load-test-metrics.yml"
-
-        if [[ -n ${ARTIFACT_DIR:-} ]]; then
-            local prefetcher_help="$ARTIFACT_DIR/image-pre-fetcher-${name}-failure-summary.html"
-            cat > "${prefetcher_help}" <<- EOM
-                <html>
-                <head>
-                <title>Image pre-fetcher ${name} failure</title>
-                <style>
-                  body { color: #e8e8e8; background-color: #424242; font-family: "Roboto", "Helvetica", "Arial", sans-serif }
-                  a { color: #ff8caa }
-                  a:visited { color: #ff8caa }
-                </style>
-                </head>
-                <body>
-
-                Waiting for image prefetcher set ${name} to complete timed out.<br>
-                To investigate closer, go to <a target="_blank" href="https://console.cloud.google.com/bigquery">BigQuery</a> and run a query such as the following:
-                <br>
-                <pre>
-EOM
-            cat >> "${prefetcher_help}" "${query}"
-            cat >> "${prefetcher_help}" <<- EOM
-                </pre>
-                Note: The data is imported into the table <a target="_blank" href="https://github.com/stackrox/stackrox/actions/workflows/batch-load-test-metrics.yml">periodically</a>.
-                <br><br>
-                </body>
-                </html>
-EOM
-        fi
-        rm -f "${query}"
-    fi
-    info "Now retrieving prefetcher metrics..."
-    local attempt=0
-    local service="service/${name}-metrics"
-    while [[ -z $(kubectl -n "${ns}" get "${service}" -o jsonpath="{.status.loadBalancer.ingress}" 2>/dev/null) ]]; do
-        if [ "$attempt" -lt "60" ]; then
-            info "Waiting for ${service} to obtain endpoint ..."
-            ((attempt++))
-            sleep 10
-        else
-            info "Something is wrong with the ${service} service. See the following 'describe' output."
-            kubectl -n "${ns}" describe "${service}" || true
-            die "Timeout waiting for ${service} to obtain endpoint!"
-        fi
-    done
-    local endpoint
-    endpoint="$(kubectl -n "${ns}" get "${service}" -o json | service_get_endpoint)"
-    local fetcher_metrics
-    fetcher_metrics="$(mktemp --suffix=.csv)"
-    local fetcher_metrics_json
-    fetcher_metrics_json="$(mktemp --suffix=.json)"
-    local metrics_url="http://${endpoint}:8080/metrics"
-    if ! curl --silent --show-error --fail --retry 3 --retry-connrefused "${metrics_url}" > "${fetcher_metrics_json}"; then
-        die "Failed to fetch prefetcher metrics from ${metrics_url}"
-    fi
-    # See the stackrox_image_prefetches table definition in https://github.com/stackrox/automation-iac/blob/main/resources/testing/stackrox-ci/metrics.tf
-    # for the order of columns.
-    if ! jq --raw-output \
-      --argjson cols '["attempt_id", "started_at", "image", "duration_ms", "node", "size_bytes", "error", "build_id", "job_name", "orchestrator", "build_tag"]' \
-      --argjson extra "${extra_fields}" \
-      'map(.started_at = (.started_at | todate) | ($extra+.) as $row | $cols | map($row[.])) as $rows | $cols, $rows[] | @csv' \
-      "${fetcher_metrics_json}" > "${fetcher_metrics}"; then
-        info "WARNING: Failed to convert image prefetcher metrics to CSV with extra fields ${extra_fields}"
-        info "Dumping the input JSON file:"
-        jq . < "${fetcher_metrics_json}"
-        die "Failed to convert image prefetcher metrics to CSV, aborting."
-    fi
-    rm -f "${fetcher_metrics_json}"
-
-    if save_image_prefetches_metrics "${fetcher_metrics}"; then
-        info "Image pre-fetcher metrics retrieved and saved."
-    else
-        info "WARNING: failed to save image pre-fetcher metrics."
-    fi
-    rm -f "${fetcher_metrics}"
-}
-
-service_get_endpoint() {
-    jq -r '.status.loadBalancer.ingress // error("List of ingress points of LB " + .metadata.name + " is empty.") | .[0] | .hostname // .ip'
-}
-
-populate_prefetcher_image_list() {
-    local name="$1"
-    local image_list="$2"
-
-    case "$name" in
-    stackrox-images)
-        local image_tag_list
-        image_tag_list=$(mktemp)
-        populate_stackrox_image_list "${image_tag_list}"
-        # convert format from "basename tag" to "quay.io/.../basename:tag" expected by pre-fetcher
-        awk '{print "quay.io/rhacs-eng/" $1 ":" $2}' "$image_tag_list" > "$image_list"
-        rm -f "$image_tag_list"
-        ;;
-    operator-e2e)
-        # shellcheck disable=SC2046
-        grep PREFETCH-THIS-IMAGE -h -A 1 $(find operator/tests/ -type f) | awk '$1 == "image:" {print $2}' | sort -u > "$image_list"
-        ;;
-    qa-e2e)
-        cp "$SCRIPTS_ROOT/qa-tests-backend/scripts/images-to-prefetch.txt" "$image_list"
-        ;;
-    *)
-        die "ERROR: An unsupported image prefetcher target was requested: $name"
-        ;;
-    esac
-}
-
-populate_stackrox_image_list() {
-    local image_list="$1"
-
-    local tag
-    tag="${MAIN_IMAGE_TAG:-$(make --quiet --no-print-directory tag)}"
-    local operator_metadata_tag
-    operator_metadata_tag="$(echo "v${tag}" | sed 's,x,0,')"
-    local operator_controller_tag="${tag//x/0}"
-
-    # Require images based on the job
-    case "$CI_JOB_NAME" in
-        *-operator-e2e-tests)
-            cat >> "${image_list}" << END
-stackrox-operator ${operator_controller_tag}
-stackrox-operator-bundle ${operator_metadata_tag}
-stackrox-operator-index ${operator_metadata_tag}
-main ${tag}
-central-db ${tag}
-collector ${tag}
-scanner ${tag}
-scanner-db ${tag}
-scanner-v4 ${tag}
-scanner-v4-db ${tag}
-END
-            ;;
-        *-race-condition-qa-e2e-tests)
-            cat >> "${image_list}" << END
-central-db ${tag}
-main ${tag}-rcd
-roxctl ${tag}
-END
-            if is_in_PR_context && ! pr_has_label "ci-build-race-condition-debug"; then
-                echo "ERROR: Your PR is missing the \"ci-build-race-condition-debug\" label."
-                echo "ERROR: This label is required to build the images for $CI_JOB_NAME."
-                # Quietly continue to allow labels added after tests start.
-                # Otherwise this message will surface in the Prow log when
-                # images timeout out below.
-            fi
-            ;;
-        *-scanner-v4-install-tests)
-            cat >> "${image_list}" << END
-stackrox-operator ${operator_controller_tag}
-stackrox-operator-bundle ${operator_metadata_tag}
-stackrox-operator-index ${operator_metadata_tag}
-main ${tag}
-central-db ${tag}
-collector ${tag}
-scanner ${tag}
-scanner-db ${tag}
-scanner-v4 ${tag}
-scanner-v4-db ${tag}
-roxctl ${tag}
-END
-            ;;
-        *)
-            cat >> "${image_list}" << END
-central-db ${tag}
-main ${tag}
-roxctl ${tag}
-END
-            ;;
-    esac
-
-    if [[ "${DEPLOY_STACKROX_VIA_OPERATOR:-}" == "true" ]]; then
-            cat >> "${image_list}" << END
-stackrox-operator ${operator_controller_tag}
-stackrox-operator-bundle ${operator_metadata_tag}
-stackrox-operator-index ${operator_metadata_tag}
-END
-    fi
-
-    # Remove duplicates.
-    unique="$(mktemp)"
-    sort -u "${image_list}" > "${unique}"
-    cat "${unique}" > "${image_list}"
-    rm -f "${unique}"
-}
-
-check_rhacs_eng_image_exists() {
-    local name="$1"
-    local tag="$2"
-
-    local url="https://quay.io/api/v1/repository/rhacs-eng/$name/tag?specificTag=$tag"
-    info "Checking for $name using $url"
-    local check
-    local extra_args=()
-    local public_images=("stackrox-operator-index")
-    if [[ -n "${QUAY_RHACS_ENG_BEARER_TOKEN:-}" ]]; then
-        extra_args+=("-H" "Authorization: Bearer ${QUAY_RHACS_ENG_BEARER_TOKEN}")
-    else
-        # shellcheck disable=SC2076
-        if [[ ! " ${public_images[*]} " =~ " ${name} " ]]; then
-            info "Warning: Checking for image existence without QUAY_RHACS_ENG_BEARER_TOKEN is not supported for image ${name}:${tag}"
-            info "Warning: It is only supported for the following public image repositories: ${public_images[*]}"
-        fi
-    fi
-    check=$(curl --location -sS "${extra_args[@]}" "$url")
-    echo "$check"
-    [[ "$(jq -r '.tags | first | .name' <<<"$check")" == "$tag" ]]
-}
-
-check_build_workflows() {
-    local commit_sha="$1"
-
-    {
-        echo
-        info "GitHub Actions workflow status for build.yaml:"
-        check-workflow-run \
-            --workflow=build.yaml \
-            --head-SHA="${commit_sha}"
-
-        echo
-        info "GitHub Actions workflow status for scanner-build.yaml:"
-        check-workflow-run \
-            --workflow=scanner-build.yaml \
-            --head-SHA="${commit_sha}"
-    } | tee "${STATE_BUILD_RESULTS}" || true
-}
-
-check_scanner_version() {
-    if ! is_release_version "$(make --quiet --no-print-directory scanner-tag)"; then
-        echo "::error::Scanner tag does not look like a release tag. Please update SCANNER_VERSION file before releasing."
-        exit 1
-    fi
-}
-
-check_collector_version() {
-    if ! is_release_version "$(make --quiet --no-print-directory collector-tag)"; then
-        echo "::error::Collector tag does not look like a release tag. Please update COLLECTOR_VERSION file before releasing."
-        exit 1
-    fi
-}
-
-publish_roxctl() {
-    if [[ "$#" -ne 1 ]]; then
-        die "missing arg. usage: publish_roxctl <tag>"
-    fi
-
-    local tag="$1"
-
-    echo "Push roxctl to gs://sr-roxc & gs://rhacs-openshift-mirror-src/assets" >> "${GITHUB_STEP_SUMMARY}"
-
-    local temp_dir
-    temp_dir="$(mktemp -d)"
-    "${SCRIPTS_ROOT}/scripts/ci/artifacts-publish/prepare-roxctl.sh" . "${temp_dir}"
-    "${SCRIPTS_ROOT}/scripts/ci/artifacts-publish/publish.sh" "${temp_dir}" "${tag}" "gs://sr-roxc"
-    "${SCRIPTS_ROOT}/scripts/ci/artifacts-publish/publish.sh" "${temp_dir}" "${tag}" "gs://rhacs-openshift-mirror-src/assets"
-}
-
-publish_openapispec() {
-    if [[ "$#" -ne 1 ]]; then
-        die "missing arg. usage: publish_openapispec <tag>"
-    fi
-
-    local tag="$1"
-
-    echo "Push OpenAPI spec to gs://rhacs-openshift-mirror-src/assets" >> "${GITHUB_STEP_SUMMARY}"
-
-    local temp_dir
-    temp_dir="$(mktemp -d)"
-    "${SCRIPTS_ROOT}/scripts/ci/artifacts-publish/prepare-openapispec.sh" "${temp_dir}" "${tag}"
-    "${SCRIPTS_ROOT}/scripts/ci/artifacts-publish/publish.sh" "${temp_dir}" "${tag}" "gs://rhacs-openshift-mirror-src/openapi-spec"
-}
-
-push_helm_charts() {
-    if [[ "$#" -ne 1 ]]; then
-        die "missing arg. usage: push_helm_charts <tag>"
-    fi
-
-    local tag="$1"
-
-    echo "Publish Helm charts to github repository stackrox/release-artifacts and create a PR" >> "${GITHUB_STEP_SUMMARY}"
-
-    local central_services_chart_dir
-    local secured_cluster_services_chart_dir
-    central_services_chart_dir="$(mktemp -d)"
-    secured_cluster_services_chart_dir="$(mktemp -d)"
-    roxctl helm output central-services --image-defaults=rhacs --output-dir "${central_services_chart_dir}/rhacs"
-    roxctl helm output central-services --image-defaults=opensource --output-dir "${central_services_chart_dir}/opensource"
-    roxctl helm output secured-cluster-services --image-defaults=rhacs --output-dir "${secured_cluster_services_chart_dir}/rhacs"
-    roxctl helm output secured-cluster-services --image-defaults=opensource --output-dir "${secured_cluster_services_chart_dir}/opensource"
-    "${SCRIPTS_ROOT}/scripts/ci/publish-helm-charts.sh" "${tag}" "${central_services_chart_dir}" "${secured_cluster_services_chart_dir}"
-}
-
-gitbot() {
-    git -c "user.name=${GITHUB_USERNAME}" \
-        -c "user.email=${GITHUB_EMAIL}" \
-        -c "url.https://${GITHUB_TOKEN}:x-oauth-basic@github.com/.insteadOf=https://github.com/" \
-        "${@}"
-}
-
-is_tagged() {
-    local tags
-    tags="$(git tag --contains)"
-    [[ -n "$tags" ]]
-}
-
-is_nightly_run() {
-    [[ "${BUILD_TAG:-}" =~ -nightly- ]] || [[ "${GITHUB_REF:-}" =~ nightly- ]]
-}
-
-is_in_PR_context() {
-    if is_GITHUB_ACTIONS && [[ -n "${GITHUB_BASE_REF:-}" ]]; then
-        return 0
-    elif is_OPENSHIFT_CI && [[ -n "${PULL_NUMBER:-}" ]]; then
-        return 0
-    elif is_OPENSHIFT_CI && [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
-        # bin, test-bin, images
-        local pull_request
-        pull_request=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].pulls[0].number' 2>&1) || return 1
-        [[ "$pull_request" =~ ^[0-9]+$ ]] && return 0
-    fi
-
-    return 1
-}
-
-get_PR_number() {
-    if is_OPENSHIFT_CI && [[ -n "${PULL_NUMBER:-}" ]]; then
-        echo "${PULL_NUMBER}"
-        return 0
-    elif is_OPENSHIFT_CI && [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
-        # bin, test-bin, images
-        local pull_request
-        pull_request=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].pulls[0].number' 2>&1) || {
-            echo 2>&1 "ERROR: Could not determine a PR number"
-            return 1
-        }
-        if [[ "$pull_request" =~ ^[0-9]+$ ]]; then
-            echo "$pull_request"
-            return 0
-        fi
-    elif is_GITHUB_ACTIONS; then
-        local pull_request
-        pull_request=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") || {
-            echo 2>&1 "ERROR: Could not determine a PR number"
-            return 1
-        }
-        if [[ "$pull_request" =~ ^[0-9]+$ ]]; then
-            echo "$pull_request"
-            return 0
-        fi
-    fi
-
-    echo 2>&1 "ERROR: Could not determine a PR number"
-
-    return 1
-}
-
-is_openshift_CI_rehearse_PR() {
-    [[ "$(get_repo_full_name)" == "openshift/release" ]]
-}
-
-get_branch_name() {
-    # Returns the PR branch name (sometimes that's called PR source branch), e.g. 'johndoe/ROX-23456-fix-branch-name'.
-    # For non-PRs, returns branch name where the commit happened, e.g. 'master'.
-    if is_OPENSHIFT_CI; then
-        # Prow variables doc: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables
-        if [[ -n "${PULL_HEAD_REF:-}" ]]; then
-            # presubmit runs
-            echo "${PULL_HEAD_REF}"
-        elif [[ -n "${PULL_BASE_REF:-}" ]]; then
-            # postsubmit and batch runs
-            echo "${PULL_BASE_REF}"
-        elif [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
-            # periodics - CLONEREFS_OPTIONS exists in binary_build_commands and images.
-            local base_ref
-            base_ref="$(jq -r <<<"${CLONEREFS_OPTIONS}" '.refs[0].base_ref')" || die "invalid CLONEREFS_OPTIONS yaml"
-            if [[ "$base_ref" == "null" ]]; then
-                die "expect: base_ref in CLONEREFS_OPTIONS.refs[0]"
-            fi
-            echo "${base_ref}"
-        else
-            die "Expected PULL_HEAD_REF or PULL_BASE_REF or CLONEREFS_OPTIONS"
-        fi
-    elif is_GITHUB_ACTIONS; then
-        # GHA doc: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#default-environment-variables
-        local ref="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME:-}}"
-        if [[ -z "${ref}" ]]; then
-            die "Expected GITHUB_HEAD_REF or GITHUB_REF_NAME"
-        fi
-        echo "${ref}"
-    else
-        die "unsupported"
-    fi
-}
-
-get_repo_full_name() {
-    if is_GITHUB_ACTIONS; then
-        [[ -n "${GITHUB_REPOSITORY:-}" ]] || die "expect: GITHUB_REPOSITORY"
-        echo "${GITHUB_REPOSITORY}"
-    elif is_OPENSHIFT_CI; then
-        if [[ -n "${REPO_OWNER:-}" ]]; then
-            # presubmit, postsubmit and batch runs
-            # (ref: https://github.com/kubernetes/test-infra/blob/master/prow/jobs.md#job-environment-variables)
-            [[ -n "${REPO_NAME:-}" ]] || die "expect: REPO_NAME"
-            echo "${REPO_OWNER}/${REPO_NAME}"
-        elif [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
-            # periodics - CLONEREFS_OPTIONS exists in binary_build_commands and images.
-            local org
-            local repo
-            org="$(jq -r <<<"${CLONEREFS_OPTIONS}" '.refs[0].org')" || die "invalid CLONEREFS_OPTIONS yaml"
-            repo="$(jq -r <<<"${CLONEREFS_OPTIONS}" '.refs[0].repo')" || die "invalid CLONEREFS_OPTIONS yaml"
-            if [[ "$org" == "null" ]] || [[ "$repo" == "null" ]]; then
-                die "expect: org and repo in CLONEREFS_OPTIONS.refs[0]"
-            fi
-            echo "${org}/${repo}"
-        else
-            die "Expect REPO_OWNER/NAME or CLONEREFS_OPTIONS"
-        fi
-    else
-        die "unsupported"
-    fi
-}
-
-get_commit_sha() {
-    if is_OPENSHIFT_CI; then
-        echo "${PULL_PULL_SHA:-${PULL_BASE_SHA}}"
-    elif is_GITHUB_ACTIONS; then
-        git rev-parse --verify HEAD
-    else
-        die "unsupported"
-    fi
-}
-
-pr_has_label() {
-    if [[ -z "${1:-}" ]]; then
-        die "usage: pr_has_label <expected label> [<pr details>]"
-    fi
-
-    local expected_label="$1"
-    local pr_details
-    local exitstatus=0
-    pr_details="${2:-$(get_pr_details)}" || exitstatus="$?"
-    if [[ "$exitstatus" != "0" ]]; then
-        info "Warning: checking for a label in a non PR context: details: ${pr_details}, exitstatus: ${exitstatus}"
-        return 1
-    fi
-
-    if is_openshift_CI_rehearse_PR; then
-        pr_has_label_in_body "${expected_label}" "$pr_details"
-    else
-        jq '([.labels | .[].name]  // []) | .[]' -r <<<"$pr_details" | grep -qx "${expected_label}"
-    fi
-}
-
-pr_has_label_in_body() {
-    if [[ "$#" -ne 2 ]]; then
-        die "usage: pr_has_label_in_body <expected label> <pr details>"
-    fi
-
-    local expected_label="$1"
-    local pr_details="$2"
-
-    [[ "$(jq -r '.body' <<<"$pr_details")" =~ \/label:[[:space:]]*$expected_label ]]
-}
-
-# pr_has_pragma() - returns true if a pragma exists. A pragma is a key with
-# value in the description body of a PR that influences how CI behaves.
-# e.g. /pragma gk_release_channel:rapid.
-pr_has_pragma() {
-    if [[ "$#" -ne 1 ]]; then
-        die "usage: pr_has_pragma <key>"
-    fi
-
-    local pr_details
-    if ! pr_details="$(get_pr_details)"; then
-        info "Warning: checking for a pragma in a non PR context"
-        return 0
-    fi
-
-    local key_to_check="$1"
-    [[ "$(jq -r '.body' <<<"$pr_details")" =~ \/pragma:[[:space:]]*$key_to_check: ]]
-}
-
-# pr_get_pragma() - outputs the pragma key value if it exists.
-pr_get_pragma() {
-    if [[ "$#" -ne 1 ]]; then
-        die "usage: pr_get_pragma <key>"
-    fi
-
-    local pr_details
-    if ! pr_details="$(get_pr_details)"; then
-        echo ''
-        return 0
-    fi
-
-    local key_to_check="$1"
-    while IFS= read -r line; do
-        if [[ "$line" =~ \/pragma:[[:space:]]*$key_to_check:[[:space:]]*(.+) ]]; then
-            # shellcheck disable=SC2001
-            echo "${BASH_REMATCH[1]}" | sed -e 's/[[:space:]]*$//'
-        fi
-    done <<< "$(jq -r '.body' <<<"$pr_details")"
-}
-
-# get_pr_details() from GitHub and display the result. Exits 1 if not run in CI in a PR context.
-_PR_DETAILS=""
-_PR_DETAILS_CACHE_FILE="/tmp/PR_DETAILS_CACHE.json"
-get_pr_details() {
-    local pull_request
-    local org
-    local repo
-
-    if [[ -n "${_PR_DETAILS}" ]]; then
-        echo "${_PR_DETAILS}"
-        return 0
-    fi
-    if [[ -e "${_PR_DETAILS_CACHE_FILE}" ]]; then
-        _PR_DETAILS="$(cat "${_PR_DETAILS_CACHE_FILE}")"
-        echo "${_PR_DETAILS}"
-        return 0
-    fi
-
-    _not_a_PR() {
-        echo "This does not appear to be a PR context" >&2
-        echo '{ "msg": "this is not a PR" }'
-        exit 1
-    }
-
-    if is_OPENSHIFT_CI; then
-        if [[ -n "${JOB_SPEC:-}" ]]; then
-            pull_request=$(jq -r <<<"$JOB_SPEC" '.refs.pulls[0].number')
-            org=$(jq -r <<<"$JOB_SPEC" '.refs.org')
-            repo=$(jq -r <<<"$JOB_SPEC" '.refs.repo')
-        elif [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
-            pull_request=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].pulls[0].number')
-            org=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].org')
-            repo=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].repo')
-        else
-            echo "Expect a JOB_SPEC or CLONEREFS_OPTIONS" >&2
-            exit 2
-        fi
-        [[ "${pull_request}" == "null" ]] && _not_a_PR
-    elif is_GITHUB_ACTIONS; then
-        pull_request="$(jq -r .pull_request.number "${GITHUB_EVENT_PATH}")" || _not_a_PR
-        [[ "${pull_request}" == "null" ]] && _not_a_PR
-        org="${GITHUB_REPOSITORY_OWNER}"
-        repo="${GITHUB_REPOSITORY#*/}"
-    else
-        echo "Unsupported CI" >&2
-        exit 2
-    fi
-
-    local headers url pr_details
-
-    headers=()
-    if [[ -n "${GITHUB_TOKEN:-}" ]]; then
-        headers+=(-H "Authorization: token ${GITHUB_TOKEN}")
-    fi
-
-    url="https://api.github.com/repos/${org}/${repo}/pulls/${pull_request}"
-
-    if ! pr_details=$(curl --retry 5 --retry-connrefused -sS "${headers[@]}" "${url}"); then
-        echo "Github API error: $pr_details, exit code: $?" >&2
-        exit 2
-    fi
-
-    if [[ "$(jq .id <<<"$pr_details")" == "null" ]]; then
-        # A valid PR response is expected at this point
-        echo "Invalid response from GitHub: $pr_details" >&2
-        exit 2
-    fi
-    _PR_DETAILS="$pr_details"
-    echo "$pr_details" | tee "${_PR_DETAILS_CACHE_FILE}"
-}
-
-openshift_ci_mods() {
-    info "BEGIN OpenShift CI mods"
-
-    openshift_ci_debug
-
-    info "Current Status:"
-    "$ROOT/status.sh" || true
-
-    # For ci_export(), override BASH_ENV from stackrox-test with something that is writable.
-    BASH_ENV=$(mktemp)
-    export BASH_ENV
-
-    # These are not set in the binary_build_commands or image build envs.
-    export CI=true
-    export OPENSHIFT_CI=true
-
-    if is_in_PR_context && ! is_openshift_CI_rehearse_PR; then
-        local sha
-        if [[ -n "${PULL_PULL_SHA:-}" ]]; then
-            sha="${PULL_PULL_SHA}"
-        else
-            sha=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].pulls[0].sha') || echo "WARNING: Cannot find pull sha"
-        fi
-        if [[ -n "${sha:-}" ]] && [[ "$sha" != "null" ]]; then
-            info "Will checkout SHA to match PR: $sha"
-            git checkout "$sha"
-            git submodule update
-        else
-            echo "WARNING: Could not determine a SHA for this PR, ${sha:-}"
-        fi
-    fi
-
-    # Target a tag if HEAD is tagged.
-    BUILD_TAG="$(git describe --exact-match --tags HEAD)" || echo "Warning: Cannot get tag"
-    export BUILD_TAG
-
-    # For gradle
-    export GRADLE_USER_HOME="${HOME}"
-
-    handle_nightly_runs
-
-    info "Status after mods:"
-    "$ROOT/status.sh" || true
-
-    STACKROX_BUILD_TAG=$(make --quiet --no-print-directory tag)
-    export STACKROX_BUILD_TAG
-
-    info "END OpenShift CI mods"
-}
-
-# openshift_ci_debug() - store useful state (env & git) to help debug CI.
-# NOTE: only run before any creds are imported to the environment.
-openshift_ci_debug() {
-    local debug="${ARTIFACT_DIR:-/tmp}/debug.txt"
-
-    echo "Env A-Z dump:" > "${debug}"
-    env | sort | grep -E '^[A-Z]' >> "${debug}" || true
-
-    ensure_writable_home_dir
-
-    # Prevent fatal error "detected dubious ownership in repository" from recent git.
-    git config --global --add safe.directory "$(pwd)"
-
-    echo "Git log:" >> "${debug}"
-    git log --oneline --decorate -n 20 >> "${debug}" || true
-
-    echo "Recent git refs:" >> "${debug}"
-    git for-each-ref --format='%(creatordate) %(refname)' --sort=creatordate | tail -20 >> "${debug}"
-}
-
-ensure_writable_home_dir() {
-    # Single step test jobs do not have HOME
-    if [[ -z "${HOME:-}" ]] || ! touch "${HOME}/openshift-ci-write-test"; then
-        info "HOME (${HOME:-unset}) is not set or not writeable, using mktemp dir"
-        HOME=$( mktemp -d )
-        export HOME
-        info "HOME is now $HOME"
-    fi
-}
-
-openshift_ci_import_creds() {
-    shopt -s nullglob
-    for cred in /tmp/vault/**/[A-Z]*; do
-        export "$(basename "$cred")"="$(cat "$cred")"
-    done
-}
-
-unset_namespace_env_var() {
-    # NAMESPACE is injected by OpenShift CI for the cluster that is running the
-    # tests but this can have side effects for stackrox tests due to its use as
-    # the default namespace e.g. with helm.
-    if [[ -n "${NAMESPACE:-}" ]]; then
-        export OPENSHIFT_CI_NAMESPACE="$NAMESPACE"
-        unset NAMESPACE
-    fi
-}
-
-openshift_ci_e2e_mods() {
-    unset_namespace_env_var
-
-    # The incoming KUBECONFIG is for the openshift/release cluster and not the
-    # e2e test cluster.
-    if [[ -n "${KUBECONFIG:-}" ]]; then
-        info "There is an incoming KUBECONFIG in ${KUBECONFIG}"
-        export OPENSHIFT_CI_KUBECONFIG="$KUBECONFIG"
-    fi
-    KUBECONFIG="$(mktemp)"
-    info "KUBECONFIG set: ${KUBECONFIG}"
-    export KUBECONFIG
-
-    # KUBERNETES_{PORT,SERVICE} env values interact with commandline kubectl tests
-    if env | grep -e ^KUBERNETES_; then
-        local envfile
-        envfile="$(mktemp)"
-        info "Will clear ^KUBERNETES_ env"
-        env | grep -e ^KUBERNETES_ | cut -d= -f1 | awk '{ print "unset", $1 }' > "$envfile"
-        # shellcheck disable=SC1090
-        source "$envfile"
-    fi
-
-    if [[ "${JOB_NAME:-}" =~ -eks- ]]; then
-        # Explicitly set AWS creds from the stackrox-stackrox-e2e-tests vault to
-        # override any from other vaults e.g. automation-flavors.
-        AWS_ACCESS_KEY_ID="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/AWS_ACCESS_KEY_ID)"
-        AWS_SECRET_ACCESS_KEY="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/AWS_SECRET_ACCESS_KEY)"
-        export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
-        aws sts get-caller-identity | jq -r '.Arn'
-    fi
-}
-
-operator_e2e_test_setup() {
-    # TODO(ROX-11901): pass the brand explicitly from the CI config file rather than hardcode here
-    registry_ro_login "quay.io/rhacs-eng"
-    export ROX_PRODUCT_BRANDING="RHACS_BRANDING"
-
-    # $NAMESPACE is set by OpenShift CI, but confuses `operator-sdk scorecard` which runs against
-    # a completely different cluster, where this namespace does not even exist.
-    # Note that even though unsetting the variable turns out not to be sufficient for `operator-sdk scorecard`
-    # (still gets the namespace from *somewhere*), we're keeping this here as it might affect other tools.
-    unset_namespace_env_var
-}
-
-handle_nightly_runs() {
-    if ! is_OPENSHIFT_CI; then
-        die "Only for OpenShift CI"
-    fi
-
-    local nightly_tag_prefix
-    nightly_tag_prefix="$(git describe --tags --abbrev=0 --exclude '*-nightly-*')-nightly-"
-    if ! is_in_PR_context && [[ "${JOB_NAME_SAFE:-}" =~ ^nightly- ]]; then
-        ci_export BUILD_TAG "${nightly_tag_prefix}$(date '+%Y%m%d')"
-    fi
-}
-
-handle_nightly_binary_version_mismatch() {
-    if ! is_OPENSHIFT_CI; then
-        die "Only for OpenShift CI"
-    fi
-
-    if is_in_PR_context || ! [[ "${JOB_NAME_SAFE:-}" =~ ^nightly- ]]; then
-        return 0
-    fi
-
-    # JOB_NAME_SAFE is not set in test_binary_build_commands context for
-    # periodics, so the roxctl produced in that step will cause deploy.sh to
-    # fail.
-
-    if ! is_in_PR_context; then
-        info "Debug:"
-        echo "JOB_NAME: ${JOB_NAME:-}"
-        echo "JOB_NAME_SAFE: ${JOB_NAME_SAFE:-}"
-    fi
-
-    info "Correcting binary versions for nightly e2e tests"
-    echo "Current roxctl is: $(command -v roxctl || true), version: $(roxctl version || true)"
-
-    if ! [[ "$(roxctl version || true)" =~ nightly-$(date '+%Y%m%d') ]]; then
-        make cli_host-arch upgrader
-        make cli-install
-        echo "Replacement roxctl is: $(command -v roxctl || true), version: $(roxctl version || true)"
-    fi
-}
-
-store_qa_test_results() {
-    if ! is_OPENSHIFT_CI; then
-        return
-    fi
-
-    local to="${1:-qa-tests}"
-
-    info "Copying qa-tests-backend results to $to"
-
-    for test_results in qa-tests-backend/build/test-results/*; do
-        store_test_results "$test_results" "$to"
-    done
-}
-
-remove_qa_test_results() {
-    rm -rf qa-tests-backend/build/test-results
-}
-
-stored_test_results() {
-    echo "${ARTIFACT_DIR}/junit-$1"
-}
-
-store_test_results() {
-    if [[ "$#" -ne 2 ]]; then
-        die "missing args. usage: store_test_results <from> <to>"
-    fi
-
-    if ! is_OPENSHIFT_CI; then
-        return
-    fi
-
-    local from="$1"
-    local to="$2"
-
-    info "Copying test results from $from to $to"
-
-    local dest
-    dest="$(stored_test_results "$to")"
-
-    mkdir -p "$dest"
-    cp -a "$from" "$dest" || true # (best effort)
-}
-
-post_process_test_results() {
-    if [[ "$#" -ne 2 ]]; then
-        die "missing args. usage: post_process_test_results <slack attachments file.json> <summary output.json>"
-    fi
-
-    if ! is_OPENSHIFT_CI; then
-        return 0
-    fi
-
-    if [[ -z "${ARTIFACT_DIR:-}" ]]; then
-        info "ERROR: ARTIFACT_DIR is not set which is expected in openshift CI"
-        return 0
-    fi
-
-    local slack_attachments_file="$1"
-    local summary_file="$2"
-    local csv_output
-    local extra_args=()
-    local base_link
-    local calculated_base_link
-    local create_jiras
-    local jira_project="ROX"
-    local prow_job_link
-
-    set +u
-    {
-        info "Post processing junit records to JIRA issues, BigQuery metrics and Slack attachments as appropriate"
-
-        prow_job_link="$(make_prow_job_link)"
-
-        if is_in_PR_context; then
-            if pr_has_label "ci-test-junit-processing"; then
-                create_jiras="true"
-            else
-                create_jiras="false"
-            fi
-            jira_project="RS"
-        else
-            if [[ "${PULL_BASE_REF:-unknown}" =~ ^release ]]; then
-                create_jiras="false"
-            elif [[ "${JOB_NAME:-unknown}" =~ interop ]]; then
-                create_jiras="false"
-            else
-                create_jiras="true"
-            fi
-        fi
-
-        if [[ "${create_jiras}" == "false" ]]; then
-            extra_args=(--dry-run)
-            info "Will use junit2jira to create CSV for BigQuery input"
-        else
-            info "Will create JIRA issues for junit failures found in ${ARTIFACT_DIR}"
-        fi
-
-        csv_output="$(mktemp --suffix=.csv)"
-        # We need a link to repository. In case it's not part of job spec (e.g., periodic`s)
-        # we will fallback to short commit
-        base_link="$(echo "$JOB_SPEC" | jq ".refs.base_link | select( . != null )" -r)"
-        calculated_base_link="https://github.com/stackrox/stackrox/commit/$(make --quiet --no-print-directory shortcommit)"
-        curl --retry 5 --retry-connrefused -SsfL https://github.com/stackrox/junit2jira/releases/download/v0.0.24/junit2jira -o junit2jira && \
-        chmod +x junit2jira && \
-        ./junit2jira \
-            -base-link "${base_link:-$calculated_base_link}" \
-            -build-id "${BUILD_ID}" \
-            -build-link "${prow_job_link}" \
-            -build-tag "${STACKROX_BUILD_TAG}" \
-            -csv-output "${csv_output}" \
-            -jira-project "${jira_project}" \
-            -job-name "${JOB_NAME}" \
-            -junit-reports-dir "${ARTIFACT_DIR}" \
-            -orchestrator "${ORCHESTRATOR_FLAVOR:-PROW}" \
-            -threshold 10 \
-            -html-output "$ARTIFACT_DIR/junit2jira-summary.html" \
-            -slack-output "${slack_attachments_file}" \
-            -summary-output "${summary_file}" \
-            "${extra_args[@]}"
-
-        save_test_metrics "${csv_output}"
-    } || true
-    set -u
-}
-
-gate_flaky_tests() {
-    local exit_code="$1"
-
-    if [[ "${exit_code}" == "0" ]]; then
-        exit "${exit_code}"
-    fi
-
-    # Gating flaky tests is enabled only on PRs.
-    if ! is_in_PR_context; then
-        exit "${exit_code}"
-    fi
-
-    # Prepare flakechecker
-    curl --retry 5 --retry-connrefused -SsfL https://github.com/stackrox/junit2jira/releases/download/v0.0.24/flakechecker -o /tmp/flakechecker || exit "${exit_code}"
-    chmod +x /tmp/flakechecker
-    setup_gcp || echo "setup_gcp called"
-
-    # Run flakechecker for failed test
-    local config_file="${SCRIPTS_ROOT}/scripts/ci/flakechecker/flake-config.yml"
-    if /tmp/flakechecker -config-file "${config_file}" -job-name "${JOB_NAME}" -junit-reports-dir "${ARTIFACT_DIR}"; then
-      # Flakechecker exits successfully IF AND ONLY IF it finds a NON-EMPTY set of test failures in the
-      # JUnit report, AND ALL these test failures are found to be known flaky tests defined in flake-config.yml file.
-      # And the recent failure ratio for found failed tests is below threshold defined in flake-config.yml.
-      #
-      # In this case, we change the overall exit code of the job to success, hoping that the failure was only
-      # due to the tests flaky behavior (and not some other issue in the test job).
-      exit 0
-    else
-      # Flakechecker fails in case it identified test failures which do not qualify for suppression, OR
-      # in case there were no test failures at all in the JUnit report (a sign of problems elsewhere in the test job).
-      #
-      # In this case we keep the exit code as it was.
-      exit "${exit_code}"
-    fi
-}
-
-make_prow_job_link() {
-    local prow_job_link="https://prow.ci.openshift.org/view/gs/origin-ci-test/"
-    if is_in_PR_context; then
-        prow_job_link+="pr-logs/pull/stackrox_stackrox/${PULL_NUMBER}/"
-    else
-        prow_job_link+="logs/"
-    fi
-    prow_job_link+="$JOB_NAME/$BUILD_ID"
-    echo "${prow_job_link}"
-}
-
-# There are currently two openshift-ci steps where junit failures are summarized for slack.
-JOB_SLACK_FAILURE_ATTACHMENTS="${SHARED_DIR:-/tmp}/job-slack-failure-attachments.json"
-END_SLACK_FAILURE_ATTACHMENTS="/tmp/end-slack-failure-attachments.json"
-JOB_JUNIT2JIRA_SUMMARY_FILE="${SHARED_DIR:-/tmp}/job-junit2jira-summary.json"
-END_JUNIT2JIRA_SUMMARY_FILE="/tmp/end-junit2jira-summary.json"
-
-send_slack_failure_summary() {
-    if ! is_OPENSHIFT_CI || is_nightly_run; then
-        return 0
-    fi
-
-    if [[ "${PULL_BASE_REF:-unknown}" =~ ^release ]]; then
-        info "Skipping slack message for release branches"
-        return 0
-    fi
-
-    if [[ "${JOB_TYPE:-unknown}" == "periodic" ]]; then
-        info "Skipping slack message for periodics (scheduled prow jobs)"
-        return 0
-    fi
-
-    if is_system_test_without_images; then
-        # Avoid multiple slack messages from the e2e tests waiting for images.
-        info "Skipping slack message for a system test failure when images were not found"
-        return 0
-    fi
-
-    # Check env required for the job link and slack an error message when
-    # undefined.
-    _slack_check_env "BUILD_ID"
-    _slack_check_env "JOB_NAME"
-    local prow_job_link
-    prow_job_link="$(make_prow_job_link)"
-
-    local webhook_url="${TEST_FAILURES_NOTIFY_WEBHOOK}"
-
-    _slack_check_env "PULL_BASE_SHA"
-    local commit_sha="${PULL_BASE_SHA}"
-
-    if is_in_PR_context; then
-        if pr_has_label "ci-test-junit-processing"; then
-            # Send to #acs-slack-ci-integration-testing when testing the
-            # JUNIT -> Jira, BigQuery, Slack pipeline.
-            webhook_url="${SLACK_CI_INTEGRATION_TESTING_WEBHOOK}"
-            commit_sha="${PULL_PULL_SHA}"
-        else
-            info "Skipping slack message for PRs"
-            return 0
-        fi
-    fi
-
-    local org repo
-
-    if [[ -n "${JOB_SPEC:-}" ]]; then
-        org=$(jq -r <<<"$JOB_SPEC" '.refs.org')
-        repo=$(jq -r <<<"$JOB_SPEC" '.refs.repo')
-    elif [[ -n "${CLONEREFS_OPTIONS:-}" ]]; then
-        org=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].org')
-        repo=$(jq -r <<<"$CLONEREFS_OPTIONS" '.refs[0].repo')
-    else
-        _send_slack_error "Expect a JOB_SPEC or CLONEREFS_OPTIONS"
-        return 1
-    fi
-
-    if [[ "$org" == "null" ]] || [[ "$repo" == "null" ]]; then
-        _send_slack_error "Could not determine org and/or repo"
-        return 1
-    fi
-
-    local commit_details_url="https://api.github.com/repos/${org}/${repo}/commits/${commit_sha}"
-    local exitstatus=0
-    local commit_details
-    commit_details=$(curl --retry 5 --retry-connrefused -sS "${commit_details_url}") || exitstatus="$?"
-    if [[ "$exitstatus" != "0" ]]; then
-        _send_slack_error "Cannot get commit details: ${commit_details}"
-        return 1
-    fi
-
-    _slack_check_env "JOB_NAME_SAFE"
-    local job_name="${JOB_NAME_SAFE#merge-}"
-
-    local commit_msg
-    commit_msg=$(jq -r <<<"$commit_details" '.commit.message') || exitstatus="$?"
-    commit_msg="${commit_msg%%$'\n'*}" # use first line of commit msg
-    local commit_url
-    commit_url=$(jq -r <<<"$commit_details" '.html_url') || exitstatus="$?"
-    local author_name
-    author_name=$(jq -r <<<"$commit_details" '.commit.author.name') || exitstatus="$?"
-    local author_login
-    author_login=$(jq -r <<<"$commit_details" '.author.login') || exitstatus="$?"
-    if [[ "$exitstatus" != "0" ]]; then
-        _send_slack_error "Error parsing the commit details: ${commit_details}"
-        return 1
-    fi
-
-    local mention_author=""
-    _set_mention_author
-
-    local slack_attachments=""
-    _make_slack_failure_attachments
-
-    local slack_mention=""
-    _make_slack_mention
-
-    # shellcheck disable=SC2016
-    local body='
-{
-    "text": "Prow job failure: \($job_name)",
-    "blocks": [
-        {
-            "type": "header",
-            "text": {
-                "type": "plain_text",
-                "text": "Prow job failure: \($job_name)"
-            }
-        },
-        {
-            "type": "section",
-            "text": {
-                "type": "mrkdwn",
-                "text": "*Commit:* <\($commit_url)|\($commit_msg)>\n*Repo:* \($repo)\n*Author:* \($author_name)\($slack_mention)\n*Log:* \($prow_job_link)"
-            }
-        },
-        {
-            "type": "context",
-            "elements": [
-                {
-                    "type": "mrkdwn",
-                    "text": "You got tagged but have no idea why or what to do? Check <https://docs.google.com/document/d/1d5ga073jkv4CO1kAJqp8MPGpC6E1bwyrCGZ7S5wKg3w/edit#heading=h.li2pdsxtk1hu|this document>."
-                }
-            ]
-        },
-        {
-            "type": "divider"
-        }
-    ],
-    "attachments": $slack_attachments
-}
-'
-
-    local payload
-    if ! payload="$(jq --null-input \
-                       --arg job_name "$job_name" \
-                       --arg commit_url "$commit_url" \
-                       --arg commit_msg "$commit_msg" \
-                       --arg repo "$repo" \
-                       --arg author_name "$author_name" \
-                       --arg slack_mention "$slack_mention" \
-                       --arg prow_job_link "$prow_job_link" \
-                       --argjson slack_attachments "$slack_attachments" \
-                       "$body")"; then
-        _send_slack_error "Error formatting slack message: [${slack_attachments}/${payload}/$?]"
-        return 1
-    fi
-
-    echo -e "About to post:\n$payload"
-
-    local post_output
-    if ! post_output="$(echo "$payload" | \
-                        curl --location --silent --show-error --fail \
-                             --data @- --header 'Content-Type: application/json' \
-                             "$webhook_url")"; then
-        _send_slack_error "Error posting to Slack: [${post_output}/$?]"
-        return 1
-    fi
-}
-
-_set_mention_author() {
-    mention_author="false"
-
-    # Mention the commit author if new JIRA issues were created
-    if [[ -f "${JOB_JUNIT2JIRA_SUMMARY_FILE}" && \
-        "$(jq -r '.newJIRAs' "${JOB_JUNIT2JIRA_SUMMARY_FILE}")" != "0" ]]; then
-        mention_author="true"
-    fi
-    if [[ -f "${END_JUNIT2JIRA_SUMMARY_FILE}" && \
-        "$(jq -r '.newJIRAs' "${END_JUNIT2JIRA_SUMMARY_FILE}")" != "0" ]]; then
-        mention_author="true"
-    fi
-}
-
-_make_slack_mention() {
-    if [[ "${mention_author}" == "true" && "${author_login}" != "dependabot[bot]" ]]; then
-        slack_mention="$("$SCRIPTS_ROOT"/scripts/ci/get-slack-user-id.sh "$author_login")"
-        if [[ -n "$slack_mention" ]]; then
-            slack_mention=", <@${slack_mention}>"
-        else
-            slack_mention=", _unable to resolve Slack user for GitHub login ${author_login}_"
-        fi
-    fi
-}
-
-_make_slack_failure_attachments() {
-    info "Converting junit failures to slack attachments"
-
-    slack_attachments=""
-    if [[ ! -f "${JOB_SLACK_FAILURE_ATTACHMENTS}" ]]; then
-        if [[ "${CREATE_CLUSTER_OUTCOME:-}" == "${OUTCOME_PASSED}" ]]; then
-            slack_attachments+="$(_make_slack_failure_plain_text_block \
-                "Could not parse junit in main test step. Check build logs for more information.")"
-        fi
-    else
-        slack_attachments+="$(cat "${JOB_SLACK_FAILURE_ATTACHMENTS}")"
-    fi
-    if [[ ! -f "${END_SLACK_FAILURE_ATTACHMENTS}" ]]; then
-        slack_attachments+="$(_make_slack_failure_plain_text_block \
-            "Could not parse junit in final test step. Check build logs for more information.")"
-    else
-        slack_attachments+="$(cat "${END_SLACK_FAILURE_ATTACHMENTS}")"
-    fi
-
-    slack_attachments="$(echo "${slack_attachments}" | jq '.[]' | jq -s '.')"
-
-    if [[ "$(echo "${slack_attachments}" | jq 'length')" == "0" ]]; then
-        msg="No junit records were found for this failure. Check build logs \
-and artifacts for more information. Consider adding an \
-issue to improve CI to detect this failure pattern. (Add a CI_Fail_Better label)."
-        slack_attachments="$(_make_slack_failure_plain_text_block "${msg}")"
-
-        # Mention the commit author when the job failed with no JUNIT records
-        mention_author="true"
-    fi
-}
-
-_make_slack_failure_block() {
-    # shellcheck disable=SC2016
-    local body='
-[
-  {
-    "color": "#bb2124",
-    "blocks": [
-      {
-        "type": "section",
-        "text": {
-          "type": "\($section_type)",
-          "text": "\($content)"
-        }
-      }
-    ]
-  }
-]
-'
-    jq --null-input \
-       --arg section_type "$1" \
-       --arg content "$2" \
-       "$body"
-}
-
-_make_slack_failure_plain_text_block() {
-    _make_slack_failure_block "plain_text" "$1"
-}
-
-_make_slack_failure_markdown_block() {
-    _make_slack_failure_block "mrkdwn" "$1"
-}
-
-_send_slack_error() {
-    echo "ERROR: $1"
-    curl -XPOST -d @- -H 'Content-Type: application/json' "${webhook_url}" << __EOM__
-{ "text": "*An error occurred dealing with a job failure:*\n\t- Job: ${prow_job_link}.\n\t- $1." }
-__EOM__
-}
-
-_slack_check_env() {
-    (
-        set +u
-        if [[ -z "$(eval echo "\$$1")" ]]; then
-            _send_slack_error "An expected environment variable is unset/empty: $1"
-            return 1
-        fi
-    )
-}
-
-slack_workflow_failure() {
-    # shellcheck disable=SC2153
-    local github_context="${GITHUB_CONTEXT}"
-    local webhook_url="${TEST_FAILURES_NOTIFY_WEBHOOK}"
-
-    if is_in_PR_context; then
-        if pr_has_label "ci-test-github-action-slack-messages"; then
-            # Send to #acs-slack-ci-integration-testing when testing.
-            webhook_url="${SLACK_CI_INTEGRATION_TESTING_WEBHOOK}"
-        else
-            info "Skipping slack message for PRs"
-            return 0
-        fi
-    fi
-
-    local workflow_name commit_msg commit_url repo author_name author_login repo_url run_id
-    workflow_name=$(jq -r <<<"${github_context}" '.workflow')
-    event_name=$(jq -r <<<"${github_context}" '.event_name')
-    if [[ "${event_name}" == "push" ]]; then
-        commit_msg=$(jq -r <<<"${github_context}" '.event.head_commit.message')
-        commit_msg="${commit_msg%%$'\n'*}" # use first line of commit msg
-        commit_url=$(jq -r <<<"${github_context}" '.event.head_commit.url')
-        author_name=$(jq -r <<<"${github_context}" '.event.head_commit.author.name')
-        author_login=$(jq -r <<<"${github_context}" '.event.head_commit.author.username')
-        repo_url=$(jq -r <<<"${github_context}" '.event.repository.url')
-    else
-        commit_msg="This is a test slack message"
-        commit_url=$(jq -r <<<"${github_context}" '.event.pull_request.diff_url')
-        author_name=$(jq -r <<<"${github_context}" '.actor')
-        author_login=$(jq -r <<<"${github_context}" '.actor')
-        repo_url=$(jq -r <<<"${github_context}" '.event.pull_request.base.repo.html_url')
-    fi
-    repo=$(jq -r <<<"${github_context}" '.repository')
-    run_id=$(jq -r <<<"${github_context}" '.run_id')
-
-    # If global "mention_author" is set use that value.
-    local mention_author="${mention_author:-true}"
-    local slack_mention=""
-    _make_slack_mention
-
-    local attachments=""
-    local job_name job_url
-    IFS=$'\n'
-    for job in $(gh run view --jq '.jobs[] | select(.conclusion == "failure")' --json 'jobs' -R "${repo}" "${run_id}" | jq -sc '.[]')
-    do
-        job_name=$(jq -r <<<"${job}" '.name')
-        job_url=$(jq -r <<<"${job}" '.url')
-        attachments+="$(_make_slack_failure_markdown_block "Job: <${job_url}|${job_name}>")"
-    done
-    attachments="$(echo "${attachments}" | jq '.[]' | jq -s '.')"
-
-    # shellcheck disable=SC2016
-    local body='
-{
-    "text": "\($workflow_name) failed.
-Commit: \($commit_msg).
-Author: \($author_name)\($slack_mention).",
-    "blocks": [
-        {
-            "type": "header",
-            "text": {
-                "type": "plain_text",
-                "text": "\($workflow_name) failed."
-            }
-        },
-        {
-            "type": "section",
-            "text": {
-                "type": "mrkdwn",
-                "text": "
-*Commit:* <\($commit_url)|\($commit_msg)>.
-*Repo:* \($repo).
-*Author:* \($author_name)\($slack_mention).
-*Workflow:* \($repo_url)/actions/runs/\($run_id)"
-            }
-        },
-        {
-            "type": "divider"
-        }
-    ],
-    "attachments": $attachments
-}
-'
-
-    local payload
-    payload="$(jq --null-input \
-        --arg workflow_name "${workflow_name}" \
-        --arg commit_url "$commit_url" \
-        --arg commit_msg "$commit_msg" \
-        --arg repo "$repo" \
-        --arg author_name "$author_name" \
-        --arg slack_mention "$slack_mention" \
-        --arg repo_url "$repo_url" \
-        --arg run_id "$run_id" \
-        --argjson attachments "$attachments" \
-       "$body")"
-
-    echo -e "About to post:\n$payload"
-
-    echo "$payload" | curl --location --silent --show-error --fail \
-         --data @- --header 'Content-Type: application/json' \
-         "${webhook_url}"
-}
-
-# junit_wrap() - runs a command and creates a JUNIT record if the command
-# succeeds or fails. Some output of the command is included in the failure
-# JUNIT.
-#
-# WARNING: If this is used to wrap a bash function and not a separate binary
-# or script file there are two side effects that may not be expected:
-#
-# 1. errexit is not propagated to the function context and the function will
-#    continue on error. This is contrary to the typical approach from `set -e`
-#    used throughout this repo.
-# 2. exports are not propagated back to the calling context because the command
-#    runs in a subshell.
-
-junit_wrap() {
-    if [[ "$#" -lt 4 ]]; then
-        die "missing args. usage: junit_wrap <class> <description> <failure_message> <command> [ args ]"
-    fi
-
-    local class="$1"; shift
-    local description="$1"; shift
-    local failure_message="$1"; shift
-    local command_output_file
-    command_output_file="$(mktemp)"
-
-    if "$@" 2>&1 | tee "${command_output_file}"; then
-        save_junit_success "${class}" "${description}"
-        rm -f "${command_output_file}"
-    else
-        local ret_code="$?"
-        local failure_body=""
-        if [[ -n "$failure_message" ]]; then
-            failure_body="${failure_message}
-"
-        fi
-        failure_body="${failure_body}Command output: $(tail --bytes=512 "${command_output_file}")"
-
-        save_junit_failure "${class}" "${description}" "${failure_body}"
-        rm -f "${command_output_file}"
-
-        return ${ret_code}
-    fi
-}
-
-junit_contains_failure() {
-    local dir="$1"
-    if [[ ! -d $dir ]]; then
-        return 1
-    fi
-    # There should be few files in such dir, and they should have well-behaved names,
-    # and "return" does not mix with piping to "while read", so we use a "for" over find.
-    # shellcheck disable=SC2044
-    for f in $(find "$dir" -type f -iname '*.xml'); do
-        if grep -q '<failure ' "$f"; then
-            return 0
-        fi
-    done
-    return 1
-}
-
-get_junit_misc_dir() {
-    echo "${ARTIFACT_DIR}/junit-misc"
-}
-
-_JUNIT_RESULT_SUCCESS="SUCCESS"
-_JUNIT_RESULT_FAILURE="FAILURE"
-_JUNIT_RESULT_SKIPPED="SKIPPED"
-
-save_junit_success() {
-    if [[ "$#" -ne 2 ]]; then
-        die "missing args. usage: save_junit_success <class> <description>"
-    fi
-
-    _save_junit_record "${_JUNIT_RESULT_SUCCESS}" "$@"
-}
-
-save_junit_failure() {
-    if [[ "$#" -ne 3 ]]; then
-        die "missing args. usage: save_junit_failure <class> <description> <details>"
-    fi
-
-    _save_junit_record "${_JUNIT_RESULT_FAILURE}" "$@"
-}
-
-save_junit_skipped() {
-    if [[ "$#" -ne 2 ]]; then
-        die "missing args. usage: save_junit_skipped <class> <description>"
-    fi
-
-    _save_junit_record "${_JUNIT_RESULT_SKIPPED}" "$@"
-}
-
-remove_junit_record() {
-    local class="$1"
-    local junit_dir
-    junit_dir="$(get_junit_misc_dir)"
-    local junit_file="${junit_dir}/junit-${class}.xml"
-    rm -f "${junit_file}"
-}
-
-_save_junit_record() {
-    local disposition="$1"
-    local class="$2"
-    local description="$3"
-    local details="${4:-}"
-
-    if [[ -z "${ARTIFACT_DIR:-}" ]]; then
-        info "Warning: save_junit_success() requires the \$ARTIFACT_DIR variable to be set"
-        return
-    fi
-
-    local junit_dir
-    junit_dir="$(get_junit_misc_dir)"
-    mkdir -p "${junit_dir}/db"
-
-    # base64 encode failure details to condense multilines
-    if [[ $details != "SUCCESS" ]]; then
-        details="$(base64 -w0 <<< "$details")"
-    fi
-
-    # record this instance
-    local record_length=3
-    local record="${junit_dir}/db/${class}.txt"
-    {
-        echo "${description}"
-        echo "${disposition}"
-        echo "${details}"
-     } >> "${record}"
-
-    local tests
-    tests=$(( "$(wc -l < "${record}")" / "${record_length}" ))
-
-    local failures=0
-    local skipped=0
-    local lines
-    readarray -t lines < "${record}"
-    while (( ${#lines[@]} ))
-    do
-        local result="${lines[1]}"
-        if [[ "${result}" == "${_JUNIT_RESULT_FAILURE}" ]]; then
-            failures=$(( failures+1 ))
-        fi
-        if [[ "${result}" == "${_JUNIT_RESULT_SKIPPED}" ]]; then
-            skipped=$(( skipped+1 ))
-        fi
-        lines=( "${lines[@]:${record_length}}" )
-    done
-
-    local junit_file="${junit_dir}/junit-${class}.xml"
-
-    cat << _EO_SUITE_HEADER_ > "${junit_file}"
-<testsuite name="${class}" tests="${tests}" skipped="${skipped}" failures="${failures}" errors="0">
-_EO_SUITE_HEADER_
-
-    readarray -t lines < "${record}"
-    while (( ${#lines[@]} ))
-    do
-        local description="${lines[0]}"
-        local result="${lines[1]}"
-        local details="${lines[2]}"
-
-        # XML escape description
-        description="${description//&/&amp;}"
-        description="${description//\"/&quot;}"
-        description="${description//\'/&#39;}"
-        description="${description//</&lt;}"
-        description="${description//>/&gt;}"
-
-        cat << _EO_CASE_HEADER_ >> "${junit_file}"
-        <testcase name="${description}" classname="${class}">
-_EO_CASE_HEADER_
-
-        if [[ "$result" == "${_JUNIT_RESULT_FAILURE}" ]]; then
-            details="$(base64 --decode <<< "$details")"
-        cat << _EO_FAILURE_ >> "${junit_file}"
-            <failure><![CDATA[${details}]]></failure>
-_EO_FAILURE_
-        fi
-        if [[ "$result" == "${_JUNIT_RESULT_SKIPPED}" ]]; then
-        cat << _EO_SKIPPED_ >> "${junit_file}"
-            <skipped/>
-_EO_SKIPPED_
-        fi
-
-        echo "        </testcase>" >> "${junit_file}"
-
-        lines=( "${lines[@]:3}" )
-    done
-
-    echo "</testsuite>" >> "${junit_file}"
-}
-
-add_build_comment_to_pr() {
-    info "Adding a comment with the build tag to the PR"
-
-    local pr_details
-    local exitstatus=0
-    pr_details="$(get_pr_details)" || exitstatus="$?"
-    if [[ "$exitstatus" != "0" ]]; then
-        echo "DEBUG: Unable to get the PR details from GitHub: $exitstatus"
-        echo "DEBUG: PR details: ${pr_details}"
-        info "Will continue without commenting on the PR"
-        return
-    fi
-
-    # hub-comment is tied to Circle CI env
-    local url
-    url=$(jq -r '.html_url' <<<"$pr_details")
-    export CIRCLE_PULL_REQUEST="$url"
-
-    local sha
-    sha=$(jq -r '.head.sha' <<<"$pr_details")
-    sha=${sha:0:7}
-    export _SHA="$sha"
-
-    local tag
-    tag=$(make tag)
-    export _TAG="$tag"
-
-    local tmpfile
-    tmpfile=$(mktemp)
-    cat > "$tmpfile" <<- EOT
-Images are ready for the commit at {{.Env._SHA}}.
-
-To use with deploy scripts, first \`export MAIN_IMAGE_TAG={{.Env._TAG}}\`.
-EOT
-
-    hub-comment -type build -template-file "$tmpfile"
-}
-
-is_system_test_without_images() {
-    case "${JOB_NAME:-missing}" in
-        *-e2e-tests|*-upgrade-tests|*-version-compatibility-tests)
-            [[ ! -f "${STATE_IMAGES_AVAILABLE}" ]]
-            ;;
-        *)
-            false
-            ;;
-    esac
-}
-
-handle_gha_tagged_build() {
-    if [[ -z "${GITHUB_REF:-}" ]]; then
-        echo "No GITHUB_REF in env"
-        exit 0
-    fi
-    echo "GITHUB_REF: ${GITHUB_REF}"
-    if [[ "${GITHUB_REF:-}" =~ ^refs/tags/ ]]; then
-        tag="${GITHUB_REF#refs/tags/*}"
-        echo "This is a tagged build: $tag"
-        echo "BUILD_TAG=$tag" >> "$GITHUB_ENV"
-    else
-        echo "This is not a tagged build"
-    fi
-}
-
-slack_prow_notice() {
-    info "Slack a notice that prow tests have started"
-
-    if [[ "$#" -lt 1 ]]; then
-        die "missing arg. usage: slack_prow_notice <tag>"
-    fi
-
-    local tag="$1"
-
-    [[ "$tag" =~ $RELEASE_RC_TAG_BASH_REGEX ]] || is_nightly_run || {
-        info "Skipping step as this is not a release, RC or nightly build"
-        return 0
-    }
-
-    local build_url
-    local webhook_url
-    if [[ "$tag" =~ $RELEASE_RC_TAG_BASH_REGEX ]]; then
-        local release
-        release="$(get_release_stream "$tag")"
-        build_url="https://prow.ci.openshift.org/?repo=stackrox%2Fstackrox&job=*release-$release*"
-        if is_release_test_stream "$tag"; then
-            # send to #acs-slack-integration-testing when testing the release process
-            webhook_url="${SLACK_MAIN_WEBHOOK}"
-        else
-            # send to #acs-release-notifications
-            webhook_url="${RELEASE_WORKFLOW_NOTIFY_WEBHOOK}"
-        fi
-    else
-        die "unexpected"
-    fi
-
-    local github_url="https://github.com/stackrox/stackrox/releases/tag/$tag"
-
-    jq -n \
-    --arg build_url "$build_url" \
-    --arg tag "$tag" \
-    --arg github_url "$github_url" \
-    '{"text": ":prow: Prow CI for tag <\($github_url)|\($tag)> started! Check the status of the tests under the following URL: \($build_url)"}' \
-| curl -XPOST -d @- -H 'Content-Type: application/json' "$webhook_url"
-}
-
-gather_debug_for_cluster_under_test() {
-    highlight_cluster_versions
-    record_cluster_info
-}
-
-highlight_cluster_versions() {
-    if [[ -z "${ARTIFACT_DIR:-}" ]]; then
-        info "No place for artifacts, skipping cluster version dump"
-        return
-    fi
-
-    artifact_file="$ARTIFACT_DIR/cluster-version.html"
-
-    cat > "$artifact_file" <<- HEAD
-<html>
-    <head>
-        <title>Cluster Versions</title>
-        <style>
-          body { color: #e8e8e8; background-color: #424242; font-family: "Roboto", "Helvetica", "Arial", sans-serif }
-          a { color: #ff8caa }
-          a:visited { color: #ff8caa }
-        </style>
-    </head>
-    <body>
-HEAD
-
-    local nodes
-    nodes="$(kubectl get nodes -o wide 2>&1 || true)"
-    local kubectl_version
-    kubectl_version="$(kubectl version -o json 2>&1 || true)"
-    local oc_version
-    oc_version="$(oc version -o json 2>&1 || true)"
-
-    cat >> "$artifact_file" << DETAILS
-      <h3>Nodes:</h3>
-      kubectl get nodes -o wide
-      <pre>$nodes</pre>
-      <h3>Versions:</h3>
-      kubectl version -o json
-      <pre>$kubectl_version</pre>
-      oc version -o json
-      <pre>$oc_version</pre>
-DETAILS
-
-    cat >> "$artifact_file" <<- FOOT
-    <br />
-    <br />
-  </body>
-</html>
-FOOT
-}
-
-record_cluster_info() {
-    _record_cluster_info || {
-        # Failure to gather metrics is not a test failure
-        info "WARNING: Recording cluster info failed"
-    }
-}
-
-_record_cluster_info() {
-    info "Record some cluster info"
-
-    # Assumes (a) there is a single cluster under test (cut_*) and (b) all nodes
-    # in the cluster are homogeneous.
-
-    # Product version. Currently used for OpenShift version. Could cover cloud
-    # provider versions for example.
-    local oc_version
-    oc_version="$(oc version -o json 2>&1 || true)"
-    local openshiftVersion
-    openshiftVersion=$(jq -r <<<"$oc_version" '.openshiftVersion')
-    set_ci_shared_export "cut_product_version" "$openshiftVersion"
-
-    # K8s version.
-    local kubectl_version
-    kubectl_version="$(kubectl version -o json 2>&1 || true)"
-    local serverGitVersion
-    serverGitVersion=$(jq -r <<<"$kubectl_version" '.serverVersion.gitVersion')
-    set_ci_shared_export "cut_k8s_version" "$serverGitVersion"
-
-    # Node info: OS, Kernel & Container Runtime.
-    local nodes
-    nodes="$(kubectl get nodes -o json 2>&1 || true)"
-    local osImage
-    osImage=$(jq -r <<<"$nodes" '.items[0].status.nodeInfo.osImage')
-    set_ci_shared_export "cut_os_image" "$osImage"
-
-    local kernelVersion
-    kernelVersion=$(jq -r <<<"$nodes" '.items[0].status.nodeInfo.kernelVersion')
-    set_ci_shared_export "cut_kernel_version" "$kernelVersion"
-
-    local containerRuntimeVersion
-    containerRuntimeVersion=$(jq -r <<<"$nodes" '.items[0].status.nodeInfo.containerRuntimeVersion')
-    set_ci_shared_export "cut_container_runtime_version" "$containerRuntimeVersion"
-}
-
-get_infra_cluster_files() {
-    local cluster_name="${1}"
-    local data_dir="/tmp/artifacts-${cluster_name}"
-    set -x
-    ls -la "${SHARED_DIR:-/tmp}" || true
-    grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
-    echo "$PATH"
-    infractl --version \
-      || { curl --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \
-        | jq -r ".result.fileChunk" \
-        | base64 -d > ./infractl;
-          file ./infractl || true;
-          install infractl ${GOPATH:-/opt}/bin/infractl || install infractl /usr/local/bin/; }
-    infractl whoami
-    for I in {1..5}; do
-        infractl artifacts ${cluster_name} \
-          --download-dir "${data_dir}"
-        if [[ -d /tmp/artifacts-${cluster_name} ]]; then
-          break
-        fi
-        printf "%s - Waiting for infra cluster [${cluster_name}]" "$(date -u)"
-        sleep 60
-    done
-    echo "$cluster_name"
-    cp "${data_dir}/kubeconfig" "${SHARED_DIR}/kubeconfig" \
-        || cp "${data_dir}/auth/kubeconfig" "${SHARED_DIR}/kubeconfig" || true
-
-    cp "${data_dir}/dotenv" "${SHARED_DIR}/dotenv" || true
-    ls -la "${SHARED_DIR:-/tmp}" || true
-    grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"* || true
-
-    export KUBECONFIG="${SHARED_DIR}/kubeconfig"
-    kubectl get nodes -o wide
-}
-
-test_on_infra() {
-    local event_json body
-    export PULL_NUMBER="${PULL_NUMBER:-${GITHUB_REF_NAME:+${GITHUB_REF_NAME%%/merge}}}"
-    echo "PULL_NUMBER=${PULL_NUMBER}"
-    export SHARED_DIR="${SHARED_DIR:-${GITHUB_WORKSPACE:+${RUNNER_TEMP}}}"
-    if ! touch "${SHARED_DIR}/file_write_test"; then
-      export SHARED_DIR="/tmp/"
-      touch "${SHARED_DIR}/file_write_test"
-    fi
-    echo "SHARED_DIR=${SHARED_DIR}"
-    REPO_NAME=${REPO_NAME:-stackrox}
-    REPO_OWNER=${REPO_OWNER:-stackrox}
-    local tries=4
-    event_json=''
-    while [[ -z "$event_json" ]]; do
-    event_json=$(set -x; curl --silent \
-      -H "X-GitHub-Api-Version: 2022-11-28" \
-      -H "Accept: application/vnd.github+json" \
-      -o - "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls/${PULL_NUMBER}")
-      if [[ $tries -gt 0 && -z "$event_json" ]]; then
-        sleep 30
-        echo $(( tries-- ))
-      else
-        break
-      fi
-    done
-    body=$(jq -r '.body' <<<"${event_json}" \
-      | tee >(cat >&2) | grep '^/test-on-infra') || return
-    while read -r cmd cluster_name job_name_match; do
-      job_name_match=${job_name_match%%]*}
-      job_name_match=${job_name_match##*[}
-      echo "Comparing job_name string '${JOB_NAME:-}' to *${job_name_match}* for infra cluster ${cluster_name}."
-      if [[ "${JOB_NAME:-}" == *"$job_name_match"* ]]; then
-        echo "Match found. Tests will run on ${cluster_name} instead of starting a new cluster."
-        echo "https://infra.rox.systems/cluster/${cluster_name}"
-        get_infra_cluster_files "${cluster_name}"
-        echo "CLUSTER_NAME=${cluster_name}" >> "${SHARED_DIR}/shared_env" || true
-        echo "KUBECONFIG=${KUBECONFIG}" >> "${SHARED_DIR}/shared_env" || true
-        break
-      fi
-    done <<<"$body"
-}
-
-if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
-    if [[ "$#" -lt 1 ]]; then
-        die "When invoked at the command line a method is required."
-    fi
-    fn="$1"
-    shift
-    "$fn" "$@"
-fi

From 4afdf87aa3563c7e17eafe09b4d63991aba90b04 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:19:50 -0600
Subject: [PATCH 117/256] filter and 8k

---
 .github/workflows/e2e-tests.yaml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 03598f9c4d23b..a40a0c8732422 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -48,18 +48,13 @@ jobs:
           show-progress: false
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
-      - name: Fetch tags
-        run: |
-          git fetch origin --jobs=2 --shallow-since=2025-01-01 'refs/tags/*.x:refs/tags/*.x' '^refs/tags/3.*'
-          git version
-
       - name: Check tag lookup
         continue-on-error: true
         run: |
           git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
 
       - run: |
-          nohup git fetch --quiet --depth=5000 origin +refs/tags/*:refs/tags/* >>/tmp/git.fetching.log 2>&1 &
+          nohup git fetch --quiet --depth=8000 origin +refs/tags/*.x:refs/tags/*.x >>/tmp/git.fetching.log 2>&1 &
           echo $! | tee /tmp/git.fetching.pid
 
       - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)

From 0e39fa59244b2040a1789902abb5296d12e8f111 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:21:57 -0600
Subject: [PATCH 118/256] tag lookup before deploy

---
 .github/workflows/e2e-tests.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index a40a0c8732422..774a068d84ce0 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -73,18 +73,12 @@ jobs:
         run: |
           set -x
 
-          # example: 4.9.x-457-g9867327a52
-          MAIN_IMAGE_TAG=$(git describe --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG:-master}")
-
           JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
 
           ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/tmp/}"
           mkdir -p "$ARTIFACT_DIR"
 
           tee -a "$GITHUB_ENV" <<EOF
-          MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}
-          TAG=${MAIN_IMAGE_TAG}
-
           CI=true
           SHARED_DIR=${SHARED_DIR:-/tmp}
 
@@ -130,6 +124,12 @@ jobs:
             # default is "K8S"
           fi
 
+      - run: |
+          # example: 4.9.x-457-g9867327a52
+          MAIN_IMAGE_TAG=$(git describe --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG:-master}")
+          echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
+          echo "TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
+
       - name: Find or Deploy Stackrox
         id: rox-installed
         if: ${{ steps.cluster-exists.outcome != 'failure' }}

From dad6e4df69632884c363916ea02b6cb1aa738479 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:23:24 -0600
Subject: [PATCH 119/256] ignore not found

---
 .github/workflows/e2e-tests.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 774a068d84ce0..5c605e791f379 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -57,7 +57,7 @@ jobs:
           nohup git fetch --quiet --depth=8000 origin +refs/tags/*.x:refs/tags/*.x >>/tmp/git.fetching.log 2>&1 &
           echo $! | tee /tmp/git.fetching.pid
 
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
 
       - uses: ./.github/actions/job-preamble
         if: ${{ !env.ACT }}
@@ -90,7 +90,7 @@ jobs:
           ARTIFACT_DIR=${ARTIFACT_DIR}
           EOF
 
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
 
       - name: Check tag lookup
         continue-on-error: true
@@ -105,7 +105,7 @@ jobs:
           source scripts/ci/lib.sh
           test_on_infra | tee -a "$GITHUB_ENV"
 
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
 
       - name: Check tag lookup
         continue-on-error: true
@@ -145,7 +145,7 @@ jobs:
           kubectl get namespace stackrox \
             || ./deploy/k8s/deploy.sh
 
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
 
       - name: Check tag lookup
         continue-on-error: true
@@ -188,7 +188,7 @@ jobs:
           fi
           echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
 
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
 
       - name: Stackrox API test
         if: ${{ steps.stackrox-api.outcome != 'failure' }}
@@ -200,7 +200,7 @@ jobs:
 
           curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.' || true
 
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
 
       - name: Get secured cluster name
         if: ${{ steps.stackrox-api.outcome != 'failure' }}

From 23367e4a310204c9f2b1031a83489875fbc3778a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:25:01 -0600
Subject: [PATCH 120/256] or true

---
 .github/workflows/e2e-tests.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 5c605e791f379..039d4c3dc2040 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -57,7 +57,7 @@ jobs:
           nohup git fetch --quiet --depth=8000 origin +refs/tags/*.x:refs/tags/*.x >>/tmp/git.fetching.log 2>&1 &
           echo $! | tee /tmp/git.fetching.pid
 
-      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
 
       - uses: ./.github/actions/job-preamble
         if: ${{ !env.ACT }}
@@ -90,7 +90,7 @@ jobs:
           ARTIFACT_DIR=${ARTIFACT_DIR}
           EOF
 
-      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
 
       - name: Check tag lookup
         continue-on-error: true
@@ -105,7 +105,7 @@ jobs:
           source scripts/ci/lib.sh
           test_on_infra | tee -a "$GITHUB_ENV"
 
-      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
 
       - name: Check tag lookup
         continue-on-error: true
@@ -145,7 +145,7 @@ jobs:
           kubectl get namespace stackrox \
             || ./deploy/k8s/deploy.sh
 
-      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
 
       - name: Check tag lookup
         continue-on-error: true
@@ -188,7 +188,7 @@ jobs:
           fi
           echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
 
-      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
 
       - name: Stackrox API test
         if: ${{ steps.stackrox-api.outcome != 'failure' }}
@@ -200,7 +200,7 @@ jobs:
 
           curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.' || true
 
-      - run: set +e; ps -f -p $(pgrep -F /tmp/git.fetching.pid)
+      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
 
       - name: Get secured cluster name
         if: ${{ steps.stackrox-api.outcome != 'failure' }}

From 5bb12ea556394e7e89cadb01c7c034d04851fc8d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:27:04 -0600
Subject: [PATCH 121/256] fetch all

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 039d4c3dc2040..8f8717bbfdaca 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -54,7 +54,7 @@ jobs:
           git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
 
       - run: |
-          nohup git fetch --quiet --depth=8000 origin +refs/tags/*.x:refs/tags/*.x >>/tmp/git.fetching.log 2>&1 &
+          nohup git fetch --depth=10000 origin '+refs/tags/*:refs/tags/*' >>/tmp/git.fetching.log 2>&1 &
           echo $! | tee /tmp/git.fetching.pid
 
       - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true

From 721bc21520e74c2e20224ed1b4754a5be120be67 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:29:42 -0600
Subject: [PATCH 122/256] cat the admin-password file

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 8f8717bbfdaca..3c14c0fb01826 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -163,7 +163,7 @@ jobs:
           ls -latr /tmp/
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
-              password="${ROX_ADMIN_PASSWORD:-$(find "${KUBECONFIG%%kubeconfig}" -name admin-password -exec cat {} \;)}"
+              password="${ROX_ADMIN_PASSWORD:-$(cat "${KUBECONFIG%%kubeconfig}"/admin-password)}"
               password="${ROX_ADMIN_PASSWORD:-$(kubectl -n stackrox get secret admin-pass -o json | jq -er '.data.password // ""' | base64 -d)}"
               password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password // ""' | base64 -d)}"
               echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"

From c98fb9db9ffb8a174f9f85ff0206e4e3d32e33d2 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:30:25 -0600
Subject: [PATCH 123/256] tag lookup later

---
 .github/workflows/e2e-tests.yaml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 3c14c0fb01826..7bfbe10771936 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -188,6 +188,11 @@ jobs:
           fi
           echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
 
+      - name: Check tag lookup
+        continue-on-error: true
+        run: |
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
+
       - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
 
       - name: Stackrox API test
@@ -210,8 +215,18 @@ jobs:
           CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
           echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
 
+      - name: Check tag lookup
+        continue-on-error: true
+        run: |
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
+
       - run: go get github.com/stackrox/scanner
 
+      - name: Check tag lookup
+        continue-on-error: true
+        run: |
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
+
       - name: install oc cli
         if: ${{ env.CLUSTER == 'OPENSHIFT' }}
         run: |
@@ -222,6 +237,11 @@ jobs:
           install --mode 755 /tmp/oc /usr/local/bin/
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
+      - name: Check tag lookup
+        continue-on-error: true
+        run: |
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
+
       - timeout-minutes: 10
         continue-on-error: true
         run: |

From 38de9394e6284f33c6c22b3381436b18f0213e0b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:36:31 -0600
Subject: [PATCH 124/256] try wait for pid

---
 .github/workflows/e2e-tests.yaml | 62 ++++----------------------------
 1 file changed, 7 insertions(+), 55 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 7bfbe10771936..0c3e57c24ac19 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -48,27 +48,15 @@ jobs:
           show-progress: false
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
-      - name: Check tag lookup
-        continue-on-error: true
-        run: |
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
-
       - run: |
           nohup git fetch --depth=10000 origin '+refs/tags/*:refs/tags/*' >>/tmp/git.fetching.log 2>&1 &
           echo $! | tee /tmp/git.fetching.pid
 
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
-
       - uses: ./.github/actions/job-preamble
         if: ${{ !env.ACT }}
         with:
           gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
 
-      - name: Check tag lookup
-        continue-on-error: true
-        run: |
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
-
       - name: Env variables
         run: |
           set -x
@@ -90,13 +78,6 @@ jobs:
           ARTIFACT_DIR=${ARTIFACT_DIR}
           EOF
 
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
-
-      - name: Check tag lookup
-        continue-on-error: true
-        run: |
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
-
       - name: Check if testing on Infra cluster
         id: cluster-exists
         timeout-minutes: 5
@@ -105,13 +86,6 @@ jobs:
           source scripts/ci/lib.sh
           test_on_infra | tee -a "$GITHUB_ENV"
 
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
-
-      - name: Check tag lookup
-        continue-on-error: true
-        run: |
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
-
       - name: Determine cluster flavor
         if: ${{ steps.cluster-exists.outcome != 'failure' }}
         timeout-minutes: 5
@@ -124,6 +98,13 @@ jobs:
             # default is "K8S"
           fi
 
+      - name: Check tag lookup
+        timeout-minutes: 0.5
+        continue-on-error: true
+        run: |
+          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' \
+            || wait $(cat /tmp/git.fetching.pid)
+
       - run: |
           # example: 4.9.x-457-g9867327a52
           MAIN_IMAGE_TAG=$(git describe --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG:-master}")
@@ -145,13 +126,6 @@ jobs:
           kubectl get namespace stackrox \
             || ./deploy/k8s/deploy.sh
 
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
-
-      - name: Check tag lookup
-        continue-on-error: true
-        run: |
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
-
       - name: Stackrox service and auth lookup
         id: stackrox-api
         if: ${{ steps.rox-installed.outcome != 'failure' }}
@@ -188,13 +162,6 @@ jobs:
           fi
           echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
 
-      - name: Check tag lookup
-        continue-on-error: true
-        run: |
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
-
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
-
       - name: Stackrox API test
         if: ${{ steps.stackrox-api.outcome != 'failure' }}
         run: |
@@ -215,18 +182,8 @@ jobs:
           CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
           echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
 
-      - name: Check tag lookup
-        continue-on-error: true
-        run: |
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
-
       - run: go get github.com/stackrox/scanner
 
-      - name: Check tag lookup
-        continue-on-error: true
-        run: |
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
-
       - name: install oc cli
         if: ${{ env.CLUSTER == 'OPENSHIFT' }}
         run: |
@@ -237,11 +194,6 @@ jobs:
           install --mode 755 /tmp/oc /usr/local/bin/
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
-      - name: Check tag lookup
-        continue-on-error: true
-        run: |
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' || true
-
       - timeout-minutes: 10
         continue-on-error: true
         run: |

From 9ab8322e4c5702a14e6e9b17c73c6791ca70b871 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:38:25 -0600
Subject: [PATCH 125/256] remove extra slash

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 0c3e57c24ac19..80c70b2802ff6 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -137,7 +137,7 @@ jobs:
           ls -latr /tmp/
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
-              password="${ROX_ADMIN_PASSWORD:-$(cat "${KUBECONFIG%%kubeconfig}"/admin-password)}"
+              password="${ROX_ADMIN_PASSWORD:-$(cat "${KUBECONFIG%%kubeconfig}admin-password")}"
               password="${ROX_ADMIN_PASSWORD:-$(kubectl -n stackrox get secret admin-pass -o json | jq -er '.data.password // ""' | base64 -d)}"
               password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password // ""' | base64 -d)}"
               echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"

From 36f0c7f7ce9d259d10d0acbaec294dce6d7469a5 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:39:27 -0600
Subject: [PATCH 126/256] remove extra ps check

---
 .github/workflows/e2e-tests.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 80c70b2802ff6..f169ee0ca433d 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -172,8 +172,6 @@ jobs:
 
           curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.' || true
 
-      - run: ps -f -p $(pgrep -F /tmp/git.fetching.pid) || true
-
       - name: Get secured cluster name
         if: ${{ steps.stackrox-api.outcome != 'failure' }}
         run: |

From 7fe5c6b156c09a16fca6a0551694a9a4680b796a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:40:36 -0600
Subject: [PATCH 127/256] pgrep and sleep

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index f169ee0ca433d..47e8ae5cf875c 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -103,7 +103,7 @@ jobs:
         continue-on-error: true
         run: |
           git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' \
-            || wait $(cat /tmp/git.fetching.pid)
+            || { while pgrep -F /tmp/git.fetching.pid; do sleep 1; done; }
 
       - run: |
           # example: 4.9.x-457-g9867327a52

From c2b39a8972525cad6fef74b953478869eebf0cc2 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 6 Aug 2025 23:43:36 -0600
Subject: [PATCH 128/256] fix overwrite after pswd from file

---
 .github/workflows/e2e-tests.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 47e8ae5cf875c..4e0e82a00ddf6 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -137,8 +137,8 @@ jobs:
           ls -latr /tmp/
           if [[ -e "${KUBECONFIG:-}" ]]; then
             if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
-              password="${ROX_ADMIN_PASSWORD:-$(cat "${KUBECONFIG%%kubeconfig}admin-password")}"
-              password="${ROX_ADMIN_PASSWORD:-$(kubectl -n stackrox get secret admin-pass -o json | jq -er '.data.password // ""' | base64 -d)}"
+              password="${ROX_ADMIN_PASSWORD:-$(cat "${KUBECONFIG%kubeconfig}admin-password")}"
+              password="${password:-$(kubectl -n stackrox get secret admin-pass -o json | jq -er '.data.password // ""' | base64 -d)}"
               password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password // ""' | base64 -d)}"
               echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
               echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"

From 20364dd6618d72df6645a17383b7a341dc994946 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 00:02:27 -0600
Subject: [PATCH 129/256] fail if api test fails

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 4e0e82a00ddf6..5c8dfc715e59c 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -170,7 +170,7 @@ jobs:
             ps -ef | grep kube || true
           fi
 
-          curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.' || true
+          curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.'
 
       - name: Get secured cluster name
         if: ${{ steps.stackrox-api.outcome != 'failure' }}

From 8f21e70905ff8abb84757fc218ec4b2bd3ff3c8d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 07:18:09 -0600
Subject: [PATCH 130/256] bg tests and wait on pids

---
 .github/workflows/e2e-tests.yaml | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 5c8dfc715e59c..125cdcfedcc65 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -199,16 +199,33 @@ jobs:
 
       - timeout-minutes: 35
         continue-on-error: true
+        # TODO: add gcr secret, possibly others for test req's
         run: |
-          make -C qa-tests-backend smoke-test
+          test_suite="smoke-test"
+          make -C qa-tests-backend "${test_suite}" 2>&1 >> "/tmp/test.${test_suite}.log" &
+          echo $! | tee -a /tmp/test.pids
 
       - timeout-minutes: 35
         continue-on-error: true
         run: |
-          make -C qa-tests-backend bat-test
+          test_suite="bat-test"
+          make -C qa-tests-backend "${test_suite}" 2>&1 >> "/tmp/test.${test_suite}.log" &
+          echo $! | tee -a /tmp/test.pids
           # TEST_TARGET=bat-test ./gradlew testBegin testParallelBAT testBAT --fail-fast
 
       - timeout-minutes: 35
         continue-on-error: true
         run: |
-          make -C qa-tests-backend test
+          test_suite="test"
+          make -C qa-tests-backend "${test_suite}" 2>&1 >> "/tmp/test.${test_suite}.log" &
+          echo $! | tee -a /tmp/test.pids
+
+      - timeout-minutes: 35
+        run: |
+          while pgrep -F /tmp/test.pids; do sleep 1; done;
+
+      - name: Publish Test Report
+        uses: test-summary/action@v2
+        if: always()
+        with:
+          paths: 'qa-tests-backend/build/test-results/**/TEST-*.xml'

From 0ab1554c2cbeb3ae727c0c249cda64398e1d6c30 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 08:20:01 -0600
Subject: [PATCH 131/256] 1s loop check pgrep

---
 .github/workflows/e2e-tests.yaml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 125cdcfedcc65..6e42fc0e8a2d3 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -220,9 +220,18 @@ jobs:
           make -C qa-tests-backend "${test_suite}" 2>&1 >> "/tmp/test.${test_suite}.log" &
           echo $! | tee -a /tmp/test.pids
 
-      - timeout-minutes: 35
+      - timeout-minutes: 30
         run: |
-          while pgrep -F /tmp/test.pids; do sleep 1; done;
+          step=1
+          total=1800
+          steps=$(( total / step ))
+          for (( i=0; i<steps; i++; )); do
+            if [[ $(pgrep -F /tmp/test.pids | wc -l) -eq 0 ]]; then
+              echo 'All tasks completed in the background.'
+              break
+            fi
+            sleep "$step"
+          done
 
       - name: Publish Test Report
         uses: test-summary/action@v2

From cb4ac54c079994bac2e019f1a9632ef62008437d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 08:24:50 -0600
Subject: [PATCH 132/256] list all files under qa-tests

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 6e42fc0e8a2d3..0a1f07fd82274 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -232,6 +232,7 @@ jobs:
             fi
             sleep "$step"
           done
+          du qa-tests-backend/build/*
 
       - name: Publish Test Report
         uses: test-summary/action@v2

From 3c3b9ca2cc94a9ce8dc1d6e50304b5cdcc976777 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 08:37:43 -0600
Subject: [PATCH 133/256] extra semicolon

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 0a1f07fd82274..14a38c7998149 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -225,7 +225,7 @@ jobs:
           step=1
           total=1800
           steps=$(( total / step ))
-          for (( i=0; i<steps; i++; )); do
+          for (( i=0; i<steps; i++ )); do
             if [[ $(pgrep -F /tmp/test.pids | wc -l) -eq 0 ]]; then
               echo 'All tasks completed in the background.'
               break

From ed4b1e74d3202b6522d84a99932c39a9e49e7b51 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 08:46:02 -0600
Subject: [PATCH 134/256] try grep for gradle

---
 .github/workflows/e2e-tests.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 14a38c7998149..0778b3e04ad78 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -222,11 +222,12 @@ jobs:
 
       - timeout-minutes: 30
         run: |
-          step=1
+          step=30
           total=1800
           steps=$(( total / step ))
           for (( i=0; i<steps; i++ )); do
-            if [[ $(pgrep -F /tmp/test.pids | wc -l) -eq 0 ]]; then
+            #if [[ $(pgrep -F /tmp/test.pids | wc -l) -eq 0 ]]; then
+            if !pgrep gradle; then
               echo 'All tasks completed in the background.'
               break
             fi

From e80a9f1a197d5c084fee8ff13dc593d7385944d3 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 08:51:10 -0600
Subject: [PATCH 135/256] space!

---
 .github/workflows/e2e-tests.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 0778b3e04ad78..455ebbfcf41e0 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -227,7 +227,9 @@ jobs:
           steps=$(( total / step ))
           for (( i=0; i<steps; i++ )); do
             #if [[ $(pgrep -F /tmp/test.pids | wc -l) -eq 0 ]]; then
-            if !pgrep gradle; then
+            ps -ef | grep java || true
+            ps -ef | grep gradle || true
+            if ! pgrep gradle; then
               echo 'All tasks completed in the background.'
               break
             fi

From 6acefefedd55f4183c8ff469aa1e701f633eb445 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 09:00:33 -0600
Subject: [PATCH 136/256] back to pids

---
 .github/workflows/e2e-tests.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 455ebbfcf41e0..b2fa5a63d0e77 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -222,20 +222,23 @@ jobs:
 
       - timeout-minutes: 30
         run: |
+          set -x
           step=30
           total=1800
           steps=$(( total / step ))
           for (( i=0; i<steps; i++ )); do
-            #if [[ $(pgrep -F /tmp/test.pids | wc -l) -eq 0 ]]; then
             ps -ef | grep java || true
             ps -ef | grep gradle || true
-            if ! pgrep gradle; then
+            if [[ $(pgrep -F /tmp/test.pids | wc -l) -eq 0 ]]; then
               echo 'All tasks completed in the background.'
               break
             fi
             sleep "$step"
           done
           du qa-tests-backend/build/*
+          for f in /tmp/test.*.log; do
+            cat "$f"
+          done
 
       - name: Publish Test Report
         uses: test-summary/action@v2

From 03978ae31bd915a409aef22bfa6d21762ab59509 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 09:11:37 -0600
Subject: [PATCH 137/256] sequential

---
 .github/workflows/e2e-tests.yaml | 30 +++---------------------------
 1 file changed, 3 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index b2fa5a63d0e77..cc5dc6e9a10f0 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -202,43 +202,19 @@ jobs:
         # TODO: add gcr secret, possibly others for test req's
         run: |
           test_suite="smoke-test"
-          make -C qa-tests-backend "${test_suite}" 2>&1 >> "/tmp/test.${test_suite}.log" &
-          echo $! | tee -a /tmp/test.pids
+          make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
 
       - timeout-minutes: 35
         continue-on-error: true
         run: |
           test_suite="bat-test"
-          make -C qa-tests-backend "${test_suite}" 2>&1 >> "/tmp/test.${test_suite}.log" &
-          echo $! | tee -a /tmp/test.pids
-          # TEST_TARGET=bat-test ./gradlew testBegin testParallelBAT testBAT --fail-fast
+          make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
 
       - timeout-minutes: 35
         continue-on-error: true
         run: |
           test_suite="test"
-          make -C qa-tests-backend "${test_suite}" 2>&1 >> "/tmp/test.${test_suite}.log" &
-          echo $! | tee -a /tmp/test.pids
-
-      - timeout-minutes: 30
-        run: |
-          set -x
-          step=30
-          total=1800
-          steps=$(( total / step ))
-          for (( i=0; i<steps; i++ )); do
-            ps -ef | grep java || true
-            ps -ef | grep gradle || true
-            if [[ $(pgrep -F /tmp/test.pids | wc -l) -eq 0 ]]; then
-              echo 'All tasks completed in the background.'
-              break
-            fi
-            sleep "$step"
-          done
-          du qa-tests-backend/build/*
-          for f in /tmp/test.*.log; do
-            cat "$f"
-          done
+          make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
 
       - name: Publish Test Report
         uses: test-summary/action@v2

From 8b5160bcdf7e74f0e78abe2166fef2b63db7766c Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 10:48:46 -0600
Subject: [PATCH 138/256] log each Env.mustGet lookup

---
 qa-tests-backend/src/main/groovy/util/Env.groovy | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa-tests-backend/src/main/groovy/util/Env.groovy b/qa-tests-backend/src/main/groovy/util/Env.groovy
index 7859d9643fc55..2f668d9e97c48 100644
--- a/qa-tests-backend/src/main/groovy/util/Env.groovy
+++ b/qa-tests-backend/src/main/groovy/util/Env.groovy
@@ -51,6 +51,7 @@ class Env {
     }
 
     static String mustGet(String key) {
+        LOG.info("Env.mustGet(\"${key}\")")
         return INSTANCE.mustGetInternal(key)
     }
 

From 28efa5b532a5920bfb882034dc24404b7c6bf83e Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 11:01:16 -0600
Subject: [PATCH 139/256] verify gcr creds

---
 .github/workflows/e2e-tests.yaml | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index cc5dc6e9a10f0..700fac1aff7a4 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -63,19 +63,20 @@ jobs:
 
           JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
 
-          ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/tmp/}"
+          ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/artifacts/}"
           mkdir -p "$ARTIFACT_DIR"
 
           tee -a "$GITHUB_ENV" <<EOF
           CI=true
           SHARED_DIR=${SHARED_DIR:-/tmp}
+          ARTIFACT_DIR=${ARTIFACT_DIR}
 
           CI_JOB_NAME=${JOB_NAME}
           JOB_NAME=${JOB_NAME}
           JOB_NAME_SAFE=qa-e2e-tests
           GRADLE_USER_HOME=${HOME}
 
-          ARTIFACT_DIR=${ARTIFACT_DIR}
+          GOOGLE_CREDENTIALS_GCR_SCANNER_V2=$(echo -e "${GOOGLE_CREDENTIALS_GCR_SCANNER_V2}")
           EOF
 
       - name: Check if testing on Infra cluster
@@ -197,9 +198,14 @@ jobs:
         run: |
           make -C qa-tests-backend compile
 
+      - timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          cd qa-tests-backend
+          ./gradlew test --tests=LocalQaPropsTest
+
       - timeout-minutes: 35
         continue-on-error: true
-        # TODO: add gcr secret, possibly others for test req's
         run: |
           test_suite="smoke-test"
           make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
@@ -216,6 +222,13 @@ jobs:
           test_suite="test"
           make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
 
+      - name: Check shared dir and artifacts
+        if: always()
+        run: |
+          set -x
+          ls -latr ${ARTIFACT_DIR}/**
+          ls -latr ${SHARED_DIR}/**
+
       - name: Publish Test Report
         uses: test-summary/action@v2
         if: always()

From 230fd76d83ef913b231f4f796e5eaddb6cff6ff6 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 11:06:49 -0600
Subject: [PATCH 140/256] unset CI

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 700fac1aff7a4..77006576840f8 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -202,6 +202,7 @@ jobs:
         continue-on-error: true
         run: |
           cd qa-tests-backend
+          unset CI
           ./gradlew test --tests=LocalQaPropsTest
 
       - timeout-minutes: 35

From e8382e28c363f110d94d0b54b73fb0445049d4bc Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 11:42:58 -0600
Subject: [PATCH 141/256] replace stackrox

---
 .github/workflows/e2e-tests.yaml | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 77006576840f8..4d8a3972bb3e4 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -112,6 +112,20 @@ jobs:
           echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
           echo "TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
 
+      - name: Remove existing Stackrox
+        if: ${{ steps.cluster-exists.outcome != 'failure' }}
+        env:
+          DEPLOY_STACKROX_VIA_OPERATOR: "false"
+          LOAD_BALANCER: "lb"
+          SCANNER_SUPPORT: "true"
+          MONITORING_SUPPORT: "false"
+          LOCAL_DEPLOYMENT: "true"
+          ROX_BASELINE_GENERATION_DURATION: "1m"
+          ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
+        run: |
+          kubectl delete namespace stackrox || true
+          ./deploy/k8s/deploy.sh
+
       - name: Find or Deploy Stackrox
         id: rox-installed
         if: ${{ steps.cluster-exists.outcome != 'failure' }}
@@ -226,9 +240,10 @@ jobs:
       - name: Check shared dir and artifacts
         if: always()
         run: |
+          set +e
           set -x
-          ls -latr ${ARTIFACT_DIR}/**
-          ls -latr ${SHARED_DIR}/**
+          ls -latr ${ARTIFACT_DIR}**
+          ls -latr ${SHARED_DIR}**
 
       - name: Publish Test Report
         uses: test-summary/action@v2

From 63fa3c59182e08a192b56353b0fef5fa5d20426c Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 12:07:06 -0600
Subject: [PATCH 142/256] log scannerv4 set

---
 .github/workflows/e2e-tests.yaml | 4 +++-
 deploy/k8s/deploy.sh             | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 4d8a3972bb3e4..f77babe58dad9 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -124,7 +124,9 @@ jobs:
           ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
         run: |
           kubectl delete namespace stackrox || true
-          ./deploy/k8s/deploy.sh
+          echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
+          ./deploy/k8s/deploy.sh \
+            || kubectl -n stackrox get deploy
 
       - name: Find or Deploy Stackrox
         id: rox-installed
diff --git a/deploy/k8s/deploy.sh b/deploy/k8s/deploy.sh
index a53b47fd1d6b0..e4ce6f217e501 100755
--- a/deploy/k8s/deploy.sh
+++ b/deploy/k8s/deploy.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
+echo "(deploy.sh) ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
 K8S_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
 # shellcheck source=./central.sh
 source "${K8S_DIR}/central.sh"

From 403b1d160130c64d10a71c1a258906371f44b525 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 12:09:09 -0600
Subject: [PATCH 143/256] and tail git fetch bg

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index f77babe58dad9..bd86ba8fb9045 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -244,6 +244,7 @@ jobs:
         run: |
           set +e
           set -x
+          tail /tmp/git.fetching.log
           ls -latr ${ARTIFACT_DIR}**
           ls -latr ${SHARED_DIR}**
 

From 74fe561a62e7f7c24ac10d905afb69a0c05ceced Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 13:34:31 -0600
Subject: [PATCH 144/256] use deploy password if deploy.sh

---
 .github/workflows/e2e-tests.yaml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index bd86ba8fb9045..55d38e74ba8e8 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -140,8 +140,13 @@ jobs:
           ROX_BASELINE_GENERATION_DURATION: "1m"
           ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
         run: |
+          set +e
           kubectl get namespace stackrox \
-            || ./deploy/k8s/deploy.sh
+            || { 
+              ./deploy/k8s/deploy.sh;
+              find deploy -name password;
+              echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";
+            }
 
       - name: Stackrox service and auth lookup
         id: stackrox-api
@@ -173,6 +178,7 @@ jobs:
                 echo $! | tee /tmp/forward.pid
                 API_PORT=8000
               fi
+              API_PORT=${API_PORT:-443}
               echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
               echo "API_PORT=${API_PORT}" | tee -a "$GITHUB_ENV"
             fi

From cb5d6750af58cfeb220fb5c5ef548bfd2fbe9aa0 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 7 Aug 2025 13:40:27 -0600
Subject: [PATCH 145/256] remove duplicate deploy

---
 .github/workflows/e2e-tests.yaml | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 55d38e74ba8e8..303fe34babeee 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -114,19 +114,8 @@ jobs:
 
       - name: Remove existing Stackrox
         if: ${{ steps.cluster-exists.outcome != 'failure' }}
-        env:
-          DEPLOY_STACKROX_VIA_OPERATOR: "false"
-          LOAD_BALANCER: "lb"
-          SCANNER_SUPPORT: "true"
-          MONITORING_SUPPORT: "false"
-          LOCAL_DEPLOYMENT: "true"
-          ROX_BASELINE_GENERATION_DURATION: "1m"
-          ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
         run: |
           kubectl delete namespace stackrox || true
-          echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
-          ./deploy/k8s/deploy.sh \
-            || kubectl -n stackrox get deploy
 
       - name: Find or Deploy Stackrox
         id: rox-installed
@@ -141,6 +130,7 @@ jobs:
           ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
         run: |
           set +e
+          echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
           kubectl get namespace stackrox \
             || { 
               ./deploy/k8s/deploy.sh;

From 2d01dfca9e972ed34fc69902c06b1051b63efda6 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 11 Aug 2025 15:20:14 -0600
Subject: [PATCH 146/256] xtrace in deploy

---
 deploy/common/k8sbased.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deploy/common/k8sbased.sh b/deploy/common/k8sbased.sh
index 4277eee9be2b4..fd5910575c1ba 100644
--- a/deploy/common/k8sbased.sh
+++ b/deploy/common/k8sbased.sh
@@ -553,6 +553,7 @@ function launch_central {
     # to ongoing modifications to the central deployment. This port-forward dies and the script hangs "Waiting for
     # Central to respond" until it times out. Waiting for rollout status should help not get into such situation.
     rollout_wait_timeout="10m"
+    set -x
     kubectl -n "${central_namespace}" rollout status deploy/central --timeout="${rollout_wait_timeout}"
 
     # if we have specified that we want to use a load balancer, then use that endpoint instead of localhost

From f5a6571e168507a6e1e38edfe9af0fe1cf6a5000 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 11 Aug 2025 15:20:54 -0600
Subject: [PATCH 147/256] CI=true

---
 .github/workflows/e2e-tests.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 303fe34babeee..5a4ff1a6e14df 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -19,6 +19,7 @@ jobs:
   e2e-tests:
     runs-on: ubuntu-latest
     env:
+      CI: "true"
       INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
       POD_SECURITY_POLICIES: "false"
       ROX_VULN_MGMT_UNIFIED_CVE_DEFERRAL: "true"
@@ -133,7 +134,7 @@ jobs:
           echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
           kubectl get namespace stackrox \
             || { 
-              ./deploy/k8s/deploy.sh;
+              CI=true ./deploy/k8s/deploy.sh;
               find deploy -name password;
               echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";
             }

From c7f17e90b2e497e922c8a571108ffc36a4966b3a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 11 Aug 2025 15:31:01 -0600
Subject: [PATCH 148/256] scc setup for tests

---
 .github/workflows/e2e-tests.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 5a4ff1a6e14df..6b135b0866d15 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -134,6 +134,9 @@ jobs:
           echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
           kubectl get namespace stackrox \
             || { 
+              if [[ "${CLUSTER,,}" == "openshift" ]]; then
+                  kubectl get scc qatest-anyuid || kubectl create -f "qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml"
+              fi
               CI=true ./deploy/k8s/deploy.sh;
               find deploy -name password;
               echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";

From b78a3ecfe2acab55c12fd1c8864f5f3405e38b1a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 11 Aug 2025 15:37:19 -0600
Subject: [PATCH 149/256] create ns before scc patch

---
 .github/workflows/e2e-tests.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 6b135b0866d15..bb80dd4e36bb3 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -135,7 +135,11 @@ jobs:
           kubectl get namespace stackrox \
             || { 
               if [[ "${CLUSTER,,}" == "openshift" ]]; then
-                  kubectl get scc qatest-anyuid || kubectl create -f "qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml"
+                  kubectl get scc qatest-anyuid \
+                    || {
+                      kubectl create ns stackrox;
+                      kubectl create -f "qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml";
+                    }
               fi
               CI=true ./deploy/k8s/deploy.sh;
               find deploy -name password;

From ed8d38b28499a67af95e2d80ebc471c887f521f5 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 11 Aug 2025 15:47:36 -0600
Subject: [PATCH 150/256] try direct in scc setup

---
 qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml b/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml
index c28290f809c30..749689d2d1c1d 100644
--- a/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml
+++ b/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml
@@ -19,14 +19,15 @@ metadata:
       harness augments qatest-anyuid capabilities in the test environment setup
       stage.
     release.openshift.io/create-only: "false"
-allowHostDirVolumePlugin: false
+allowHostDirVolumePlugin: true
 allowHostIPC: false
-allowHostNetwork: false
+allowHostNetwork: true
 allowHostPID: false
-allowHostPorts: false
+allowHostPorts: true
 allowPrivilegeEscalation: true
-allowPrivilegedContainer: false
-allowedCapabilities: null
+allowPrivilegedContainer: true
+allowedCapabilities: ["*"]
+allowedUnsafeSysctls: ["*"]
 defaultAddCapabilities: null
 fsGroup:
   type: RunAsAny
@@ -42,7 +43,7 @@ seLinuxContext:
   type: MustRunAs
 supplementalGroups:
   type: RunAsAny
-users: []
+users: [system:serviceaccount:stackrox:default]
 volumes:
 - configMap
 - downwardAPI

From 7bf2a396bf9cf1ee64edf127b8e0311ca9682f51 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 11 Aug 2025 15:49:22 -0600
Subject: [PATCH 151/256] requiredDrop null

---
 qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml b/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml
index 749689d2d1c1d..95094f6e52fe7 100644
--- a/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml
+++ b/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml
@@ -35,8 +35,7 @@ groups:
 - system:cluster-admins
 priority: 10
 readOnlyRootFilesystem: false
-requiredDropCapabilities:
-- MKNOD
+requiredDropCapabilities: null
 runAsUser:
   type: RunAsAny
 seLinuxContext:

From 04fb7c228b679b24fac679ac7d09e129376b2f7b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 09:27:25 -0600
Subject: [PATCH 152/256] do not always remove

---
 .github/workflows/e2e-tests.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index bb80dd4e36bb3..c2ad77dc2c241 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -113,10 +113,10 @@ jobs:
           echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
           echo "TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
 
-      - name: Remove existing Stackrox
-        if: ${{ steps.cluster-exists.outcome != 'failure' }}
-        run: |
-          kubectl delete namespace stackrox || true
+      #- name: Remove existing Stackrox
+      #  if: ${{ steps.cluster-exists.outcome != 'failure' }}
+      #  run: |
+      #    kubectl delete namespace stackrox || true
 
       - name: Find or Deploy Stackrox
         id: rox-installed

From 0bac9f95536d760fc977969d2698a2e9e73e34ff Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 11:04:09 -0600
Subject: [PATCH 153/256] stop logging var setup

---
 .github/workflows/e2e-tests.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index c2ad77dc2c241..8632ffcfecc04 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -60,14 +60,15 @@ jobs:
 
       - name: Env variables
         run: |
-          set -x
-
           JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
 
           ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/artifacts/}"
           mkdir -p "$ARTIFACT_DIR"
 
-          tee -a "$GITHUB_ENV" <<EOF
+          GOOGLE_CREDENTIALS_GCR_SCANNER_V2=$(printf "%s" "${GOOGLE_CREDENTIALS_GCR_SCANNER_V2}")
+          echo "Does this have escapes?!"
+          echo "${GOOGLE_CREDENTIALS_GCR_SCANNER_V2:0:10}"
+          cat >> "$GITHUB_ENV" <<EOF
           CI=true
           SHARED_DIR=${SHARED_DIR:-/tmp}
           ARTIFACT_DIR=${ARTIFACT_DIR}
@@ -77,7 +78,7 @@ jobs:
           JOB_NAME_SAFE=qa-e2e-tests
           GRADLE_USER_HOME=${HOME}
 
-          GOOGLE_CREDENTIALS_GCR_SCANNER_V2=$(echo -e "${GOOGLE_CREDENTIALS_GCR_SCANNER_V2}")
+          GOOGLE_CREDENTIALS_GCR_SCANNER_V2=${GOOGLE_CREDENTIALS_GCR_SCANNER_V2}
           EOF
 
       - name: Check if testing on Infra cluster

From e4143f8d063748c8748b7766a6c83bd8a85cc8bb Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 15:26:42 -0600
Subject: [PATCH 154/256] without re-setting in env

---
 .github/workflows/e2e-tests.yaml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 8632ffcfecc04..235807605425e 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -65,7 +65,6 @@ jobs:
           ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/artifacts/}"
           mkdir -p "$ARTIFACT_DIR"
 
-          GOOGLE_CREDENTIALS_GCR_SCANNER_V2=$(printf "%s" "${GOOGLE_CREDENTIALS_GCR_SCANNER_V2}")
           echo "Does this have escapes?!"
           echo "${GOOGLE_CREDENTIALS_GCR_SCANNER_V2:0:10}"
           cat >> "$GITHUB_ENV" <<EOF
@@ -77,8 +76,6 @@ jobs:
           JOB_NAME=${JOB_NAME}
           JOB_NAME_SAFE=qa-e2e-tests
           GRADLE_USER_HOME=${HOME}
-
-          GOOGLE_CREDENTIALS_GCR_SCANNER_V2=${GOOGLE_CREDENTIALS_GCR_SCANNER_V2}
           EOF
 
       - name: Check if testing on Infra cluster

From 2a9899cc138a5cc85cbff65fd33caa2db3d53126 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 15:53:00 -0600
Subject: [PATCH 155/256] check for null pr body also

---
 scripts/ci/lib.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 4fb8ce2def41e..cf26286d598e3 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2450,7 +2450,8 @@ test_on_infra() {
     REPO_OWNER=${REPO_OWNER:-stackrox}
     local tries=4
     event_json=''
-    while [[ -z "$event_json" ]]; do
+    body='null'
+    while [[ -z "$event_json" || "$body" == 'null' ]]; do
       event_json=$(set -x; curl --silent \
         -H "X-GitHub-Api-Version: 2022-11-28" \
         -H "Accept: application/vnd.github+json" \
@@ -2459,11 +2460,11 @@ test_on_infra() {
         sleep 30
         echo $(( tries-- )) >&2
       else
+        body=$(jq -r '.body' <<<"${event_json}" \
+          | tee >(cat >&2) | grep '^/test-on-infra') || return
         break
       fi
     done
-    body=$(jq -r '.body' <<<"${event_json}" \
-      | tee >(cat >&2) | grep '^/test-on-infra') || return
     while read -r cmd cluster_name job_name_match; do
       job_name_match=${job_name_match%%]*}
       job_name_match=${job_name_match##*[}

From 51d903a3b056a1d8f9d2ef2876e03d044f0feeef Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 16:01:19 -0600
Subject: [PATCH 156/256] retry null desc also

---
 scripts/ci/lib.sh | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index cf26286d598e3..63ba5b319c1ef 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2446,22 +2446,32 @@ test_on_infra() {
       touch "${SHARED_DIR}/file_write_test"
     fi
     echo "SHARED_DIR=${SHARED_DIR}" >&2
-    REPO_NAME=${REPO_NAME:-stackrox}
-    REPO_OWNER=${REPO_OWNER:-stackrox}
+    local REPO_NAME=${REPO_NAME:-stackrox}
+    local REPO_OWNER=${REPO_OWNER:-stackrox}
     local tries=4
     event_json=''
     body='null'
-    while [[ -z "$event_json" || "$body" == 'null' ]]; do
+    for (( tries=4; tries>0; tries-- )); do
       event_json=$(set -x; curl --silent \
         -H "X-GitHub-Api-Version: 2022-11-28" \
         -H "Accept: application/vnd.github+json" \
         -o - "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/pulls/${PULL_NUMBER}")
-      if [[ $tries -gt 0 && -z "$event_json" ]]; then
-        sleep 30
-        echo $(( tries-- )) >&2
+      if [[ -z "$event_json" ]]; then
+        echo 'No pr description found.' .&2
       else
         body=$(jq -r '.body' <<<"${event_json}" \
-          | tee >(cat >&2) | grep '^/test-on-infra') || return
+          | tee >(cat >&2) | grep '^/test-on-infra') || true
+        if [[ "$body" == 'null' ]]; then
+          echo 'No PR description found.' .&2
+        else
+          echo 'PR description found.' .&2
+          break
+        fi
+      fi
+      if [[ $tries -gt 0 ]]; then
+        sleep 30
+        echo "Retry fetch github pr description $(( tries-- ))" >&2
+      else
         break
       fi
     done

From 5915398dc5035ec532f41e611d9fa91c0fc877c8 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 16:01:46 -0600
Subject: [PATCH 157/256] remove var check

---
 .github/workflows/e2e-tests.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 235807605425e..763e8887418df 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -65,8 +65,6 @@ jobs:
           ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/artifacts/}"
           mkdir -p "$ARTIFACT_DIR"
 
-          echo "Does this have escapes?!"
-          echo "${GOOGLE_CREDENTIALS_GCR_SCANNER_V2:0:10}"
           cat >> "$GITHUB_ENV" <<EOF
           CI=true
           SHARED_DIR=${SHARED_DIR:-/tmp}

From cf7cf828ac8b7e2f37151c52b35ef17cd361616a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 16:03:02 -0600
Subject: [PATCH 158/256] fix redirect

---
 scripts/ci/lib.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 63ba5b319c1ef..628fe0c2a6b19 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2462,9 +2462,9 @@ test_on_infra() {
         body=$(jq -r '.body' <<<"${event_json}" \
           | tee >(cat >&2) | grep '^/test-on-infra') || true
         if [[ "$body" == 'null' ]]; then
-          echo 'No PR description found.' .&2
+          echo 'No PR description found.' >&2
         else
-          echo 'PR description found.' .&2
+          echo 'PR description found.' >&2
           break
         fi
       fi

From 54c52deb3c3da13d0915608882ab9b6c4671cf15 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 16:31:30 -0600
Subject: [PATCH 159/256] try new sha

---
 qa-tests-backend/src/test/groovy/LocalQaPropsTest.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa-tests-backend/src/test/groovy/LocalQaPropsTest.groovy b/qa-tests-backend/src/test/groovy/LocalQaPropsTest.groovy
index 48353aa789692..b5c4a53823f03 100644
--- a/qa-tests-backend/src/test/groovy/LocalQaPropsTest.groovy
+++ b/qa-tests-backend/src/test/groovy/LocalQaPropsTest.groovy
@@ -27,7 +27,7 @@ class LocalQaPropsTest extends Specification {
         def canonicalJson = JsonOutput.toJson(rawData)
         def canonicalJsonSha256 = DigestUtils.sha256Hex(canonicalJson)
         then:
-        canonicalJsonSha256 == 'f75d8cf9ea0c7886293f689478daafe75126c719313b4366c02bd41d69bb05e5'
+        canonicalJsonSha256 == '86fc788697b3f201422cda5ee1e7a98882f8929c02e668599c6c190d080230c2'
     }
 
     def "CheckPropertyFileInputValue > GOOGLE_CREDENTIALS_GCR_NO_ACCESS_KEY_V2"() {

From 60ada4c8d2c83b9cf586159e0a60f8f6b2cc3853 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 17:41:01 -0600
Subject: [PATCH 160/256] gradle fail fast

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 763e8887418df..56954c6722a70 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -20,6 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       CI: "true"
+      FAIL_FAST: "true"
       INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
       POD_SECURITY_POLICIES: "false"
       ROX_VULN_MGMT_UNIFIED_CVE_DEFERRAL: "true"

From 160a9f9ed9bcd3e9b96ffa24b307702554c5fde6 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 22:04:22 -0600
Subject: [PATCH 161/256] show all in summary table

---
 .github/workflows/e2e-tests.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 56954c6722a70..604d57edfba03 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -251,6 +251,7 @@ jobs:
 
       - name: Publish Test Report
         uses: test-summary/action@v2
-        if: always()
+        if: !cancelled()
         with:
           paths: 'qa-tests-backend/build/test-results/**/TEST-*.xml'
+          show: "all"

From 35ae053f153c1cd54d587b2b2fd4d308ae311609 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 22:27:03 -0600
Subject: [PATCH 162/256] manual test summary

---
 .github/workflows/e2e-tests.yaml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 604d57edfba03..5e00b885d7594 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -250,8 +250,21 @@ jobs:
           ls -latr ${SHARED_DIR}**
 
       - name: Publish Test Report
+        id: test-results
         uses: test-summary/action@v2
+        continue-on-error: true
         if: !cancelled()
         with:
           paths: 'qa-tests-backend/build/test-results/**/TEST-*.xml'
           show: "all"
+
+      - name: Determine cluster flavor
+        if: ${{ steps.test-results.outcome == 'failure' }}
+        timeout-minutes: 5
+        run: |
+          tee -a "$GITHUB_STEP_SUMMARY" <<EOF
+          :boom: ${{ steps.test_summary.outputs.failed }}/${{ steps.test_summary.outputs.total }} failed
+          :seedling: ${{ steps.test_summary.outputs.passed }}/${{ steps.test_summary.outputs.total }} passed
+          :zzz: ${{ steps.test_summary.outputs.skipped }}/${{ steps.test_summary.outputs.total }} skipped
+          EOF
+

From 86610869fe373c233481986436065c51b1556bab Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 22:40:42 -0600
Subject: [PATCH 163/256] wrap !cancelled

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 5e00b885d7594..d4af3a8f723e6 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -253,7 +253,7 @@ jobs:
         id: test-results
         uses: test-summary/action@v2
         continue-on-error: true
-        if: !cancelled()
+        if: ${{ !cancelled() }}
         with:
           paths: 'qa-tests-backend/build/test-results/**/TEST-*.xml'
           show: "all"

From 9d8510de6b9d3b9ee70e734fa678270f33deb801 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 22:49:18 -0600
Subject: [PATCH 164/256] remove debug log of collect vars

---
 qa-tests-backend/src/main/groovy/util/Helpers.groovy | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/qa-tests-backend/src/main/groovy/util/Helpers.groovy b/qa-tests-backend/src/main/groovy/util/Helpers.groovy
index 79a0fd9869890..7a72a27b544df 100644
--- a/qa-tests-backend/src/main/groovy/util/Helpers.groovy
+++ b/qa-tests-backend/src/main/groovy/util/Helpers.groovy
@@ -88,7 +88,6 @@ class Helpers {
 
     @CompileDynamic
     static void collectDebugForFailure(Throwable exception) {
-        log.info("collectDebugForFailure" + exception)
         if (!collectDebug()) {
             return
         }
@@ -183,10 +182,6 @@ class Helpers {
     }
 
     private static boolean collectDebug() {
-        log.warn("Debug collection skip check: "+
-                 "(CI: ${Env.IN_CI} ||"+
-                 " GATHER_QA_TEST_DEBUG_LOGS: ${Env.GATHER_QA_TEST_DEBUG_LOGS}) && "+
-                 " QA_TEST_DEBUG_LOGS: ${Env.QA_TEST_DEBUG_LOGS} != ''")
         if ((Env.IN_CI || Env.GATHER_QA_TEST_DEBUG_LOGS) && (Env.QA_TEST_DEBUG_LOGS != "")) {
             return true
         }

From 0a063f148fe6911fc01a735ce8c852946171e9eb Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 12 Aug 2025 23:02:44 -0600
Subject: [PATCH 165/256] quiet gradle

---
 .github/workflows/e2e-tests.yaml | 8 --------
 qa-tests-backend/Makefile        | 2 +-
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index d4af3a8f723e6..46fa0d645a551 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -215,13 +215,6 @@ jobs:
         run: |
           make -C qa-tests-backend compile
 
-      - timeout-minutes: 5
-        continue-on-error: true
-        run: |
-          cd qa-tests-backend
-          unset CI
-          ./gradlew test --tests=LocalQaPropsTest
-
       - timeout-minutes: 35
         continue-on-error: true
         run: |
@@ -267,4 +260,3 @@ jobs:
           :seedling: ${{ steps.test_summary.outputs.passed }}/${{ steps.test_summary.outputs.total }} passed
           :zzz: ${{ steps.test_summary.outputs.skipped }}/${{ steps.test_summary.outputs.total }} skipped
           EOF
-
diff --git a/qa-tests-backend/Makefile b/qa-tests-backend/Makefile
index 10861a71f6079..1c5ac001dc89b 100644
--- a/qa-tests-backend/Makefile
+++ b/qa-tests-backend/Makefile
@@ -6,7 +6,7 @@ GRADLE_TEST_ARGS =
 # Stop testing after the first test failure by defining FAIL_FAST. (Can be
 # achieved on a PR with a ci-fail-fast label)
 ifdef FAIL_FAST
-GRADLE_TEST_ARGS = --fail-fast
+GRADLE_TEST_ARGS = --fail-fast --quiet
 endif
 
 ###########

From 518027a40ac24e201b464c7d7e6871b9bd64b907 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 08:11:49 -0600
Subject: [PATCH 166/256] wait for tag match

---
 .github/workflows/e2e-tests.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 46fa0d645a551..7111ea805b3b0 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -101,8 +101,11 @@ jobs:
         timeout-minutes: 0.5
         continue-on-error: true
         run: |
-          git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*' \
-            || { while pgrep -F /tmp/git.fetching.pid; do sleep 1; done; }
+          while [[ "$(git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*')" \
+            != *".x-"* ]]; do
+            sleep 1
+            pgrep -F /tmp/git.fetching.pid || break
+          done
 
       - run: |
           # example: 4.9.x-457-g9867327a52

From 5d61726b15e63e93ba65d93040099e08ba82cc8d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 08:14:06 -0600
Subject: [PATCH 167/256] git describe always

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 7111ea805b3b0..9e5603ec764ec 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -101,7 +101,7 @@ jobs:
         timeout-minutes: 0.5
         continue-on-error: true
         run: |
-          while [[ "$(git describe --tags --abbrev=10 --dirty --long --exclude '*-nightly-*')" \
+          while [[ "$(git describe --tags --abbrev=10 --always --dirty --long --exclude '*-nightly-*')" \
             != *".x-"* ]]; do
             sleep 1
             pgrep -F /tmp/git.fetching.pid || break

From ab0d608022b188da4212c22efc78c622e6002931 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 08:16:13 -0600
Subject: [PATCH 168/256] check gradle daemon

---
 .github/workflows/e2e-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 9e5603ec764ec..b9c0713e3f5e7 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -217,10 +217,12 @@ jobs:
         continue-on-error: true
         run: |
           make -C qa-tests-backend compile
+          gradle --status || true
 
       - timeout-minutes: 35
         continue-on-error: true
         run: |
+          gradle --status || true
           test_suite="smoke-test"
           make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
 

From 3b8f028bf461852e79d124948ebc9d1d1cad15f6 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 08:17:28 -0600
Subject: [PATCH 169/256] xtrace tag check

---
 .github/workflows/e2e-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index b9c0713e3f5e7..b00adb98e4960 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -101,11 +101,13 @@ jobs:
         timeout-minutes: 0.5
         continue-on-error: true
         run: |
+          set -x
           while [[ "$(git describe --tags --abbrev=10 --always --dirty --long --exclude '*-nightly-*')" \
             != *".x-"* ]]; do
             sleep 1
             pgrep -F /tmp/git.fetching.pid || break
           done
+          git describe --tags --abbrev=10 --always --dirty --long --exclude '*-nightly-*' || true
 
       - run: |
           # example: 4.9.x-457-g9867327a52

From 88f1c83951727b530518deb7b54836a13d56a5c7 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 08:23:16 -0600
Subject: [PATCH 170/256] match in git describe

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index b00adb98e4960..a29c7f6777182 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -102,7 +102,7 @@ jobs:
         continue-on-error: true
         run: |
           set -x
-          while [[ "$(git describe --tags --abbrev=10 --always --dirty --long --exclude '*-nightly-*')" \
+          while [[ "$(git describe --match='*.x' --tags --abbrev=10 --always --dirty --long --exclude '*-nightly-*')" \
             != *".x-"* ]]; do
             sleep 1
             pgrep -F /tmp/git.fetching.pid || break

From a2464d5a8ef6db247c74c2e5adb4538fe77cebd6 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 08:25:34 -0600
Subject: [PATCH 171/256] fetch all in bg

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index a29c7f6777182..4567da06ce1ae 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -51,7 +51,7 @@ jobs:
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
       - run: |
-          nohup git fetch --depth=10000 origin '+refs/tags/*:refs/tags/*' >>/tmp/git.fetching.log 2>&1 &
+          nohup git fetch origin '+refs/tags/*:refs/tags/*' >>/tmp/git.fetching.log 2>&1 &
           echo $! | tee /tmp/git.fetching.pid
 
       - uses: ./.github/actions/job-preamble

From bce60d1f9b2a5ac6b3ee81a9dd3cbdf1e8e0da0a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 08:31:52 -0600
Subject: [PATCH 172/256] no fail on image tag

---
 .github/workflows/e2e-tests.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 4567da06ce1ae..05110e1d39992 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -108,8 +108,7 @@ jobs:
             pgrep -F /tmp/git.fetching.pid || break
           done
           git describe --tags --abbrev=10 --always --dirty --long --exclude '*-nightly-*' || true
-
-      - run: |
+          set +e
           # example: 4.9.x-457-g9867327a52
           MAIN_IMAGE_TAG=$(git describe --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG:-master}")
           echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"

From 52d6086de2e609992848cc0d4b355b3b098e5621 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 08:56:07 -0600
Subject: [PATCH 173/256] use gradle wrapper

---
 .github/workflows/e2e-tests.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 05110e1d39992..9a698bde2772f 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -218,12 +218,12 @@ jobs:
         continue-on-error: true
         run: |
           make -C qa-tests-backend compile
-          gradle --status || true
+          { pushd qa-tests-backend; ./gradlew --status || true; popd; }
 
       - timeout-minutes: 35
         continue-on-error: true
         run: |
-          gradle --status || true
+          { pushd qa-tests-backend; ./gradlew --status || true; popd; }
           test_suite="smoke-test"
           make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
 

From 9b426e176355364e3e6b7c9512362cac3752f736 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 09:05:44 -0600
Subject: [PATCH 174/256] use gha gradle action

---
 .github/workflows/e2e-tests.yaml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 9a698bde2772f..1125778b1c758 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -16,6 +16,12 @@ env:
   ROX_PRODUCT_BRANDING: stackrox
 
 jobs:
+  gradle-validation:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: gradle/actions/wrapper-validation@v4
+
   e2e-tests:
     runs-on: ubuntu-latest
     env:
@@ -214,6 +220,12 @@ jobs:
           install --mode 755 /tmp/oc /usr/local/bin/
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          gradle-version: '8.50'
+          # TODO: enable on master: cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
+
       - timeout-minutes: 10
         continue-on-error: true
         run: |

From 5ab984045345092aef5b51c35c9bb9cc9897cf3f Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 10:00:53 -0600
Subject: [PATCH 175/256] enable echo on compile

---
 qa-tests-backend/Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/qa-tests-backend/Makefile b/qa-tests-backend/Makefile
index 1c5ac001dc89b..fd7dffaa48ad7 100644
--- a/qa-tests-backend/Makefile
+++ b/qa-tests-backend/Makefile
@@ -15,7 +15,7 @@ endif
 .PHONY: style
 style: compile
 	@echo "+ $@"
-	@$(GRADLE) --continue codenarcMain codenarcTest ; \
+	$(GRADLE) --continue codenarcMain codenarcTest ; \
 	STATUS=$$? ; \
 	if [ $$STATUS -ne 0 ]; then \
 	    cat build/reports/codenarc/main.txt build/reports/codenarc/test.txt >&2 2>/dev/null ; \
@@ -25,12 +25,12 @@ style: compile
 .PHONY: compile
 compile: build/generated
 	@echo "+ $@"
-	@$(GRADLE) assemble testClasses
+	$(GRADLE) assemble testClasses
 
 .PHONY: style-fix
 style-fix:
 	@echo "+ $@"
-	@scripts/fix_lint.py .
+	scripts/fix_lint.py .
 
 .PHONY: proto-generated-srcs
 proto-generated-srcs: build/generated
@@ -42,7 +42,7 @@ build/generated: src/main/proto/scanner
 src/main/proto/scanner: ../go.mod
 	@echo "+ copy scanner protos"
 	-rm -rf src/main/proto/scanner
-	@scripts/copy_scanner_protos.sh
+	scripts/copy_scanner_protos.sh
 
 .PHONY: clean-generated-srcs
 clean-generated-srcs:

From 93c941d69c55dbf156ee1b2c23455cf6ed9f42d7 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 10:12:48 -0600
Subject: [PATCH 176/256] use wrapper gradle version

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 1125778b1c758..e8701f9992cfb 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -223,7 +223,7 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4
         with:
-          gradle-version: '8.50'
+          gradle-version: wrapper
           # TODO: enable on master: cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
 
       - timeout-minutes: 10

From 4d9fae4c48eba971636cf74d511f2b416802d54c Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 10:21:59 -0600
Subject: [PATCH 177/256] try gradle 8.14

---
 .github/workflows/e2e-tests.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index e8701f9992cfb..67bb77f7e2621 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -226,6 +226,10 @@ jobs:
           gradle-version: wrapper
           # TODO: enable on master: cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
 
+      - run: |
+          pushd qa-tests-backend
+          ./gradlew wrapper --gradle-version "8.14" || true
+
       - timeout-minutes: 10
         continue-on-error: true
         run: |

From c6375248f9625ee5d20f24d1004ef46e85a74997 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 10:26:12 -0600
Subject: [PATCH 178/256] update before setup

---
 .github/workflows/e2e-tests.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 67bb77f7e2621..ebef6441a7692 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -220,16 +220,16 @@ jobs:
           install --mode 755 /tmp/oc /usr/local/bin/
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
+      - run: |
+          pushd qa-tests-backend
+          ./gradlew wrapper --gradle-version "8.14" || true
+
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4
         with:
           gradle-version: wrapper
           # TODO: enable on master: cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
 
-      - run: |
-          pushd qa-tests-backend
-          ./gradlew wrapper --gradle-version "8.14" || true
-
       - timeout-minutes: 10
         continue-on-error: true
         run: |

From 4dc5aebf0eb2b00d525376852ad419c09f921e3f Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 10:31:46 -0600
Subject: [PATCH 179/256] log=warn

---
 qa-tests-backend/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa-tests-backend/Makefile b/qa-tests-backend/Makefile
index fd7dffaa48ad7..1f3aa3248d975 100644
--- a/qa-tests-backend/Makefile
+++ b/qa-tests-backend/Makefile
@@ -6,7 +6,7 @@ GRADLE_TEST_ARGS =
 # Stop testing after the first test failure by defining FAIL_FAST. (Can be
 # achieved on a PR with a ci-fail-fast label)
 ifdef FAIL_FAST
-GRADLE_TEST_ARGS = --fail-fast --quiet
+GRADLE_TEST_ARGS = --fail-fast --warn
 endif
 
 ###########

From 65796dddfbddc2657824f9975584f243b2bc6e74 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 10:37:30 -0600
Subject: [PATCH 180/256] and check if the gradle wrapper changed

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index ebef6441a7692..24d2b8c05afe1 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -223,6 +223,7 @@ jobs:
       - run: |
           pushd qa-tests-backend
           ./gradlew wrapper --gradle-version "8.14" || true
+          git diff || true
 
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4

From 02350016c9fc50f7764fc4c94b39645d468b5004 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 12:05:38 -0600
Subject: [PATCH 181/256] fail-fast steps also

---
 .github/workflows/e2e-tests.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 24d2b8c05afe1..3ad94a50c57ab 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -245,13 +245,11 @@ jobs:
           make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
 
       - timeout-minutes: 35
-        continue-on-error: true
         run: |
           test_suite="bat-test"
           make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
 
       - timeout-minutes: 35
-        continue-on-error: true
         run: |
           test_suite="test"
           make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"

From 988cf1d4615387e4028c4cca914dcf946954422d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 12:07:05 -0600
Subject: [PATCH 182/256] cancel old jobs

---
 .github/workflows/e2e-tests.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 3ad94a50c57ab..14f95a961ab76 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -15,6 +15,10 @@ on:
 env:
   ROX_PRODUCT_BRANDING: stackrox
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
   gradle-validation:
     runs-on: ubuntu-latest

From 256ae341bf1f87634ef3ae20f215d579f19474b4 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 12:07:37 -0600
Subject: [PATCH 183/256] always run test summary

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 14f95a961ab76..e626ef8722767 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -271,7 +271,7 @@ jobs:
         id: test-results
         uses: test-summary/action@v2
         continue-on-error: true
-        if: ${{ !cancelled() }}
+        if: always()
         with:
           paths: 'qa-tests-backend/build/test-results/**/TEST-*.xml'
           show: "all"

From 263dda1f0e31697f7f5e04a75eb7677beebaa620 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 12:28:52 -0600
Subject: [PATCH 184/256] no continue on test steps

---
 .github/workflows/e2e-tests.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index e626ef8722767..b327a1644660a 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -236,13 +236,11 @@ jobs:
           # TODO: enable on master: cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
 
       - timeout-minutes: 10
-        continue-on-error: true
         run: |
           make -C qa-tests-backend compile
           { pushd qa-tests-backend; ./gradlew --status || true; popd; }
 
       - timeout-minutes: 35
-        continue-on-error: true
         run: |
           { pushd qa-tests-backend; ./gradlew --status || true; popd; }
           test_suite="smoke-test"

From cd1a2949b534a2bf2937fe0c57d3989ffe3ed460 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 12:44:22 -0600
Subject: [PATCH 185/256] check daemon at end

---
 .github/workflows/e2e-tests.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index b327a1644660a..a897a01b2eb52 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -238,11 +238,9 @@ jobs:
       - timeout-minutes: 10
         run: |
           make -C qa-tests-backend compile
-          { pushd qa-tests-backend; ./gradlew --status || true; popd; }
 
       - timeout-minutes: 35
         run: |
-          { pushd qa-tests-backend; ./gradlew --status || true; popd; }
           test_suite="smoke-test"
           make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
 
@@ -264,6 +262,8 @@ jobs:
           tail /tmp/git.fetching.log
           ls -latr ${ARTIFACT_DIR}**
           ls -latr ${SHARED_DIR}**
+          echo "check daemon:"
+          { pushd qa-tests-backend; ./gradlew --status || true; popd; }
 
       - name: Publish Test Report
         id: test-results

From 4f5520c2b5e342bb93dc3dd70eeb9f19f8ab510d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 13:02:21 -0600
Subject: [PATCH 186/256] try project-dir

---
 .github/workflows/e2e-tests.yaml | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index a897a01b2eb52..66279b5eaa173 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -241,18 +241,21 @@ jobs:
 
       - timeout-minutes: 35
         run: |
-          test_suite="smoke-test"
-          make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
+          export TEST_TARGET="smoke-test"
+          gradle testSMOKE --project-dir qa-tests-backend --fail-fast --warn \
+            || qa-tests-backend/gradlew testSMOKE --project-dir qa-tests-backend --fail-fast --warn
 
       - timeout-minutes: 35
         run: |
-          test_suite="bat-test"
-          make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
+          pushd qa-tests-backend
+          export TEST_TARGET="bat-test"
+          ./gradlew testParallelBAT testBAT --fail-fast --warn
 
       - timeout-minutes: 35
         run: |
-          test_suite="test"
-          make -C qa-tests-backend "${test_suite}" 2>&1 | tee -a "/tmp/test.${test_suite}.log"
+          pushd qa-tests-backend
+          export TEST_TARGET="test"
+          ./gradlew testParallel testRest --fail-fast --warn
 
       - name: Check shared dir and artifacts
         if: always()

From 20ce0f751860a3f545f5bbeca92e3fb5d263abb7 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 13:04:59 -0600
Subject: [PATCH 187/256] gradle wrapper direct

---
 .github/workflows/e2e-tests.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 66279b5eaa173..1852168c29e53 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -225,8 +225,7 @@ jobs:
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
       - run: |
-          pushd qa-tests-backend
-          ./gradlew wrapper --gradle-version "8.14" || true
+          gradle wrapper --project-dir qa-tests-backend --gradle-version "8.14" || true
           git diff || true
 
       - name: Setup Gradle

From 5f36ac5b102e81dcf529f6af3cad8f96542bffc0 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 13:07:08 -0600
Subject: [PATCH 188/256] gradle cache use

---
 .github/workflows/e2e-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 1852168c29e53..2009fbfa54f60 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -232,6 +232,8 @@ jobs:
         uses: gradle/actions/setup-gradle@v4
         with:
           gradle-version: wrapper
+          cache-read-only: false
+          cache-overwrite-existing: true
           # TODO: enable on master: cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
 
       - timeout-minutes: 10

From b1a1d8452371fc6a3995a1a8ad17050ef7f51afa Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 13:10:14 -0600
Subject: [PATCH 189/256] use gradlew for update

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 2009fbfa54f60..af90cc2213c3c 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -225,7 +225,7 @@ jobs:
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
       - run: |
-          gradle wrapper --project-dir qa-tests-backend --gradle-version "8.14" || true
+          cd qa-tests-backend && ./gradlew wrapper --gradle-version "8.14" || true
           git diff || true
 
       - name: Setup Gradle

From f64eef83bd12adc4d6a46b38182ec6fc11e4940d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 13:15:48 -0600
Subject: [PATCH 190/256] cd first

---
 .github/workflows/e2e-tests.yaml | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index af90cc2213c3c..c36f7656728d1 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -243,20 +243,17 @@ jobs:
       - timeout-minutes: 35
         run: |
           export TEST_TARGET="smoke-test"
-          gradle testSMOKE --project-dir qa-tests-backend --fail-fast --warn \
-            || qa-tests-backend/gradlew testSMOKE --project-dir qa-tests-backend --fail-fast --warn
+          cd qa-tests-backend && ./gradlew testSMOKE --fail-fast --warn
 
       - timeout-minutes: 35
         run: |
-          pushd qa-tests-backend
           export TEST_TARGET="bat-test"
-          ./gradlew testParallelBAT testBAT --fail-fast --warn
+          cd qa-tests-backend && ./gradlew testParallelBAT testBAT --fail-fast --warn
 
       - timeout-minutes: 35
         run: |
-          pushd qa-tests-backend
           export TEST_TARGET="test"
-          ./gradlew testParallel testRest --fail-fast --warn
+          cd qa-tests-backend && ./gradlew testParallel testRest --fail-fast --warn
 
       - name: Check shared dir and artifacts
         if: always()

From 9ecaae7a6d198898d573ac4020d5ba266a125ff9 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 13:16:37 -0600
Subject: [PATCH 191/256] skip gradle upgrade

---
 .github/workflows/e2e-tests.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index c36f7656728d1..7b01f5cbc40a4 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -224,9 +224,9 @@ jobs:
           install --mode 755 /tmp/oc /usr/local/bin/
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
-      - run: |
-          cd qa-tests-backend && ./gradlew wrapper --gradle-version "8.14" || true
-          git diff || true
+      #- run: |
+      #    cd qa-tests-backend && ./gradlew wrapper --gradle-version "8.14" || true
+      #    git diff || true
 
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4

From 0720e922209856ab25160132ea7e1fbd20a4d6e8 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 13:21:07 -0600
Subject: [PATCH 192/256] retry infractl download

---
 scripts/ci/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 628fe0c2a6b19..8657a5e18c3b5 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2411,7 +2411,7 @@ get_infra_cluster_files() {
     local data_dir="${SHARED_DIR:-/tmp}"
     ls -la "${data_dir}" || true
     infractl --version \
-      || { set -x; curl --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \
+      || { set -x; curl --retry 8 --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \
         | jq -r ".result.fileChunk" \
         | base64 -d > ./infractl;
           file ./infractl || true;

From 22475b9e2ab3960945244f5f2839a07b0c4dd2fc Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 13 Aug 2025 14:36:01 -0600
Subject: [PATCH 193/256] retry whoami

---
 scripts/ci/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 8657a5e18c3b5..11aab4749c3c6 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2416,7 +2416,7 @@ get_infra_cluster_files() {
         | base64 -d > ./infractl;
           file ./infractl || true;
           install infractl ${GOPATH:-/opt}/bin/infractl || install infractl /usr/local/bin/; }
-    infractl whoami
+    retry 5 true infractl whoami
     for I in {1..5}; do
         infractl artifacts ${cluster_name} \
           --download-dir "${data_dir}"

From a2f4e295797c7f1972310383842e8e86120c6738 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 14 Aug 2025 08:09:43 -0600
Subject: [PATCH 194/256] empty commit


From bc01ff42f62eb0fa8ecef77aedf6a60a948c9afb Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 14 Aug 2025 09:08:13 -0600
Subject: [PATCH 195/256] log more in smoke test

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 7b01f5cbc40a4..29412438c7ae3 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -243,7 +243,7 @@ jobs:
       - timeout-minutes: 35
         run: |
           export TEST_TARGET="smoke-test"
-          cd qa-tests-backend && ./gradlew testSMOKE --fail-fast --warn
+          cd qa-tests-backend && ./gradlew testSMOKE --fail-fast
 
       - timeout-minutes: 35
         run: |

From 3ed1d7e0ab2f7929731b6ea4ab30474cace367c2 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 14 Aug 2025 12:53:33 -0600
Subject: [PATCH 196/256] try use master cache

---
 .github/workflows/e2e-tests.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 29412438c7ae3..738cd406a4f19 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -232,9 +232,9 @@ jobs:
         uses: gradle/actions/setup-gradle@v4
         with:
           gradle-version: wrapper
-          cache-read-only: false
+          #cache-read-only: false
           cache-overwrite-existing: true
-          # TODO: enable on master: cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
+          cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
 
       - timeout-minutes: 10
         run: |

From 01131b6d4094d56039c0a972f12ecb29c7afc4f7 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 14 Aug 2025 12:59:10 -0600
Subject: [PATCH 197/256] fetch all

---
 .github/workflows/e2e-tests.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 738cd406a4f19..758ead9ac403b 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -54,9 +54,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 1  # ${{ github.event.pull_request.commits }}
+          fetch-depth: 0  # ${{ github.event.pull_request.commits }}
           # TODO: clone both test and tested as repo's?
-          fetch-tags: false
+          fetch-tags: true
           show-progress: false
           ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 

From 1f0f8087e54492550b84600b4c5047d064a34375 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 14 Aug 2025 13:02:07 -0600
Subject: [PATCH 198/256] try default checkout

---
 .github/workflows/e2e-tests.yaml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 758ead9ac403b..25a300f57b2f4 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -53,12 +53,6 @@ jobs:
       ORCHESTRATOR_FLAVOR: "openshift"
     steps:
       - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0  # ${{ github.event.pull_request.commits }}
-          # TODO: clone both test and tested as repo's?
-          fetch-tags: true
-          show-progress: false
-          ref: ${{ inputs.commit || github.event.pull_request.head.sha }}
 
       - run: |
           nohup git fetch origin '+refs/tags/*:refs/tags/*' >>/tmp/git.fetching.log 2>&1 &

From 09a8370c0e144428369060731e934a8fbc6eeadf Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 14 Aug 2025 13:31:38 -0600
Subject: [PATCH 199/256] empty commit


From 1a61e5116b3a7fb3ab589b6f1ff0c067293f2d6d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 14 Aug 2025 14:02:07 -0600
Subject: [PATCH 200/256] try tag match

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 25a300f57b2f4..405c2d32a00bc 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -114,7 +114,7 @@ jobs:
           git describe --tags --abbrev=10 --always --dirty --long --exclude '*-nightly-*' || true
           set +e
           # example: 4.9.x-457-g9867327a52
-          MAIN_IMAGE_TAG=$(git describe --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG:-master}")
+          MAIN_IMAGE_TAG=$(git describe --match="*.x" --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG:-master}")
           echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
           echo "TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
 

From b226e5b4ebae33f6b7d71535578ca3625373e6bd Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 14 Aug 2025 14:05:47 -0600
Subject: [PATCH 201/256] fetch tags

---
 .github/workflows/e2e-tests.yaml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 405c2d32a00bc..37fbd54a072a9 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -20,12 +20,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  gradle-validation:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: gradle/actions/wrapper-validation@v4
-
   e2e-tests:
     runs-on: ubuntu-latest
     env:
@@ -53,6 +47,10 @@ jobs:
       ORCHESTRATOR_FLAVOR: "openshift"
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # ${{ github.event.pull_request.commits }}
+          fetch-tags: true
+          show-progress: false
 
       - run: |
           nohup git fetch origin '+refs/tags/*:refs/tags/*' >>/tmp/git.fetching.log 2>&1 &

From b74eff37f689e008ed8ca328a71ebf875846a7b9 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 14 Aug 2025 16:12:25 -0600
Subject: [PATCH 202/256] xtrace on ci_export

---
 tests/e2e/lib.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 51bb24753aa95..018793fbf3286 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -203,9 +203,11 @@ export_test_environment() {
         ci_export ROX_NON_AGGREGATED_NETWORKS "${ROX_NON_AGGREGATED_NETWORKS:-34.118.224.0/20}"
     fi
 
+    set -x
     ci_export CLUSTER_API_ENDPOINT "$CLUSTER_API_ENDPOINT"
     ci_export CLUSTER_USERNAME "$CLUSTER_USERNAME"
     ci_export CLUSTER_PASSWORD "$CLUSTER_PASSWORD"
+    set +x
     
     if [[ "${CI_JOB_NAME:-}" =~ ocp-4-[0-9]+-ui-e2e-tests ]]; then
         info "Exposing OCP cluster information for UI e2e tests"

From 82f0c3000e8732c312d6a145e52d59ef2fefd01b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Thu, 14 Aug 2025 16:50:49 -0600
Subject: [PATCH 203/256] var for ocp like osd

---
 .openshift-ci/dispatch.sh |  2 +-
 tests/e2e/lib.sh          | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/.openshift-ci/dispatch.sh b/.openshift-ci/dispatch.sh
index 4c939d8602bcb..65ca11c21dfdb 100755
--- a/.openshift-ci/dispatch.sh
+++ b/.openshift-ci/dispatch.sh
@@ -49,7 +49,7 @@ case "$ci_job" in
 esac
 
 case "$ci_job" in
-    eks-qa-e2e-tests|osd*qa-e2e-tests)
+    eks-qa-e2e-tests|osd*qa-e2e-tests|ocp-4-*-ui-e2e-tests)
         setup_automation_flavor_e2e_cluster "$ci_job"
         ;;
 esac
diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 018793fbf3286..7ba87b99fce13 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -1504,6 +1504,30 @@ setup_automation_flavor_e2e_cluster() {
                 --username "$CLUSTER_USERNAME" \
                 --password "$CLUSTER_PASSWORD" \
                 --insecure-skip-tls-verify=true
+    elif [[ "$ci_job" =~ ocp-4-.*-ui-e2e-tests ]]; then
+        info "Setting up OCP cluster information for UI e2e tests"
+        if [[ -f "${SHARED_DIR}/dotenv" ]]; then
+            source "${SHARED_DIR}/dotenv"
+        fi
+        
+        # Read console URL from url file if available, or use OPENSHIFT_CONSOLE_URL from dotenv
+        if [[ -f "${SHARED_DIR}/url" ]]; then
+            CLUSTER_API_ENDPOINT="$(cat "${SHARED_DIR}/url")"
+            export CLUSTER_API_ENDPOINT
+        elif [[ -n "${OPENSHIFT_CONSOLE_URL:-}" ]]; then
+            export CLUSTER_API_ENDPOINT="$OPENSHIFT_CONSOLE_URL"
+        fi
+        
+        # Map OpenShift console credentials to CLUSTER_* variables for cypress
+        if [[ -n "${OPENSHIFT_CONSOLE_USERNAME:-}" ]]; then
+            export CLUSTER_USERNAME="$OPENSHIFT_CONSOLE_USERNAME"
+        fi
+        if [[ -n "${OPENSHIFT_CONSOLE_PASSWORD:-}" ]]; then
+            export CLUSTER_PASSWORD="$OPENSHIFT_CONSOLE_PASSWORD"
+        fi
+        
+        # Set ORCHESTRATOR_FLAVOR for cypress
+        export ORCHESTRATOR_FLAVOR="openshift"
     fi
 }
 

From 0f62d520c802de9f3efa55248487b99836b58fc7 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 09:52:47 -0600
Subject: [PATCH 204/256] simplify

---
 .openshift-ci/dispatch.sh |  4 ++--
 tests/e2e/lib.sh          | 28 +++++++---------------------
 2 files changed, 9 insertions(+), 23 deletions(-)

diff --git a/.openshift-ci/dispatch.sh b/.openshift-ci/dispatch.sh
index 65ca11c21dfdb..a27c2264dd44c 100755
--- a/.openshift-ci/dispatch.sh
+++ b/.openshift-ci/dispatch.sh
@@ -14,7 +14,7 @@ set -euo pipefail
 
 if [[ -n "${SHARED_DIR:-}" ]]; then
     echo "SHARED_DIR: ${SHARED_DIR}"
-    ls -l "${SHARED_DIR}"
+    ls -latr "${SHARED_DIR}"
     if [[ -f "${SHARED_DIR}/shared_env" ]]; then
         cat "${SHARED_DIR:-}/shared_env"
         # shellcheck disable=SC1091
@@ -49,7 +49,7 @@ case "$ci_job" in
 esac
 
 case "$ci_job" in
-    eks-qa-e2e-tests|osd*qa-e2e-tests|ocp-4-*-ui-e2e-tests)
+    eks-qa-e2e-tests|osd*qa-e2e-tests|ocp-*-ui-e2e-tests)
         setup_automation_flavor_e2e_cluster "$ci_job"
         ;;
 esac
diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 7ba87b99fce13..7e25ff11cae64 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -1496,29 +1496,13 @@ setup_automation_flavor_e2e_cluster() {
     ls -l "${SHARED_DIR}"
     export KUBECONFIG="${SHARED_DIR}/kubeconfig"
 
-    if [[ "$ci_job" =~ ^osd ]]; then
-        info "Logging in to an OSD cluster"
+    if [[ "$ci_job" =~ ^(osd|ocp) ]]; then
+        info "Logging in to the ${ci_job:0:3} cluster"
         source "${SHARED_DIR}/dotenv"
 
-        oc login "$CLUSTER_API_ENDPOINT" \
-                --username "$CLUSTER_USERNAME" \
-                --password "$CLUSTER_PASSWORD" \
-                --insecure-skip-tls-verify=true
-    elif [[ "$ci_job" =~ ocp-4-.*-ui-e2e-tests ]]; then
-        info "Setting up OCP cluster information for UI e2e tests"
-        if [[ -f "${SHARED_DIR}/dotenv" ]]; then
-            source "${SHARED_DIR}/dotenv"
-        fi
-        
-        # Read console URL from url file if available, or use OPENSHIFT_CONSOLE_URL from dotenv
-        if [[ -f "${SHARED_DIR}/url" ]]; then
-            CLUSTER_API_ENDPOINT="$(cat "${SHARED_DIR}/url")"
-            export CLUSTER_API_ENDPOINT
-        elif [[ -n "${OPENSHIFT_CONSOLE_URL:-}" ]]; then
+        if [[ -n "${OPENSHIFT_CONSOLE_URL:-}" ]]; then
             export CLUSTER_API_ENDPOINT="$OPENSHIFT_CONSOLE_URL"
         fi
-        
-        # Map OpenShift console credentials to CLUSTER_* variables for cypress
         if [[ -n "${OPENSHIFT_CONSOLE_USERNAME:-}" ]]; then
             export CLUSTER_USERNAME="$OPENSHIFT_CONSOLE_USERNAME"
         fi
@@ -1526,8 +1510,10 @@ setup_automation_flavor_e2e_cluster() {
             export CLUSTER_PASSWORD="$OPENSHIFT_CONSOLE_PASSWORD"
         fi
         
-        # Set ORCHESTRATOR_FLAVOR for cypress
-        export ORCHESTRATOR_FLAVOR="openshift"
+        oc login "$CLUSTER_API_ENDPOINT" \
+                --username "$CLUSTER_USERNAME" \
+                --password "$CLUSTER_PASSWORD" \
+                --insecure-skip-tls-verify=true
     fi
 }
 

From 8975d8d2e1d7cd67e42191c092555683bd39c20a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 09:58:20 -0600
Subject: [PATCH 205/256] lint

---
 tests/e2e/lib.sh | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 7e25ff11cae64..4176ab4b135ad 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -204,17 +204,11 @@ export_test_environment() {
     fi
 
     set -x
-    ci_export CLUSTER_API_ENDPOINT "$CLUSTER_API_ENDPOINT"
-    ci_export CLUSTER_USERNAME "$CLUSTER_USERNAME"
-    ci_export CLUSTER_PASSWORD "$CLUSTER_PASSWORD"
+    echo "Exporting OCP cluster information for UI e2e tests"
+    ci_export CLUSTER_API_ENDPOINT "${CLUSTER_API_ENDPOINT:-}"
+    ci_export CLUSTER_USERNAME "${CLUSTER_USERNAME:-}"
+    ci_export CLUSTER_PASSWORD "${CLUSTER_PASSWORD:-}"
     set +x
-    
-    if [[ "${CI_JOB_NAME:-}" =~ ocp-4-[0-9]+-ui-e2e-tests ]]; then
-        info "Exposing OCP cluster information for UI e2e tests"
-        # Expose OCP cluster information for UI e2e tests
-     else
-        info "Not exposing OCP cluster information for UI e2e tests"
-    fi
 }
 
 deploy_stackrox_operator() {
@@ -1509,7 +1503,7 @@ setup_automation_flavor_e2e_cluster() {
         if [[ -n "${OPENSHIFT_CONSOLE_PASSWORD:-}" ]]; then
             export CLUSTER_PASSWORD="$OPENSHIFT_CONSOLE_PASSWORD"
         fi
-        
+
         oc login "$CLUSTER_API_ENDPOINT" \
                 --username "$CLUSTER_USERNAME" \
                 --password "$CLUSTER_PASSWORD" \

From 8a96f63d1762948866ee6212fa2a8beddf434762 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 10:01:15 -0600
Subject: [PATCH 206/256] simplify cluster auth vars

---
 tests/e2e/lib.sh | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 4176ab4b135ad..8c1b763c7b6ce 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -1494,15 +1494,11 @@ setup_automation_flavor_e2e_cluster() {
         info "Logging in to the ${ci_job:0:3} cluster"
         source "${SHARED_DIR}/dotenv"
 
-        if [[ -n "${OPENSHIFT_CONSOLE_URL:-}" ]]; then
-            export CLUSTER_API_ENDPOINT="$OPENSHIFT_CONSOLE_URL"
-        fi
-        if [[ -n "${OPENSHIFT_CONSOLE_USERNAME:-}" ]]; then
-            export CLUSTER_USERNAME="$OPENSHIFT_CONSOLE_USERNAME"
-        fi
-        if [[ -n "${OPENSHIFT_CONSOLE_PASSWORD:-}" ]]; then
-            export CLUSTER_PASSWORD="$OPENSHIFT_CONSOLE_PASSWORD"
-        fi
+        # OSD and OCP require one of (CLUSTER_|OPENSHIFT_CONSOLE_) var groups.
+        # Fail if neither are found from the dotenv.
+        export CLUSTER_API_ENDPOINT="${CLUSTER_API_ENDPOINT:-${OPENSHIFT_CONSOLE_URL}}"
+        export CLUSTER_USERNAME="${CLUSTER_USERNAME:-${OPENSHIFT_CONSOLE_USERNAME}}"
+        export CLUSTER_PASSWORD="${CLUSTER_PASSWORD:-${OPENSHIFT_CONSOLE_PASSWORD}}"
 
         oc login "$CLUSTER_API_ENDPOINT" \
                 --username "$CLUSTER_USERNAME" \

From 8e0fc15a7637c32c3867349c94ed80057ecb8b10 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 10:14:06 -0600
Subject: [PATCH 207/256] deploy-local

---
 .github/workflows/e2e-tests.yaml | 278 +++++++++++++++++++++++++++++++
 1 file changed, 278 insertions(+)
 create mode 100644 .github/workflows/e2e-tests.yaml

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
new file mode 100644
index 0000000000000..ed27e2d12b099
--- /dev/null
+++ b/.github/workflows/e2e-tests.yaml
@@ -0,0 +1,278 @@
+name: E2E tests
+on:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      commit:
+        description: 'Commit SHA of tests'
+        required: true
+        type: string
+      tested_commit:
+        description: 'Commit SHA to test'
+        required: true
+        type: string
+  workflow_call:
+env:
+  ROX_PRODUCT_BRANDING: stackrox
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  e2e-tests:
+    runs-on: ubuntu-latest
+    env:
+      CI: "true"
+      FAIL_FAST: "true"
+      INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
+      POD_SECURITY_POLICIES: "false"
+      ROX_VULN_MGMT_UNIFIED_CVE_DEFERRAL: "true"
+      ROX_PROCESSES_LISTENING_ON_PORT: "false"
+      REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
+      REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
+      GOOGLE_CREDENTIALS_GCR_SCANNER_V2: ${{ secrets.GOOGLE_CREDENTIALS_GCR_SCANNER_V2 }}
+      GCP_SERVICE_ACCOUNT_STACKROX_CI: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
+      QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
+      QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
+      QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }}
+      MAIN_IMAGE_TAG: ${{ inputs.tested_commit || github.event.pull_request.base.sha }}
+      TAG: ${{ inputs.tested_commit || github.event.pull_request.base.sha }}
+      GATHER_QA_TEST_DEBUG_LOGS: "false"
+      QA_TEST_DEBUG_LOGS: ""
+      SHARED_DIR: "/tmp"
+      CLUSTER: 'OPENSHIFT'
+      SENSOR_HELM_DEPLOY: "false"
+      ROX_SCANNER_V4: "false"
+      ORCHESTRATOR_FLAVOR: "openshift"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # ${{ github.event.pull_request.commits }}
+          fetch-tags: true
+          show-progress: false
+
+      - run: |
+          nohup git fetch origin '+refs/tags/*:refs/tags/*' >>/tmp/git.fetching.log 2>&1 &
+          echo $! | tee /tmp/git.fetching.pid
+
+      - uses: ./.github/actions/job-preamble
+        if: ${{ !env.ACT }}
+        with:
+          gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
+
+      - name: Env variables
+        run: |
+          JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
+
+          ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/artifacts/}"
+          mkdir -p "$ARTIFACT_DIR"
+
+          cat >> "$GITHUB_ENV" <<EOF
+          CI=true
+          SHARED_DIR=${SHARED_DIR:-/tmp}
+          ARTIFACT_DIR=${ARTIFACT_DIR}
+
+          CI_JOB_NAME=${JOB_NAME}
+          JOB_NAME=${JOB_NAME}
+          JOB_NAME_SAFE=qa-e2e-tests
+          GRADLE_USER_HOME=${HOME}
+          EOF
+
+      - name: Check if testing on Infra cluster
+        id: cluster-exists
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          source scripts/ci/lib.sh
+          test_on_infra | tee -a "$GITHUB_ENV"
+
+      - name: Determine cluster flavor
+        if: ${{ steps.cluster-exists.outcome != 'failure' }}
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          set -x
+          # groovy tests require CLUSTER='OPENSHIFT' for ocp or ='K8S' for GKE.
+          if [[ "$(infractl get "$CLUSTER_NAME" --json | jq -r '.Flavor')" == *"openshift"* ]]; then
+            echo "CLUSTER=OPENSHIFT" | tee -a "$GITHUB_ENV"
+            # default is "K8S"
+          fi
+
+      - name: Check tag lookup
+        timeout-minutes: 0.5
+        continue-on-error: true
+        run: |
+          set -x
+          while [[ "$(git describe --match='*.x' --tags --abbrev=10 --always --dirty --long --exclude '*-nightly-*')" \
+            != *".x-"* ]]; do
+            sleep 1
+            pgrep -F /tmp/git.fetching.pid || break
+          done
+          git describe --tags --abbrev=10 --always --dirty --long --exclude '*-nightly-*' || true
+          set +e
+          # example: 4.9.x-457-g9867327a52
+          MAIN_IMAGE_TAG=$(git describe --match="*.x" --tags --abbrev=10 --long --exclude '*-nightly-*' "${MAIN_IMAGE_TAG:-master}")
+          echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
+          echo "TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
+
+      #- name: Remove existing Stackrox
+      #  if: ${{ steps.cluster-exists.outcome != 'failure' }}
+      #  run: |
+      #    kubectl delete namespace stackrox || true
+
+      - name: Find or Deploy Stackrox
+        id: rox-installed
+        if: ${{ steps.cluster-exists.outcome != 'failure' }}
+        env:
+          DEPLOY_STACKROX_VIA_OPERATOR: "false"
+          # LOAD_BALANCER: "lb"
+          # SCANNER_SUPPORT: "true"
+          MONITORING_SUPPORT: "false"
+          LOCAL_DEPLOYMENT: "true"
+          ROX_BASELINE_GENERATION_DURATION: "1m"
+          ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
+        run: |
+          set +e
+          echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
+          kubectl get namespace stackrox \
+            || { 
+              if [[ "${CLUSTER,,}" == "openshift" ]]; then
+                  kubectl get scc qatest-anyuid \
+                    || {
+                      kubectl create ns stackrox;
+                      kubectl create -f "qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml";
+                    }
+              fi
+              CI=true ./deploy/deploy-local.sh;
+              find deploy -name password;
+              echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";
+            }
+
+      - name: Stackrox service and auth lookup
+        id: stackrox-api
+        if: ${{ steps.rox-installed.outcome != 'failure' }}
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          set +e
+          set -x
+          ls -latr /tmp/
+          if [[ -e "${KUBECONFIG:-}" ]]; then
+            if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
+              password="${ROX_ADMIN_PASSWORD:-$(cat "${KUBECONFIG%kubeconfig}admin-password")}"
+              password="${password:-$(kubectl -n stackrox get secret admin-pass -o json | jq -er '.data.password // ""' | base64 -d)}"
+              password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password // ""' | base64 -d)}"
+              echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
+              echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
+            fi
+            if [[ -z "${API_HOSTNAME:-}" || ( $API_HOSTNAME == 'localhost' && -z "$(pgrep -F /tmp/forward.pid)" ) ]]; then
+              API_HOSTNAME=$(kubectl -n stackrox get route central -o jsonpath='{.status.ingress[0].host}')
+              API_HOSTNAME=${API_HOSTNAME:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].ip}')}
+              API_HOSTNAME=${API_HOSTNAME:-localhost}
+              echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
+              API_PORT=${API_PORT:-$(kubectl -n stackrox get route central -o jsonpath='{.spec.port.targetPort}')}
+              API_PORT=${API_PORT:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].port}')}
+              API_PORT=${API_PORT//https/443}
+              if [[ "$API_HOSTNAME" == 'localhost' ]]; then
+                nohup kubectl -n stackrox port-forward svc/central 8000:443 &
+                echo $! | tee /tmp/forward.pid
+                API_PORT=8000
+              fi
+              API_PORT=${API_PORT:-443}
+              echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
+              echo "API_PORT=${API_PORT}" | tee -a "$GITHUB_ENV"
+            fi
+          fi
+          echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
+
+      - name: Stackrox API test
+        if: ${{ steps.stackrox-api.outcome != 'failure' }}
+        run: |
+          if [[ "${API_HOSTNAME:-}" == 'localhost' && -e /tmp/forward.pid ]]; then 
+            cat /tmp/forward.pid
+            ps -ef | grep kube || true
+          fi
+
+          curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.'
+
+      - name: Get secured cluster name
+        if: ${{ steps.stackrox-api.outcome != 'failure' }}
+        run: |
+          echo "Groovy tests expect a secured cluster. Default name is 'remote' but the Infra demo clusters set names like 'production'. Look up what secured clusters exist:"
+          set -x
+          CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
+          echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
+
+      - run: go get github.com/stackrox/scanner
+
+      - name: install oc cli
+        if: ${{ env.CLUSTER == 'OPENSHIFT' }}
+        run: |
+          # Some tests require `oc`.
+          curl --retry 5 --retry-all-errors -L -o /tmp/openshift-client-linux.tar.gz \
+            https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz
+          tar -C /tmp -xzf /tmp/openshift-client-linux.tar.gz
+          install --mode 755 /tmp/oc /usr/local/bin/
+          install --mode 755 /tmp/kubectl /usr/local/bin/
+
+      #- run: |
+      #    cd qa-tests-backend && ./gradlew wrapper --gradle-version "8.14" || true
+      #    git diff || true
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          gradle-version: wrapper
+          #cache-read-only: false
+          cache-overwrite-existing: true
+          cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
+
+      - timeout-minutes: 10
+        run: |
+          make -C qa-tests-backend compile
+
+      - timeout-minutes: 35
+        run: |
+          export TEST_TARGET="smoke-test"
+          cd qa-tests-backend && ./gradlew testSMOKE --fail-fast
+
+      - timeout-minutes: 35
+        run: |
+          export TEST_TARGET="bat-test"
+          cd qa-tests-backend && ./gradlew testParallelBAT testBAT --fail-fast --warn
+
+      - timeout-minutes: 35
+        run: |
+          export TEST_TARGET="test"
+          cd qa-tests-backend && ./gradlew testParallel testRest --fail-fast --warn
+
+      - name: Check shared dir and artifacts
+        if: always()
+        run: |
+          set +e
+          set -x
+          tail /tmp/git.fetching.log
+          ls -latr ${ARTIFACT_DIR}**
+          ls -latr ${SHARED_DIR}**
+          echo "check daemon:"
+          { pushd qa-tests-backend; ./gradlew --status || true; popd; }
+
+      - name: Publish Test Report
+        id: test-results
+        uses: test-summary/action@v2
+        continue-on-error: true
+        if: always()
+        with:
+          paths: 'qa-tests-backend/build/test-results/**/TEST-*.xml'
+          show: "all"
+
+      - name: Determine cluster flavor
+        if: ${{ steps.test-results.outcome == 'failure' }}
+        timeout-minutes: 5
+        run: |
+          tee -a "$GITHUB_STEP_SUMMARY" <<EOF
+          :boom: ${{ steps.test_summary.outputs.failed }}/${{ steps.test_summary.outputs.total }} failed
+          :seedling: ${{ steps.test_summary.outputs.passed }}/${{ steps.test_summary.outputs.total }} passed
+          :zzz: ${{ steps.test_summary.outputs.skipped }}/${{ steps.test_summary.outputs.total }} skipped
+          EOF

From dcb59f7ee10e0565e99439441ad9b9e45cd6806e Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 11:13:55 -0600
Subject: [PATCH 208/256] login to ocp cluster in gha

---
 .github/workflows/e2e-tests.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index ed27e2d12b099..3453ec8602e9b 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -216,9 +216,11 @@ jobs:
           install --mode 755 /tmp/oc /usr/local/bin/
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
-      #- run: |
-      #    cd qa-tests-backend && ./gradlew wrapper --gradle-version "8.14" || true
-      #    git diff || true
+      - run: |
+          set +e
+          set -x
+          ls -latr ${SHARED_DIR}**
+          setup_automation_flavor_e2e_cluster "${CI_JOB_NAME}"
 
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4

From 0aeae04bb6b162b712c33612b1fbafbbb81e045f Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 11:26:39 -0600
Subject: [PATCH 209/256] source the e2e lib

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 3453ec8602e9b..ef6582befcd4a 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -220,6 +220,7 @@ jobs:
           set +e
           set -x
           ls -latr ${SHARED_DIR}**
+          source tests/e2e/lib.sh
           setup_automation_flavor_e2e_cluster "${CI_JOB_NAME}"
 
       - name: Setup Gradle

From 9db056584c9ec22a89ec8ea37d93508894ee078b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 12:08:35 -0600
Subject: [PATCH 210/256] test client certs

---
 tests/e2e/lib.sh | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 51bb24753aa95..b2c9ee4d4c5ed 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -202,17 +202,6 @@ export_test_environment() {
         # GKE uses this network for services. Consider it as a private subnet.
         ci_export ROX_NON_AGGREGATED_NETWORKS "${ROX_NON_AGGREGATED_NETWORKS:-34.118.224.0/20}"
     fi
-
-    ci_export CLUSTER_API_ENDPOINT "$CLUSTER_API_ENDPOINT"
-    ci_export CLUSTER_USERNAME "$CLUSTER_USERNAME"
-    ci_export CLUSTER_PASSWORD "$CLUSTER_PASSWORD"
-    
-    if [[ "${CI_JOB_NAME:-}" =~ ocp-4-[0-9]+-ui-e2e-tests ]]; then
-        info "Exposing OCP cluster information for UI e2e tests"
-        # Expose OCP cluster information for UI e2e tests
-     else
-        info "Not exposing OCP cluster information for UI e2e tests"
-    fi
 }
 
 deploy_stackrox_operator() {
@@ -1497,11 +1486,15 @@ setup_automation_flavor_e2e_cluster() {
     if [[ "$ci_job" =~ ^osd ]]; then
         info "Logging in to an OSD cluster"
         source "${SHARED_DIR}/dotenv"
-
         oc login "$CLUSTER_API_ENDPOINT" \
                 --username "$CLUSTER_USERNAME" \
                 --password "$CLUSTER_PASSWORD" \
                 --insecure-skip-tls-verify=true
+    elif [[ "$ci_job" == ocp* ]]; then
+        info "Testing client certs for OCP cluster"
+        echo "KUBECONFIG:${KUBECONFIG:-}"
+        oc whoami || true
+        oc version
     fi
 }
 

From 1678267422023e26031865b6e26bb66d1d99ab6f Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 12:10:46 -0600
Subject: [PATCH 211/256] show console url

---
 .openshift-ci/dispatch.sh | 15 +++------------
 tests/e2e/lib.sh          |  2 +-
 2 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/.openshift-ci/dispatch.sh b/.openshift-ci/dispatch.sh
index 4c939d8602bcb..968ff14e0c72d 100755
--- a/.openshift-ci/dispatch.sh
+++ b/.openshift-ci/dispatch.sh
@@ -12,18 +12,9 @@ source "$ROOT/tests/e2e/lib.sh"
 
 set -euo pipefail
 
-if [[ -n "${SHARED_DIR:-}" ]]; then
-    echo "SHARED_DIR: ${SHARED_DIR}"
-    ls -l "${SHARED_DIR}"
-    if [[ -f "${SHARED_DIR}/shared_env" ]]; then
-        cat "${SHARED_DIR:-}/shared_env"
-        # shellcheck disable=SC1091
-        source "${SHARED_DIR}/shared_env"
-    else
-        echo "shared_env file does not exist in ${SHARED_DIR}"
-    fi
-else
-    echo "SHARED_DIR is not set"
+if [[ -f "${SHARED_DIR:-}/shared_env" ]]; then
+    # shellcheck disable=SC1091
+    source "${SHARED_DIR:-}/shared_env"
 fi
 
 openshift_ci_mods
diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index b2c9ee4d4c5ed..d4591cdc41926 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -1493,7 +1493,7 @@ setup_automation_flavor_e2e_cluster() {
     elif [[ "$ci_job" == ocp* ]]; then
         info "Testing client certs for OCP cluster"
         echo "KUBECONFIG:${KUBECONFIG:-}"
-        oc whoami || true
+        oc whoami --show-console || true
         oc version
     fi
 }

From 76809b354dc083a5834ecd642d387c100eaaab06 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 15:41:16 -0600
Subject: [PATCH 212/256] try start-ocp bridge

---
 .openshift-ci/dispatch.sh | 2 +-
 tests/e2e/lib.sh          | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/.openshift-ci/dispatch.sh b/.openshift-ci/dispatch.sh
index 968ff14e0c72d..39ee8bf999325 100755
--- a/.openshift-ci/dispatch.sh
+++ b/.openshift-ci/dispatch.sh
@@ -40,7 +40,7 @@ case "$ci_job" in
 esac
 
 case "$ci_job" in
-    eks-qa-e2e-tests|osd*qa-e2e-tests)
+    eks-qa-e2e-tests|osd*qa-e2e-tests|ocp*e2e-tests)
         setup_automation_flavor_e2e_cluster "$ci_job"
         ;;
 esac
diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index d4591cdc41926..5d467b214e94f 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -1495,6 +1495,11 @@ setup_automation_flavor_e2e_cluster() {
         echo "KUBECONFIG:${KUBECONFIG:-}"
         oc whoami --show-console || true
         oc version
+        if [[ "$ci_job" == *ui-e2e* ]]; then
+            export ACS_API_SERVICE_URL="${API_ENDPOINT:-http://localhost:8000}"
+            ./ui/apps/platform/scripts/start-ocp-console.sh
+            curl --retry=5 'http://localhost:9000' || true
+        fi
     fi
 }
 

From f8d3b27b4eac81983c53f690222e1e9672f06a86 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 15:54:24 -0600
Subject: [PATCH 213/256] set acs_api var

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index ef6582befcd4a..99c0ac297331d 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -221,6 +221,7 @@ jobs:
           set -x
           ls -latr ${SHARED_DIR}**
           source tests/e2e/lib.sh
+          ACS_API_SERVICE_URL="${API_ENDPOINT}"
           setup_automation_flavor_e2e_cluster "${CI_JOB_NAME}"
 
       - name: Setup Gradle

From 9833e233284e76978824bcc005ba79abaf308d19 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 17:43:48 -0600
Subject: [PATCH 214/256] merge fail

---
 tests/e2e/lib.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index f35456a382246..2686fa18b86aa 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -202,7 +202,6 @@ export_test_environment() {
         # GKE uses this network for services. Consider it as a private subnet.
         ci_export ROX_NON_AGGREGATED_NETWORKS "${ROX_NON_AGGREGATED_NETWORKS:-34.118.224.0/20}"
     fi
-<<<<<<< HEAD
 
     set -x
     echo "Exporting OCP cluster information for UI e2e tests"
@@ -210,8 +209,6 @@ export_test_environment() {
     ci_export CLUSTER_USERNAME "${CLUSTER_USERNAME:-}"
     ci_export CLUSTER_PASSWORD "${CLUSTER_PASSWORD:-}"
     set +x
-=======
->>>>>>> rox-29840-access-ocp-console
 }
 
 deploy_stackrox_operator() {

From 9492b83075383b101a2808b12cd15a11ae0dcfea Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 20:10:20 -0600
Subject: [PATCH 215/256] lookup api with kubeconfig

---
 tests/e2e/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 2686fa18b86aa..bc9e8f1033626 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -1504,7 +1504,7 @@ setup_automation_flavor_e2e_cluster() {
 
         # OSD and OCP require one of (CLUSTER_|OPENSHIFT_CONSOLE_) var groups.
         # Fail if neither are found from the dotenv.
-        export CLUSTER_API_ENDPOINT="${CLUSTER_API_ENDPOINT:-${OPENSHIFT_CONSOLE_URL}}"
+        export CLUSTER_API_ENDPOINT="${CLUSTER_API_ENDPOINT:-$(oc whoami --show-server)}"
         export CLUSTER_USERNAME="${CLUSTER_USERNAME:-${OPENSHIFT_CONSOLE_USERNAME}}"
         export CLUSTER_PASSWORD="${CLUSTER_PASSWORD:-${OPENSHIFT_CONSOLE_PASSWORD}}"
 

From 03822c43a4462a380e1f2cd352ebfb4e8741409a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 20:18:33 -0600
Subject: [PATCH 216/256] try ui e2e

---
 .github/workflows/e2e-tests.yaml | 2 +-
 tests/e2e/lib.sh                 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 99c0ac297331d..5ba1645adc0cb 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -63,7 +63,7 @@ jobs:
 
       - name: Env variables
         run: |
-          JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
+          JOB_NAME="${JOB_NAME:-ocp-4-19-ui-e2e-tests}"
 
           ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/artifacts/}"
           mkdir -p "$ARTIFACT_DIR"
diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index bc9e8f1033626..907d8f9e6c55f 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -1505,7 +1505,7 @@ setup_automation_flavor_e2e_cluster() {
         # OSD and OCP require one of (CLUSTER_|OPENSHIFT_CONSOLE_) var groups.
         # Fail if neither are found from the dotenv.
         export CLUSTER_API_ENDPOINT="${CLUSTER_API_ENDPOINT:-$(oc whoami --show-server)}"
-        export CLUSTER_USERNAME="${CLUSTER_USERNAME:-${OPENSHIFT_CONSOLE_USERNAME}}"
+        export CLUSTER_USERNAME="${CLUSTER_USERNAME:-${OPENSHIFT_CONSOLE_USERNAME:kubeadmin}}"
         export CLUSTER_PASSWORD="${CLUSTER_PASSWORD:-${OPENSHIFT_CONSOLE_PASSWORD}}"
 
         oc login "$CLUSTER_API_ENDPOINT" \

From 7b921ffa137081e799f8b86049a493b0714b797b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 20:20:35 -0600
Subject: [PATCH 217/256] missing dash

---
 tests/e2e/lib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 907d8f9e6c55f..e37c39f2b5fd6 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -1505,7 +1505,7 @@ setup_automation_flavor_e2e_cluster() {
         # OSD and OCP require one of (CLUSTER_|OPENSHIFT_CONSOLE_) var groups.
         # Fail if neither are found from the dotenv.
         export CLUSTER_API_ENDPOINT="${CLUSTER_API_ENDPOINT:-$(oc whoami --show-server)}"
-        export CLUSTER_USERNAME="${CLUSTER_USERNAME:-${OPENSHIFT_CONSOLE_USERNAME:kubeadmin}}"
+        export CLUSTER_USERNAME="${CLUSTER_USERNAME:-${OPENSHIFT_CONSOLE_USERNAME:-kubeadmin}}"
         export CLUSTER_PASSWORD="${CLUSTER_PASSWORD:-${OPENSHIFT_CONSOLE_PASSWORD}}"
 
         oc login "$CLUSTER_API_ENDPOINT" \

From 8ea96ad7ddea93b9b6872a4770ca1b47ddb45ba1 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 21:13:35 -0600
Subject: [PATCH 218/256] without bridge

---
 tests/e2e/lib.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index e37c39f2b5fd6..ef7dbc3e88552 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -1517,11 +1517,12 @@ setup_automation_flavor_e2e_cluster() {
         echo "KUBECONFIG:${KUBECONFIG:-}"
         oc whoami --show-console || true
         oc version
-        if [[ "$ci_job" == *ui-e2e* ]]; then
-            export ACS_API_SERVICE_URL="${API_ENDPOINT:-http://localhost:8000}"
-            ./ui/apps/platform/scripts/start-ocp-console.sh
-            curl --retry=5 'http://localhost:9000' || true
-        fi
+        #if [[ "$ci_job" == *ui-e2e* ]]; then
+        #    export ACS_API_SERVICE_URL="${API_ENDPOINT:-http://localhost:8000}"
+        #    ./ui/apps/platform/scripts/start-ocp-console.sh 2>&1 > &
+        #    echo $! > /tmp/
+        #    curl --retry=5 'http://localhost:9000' || true
+        #fi
     fi
 }
 

From c78b599e6612f0bbb6fcc9dc46d9f22fbc4c7417 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 21:34:51 -0600
Subject: [PATCH 219/256] run ui-e2e

---
 .github/workflows/e2e-tests.yaml | 216 +++++++++++++++++--------------
 1 file changed, 116 insertions(+), 100 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 5ba1645adc0cb..5091fe1e0e26a 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -121,88 +121,88 @@ jobs:
       #  run: |
       #    kubectl delete namespace stackrox || true
 
-      - name: Find or Deploy Stackrox
-        id: rox-installed
-        if: ${{ steps.cluster-exists.outcome != 'failure' }}
-        env:
-          DEPLOY_STACKROX_VIA_OPERATOR: "false"
-          # LOAD_BALANCER: "lb"
-          # SCANNER_SUPPORT: "true"
-          MONITORING_SUPPORT: "false"
-          LOCAL_DEPLOYMENT: "true"
-          ROX_BASELINE_GENERATION_DURATION: "1m"
-          ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
-        run: |
-          set +e
-          echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
-          kubectl get namespace stackrox \
-            || { 
-              if [[ "${CLUSTER,,}" == "openshift" ]]; then
-                  kubectl get scc qatest-anyuid \
-                    || {
-                      kubectl create ns stackrox;
-                      kubectl create -f "qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml";
-                    }
-              fi
-              CI=true ./deploy/deploy-local.sh;
-              find deploy -name password;
-              echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";
-            }
+      #- name: Find or Deploy Stackrox
+      #  id: rox-installed
+      #  if: ${{ steps.cluster-exists.outcome != 'failure' }}
+      #  env:
+      #    DEPLOY_STACKROX_VIA_OPERATOR: "false"
+      #    # LOAD_BALANCER: "lb"
+      #    # SCANNER_SUPPORT: "true"
+      #    MONITORING_SUPPORT: "false"
+      #    LOCAL_DEPLOYMENT: "true"
+      #    ROX_BASELINE_GENERATION_DURATION: "1m"
+      #    ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
+      #  run: |
+      #    set +e
+      #    echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
+      #    kubectl get namespace stackrox \
+      #      || { 
+      #        if [[ "${CLUSTER,,}" == "openshift" ]]; then
+      #            kubectl get scc qatest-anyuid \
+      #              || {
+      #                kubectl create ns stackrox;
+      #                kubectl create -f "qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml";
+      #              }
+      #        fi
+      #        CI=true ./deploy/deploy-local.sh;
+      #        find deploy -name password;
+      #        echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";
+      #      }
 
-      - name: Stackrox service and auth lookup
-        id: stackrox-api
-        if: ${{ steps.rox-installed.outcome != 'failure' }}
-        timeout-minutes: 5
-        continue-on-error: true
-        run: |
-          set +e
-          set -x
-          ls -latr /tmp/
-          if [[ -e "${KUBECONFIG:-}" ]]; then
-            if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
-              password="${ROX_ADMIN_PASSWORD:-$(cat "${KUBECONFIG%kubeconfig}admin-password")}"
-              password="${password:-$(kubectl -n stackrox get secret admin-pass -o json | jq -er '.data.password // ""' | base64 -d)}"
-              password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password // ""' | base64 -d)}"
-              echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
-              echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
-            fi
-            if [[ -z "${API_HOSTNAME:-}" || ( $API_HOSTNAME == 'localhost' && -z "$(pgrep -F /tmp/forward.pid)" ) ]]; then
-              API_HOSTNAME=$(kubectl -n stackrox get route central -o jsonpath='{.status.ingress[0].host}')
-              API_HOSTNAME=${API_HOSTNAME:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].ip}')}
-              API_HOSTNAME=${API_HOSTNAME:-localhost}
-              echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
-              API_PORT=${API_PORT:-$(kubectl -n stackrox get route central -o jsonpath='{.spec.port.targetPort}')}
-              API_PORT=${API_PORT:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].port}')}
-              API_PORT=${API_PORT//https/443}
-              if [[ "$API_HOSTNAME" == 'localhost' ]]; then
-                nohup kubectl -n stackrox port-forward svc/central 8000:443 &
-                echo $! | tee /tmp/forward.pid
-                API_PORT=8000
-              fi
-              API_PORT=${API_PORT:-443}
-              echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
-              echo "API_PORT=${API_PORT}" | tee -a "$GITHUB_ENV"
-            fi
-          fi
-          echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
+      #- name: Stackrox service and auth lookup
+      #  id: stackrox-api
+      #  if: ${{ steps.rox-installed.outcome != 'failure' }}
+      #  timeout-minutes: 5
+      #  continue-on-error: true
+      #  run: |
+      #    set +e
+      #    set -x
+      #    ls -latr /tmp/
+      #    if [[ -e "${KUBECONFIG:-}" ]]; then
+      #      if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
+      #        password="${ROX_ADMIN_PASSWORD:-$(cat "${KUBECONFIG%kubeconfig}admin-password")}"
+      #        password="${password:-$(kubectl -n stackrox get secret admin-pass -o json | jq -er '.data.password // ""' | base64 -d)}"
+      #        password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password // ""' | base64 -d)}"
+      #        echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
+      #        echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
+      #      fi
+      #      if [[ -z "${API_HOSTNAME:-}" || ( $API_HOSTNAME == 'localhost' && -z "$(pgrep -F /tmp/forward.pid)" ) ]]; then
+      #        API_HOSTNAME=$(kubectl -n stackrox get route central -o jsonpath='{.status.ingress[0].host}')
+      #        API_HOSTNAME=${API_HOSTNAME:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].ip}')}
+      #        API_HOSTNAME=${API_HOSTNAME:-localhost}
+      #        echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
+      #        API_PORT=${API_PORT:-$(kubectl -n stackrox get route central -o jsonpath='{.spec.port.targetPort}')}
+      #        API_PORT=${API_PORT:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].port}')}
+      #        API_PORT=${API_PORT//https/443}
+      #        if [[ "$API_HOSTNAME" == 'localhost' ]]; then
+      #          nohup kubectl -n stackrox port-forward svc/central 8000:443 &
+      #          echo $! | tee /tmp/forward.pid
+      #          API_PORT=8000
+      #        fi
+      #        API_PORT=${API_PORT:-443}
+      #        echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
+      #        echo "API_PORT=${API_PORT}" | tee -a "$GITHUB_ENV"
+      #      fi
+      #    fi
+      #    echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
 
-      - name: Stackrox API test
-        if: ${{ steps.stackrox-api.outcome != 'failure' }}
-        run: |
-          if [[ "${API_HOSTNAME:-}" == 'localhost' && -e /tmp/forward.pid ]]; then 
-            cat /tmp/forward.pid
-            ps -ef | grep kube || true
-          fi
+      #- name: Stackrox API test
+      #  if: ${{ steps.stackrox-api.outcome != 'failure' }}
+      #  run: |
+      #    if [[ "${API_HOSTNAME:-}" == 'localhost' && -e /tmp/forward.pid ]]; then 
+      #      cat /tmp/forward.pid
+      #      ps -ef | grep kube || true
+      #    fi
 
-          curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.'
+      #    curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.'
 
-      - name: Get secured cluster name
-        if: ${{ steps.stackrox-api.outcome != 'failure' }}
-        run: |
-          echo "Groovy tests expect a secured cluster. Default name is 'remote' but the Infra demo clusters set names like 'production'. Look up what secured clusters exist:"
-          set -x
-          CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
-          echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
+      #- name: Get secured cluster name
+      #  if: ${{ steps.stackrox-api.outcome != 'failure' }}
+      #  run: |
+      #    echo "Groovy tests expect a secured cluster. Default name is 'remote' but the Infra demo clusters set names like 'production'. Look up what secured clusters exist:"
+      #    set -x
+      #    CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
+      #    echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
 
       - run: go get github.com/stackrox/scanner
 
@@ -216,6 +216,7 @@ jobs:
           install --mode 755 /tmp/oc /usr/local/bin/
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
+      # UI e2e - Cypress tests
       - run: |
           set +e
           set -x
@@ -224,32 +225,47 @@ jobs:
           ACS_API_SERVICE_URL="${API_ENDPOINT}"
           setup_automation_flavor_e2e_cluster "${CI_JOB_NAME}"
 
-      - name: Setup Gradle
-        uses: gradle/actions/setup-gradle@v4
-        with:
-          gradle-version: wrapper
-          #cache-read-only: false
-          cache-overwrite-existing: true
-          cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
+      - run: |
+          set +e
+          set -x
+          export DEPLOY_STACKROX_VIA_OPERATOR=true
+          export INSTALL_COMPLIANCE_OPERATOR=true
+          export ORCHESTRATOR_FLAVOR=openshift
 
-      - timeout-minutes: 10
-        run: |
-          make -C qa-tests-backend compile
+          #DEPLOY_DIR="deploy/${ORCHESTRATOR_FLAVOR}"
+          #HOSTALIASES=/tmp/hostaliases
+          #UI_BASE_URL (set to either "https://central-lb:443" or "https://localhost:${LOCAL_PORT}")
+          #make -C ui test-e2e, so these environment variables will be available during that execution.
 
-      - timeout-minutes: 35
-        run: |
-          export TEST_TARGET="smoke-test"
-          cd qa-tests-backend && ./gradlew testSMOKE --fail-fast
+          tests/e2e/run-ui-e2e.sh
 
-      - timeout-minutes: 35
-        run: |
-          export TEST_TARGET="bat-test"
-          cd qa-tests-backend && ./gradlew testParallelBAT testBAT --fail-fast --warn
+      # QA e2e - groovy tests
+      #- name: Setup Gradle
+      #  uses: gradle/actions/setup-gradle@v4
+      #  with:
+      #    gradle-version: wrapper
+      #    #cache-read-only: false
+      #    cache-overwrite-existing: true
+      #    cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
 
-      - timeout-minutes: 35
-        run: |
-          export TEST_TARGET="test"
-          cd qa-tests-backend && ./gradlew testParallel testRest --fail-fast --warn
+      #- timeout-minutes: 10
+      #  run: |
+      #    make -C qa-tests-backend compile
+
+      #- timeout-minutes: 35
+      #  run: |
+      #    export TEST_TARGET="smoke-test"
+      #    cd qa-tests-backend && ./gradlew testSMOKE --fail-fast
+
+      #- timeout-minutes: 35
+      #  run: |
+      #    export TEST_TARGET="bat-test"
+      #    cd qa-tests-backend && ./gradlew testParallelBAT testBAT --fail-fast --warn
+
+      #- timeout-minutes: 35
+      #  run: |
+      #    export TEST_TARGET="test"
+      #    cd qa-tests-backend && ./gradlew testParallel testRest --fail-fast --warn
 
       - name: Check shared dir and artifacts
         if: always()

From b48d8f9e94f7a79fc85b4ed8d9921b31c7c53f87 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 21:35:43 -0600
Subject: [PATCH 220/256] always delete stackrox ns

---
 .github/workflows/e2e-tests.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 5091fe1e0e26a..70a92b1c27662 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -116,10 +116,11 @@ jobs:
           echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
           echo "TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
 
-      #- name: Remove existing Stackrox
-      #  if: ${{ steps.cluster-exists.outcome != 'failure' }}
-      #  run: |
-      #    kubectl delete namespace stackrox || true
+      - name: Remove existing Stackrox
+        continue-on-error: true
+        if: ${{ steps.cluster-exists.outcome != 'failure' }}
+        run: |
+          kubectl delete namespace stackrox || true
 
       #- name: Find or Deploy Stackrox
       #  id: rox-installed

From d4bfb3832ed3fb559890916defbefea7aae15d9f Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 21:38:38 -0600
Subject: [PATCH 221/256] api_endpoint not set

---
 .github/workflows/e2e-tests.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 70a92b1c27662..e5439acd0990a 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -223,7 +223,6 @@ jobs:
           set -x
           ls -latr ${SHARED_DIR}**
           source tests/e2e/lib.sh
-          ACS_API_SERVICE_URL="${API_ENDPOINT}"
           setup_automation_flavor_e2e_cluster "${CI_JOB_NAME}"
 
       - run: |

From f77b35651d3699a75d7901842091e1202c914924 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 22:01:18 -0600
Subject: [PATCH 222/256] no quiet on openssl

---
 tests/e2e/run-ui-e2e.sh      | 2 ++
 tests/scripts/setup-certs.sh | 2 +-
 ui/Makefile                  | 2 +-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/e2e/run-ui-e2e.sh b/tests/e2e/run-ui-e2e.sh
index b56490112d2fd..b67199974f8ee 100755
--- a/tests/e2e/run-ui-e2e.sh
+++ b/tests/e2e/run-ui-e2e.sh
@@ -56,7 +56,9 @@ run_ui_e2e_tests() {
         export UI_BASE_URL="https://localhost:${LOCAL_PORT}"
     fi
 
+    echo "Calling make"
     make -C ui test-e2e || touch FAIL
+    echo "make finished."
 
     store_test_results "ui/test-results/reports/cypress/integration/." "cy-reps"
 
diff --git a/tests/scripts/setup-certs.sh b/tests/scripts/setup-certs.sh
index ed251cae65105..fb13018f37e0f 100755
--- a/tests/scripts/setup-certs.sh
+++ b/tests/scripts/setup-certs.sh
@@ -38,7 +38,7 @@ keyUsage = critical, digitalSignature, cRLSign, keyCertSign
   basicConstraints=CA:TRUE,pathlen:1
   "
     # Root CA
-    openssl req -quiet -nodes -config <(echo "$root_ca_exts") -new -x509 -keyout ca.key -out ca.crt -subj "/CN=Root ${ca_name}"
+    openssl req -nodes -config <(echo "$root_ca_exts") -new -x509 -keyout ca.key -out ca.crt -subj "/CN=Root ${ca_name}"
 
     # Intermediate CA
     openssl genrsa -out intermediate.key 4096
diff --git a/ui/Makefile b/ui/Makefile
index b33f583f66dfe..2958d894d0567 100644
--- a/ui/Makefile
+++ b/ui/Makefile
@@ -6,7 +6,7 @@ all: deps lint test test-component build
 
 SOURCES := $(shell find apps -type d \( -name node_modules \) -prune -o -print)
 
-ROX_PRODUCT_BRANDING ?= $(shell $(MAKE) --quiet --no-print-directory -C .. product-branding)
+ROX_PRODUCT_BRANDING ?= $(shell $(MAKE) --no-print-directory -C .. product-branding)
 export ROX_PRODUCT_BRANDING := $(ROX_PRODUCT_BRANDING)
 
 deps: apps/platform/package.json apps/platform/package-lock.json

From 83795c7b3bde26f84f4ddde46b0139cd48e2b222 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 22:53:16 -0600
Subject: [PATCH 223/256] add VERSION for operator deploy

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index e5439acd0990a..b215486bc701b 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -237,6 +237,7 @@ jobs:
           #UI_BASE_URL (set to either "https://central-lb:443" or "https://localhost:${LOCAL_PORT}")
           #make -C ui test-e2e, so these environment variables will be available during that execution.
 
+          export VERSION="${MAIN_IMAGE_TAG}"  # operator/Makefile VERSION is CSV_TAG
           tests/e2e/run-ui-e2e.sh
 
       # QA e2e - groovy tests

From 5ff4973f04099fccc826b09e737fbd8f5204101a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 23:11:13 -0600
Subject: [PATCH 224/256] use oper makefile rewrite of tag

---
 .github/workflows/e2e-tests.yaml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index b215486bc701b..6453c43a54679 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -218,14 +218,18 @@ jobs:
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
       # UI e2e - Cypress tests
-      - run: |
+      - name: Env for UI E2E
+        timeout-minutes: 15
+        run: |
           set +e
           set -x
           ls -latr ${SHARED_DIR}**
           source tests/e2e/lib.sh
           setup_automation_flavor_e2e_cluster "${CI_JOB_NAME}"
 
-      - run: |
+      - name: UI E2E
+        timeout-minutes: 30
+        run: |
           set +e
           set -x
           export DEPLOY_STACKROX_VIA_OPERATOR=true
@@ -237,7 +241,7 @@ jobs:
           #UI_BASE_URL (set to either "https://central-lb:443" or "https://localhost:${LOCAL_PORT}")
           #make -C ui test-e2e, so these environment variables will be available during that execution.
 
-          export VERSION="${MAIN_IMAGE_TAG}"  # operator/Makefile VERSION is CSV_TAG
+          export VERSION=$(echo "${MAIN_IMAGE_TAG}" | sed -E 's@^(([[:digit:]]+\.)+)x(-)?@\10\3@g')  # operator/Makefile VERSION is CSV_TAG
           tests/e2e/run-ui-e2e.sh
 
       # QA e2e - groovy tests

From 7d1bfa3064dcaf03fd6a63b878cdb096479504c9 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Fri, 15 Aug 2025 23:24:19 -0600
Subject: [PATCH 225/256] store infra files in tmp not shared

---
 scripts/ci/lib.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index 11aab4749c3c6..be876b05cbfef 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2408,7 +2408,7 @@ _record_cluster_info() {
 
 get_infra_cluster_files() {
     local cluster_name="${1}"
-    local data_dir="${SHARED_DIR:-/tmp}"
+    local data_dir="${SCRATCH:-/tmp}"
     ls -la "${data_dir}" || true
     infractl --version \
       || { set -x; curl --retry 8 --fail -sL https://infra.rox.systems/v1/cli/linux/amd64/upgrade \
@@ -2427,7 +2427,9 @@ get_infra_cluster_files() {
         sleep 60
     done
     echo "$cluster_name"
-    #|| cp "${data_dir}/auth/kubeconfig" "${SHARED_DIR}/kubeconfig" || true
+    cp "${data_dir}/auth/kubeconfig" "${SHARED_DIR}/kubeconfig" || true
+    cp "${data_dir}/dotenv" "${SHARED_DIR}/" || true
+    cp "${data_dir}/admin-password" "${SHARED_DIR}/" || true
 
     ls -la "${SHARED_DIR:-/tmp}" || true
     grep -o '^[A-Z_]*=' "${SHARED_DIR:-/tmp}/"*env || true

From 86c1d31fcdbfb19d4d6100f144aa344df1554c0b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sat, 16 Aug 2025 12:17:22 -0600
Subject: [PATCH 226/256] handle no infra test

---
 scripts/ci/lib.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
index be876b05cbfef..2ec920d97d59e 100755
--- a/scripts/ci/lib.sh
+++ b/scripts/ci/lib.sh
@@ -2467,6 +2467,8 @@ test_on_infra() {
           echo 'No PR description found.' >&2
         else
           echo 'PR description found.' >&2
+          echo "$body" | grep '^/test-on-infra' \
+            || return 0
           break
         fi
       fi
@@ -2477,9 +2479,17 @@ test_on_infra() {
         break
       fi
     done
+      if [[ -z "$body" ]]; then
+        echo 'No Infra clusters listed.'
+        continue
+      fi
     while read -r cmd cluster_name job_name_match; do
       job_name_match=${job_name_match%%]*}
       job_name_match=${job_name_match##*[}
+      if [[ -z "$job_name_match" ]]; then
+        echo 'Job name match is empty'
+        continue
+      fi
       echo "Comparing job_name string '${JOB_NAME:-}' to *${job_name_match}* for infra cluster ${cluster_name}." >&2
       if [[ "${JOB_NAME:-}" == *"$job_name_match"* ]]; then
         echo "Match found. Tests will run on ${cluster_name} instead of starting a new cluster." >&2

From c9fe7999814bcbf5bf76daa61069b670f3da2562 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sat, 16 Aug 2025 22:26:08 -0600
Subject: [PATCH 227/256] use console url not api

---
 tests/e2e/lib.sh                        | 4 +++-
 ui/apps/platform/scripts/cypress-ocp.sh | 7 ++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index ef7dbc3e88552..8fcc42834a545 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -205,6 +205,7 @@ export_test_environment() {
 
     set -x
     echo "Exporting OCP cluster information for UI e2e tests"
+    ci_export CLUSTER_CONSOLE_URL "${CLUSTER_CONSOLE_URL:-}"
     ci_export CLUSTER_API_ENDPOINT "${CLUSTER_API_ENDPOINT:-}"
     ci_export CLUSTER_USERNAME "${CLUSTER_USERNAME:-}"
     ci_export CLUSTER_PASSWORD "${CLUSTER_PASSWORD:-}"
@@ -1504,6 +1505,7 @@ setup_automation_flavor_e2e_cluster() {
 
         # OSD and OCP require one of (CLUSTER_|OPENSHIFT_CONSOLE_) var groups.
         # Fail if neither are found from the dotenv.
+        export CLUSTER_CONSOLE_URL="${CLUSTER_CONSOLE_ENDPOINT:-${OPENSHIFT_CONSOLE_URL:-$(oc whoami --show-console)}}"
         export CLUSTER_API_ENDPOINT="${CLUSTER_API_ENDPOINT:-$(oc whoami --show-server)}"
         export CLUSTER_USERNAME="${CLUSTER_USERNAME:-${OPENSHIFT_CONSOLE_USERNAME:-kubeadmin}}"
         export CLUSTER_PASSWORD="${CLUSTER_PASSWORD:-${OPENSHIFT_CONSOLE_PASSWORD}}"
@@ -1515,7 +1517,7 @@ setup_automation_flavor_e2e_cluster() {
 
         info "Testing client certs for OCP cluster"
         echo "KUBECONFIG:${KUBECONFIG:-}"
-        oc whoami --show-console || true
+        oc whoami || true
         oc version
         #if [[ "$ci_job" == *ui-e2e* ]]; then
         #    export ACS_API_SERVICE_URL="${API_ENDPOINT:-http://localhost:8000}"
diff --git a/ui/apps/platform/scripts/cypress-ocp.sh b/ui/apps/platform/scripts/cypress-ocp.sh
index ba25febafeb83..1d66fdfc41cba 100755
--- a/ui/apps/platform/scripts/cypress-ocp.sh
+++ b/ui/apps/platform/scripts/cypress-ocp.sh
@@ -1,10 +1,11 @@
 #!/usr/bin/env bash
 
-env | sort
+env | grep -o '^CLUSTER.*'
+set -x
 
 # Opens cypress with environment variables for feature flags and auth
-CLUSTER_API_ENDPOINT="${CLUSTER_API_ENDPOINT:-http://localhost:9000}"
-API_PROXY_BASE_URL="${CLUSTER_API_ENDPOINT}/api/proxy/plugin/advanced-cluster-security/api-service"
+CLUSTER_CONSOLE_URL="${CLUSTER_CONSOLE_URL:-http://localhost:9000}"
+API_PROXY_BASE_URL="${CLUSTER_CONSOLE_URL}/api/proxy/plugin/advanced-cluster-security/api-service"
 
 if [[ -z "$CLUSTER_USERNAME" || -z "$CLUSTER_PASSWORD" ]]; then
     echo "CLUSTER_USERNAME and CLUSTER_PASSWORD must be set"

From a116c40e4c2d8b9570585df45d5965aa639eb8ca Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sat, 16 Aug 2025 22:30:24 -0600
Subject: [PATCH 228/256] kind cluster if no infra

---
 .github/workflows/e2e-tests.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 6453c43a54679..c597085183996 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -23,6 +23,7 @@ jobs:
   e2e-tests:
     runs-on: ubuntu-latest
     env:
+      KUBECONFIG: "/tmp/kubeconfig"
       CI: "true"
       FAIL_FAST: "true"
       INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
@@ -86,6 +87,13 @@ jobs:
         run: |
           source scripts/ci/lib.sh
           test_on_infra | tee -a "$GITHUB_ENV"
+          infractl get "$CLUSTER_NAME"
+
+      - name: Create k8s Kind Cluster
+        if: ${{ steps.cluster-exists.outcome == 'failure' }}
+        uses: helm/kind-action@v1
+        with:
+          kubeconfig: "${{ env.KUBECONFIG }}"
 
       - name: Determine cluster flavor
         if: ${{ steps.cluster-exists.outcome != 'failure' }}

From 19d1e5387907a0dfd6a19557b643d5288df71c4a Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 08:19:01 -0600
Subject: [PATCH 229/256] ocplogin

---
 ui/apps/platform/cypress/integration-ocp/smoke.test.ts | 2 +-
 ui/apps/platform/scripts/cypress-ocp.sh                | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/ui/apps/platform/cypress/integration-ocp/smoke.test.ts b/ui/apps/platform/cypress/integration-ocp/smoke.test.ts
index 485390e181c8d..d421f32bffc3f 100644
--- a/ui/apps/platform/cypress/integration-ocp/smoke.test.ts
+++ b/ui/apps/platform/cypress/integration-ocp/smoke.test.ts
@@ -2,7 +2,7 @@ import { withOcpAuth } from '../helpers/ocpAuth';
 
 describe('Basic connectivity to the OCP plugin', () => {
     it('should open the OCP web console', () => {
-        // withOcpAuth();
+        withOcpAuth();
 
         cy.visit('/');
         // TODO Handle auth/skip auth in dev
diff --git a/ui/apps/platform/scripts/cypress-ocp.sh b/ui/apps/platform/scripts/cypress-ocp.sh
index 1d66fdfc41cba..82b03ee5b81ab 100755
--- a/ui/apps/platform/scripts/cypress-ocp.sh
+++ b/ui/apps/platform/scripts/cypress-ocp.sh
@@ -4,8 +4,8 @@ env | grep -o '^CLUSTER.*'
 set -x
 
 # Opens cypress with environment variables for feature flags and auth
-CLUSTER_CONSOLE_URL="${CLUSTER_CONSOLE_URL:-http://localhost:9000}"
-API_PROXY_BASE_URL="${CLUSTER_CONSOLE_URL}/api/proxy/plugin/advanced-cluster-security/api-service"
+CLUSTER_API_ENDPOINT="${CLUSTER_API_ENDPOINT:-http://localhost:9000}"
+API_PROXY_BASE_URL="${CLUSTER_API_ENDPOINT}/api/proxy/plugin/advanced-cluster-security/api-service"
 
 if [[ -z "$CLUSTER_USERNAME" || -z "$CLUSTER_PASSWORD" ]]; then
     echo "CLUSTER_USERNAME and CLUSTER_PASSWORD must be set"
@@ -29,8 +29,8 @@ fi
 artifacts_dir="${TEST_RESULTS_OUTPUT_DIR:-cypress/test-results}/ocp-artifacts"
 export CYPRESS_VIDEOS_FOLDER="${artifacts_dir}/videos"
 export CYPRESS_SCREENSHOTS_FOLDER="${artifacts_dir}/screenshots"
-if [[ -n "${CLUSTER_API_ENDPOINT}" ]]; then
-  export CYPRESS_BASE_URL="${CLUSTER_API_ENDPOINT}"
+if [[ -n "${CLUSTER_CONSOLE_URL}" ]]; then
+  export CYPRESS_BASE_URL="${CLUSTER_CONSOLE_URL}"
 fi
 
 export CYPRESS_SPEC_PATTERN='cypress/integration-ocp/**/*.test.{js,ts}'

From 8e5fdbd0225e41f00c3d18ce242c494a1211d669 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 08:20:53 -0600
Subject: [PATCH 230/256] disable testoninfra in prow

---
 .openshift-ci/begin.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.openshift-ci/begin.sh b/.openshift-ci/begin.sh
index 68006ffe075b7..27c735730ca4e 100755
--- a/.openshift-ci/begin.sh
+++ b/.openshift-ci/begin.sh
@@ -18,7 +18,7 @@ openshift_ci_import_creds
 setup_gcp
 set_ci_shared_export started_at "$(date -u +%s)"
 
-test_on_infra
+#test_on_infra
 
 if [[ -z "${SHARED_DIR:-}" ]]; then
     echo "ERROR: There is no SHARED_DIR for step env sharing"

From a23317bac74fd4aaffdc6142311cdcd94598b57f Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 08:36:21 -0600
Subject: [PATCH 231/256] test with kubeconfig separately

---
 .github/workflows/e2e-tests.yaml | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index c597085183996..d6d3d243cea6c 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -81,13 +81,15 @@ jobs:
           EOF
 
       - name: Check if testing on Infra cluster
-        id: cluster-exists
         timeout-minutes: 5
         continue-on-error: true
         run: |
           source scripts/ci/lib.sh
           test_on_infra | tee -a "$GITHUB_ENV"
-          infractl get "$CLUSTER_NAME"
+
+      - id: cluster-exists
+        run: |
+          kubectl get nodes
 
       - name: Create k8s Kind Cluster
         if: ${{ steps.cluster-exists.outcome == 'failure' }}
@@ -124,11 +126,11 @@ jobs:
           echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
           echo "TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
 
-      - name: Remove existing Stackrox
-        continue-on-error: true
-        if: ${{ steps.cluster-exists.outcome != 'failure' }}
-        run: |
-          kubectl delete namespace stackrox || true
+      #- name: Remove existing Stackrox
+      #  continue-on-error: true
+      #  if: ${{ steps.cluster-exists.outcome != 'failure' }}
+      #  run: |
+      #    kubectl delete namespace stackrox || true
 
       #- name: Find or Deploy Stackrox
       #  id: rox-installed

From 0594695344249b5d4072962195cc8c4ceeb1ed77 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 08:48:23 -0600
Subject: [PATCH 232/256] continue if no cluster

---
 .github/workflows/e2e-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index d6d3d243cea6c..2ad33f73b24a9 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -88,6 +88,7 @@ jobs:
           test_on_infra | tee -a "$GITHUB_ENV"
 
       - id: cluster-exists
+        continue-on-error: true
         run: |
           kubectl get nodes
 

From 1aaf61320f01e7243e4b4679bcb105cc573218ce Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 09:00:04 -0600
Subject: [PATCH 233/256] cluster type check

---
 .github/workflows/e2e-tests.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 2ad33f73b24a9..0bdee102e7ce9 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -42,7 +42,6 @@ jobs:
       GATHER_QA_TEST_DEBUG_LOGS: "false"
       QA_TEST_DEBUG_LOGS: ""
       SHARED_DIR: "/tmp"
-      CLUSTER: 'OPENSHIFT'
       SENSOR_HELM_DEPLOY: "false"
       ROX_SCANNER_V4: "false"
       ORCHESTRATOR_FLAVOR: "openshift"
@@ -99,11 +98,14 @@ jobs:
           kubeconfig: "${{ env.KUBECONFIG }}"
 
       - name: Determine cluster flavor
-        if: ${{ steps.cluster-exists.outcome != 'failure' }}
         timeout-minutes: 5
         continue-on-error: true
         run: |
           set -x
+          if kubectl get scc; then
+            echo "CLUSTER=OPENSHIFT" | tee -a "$GITHUB_ENV"
+          fi 
+          exit 0
           # groovy tests require CLUSTER='OPENSHIFT' for ocp or ='K8S' for GKE.
           if [[ "$(infractl get "$CLUSTER_NAME" --json | jq -r '.Flavor')" == *"openshift"* ]]; then
             echo "CLUSTER=OPENSHIFT" | tee -a "$GITHUB_ENV"
@@ -245,7 +247,7 @@ jobs:
           set -x
           export DEPLOY_STACKROX_VIA_OPERATOR=true
           export INSTALL_COMPLIANCE_OPERATOR=true
-          export ORCHESTRATOR_FLAVOR=openshift
+          export ORCHESTRATOR_FLAVOR="${CLUSTER}"
 
           #DEPLOY_DIR="deploy/${ORCHESTRATOR_FLAVOR}"
           #HOSTALIASES=/tmp/hostaliases

From a9f779b781827ce8e9c59df54456f0bdaca095ae Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 09:11:19 -0600
Subject: [PATCH 234/256] remove logging of osd vars

---
 tests/e2e/lib.sh | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
index 8fcc42834a545..e02351b66110c 100755
--- a/tests/e2e/lib.sh
+++ b/tests/e2e/lib.sh
@@ -202,14 +202,6 @@ export_test_environment() {
         # GKE uses this network for services. Consider it as a private subnet.
         ci_export ROX_NON_AGGREGATED_NETWORKS "${ROX_NON_AGGREGATED_NETWORKS:-34.118.224.0/20}"
     fi
-
-    set -x
-    echo "Exporting OCP cluster information for UI e2e tests"
-    ci_export CLUSTER_CONSOLE_URL "${CLUSTER_CONSOLE_URL:-}"
-    ci_export CLUSTER_API_ENDPOINT "${CLUSTER_API_ENDPOINT:-}"
-    ci_export CLUSTER_USERNAME "${CLUSTER_USERNAME:-}"
-    ci_export CLUSTER_PASSWORD "${CLUSTER_PASSWORD:-}"
-    set +x
 }
 
 deploy_stackrox_operator() {

From cb10afc9aa66a522dd4599d837ff942157402d2e Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 16:24:46 -0600
Subject: [PATCH 235/256] wait 1min

---
 ui/apps/platform/cypress/helpers/ocpAuth.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ui/apps/platform/cypress/helpers/ocpAuth.ts b/ui/apps/platform/cypress/helpers/ocpAuth.ts
index 79d1536a339fd..43ec2a92b57a4 100644
--- a/ui/apps/platform/cypress/helpers/ocpAuth.ts
+++ b/ui/apps/platform/cypress/helpers/ocpAuth.ts
@@ -4,7 +4,7 @@ export function withOcpAuth() {
     }
 
     cy.session('ocp-session-auth', () => {
-        cy.visit('/');
+        cy.visit('/', { timeout: 60000 });
         cy.get('input[name="username"]').type(Cypress.env('CLUSTER_USERNAME'));
         cy.get('input[name="password"]').type(Cypress.env('CLUSTER_PASSWORD'));
         cy.get('button[type="submit"]').click();

From 2f0165c4a4a4c86537315d1ae584fe3b6ba3e64b Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 17:03:13 -0600
Subject: [PATCH 236/256] add timeout for un get

---
 ui/apps/platform/cypress/helpers/ocpAuth.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ui/apps/platform/cypress/helpers/ocpAuth.ts b/ui/apps/platform/cypress/helpers/ocpAuth.ts
index 43ec2a92b57a4..c248b1e0c4c16 100644
--- a/ui/apps/platform/cypress/helpers/ocpAuth.ts
+++ b/ui/apps/platform/cypress/helpers/ocpAuth.ts
@@ -5,7 +5,8 @@ export function withOcpAuth() {
 
     cy.session('ocp-session-auth', () => {
         cy.visit('/', { timeout: 60000 });
-        cy.get('input[name="username"]').type(Cypress.env('CLUSTER_USERNAME'));
+        // 8s timeout for get?
+        cy.get('input[name="username"]', { timeout: 60000 }).type(Cypress.env('CLUSTER_USERNAME'));
         cy.get('input[name="password"]').type(Cypress.env('CLUSTER_PASSWORD'));
         cy.get('button[type="submit"]').click();
         // TODO Handle OCP welcome modal

From 6de328a97ea8993877d6ce37f15c3c3875abb63d Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 22:48:30 -0600
Subject: [PATCH 237/256] hack ocp login to work

---
 ui/apps/platform/cypress.config.js          | 4 ++--
 ui/apps/platform/cypress/helpers/ocpAuth.ts | 8 ++++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/ui/apps/platform/cypress.config.js b/ui/apps/platform/cypress.config.js
index f077d7cd91bfc..11b622bcfa885 100644
--- a/ui/apps/platform/cypress.config.js
+++ b/ui/apps/platform/cypress.config.js
@@ -7,7 +7,7 @@
  */
 
 module.exports = {
-    blockHosts: ['*.*'], // Browser options
+    // blockHosts: ['*.*'], // Browser options
     chromeWebSecurity: false, // Browser options
     defaultCommandTimeout: 8000, // Timeouts options
     numTestsKeptInMemory: 0, // Global options
@@ -25,7 +25,7 @@ module.exports = {
 
     e2e: {
         baseUrl: 'https://localhost:3000',
-        specPattern: 'cypress/integration/**/*.test.{js,ts}',
+        specPattern: 'cypress/integration*/**/*.test.{js,ts}',
         viewportHeight: 850, // Viewport options
         viewportWidth: 1440, // Viewport options
         setupNodeEvents: (on) => {
diff --git a/ui/apps/platform/cypress/helpers/ocpAuth.ts b/ui/apps/platform/cypress/helpers/ocpAuth.ts
index c248b1e0c4c16..0d141d4e5385f 100644
--- a/ui/apps/platform/cypress/helpers/ocpAuth.ts
+++ b/ui/apps/platform/cypress/helpers/ocpAuth.ts
@@ -4,11 +4,15 @@ export function withOcpAuth() {
     }
 
     cy.session('ocp-session-auth', () => {
-        cy.visit('/', { timeout: 60000 });
+        cy.visit('/', { timeout: 6000 });
+        cy.url().should("contain", "/login?");
         // 8s timeout for get?
-        cy.get('input[name="username"]', { timeout: 60000 }).type(Cypress.env('CLUSTER_USERNAME'));
+        cy.get('input[name="username"]').type(Cypress.env('CLUSTER_USERNAME'));
         cy.get('input[name="password"]').type(Cypress.env('CLUSTER_PASSWORD'));
         cy.get('button[type="submit"]').click();
+        cy.url().should("contain", "/dashboards");
+        cy.contains('Skip tour', { timeout: 10000 }).click()
+        cy.wait(1000)
         // TODO Handle OCP welcome modal
     });
 }

From 1d30004297f4cb363d478f3138edbbb4444538e3 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 22:51:18 -0600
Subject: [PATCH 238/256] turn off xtrace for env

---
 .github/workflows/e2e-tests.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 0bdee102e7ce9..e471807bb38f3 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -235,7 +235,6 @@ jobs:
         timeout-minutes: 15
         run: |
           set +e
-          set -x
           ls -latr ${SHARED_DIR}**
           source tests/e2e/lib.sh
           setup_automation_flavor_e2e_cluster "${CI_JOB_NAME}"

From c1d4b26dae2d7171c7ea414aa03ffd91dceebeff Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 22:56:43 -0600
Subject: [PATCH 239/256] include int-ocp results

---
 tests/e2e/run-ui-e2e.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/e2e/run-ui-e2e.sh b/tests/e2e/run-ui-e2e.sh
index b67199974f8ee..675fd08c9cbb4 100755
--- a/tests/e2e/run-ui-e2e.sh
+++ b/tests/e2e/run-ui-e2e.sh
@@ -60,7 +60,7 @@ run_ui_e2e_tests() {
     make -C ui test-e2e || touch FAIL
     echo "make finished."
 
-    store_test_results "ui/test-results/reports/cypress/integration/." "cy-reps"
+    store_test_results "ui/test-results/reports/cypress/integration*/." "cy-reps"
 
     if is_OPENSHIFT_CI; then
         cp -a ui/test-results/artifacts/* "${ARTIFACT_DIR}/" || true

From 1bdce8a97a7dc32ef60c944a15ed1c06214b5a23 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 23:01:12 -0600
Subject: [PATCH 240/256] do not reinstall

---
 .github/workflows/e2e-tests.yaml |  9 ++-------
 tests/e2e/run-ui-e2e.sh          | 14 ++++++++------
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index e471807bb38f3..9a0855f4998d8 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -231,18 +231,13 @@ jobs:
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
       # UI e2e - Cypress tests
-      - name: Env for UI E2E
-        timeout-minutes: 15
+      - name: UI E2E
+        timeout-minutes: 45
         run: |
           set +e
           ls -latr ${SHARED_DIR}**
           source tests/e2e/lib.sh
           setup_automation_flavor_e2e_cluster "${CI_JOB_NAME}"
-
-      - name: UI E2E
-        timeout-minutes: 30
-        run: |
-          set +e
           set -x
           export DEPLOY_STACKROX_VIA_OPERATOR=true
           export INSTALL_COMPLIANCE_OPERATOR=true
diff --git a/tests/e2e/run-ui-e2e.sh b/tests/e2e/run-ui-e2e.sh
index 675fd08c9cbb4..a840c641af6cd 100755
--- a/tests/e2e/run-ui-e2e.sh
+++ b/tests/e2e/run-ui-e2e.sh
@@ -25,12 +25,14 @@ test_ui_e2e() {
     export_test_environment
 
     setup_deployment_env false false
-    remove_existing_stackrox_resources
-    setup_default_TLS_certs
-
-    # deploy the optional components before stackrox
-    deploy_optional_e2e_components
-    deploy_stackrox
+    kubectl get ns stackrox || {
+        remove_existing_stackrox_resources
+        setup_default_TLS_certs
+
+        # deploy the optional components before stackrox
+        deploy_optional_e2e_components
+        deploy_stackrox
+    }
 
     run_ui_e2e_tests
 }

From 4a79299e76d0aab714de57dd8124ef3aa70297f4 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 23:04:40 -0600
Subject: [PATCH 241/256] lookup rox vars

---
 .github/workflows/e2e-tests.yaml | 104 +++++++++++++++----------------
 1 file changed, 52 insertions(+), 52 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 9a0855f4998d8..21aef11335364 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -163,60 +163,60 @@ jobs:
       #        echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";
       #      }
 
-      #- name: Stackrox service and auth lookup
-      #  id: stackrox-api
-      #  if: ${{ steps.rox-installed.outcome != 'failure' }}
-      #  timeout-minutes: 5
-      #  continue-on-error: true
-      #  run: |
-      #    set +e
-      #    set -x
-      #    ls -latr /tmp/
-      #    if [[ -e "${KUBECONFIG:-}" ]]; then
-      #      if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
-      #        password="${ROX_ADMIN_PASSWORD:-$(cat "${KUBECONFIG%kubeconfig}admin-password")}"
-      #        password="${password:-$(kubectl -n stackrox get secret admin-pass -o json | jq -er '.data.password // ""' | base64 -d)}"
-      #        password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password // ""' | base64 -d)}"
-      #        echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
-      #        echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
-      #      fi
-      #      if [[ -z "${API_HOSTNAME:-}" || ( $API_HOSTNAME == 'localhost' && -z "$(pgrep -F /tmp/forward.pid)" ) ]]; then
-      #        API_HOSTNAME=$(kubectl -n stackrox get route central -o jsonpath='{.status.ingress[0].host}')
-      #        API_HOSTNAME=${API_HOSTNAME:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].ip}')}
-      #        API_HOSTNAME=${API_HOSTNAME:-localhost}
-      #        echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
-      #        API_PORT=${API_PORT:-$(kubectl -n stackrox get route central -o jsonpath='{.spec.port.targetPort}')}
-      #        API_PORT=${API_PORT:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].port}')}
-      #        API_PORT=${API_PORT//https/443}
-      #        if [[ "$API_HOSTNAME" == 'localhost' ]]; then
-      #          nohup kubectl -n stackrox port-forward svc/central 8000:443 &
-      #          echo $! | tee /tmp/forward.pid
-      #          API_PORT=8000
-      #        fi
-      #        API_PORT=${API_PORT:-443}
-      #        echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
-      #        echo "API_PORT=${API_PORT}" | tee -a "$GITHUB_ENV"
-      #      fi
-      #    fi
-      #    echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
-
-      #- name: Stackrox API test
-      #  if: ${{ steps.stackrox-api.outcome != 'failure' }}
-      #  run: |
-      #    if [[ "${API_HOSTNAME:-}" == 'localhost' && -e /tmp/forward.pid ]]; then 
-      #      cat /tmp/forward.pid
-      #      ps -ef | grep kube || true
-      #    fi
+      - name: Stackrox service and auth lookup
+        id: stackrox-api
+        if: ${{ steps.rox-installed.outcome != 'failure' }}
+        timeout-minutes: 5
+        continue-on-error: true
+        run: |
+          set +e
+          set -x
+          ls -latr /tmp/
+          if [[ -e "${KUBECONFIG:-}" ]]; then
+            if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
+              password="${ROX_ADMIN_PASSWORD:-$(cat "${KUBECONFIG%kubeconfig}admin-password")}"
+              password="${password:-$(kubectl -n stackrox get secret admin-pass -o json | jq -er '.data.password // ""' | base64 -d)}"
+              password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password // ""' | base64 -d)}"
+              echo "ROX_USERNAME=admin" | tee -a "$GITHUB_ENV"
+              echo "ROX_ADMIN_PASSWORD=${password}" >> "$GITHUB_ENV"
+            fi
+            if [[ -z "${API_HOSTNAME:-}" || ( $API_HOSTNAME == 'localhost' && -z "$(pgrep -F /tmp/forward.pid)" ) ]]; then
+              API_HOSTNAME=$(kubectl -n stackrox get route central -o jsonpath='{.status.ingress[0].host}')
+              API_HOSTNAME=${API_HOSTNAME:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].ip}')}
+              API_HOSTNAME=${API_HOSTNAME:-localhost}
+              echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
+              API_PORT=${API_PORT:-$(kubectl -n stackrox get route central -o jsonpath='{.spec.port.targetPort}')}
+              API_PORT=${API_PORT:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].port}')}
+              API_PORT=${API_PORT//https/443}
+              if [[ "$API_HOSTNAME" == 'localhost' ]]; then
+                nohup kubectl -n stackrox port-forward svc/central 8000:443 &
+                echo $! | tee /tmp/forward.pid
+                API_PORT=8000
+              fi
+              API_PORT=${API_PORT:-443}
+              echo "API_HOSTNAME=${API_HOSTNAME}" | tee -a "$GITHUB_ENV"
+              echo "API_PORT=${API_PORT}" | tee -a "$GITHUB_ENV"
+            fi
+          fi
+          echo "API_ENDPOINT=${API_HOSTNAME:-localhost}:${API_PORT:-8000}" | tee -a "$GITHUB_ENV"
+
+      - name: Stackrox API test
+        if: ${{ steps.stackrox-api.outcome != 'failure' }}
+        run: |
+          if [[ "${API_HOSTNAME:-}" == 'localhost' && -e /tmp/forward.pid ]]; then 
+            cat /tmp/forward.pid
+            ps -ef | grep kube || true
+          fi
 
-      #    curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.'
+          curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.'
 
-      #- name: Get secured cluster name
-      #  if: ${{ steps.stackrox-api.outcome != 'failure' }}
-      #  run: |
-      #    echo "Groovy tests expect a secured cluster. Default name is 'remote' but the Infra demo clusters set names like 'production'. Look up what secured clusters exist:"
-      #    set -x
-      #    CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
-      #    echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
+      - name: Get secured cluster name
+        if: ${{ steps.stackrox-api.outcome != 'failure' }}
+        run: |
+          echo "Groovy tests expect a secured cluster. Default name is 'remote' but the Infra demo clusters set names like 'production'. Look up what secured clusters exist:"
+          set -x
+          CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
+          echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
 
       - run: go get github.com/stackrox/scanner
 

From b8eb561eed3c4bf42532d1008d1a1185830c41c6 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Sun, 17 Aug 2025 23:11:25 -0600
Subject: [PATCH 242/256] disable secured cluster lookup

---
 .github/workflows/e2e-tests.yaml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 21aef11335364..8779c94418217 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -210,13 +210,14 @@ jobs:
 
           curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.'
 
-      - name: Get secured cluster name
-        if: ${{ steps.stackrox-api.outcome != 'failure' }}
-        run: |
-          echo "Groovy tests expect a secured cluster. Default name is 'remote' but the Infra demo clusters set names like 'production'. Look up what secured clusters exist:"
-          set -x
-          CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
-          echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
+      # Only needed for Groovy
+      #- name: Get secured cluster name
+      #  if: ${{ steps.stackrox-api.outcome != 'failure' }}
+      #  run: |
+      #    echo "Groovy tests expect a secured cluster. Default name is 'remote' but the Infra demo clusters set names like 'production'. Look up what secured clusters exist:"
+      #    set -x
+      #    CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
+      #    echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
 
       - run: go get github.com/stackrox/scanner
 

From ec8b6c171433499dfb53183a40819409b32a1302 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 19 Aug 2025 22:49:57 -0600
Subject: [PATCH 243/256] remove ui test debug

---
 ui/Makefile                                   |  2 +-
 ui/apps/platform/cypress.config.js            |  4 +-
 .../cypress/integration-ocp/smoke.test.ts     | 11 ----
 ui/apps/platform/package.json                 |  3 +-
 ui/apps/platform/scripts/cypress-ocp.sh       | 59 -------------------
 5 files changed, 4 insertions(+), 75 deletions(-)
 delete mode 100644 ui/apps/platform/cypress/integration-ocp/smoke.test.ts
 delete mode 100755 ui/apps/platform/scripts/cypress-ocp.sh

diff --git a/ui/Makefile b/ui/Makefile
index 2958d894d0567..b33f583f66dfe 100644
--- a/ui/Makefile
+++ b/ui/Makefile
@@ -6,7 +6,7 @@ all: deps lint test test-component build
 
 SOURCES := $(shell find apps -type d \( -name node_modules \) -prune -o -print)
 
-ROX_PRODUCT_BRANDING ?= $(shell $(MAKE) --no-print-directory -C .. product-branding)
+ROX_PRODUCT_BRANDING ?= $(shell $(MAKE) --quiet --no-print-directory -C .. product-branding)
 export ROX_PRODUCT_BRANDING := $(ROX_PRODUCT_BRANDING)
 
 deps: apps/platform/package.json apps/platform/package-lock.json
diff --git a/ui/apps/platform/cypress.config.js b/ui/apps/platform/cypress.config.js
index 11b622bcfa885..f077d7cd91bfc 100644
--- a/ui/apps/platform/cypress.config.js
+++ b/ui/apps/platform/cypress.config.js
@@ -7,7 +7,7 @@
  */
 
 module.exports = {
-    // blockHosts: ['*.*'], // Browser options
+    blockHosts: ['*.*'], // Browser options
     chromeWebSecurity: false, // Browser options
     defaultCommandTimeout: 8000, // Timeouts options
     numTestsKeptInMemory: 0, // Global options
@@ -25,7 +25,7 @@ module.exports = {
 
     e2e: {
         baseUrl: 'https://localhost:3000',
-        specPattern: 'cypress/integration*/**/*.test.{js,ts}',
+        specPattern: 'cypress/integration/**/*.test.{js,ts}',
         viewportHeight: 850, // Viewport options
         viewportWidth: 1440, // Viewport options
         setupNodeEvents: (on) => {
diff --git a/ui/apps/platform/cypress/integration-ocp/smoke.test.ts b/ui/apps/platform/cypress/integration-ocp/smoke.test.ts
deleted file mode 100644
index d421f32bffc3f..0000000000000
--- a/ui/apps/platform/cypress/integration-ocp/smoke.test.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-import { withOcpAuth } from '../helpers/ocpAuth';
-
-describe('Basic connectivity to the OCP plugin', () => {
-    it('should open the OCP web console', () => {
-        withOcpAuth();
-
-        cy.visit('/');
-        // TODO Handle auth/skip auth in dev
-        // TODO Handle OCP welcome modal
-    });
-});
diff --git a/ui/apps/platform/package.json b/ui/apps/platform/package.json
index 75425663bc417..cdf1ecb1956bf 100644
--- a/ui/apps/platform/package.json
+++ b/ui/apps/platform/package.json
@@ -107,10 +107,9 @@
         "lint:fix": "eslint --fix --quiet",
         "tsc": "tsc",
         "cypress-open": "./scripts/cypress.sh open --e2e --config defaultBrowser=chrome",
-        "cypress-open:ocp": "./scripts/cypress-ocp.sh open --e2e --config defaultBrowser=chrome",
         "cypress-spec": "./scripts/cypress.sh run --spec",
         "cypress-component": "CYPRESS_COMPONENT_TEST=true ./scripts/cypress-component.sh open --component",
-        "test-e2e": "TZ=UTC ./scripts/cypress-ocp.sh run --reporter mocha-multi-reporters --reporter-options configFile=cypress/mocha.config.js",
+        "test-e2e": "TZ=UTC ./scripts/cypress.sh run --reporter mocha-multi-reporters --reporter-options configFile=cypress/mocha.config.js",
         "test-e2e-local": "TZ=UTC ./scripts/cypress.sh run",
         "test-component": "TZ=UTC CYPRESS_COMPONENT_TEST=true ./scripts/cypress-component.sh run --reporter mocha-multi-reporters --reporter-options configFile=cypress/mocha.config.js --component",
         "generate-graphql-possible-types": "ROX_AUTH_TOKEN=$(./scripts/get-auth-token.sh) node scripts/generate-graphql-possible-types.js",
diff --git a/ui/apps/platform/scripts/cypress-ocp.sh b/ui/apps/platform/scripts/cypress-ocp.sh
deleted file mode 100755
index 82b03ee5b81ab..0000000000000
--- a/ui/apps/platform/scripts/cypress-ocp.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/env bash
-
-env | grep -o '^CLUSTER.*'
-set -x
-
-# Opens cypress with environment variables for feature flags and auth
-CLUSTER_API_ENDPOINT="${CLUSTER_API_ENDPOINT:-http://localhost:9000}"
-API_PROXY_BASE_URL="${CLUSTER_API_ENDPOINT}/api/proxy/plugin/advanced-cluster-security/api-service"
-
-if [[ -z "$CLUSTER_USERNAME" || -z "$CLUSTER_PASSWORD" ]]; then
-    echo "CLUSTER_USERNAME and CLUSTER_PASSWORD must be set"
-    exit 1
-fi
-
-curl_cfg() { # Use built-in echo to not expose $2 in the process list.
-  echo -n "$1 = \"${2//[\"\\]/\\&}\""
-}
-
-if [[ -n "$CLUSTER_PASSWORD" ]]; then
-  readarray -t arr < <(curl -sk --config <(curl_cfg user "$CLUSTER_USERNAME:$CLUSTER_PASSWORD") "${API_PROXY_BASE_URL}"/v1/featureflags | jq -cr '.featureFlags[] | {name: .envVar, enabled: .enabled}')
-  for i in "${arr[@]}"; do
-    name=$(echo "$i" | jq -rc .name)
-    val=$(echo "$i" | jq -rc .enabled)
-    export CYPRESS_"${name}"="${val}"
-  done
-fi
-
-# eventually it should be in cypress.config.js: https://github.com/cypress-io/cypress/issues/5218
-artifacts_dir="${TEST_RESULTS_OUTPUT_DIR:-cypress/test-results}/ocp-artifacts"
-export CYPRESS_VIDEOS_FOLDER="${artifacts_dir}/videos"
-export CYPRESS_SCREENSHOTS_FOLDER="${artifacts_dir}/screenshots"
-if [[ -n "${CLUSTER_CONSOLE_URL}" ]]; then
-  export CYPRESS_BASE_URL="${CLUSTER_CONSOLE_URL}"
-fi
-
-export CYPRESS_SPEC_PATTERN='cypress/integration-ocp/**/*.test.{js,ts}'
-
-# be able to skip tests that are not relevant, for example: openshift
-export CYPRESS_ORCHESTRATOR_FLAVOR="${ORCHESTRATOR_FLAVOR}"
-
-export CYPRESS_OCP_BRIDGE_AUTH_DISABLED="${OCP_BRIDGE_AUTH_DISABLED}"
-export CYPRESS_CLUSTER_USERNAME="${CLUSTER_USERNAME}"
-export CYPRESS_CLUSTER_PASSWORD="${CLUSTER_PASSWORD}"
-
-# exit if ORCHESTRATOR_FLAVOR is not 'openshift'
-if [ "${ORCHESTRATOR_FLAVOR}" != "openshift" ]; then
-    echo "ORCHESTRATOR_FLAVOR is not 'openshift', skipping cypress-ocp"
-    exit 0
-fi
-
-if [ "$2" == "--spec" ]; then
-    if [ $# -ne 3 ]; then
-        echo "usage: npm run cypress-spec <spec-file>"
-        exit 1
-    fi
-    cypress run --spec "cypress/integration-ocp/$3"
-else
-    DEBUG="cypress*" NO_COLOR=1 cypress "$@" 2> /dev/null
-fi

From 9babeb20bb7ef55580db7f019e6ab3a0648722a9 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 19 Aug 2025 22:56:44 -0600
Subject: [PATCH 244/256] test on kind

---
 .github/workflows/e2e-tests.yaml | 166 +++++++++++++++----------------
 1 file changed, 83 insertions(+), 83 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 8779c94418217..d7bc4755ed2bf 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -44,7 +44,7 @@ jobs:
       SHARED_DIR: "/tmp"
       SENSOR_HELM_DEPLOY: "false"
       ROX_SCANNER_V4: "false"
-      ORCHESTRATOR_FLAVOR: "openshift"
+      #ORCHESTRATOR_FLAVOR: "openshift"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -63,7 +63,7 @@ jobs:
 
       - name: Env variables
         run: |
-          JOB_NAME="${JOB_NAME:-ocp-4-19-ui-e2e-tests}"
+          JOB_NAME="${JOB_NAME:-ocp-4-19-qa-e2e-tests}"
 
           ARTIFACT_DIR="${ARTIFACT_DIR:-${SHARED_DIR}/artifacts/}"
           mkdir -p "$ARTIFACT_DIR"
@@ -79,12 +79,12 @@ jobs:
           GRADLE_USER_HOME=${HOME}
           EOF
 
-      - name: Check if testing on Infra cluster
-        timeout-minutes: 5
-        continue-on-error: true
-        run: |
-          source scripts/ci/lib.sh
-          test_on_infra | tee -a "$GITHUB_ENV"
+      #- name: Check if testing on Infra cluster
+      #  timeout-minutes: 5
+      #  continue-on-error: true
+      #  run: |
+      #    source scripts/ci/lib.sh
+      #    test_on_infra | tee -a "$GITHUB_ENV"
 
       - id: cluster-exists
         continue-on-error: true
@@ -135,33 +135,33 @@ jobs:
       #  run: |
       #    kubectl delete namespace stackrox || true
 
-      #- name: Find or Deploy Stackrox
-      #  id: rox-installed
-      #  if: ${{ steps.cluster-exists.outcome != 'failure' }}
-      #  env:
-      #    DEPLOY_STACKROX_VIA_OPERATOR: "false"
-      #    # LOAD_BALANCER: "lb"
-      #    # SCANNER_SUPPORT: "true"
-      #    MONITORING_SUPPORT: "false"
-      #    LOCAL_DEPLOYMENT: "true"
-      #    ROX_BASELINE_GENERATION_DURATION: "1m"
-      #    ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
-      #  run: |
-      #    set +e
-      #    echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
-      #    kubectl get namespace stackrox \
-      #      || { 
-      #        if [[ "${CLUSTER,,}" == "openshift" ]]; then
-      #            kubectl get scc qatest-anyuid \
-      #              || {
-      #                kubectl create ns stackrox;
-      #                kubectl create -f "qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml";
-      #              }
-      #        fi
-      #        CI=true ./deploy/deploy-local.sh;
-      #        find deploy -name password;
-      #        echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";
-      #      }
+      - name: Find or Deploy Stackrox
+        id: rox-installed
+        if: ${{ steps.cluster-exists.outcome != 'failure' }}
+        env:
+          DEPLOY_STACKROX_VIA_OPERATOR: "false"
+          # LOAD_BALANCER: "lb"
+          # SCANNER_SUPPORT: "true"
+          MONITORING_SUPPORT: "false"
+          LOCAL_DEPLOYMENT: "true"
+          ROX_BASELINE_GENERATION_DURATION: "1m"
+          ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
+        run: |
+          set +e
+          echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
+          kubectl get namespace stackrox \
+            || { 
+              if [[ "${CLUSTER,,}" == "openshift" ]]; then
+                  kubectl get scc qatest-anyuid \
+                    || {
+                      kubectl create ns stackrox;
+                      kubectl create -f "qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml";
+                    }
+              fi
+              CI=true ./deploy/deploy-local.sh;
+              find deploy -name password;
+              echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";
+            }
 
       - name: Stackrox service and auth lookup
         id: stackrox-api
@@ -211,13 +211,13 @@ jobs:
           curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters" | jq '.'
 
       # Only needed for Groovy
-      #- name: Get secured cluster name
-      #  if: ${{ steps.stackrox-api.outcome != 'failure' }}
-      #  run: |
-      #    echo "Groovy tests expect a secured cluster. Default name is 'remote' but the Infra demo clusters set names like 'production'. Look up what secured clusters exist:"
-      #    set -x
-      #    CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
-      #    echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
+      - name: Get secured cluster name
+        if: ${{ steps.stackrox-api.outcome != 'failure' }}
+        run: |
+          echo "Groovy tests expect a secured cluster. Default name is 'remote' but the Infra demo clusters set names like 'production'. Look up what secured clusters exist:"
+          set -x
+          CLUSTER_INFO=$(curl -sk -u "${ROX_USERNAME}:${ROX_ADMIN_PASSWORD}" "https://${API_ENDPOINT}/v1/clusters")
+          echo "REMOTE_CLUSTER_NAME=$(jq -r '(.clusters[] | select(.name == "production") | .name) // .clusters[0].name' <<<"$CLUSTER_INFO")" | tee -a "$GITHUB_ENV"
 
       - run: go get github.com/stackrox/scanner
 
@@ -231,54 +231,54 @@ jobs:
           install --mode 755 /tmp/oc /usr/local/bin/
           install --mode 755 /tmp/kubectl /usr/local/bin/
 
-      # UI e2e - Cypress tests
-      - name: UI E2E
-        timeout-minutes: 45
-        run: |
-          set +e
-          ls -latr ${SHARED_DIR}**
-          source tests/e2e/lib.sh
-          setup_automation_flavor_e2e_cluster "${CI_JOB_NAME}"
-          set -x
-          export DEPLOY_STACKROX_VIA_OPERATOR=true
-          export INSTALL_COMPLIANCE_OPERATOR=true
-          export ORCHESTRATOR_FLAVOR="${CLUSTER}"
+      ## UI e2e - Cypress tests
+      #- name: UI E2E
+      #  timeout-minutes: 45
+      #  run: |
+      #    set +e
+      #    ls -latr ${SHARED_DIR}**
+      #    source tests/e2e/lib.sh
+      #    setup_automation_flavor_e2e_cluster "${CI_JOB_NAME}"
+      #    set -x
+      #    export DEPLOY_STACKROX_VIA_OPERATOR=true
+      #    export INSTALL_COMPLIANCE_OPERATOR=true
+      #    export ORCHESTRATOR_FLAVOR="${CLUSTER}"
 
-          #DEPLOY_DIR="deploy/${ORCHESTRATOR_FLAVOR}"
-          #HOSTALIASES=/tmp/hostaliases
-          #UI_BASE_URL (set to either "https://central-lb:443" or "https://localhost:${LOCAL_PORT}")
-          #make -C ui test-e2e, so these environment variables will be available during that execution.
+      #    #DEPLOY_DIR="deploy/${ORCHESTRATOR_FLAVOR}"
+      #    #HOSTALIASES=/tmp/hostaliases
+      #    #UI_BASE_URL (set to either "https://central-lb:443" or "https://localhost:${LOCAL_PORT}")
+      #    #make -C ui test-e2e, so these environment variables will be available during that execution.
 
-          export VERSION=$(echo "${MAIN_IMAGE_TAG}" | sed -E 's@^(([[:digit:]]+\.)+)x(-)?@\10\3@g')  # operator/Makefile VERSION is CSV_TAG
-          tests/e2e/run-ui-e2e.sh
+      #    export VERSION=$(echo "${MAIN_IMAGE_TAG}" | sed -E 's@^(([[:digit:]]+\.)+)x(-)?@\10\3@g')  # operator/Makefile VERSION is CSV_TAG
+      #    tests/e2e/run-ui-e2e.sh
 
       # QA e2e - groovy tests
-      #- name: Setup Gradle
-      #  uses: gradle/actions/setup-gradle@v4
-      #  with:
-      #    gradle-version: wrapper
-      #    #cache-read-only: false
-      #    cache-overwrite-existing: true
-      #    cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
-
-      #- timeout-minutes: 10
-      #  run: |
-      #    make -C qa-tests-backend compile
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          gradle-version: wrapper
+          #cache-read-only: false
+          cache-overwrite-existing: true
+          cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/release' }}
 
-      #- timeout-minutes: 35
-      #  run: |
-      #    export TEST_TARGET="smoke-test"
-      #    cd qa-tests-backend && ./gradlew testSMOKE --fail-fast
+      - timeout-minutes: 10
+        run: |
+          make -C qa-tests-backend compile
 
-      #- timeout-minutes: 35
-      #  run: |
-      #    export TEST_TARGET="bat-test"
-      #    cd qa-tests-backend && ./gradlew testParallelBAT testBAT --fail-fast --warn
+      - timeout-minutes: 35
+        run: |
+          export TEST_TARGET="smoke-test"
+          cd qa-tests-backend && ./gradlew testSMOKE --fail-fast
 
-      #- timeout-minutes: 35
-      #  run: |
-      #    export TEST_TARGET="test"
-      #    cd qa-tests-backend && ./gradlew testParallel testRest --fail-fast --warn
+      - timeout-minutes: 35
+        run: |
+          export TEST_TARGET="bat-test"
+          cd qa-tests-backend && ./gradlew testParallelBAT testBAT --fail-fast --warn
+
+      - timeout-minutes: 35
+        run: |
+          export TEST_TARGET="test"
+          cd qa-tests-backend && ./gradlew testParallel testRest --fail-fast --warn
 
       - name: Check shared dir and artifacts
         if: always()

From 07ea504c7e851a5e784f042dcaa4c36cae3cf18e Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Tue, 19 Aug 2025 23:00:11 -0600
Subject: [PATCH 245/256] install on non-infra

---
 .github/workflows/e2e-tests.yaml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index d7bc4755ed2bf..fc0db938a10d8 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -129,15 +129,14 @@ jobs:
           echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
           echo "TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
 
-      #- name: Remove existing Stackrox
-      #  continue-on-error: true
-      #  if: ${{ steps.cluster-exists.outcome != 'failure' }}
-      #  run: |
-      #    kubectl delete namespace stackrox || true
+      - name: Remove existing Stackrox
+        continue-on-error: true
+        if: ${{ steps.cluster-exists.outcome != 'failure' }}
+        run: |
+          kubectl delete namespace stackrox
 
       - name: Find or Deploy Stackrox
         id: rox-installed
-        if: ${{ steps.cluster-exists.outcome != 'failure' }}
         env:
           DEPLOY_STACKROX_VIA_OPERATOR: "false"
           # LOAD_BALANCER: "lb"

From 8a4ce7cbf357487fb93cb9a60a6dc9c830a55b81 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 20 Aug 2025 08:53:11 -0600
Subject: [PATCH 246/256] share username

---
 .github/workflows/e2e-tests.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index fc0db938a10d8..ab247ece45d0d 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -48,7 +48,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # ${{ github.event.pull_request.commits }}
+          fetch-depth: ${{ github.event.pull_request.commits }}
           fetch-tags: true
           show-progress: false
 
@@ -159,6 +159,7 @@ jobs:
               fi
               CI=true ./deploy/deploy-local.sh;
               find deploy -name password;
+              echo "ROX_USERNAME=admin" >> "$GITHUB_ENV";
               echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";
             }
 
@@ -172,7 +173,7 @@ jobs:
           set -x
           ls -latr /tmp/
           if [[ -e "${KUBECONFIG:-}" ]]; then
-            if [[ -z "${ROX_ADMIN_PASSWORD:+set}" ]]; then
+            if [[ -z "${ROX_ADMIN_PASSWORD:-}" ]]; then
               password="${ROX_ADMIN_PASSWORD:-$(cat "${KUBECONFIG%kubeconfig}admin-password")}"
               password="${password:-$(kubectl -n stackrox get secret admin-pass -o json | jq -er '.data.password // ""' | base64 -d)}"
               password="${password:-$(kubectl -n stackrox get secret central-htpasswd -o json | jq -er '.data.password // ""' | base64 -d)}"
@@ -188,6 +189,7 @@ jobs:
               API_PORT=${API_PORT:-$(kubectl -n stackrox get service central-loadbalancer -o jsonpath='{.status.loadBalancer.ingress[0].port}')}
               API_PORT=${API_PORT//https/443}
               if [[ "$API_HOSTNAME" == 'localhost' ]]; then
+                ps -ef|grep kubectl || true
                 nohup kubectl -n stackrox port-forward svc/central 8000:443 &
                 echo $! | tee /tmp/forward.pid
                 API_PORT=8000

From 15c2d933e2ed692af5345688577ab7e77abb60f8 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 20 Aug 2025 08:54:49 -0600
Subject: [PATCH 247/256] quiet deploy

---
 deploy/common/k8sbased.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/deploy/common/k8sbased.sh b/deploy/common/k8sbased.sh
index b211454e76b5b..5fca27731ea7c 100644
--- a/deploy/common/k8sbased.sh
+++ b/deploy/common/k8sbased.sh
@@ -147,7 +147,6 @@ function launch_central {
     local common_dir="${k8s_dir}/../common"
 
     verify_orch
-    echo "CI:$CI"
     if [[ -z "$CI" ]]; then
         prompt_if_central_exists "${central_namespace}"
     fi
@@ -553,7 +552,6 @@ function launch_central {
     # to ongoing modifications to the central deployment. This port-forward dies and the script hangs "Waiting for
     # Central to respond" until it times out. Waiting for rollout status should help not get into such situation.
     rollout_wait_timeout="10m"
-    set -x
     kubectl -n "${central_namespace}" rollout status deploy/central --timeout="${rollout_wait_timeout}"
 
     # if we have specified that we want to use a load balancer, then use that endpoint instead of localhost

From 085f48a4c2e209f71442017ba3094688b6a4d42e Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 20 Aug 2025 09:01:33 -0600
Subject: [PATCH 248/256] lookup only if existed

---
 .github/workflows/e2e-tests.yaml | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index ab247ece45d0d..f94bbe4e6fd9e 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -129,14 +129,15 @@ jobs:
           echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
           echo "TAG=${MAIN_IMAGE_TAG}" | tee -a "$GITHUB_ENV"
 
-      - name: Remove existing Stackrox
-        continue-on-error: true
-        if: ${{ steps.cluster-exists.outcome != 'failure' }}
-        run: |
-          kubectl delete namespace stackrox
+      #- name: Remove existing Stackrox
+      #  continue-on-error: true
+      #  if: ${{ steps.cluster-exists.outcome != 'failure' }}
+      #  run: |
+      #    kubectl delete namespace stackrox
 
       - name: Find or Deploy Stackrox
-        id: rox-installed
+        id: rox-exists
+        continue-on-error: true
         env:
           DEPLOY_STACKROX_VIA_OPERATOR: "false"
           # LOAD_BALANCER: "lb"
@@ -161,11 +162,12 @@ jobs:
               find deploy -name password;
               echo "ROX_USERNAME=admin" >> "$GITHUB_ENV";
               echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";
+              exit 1;
             }
 
       - name: Stackrox service and auth lookup
         id: stackrox-api
-        if: ${{ steps.rox-installed.outcome != 'failure' }}
+        if: ${{ steps.rox-exists.outcome != 'failure' }}
         timeout-minutes: 5
         continue-on-error: true
         run: |

From bf68a2aab1ac134623579d6f5edca12f201f2e06 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 20 Aug 2025 09:10:04 -0600
Subject: [PATCH 249/256] fetch all

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index f94bbe4e6fd9e..f9b5e9cf3b89d 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -48,7 +48,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: ${{ github.event.pull_request.commits }}
+          fetch-depth: 0 #${{ github.event.pull_request.commits }}
           fetch-tags: true
           show-progress: false
 

From bbf6092cb29a22b2e7a405f97e8f988200f706d8 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 20 Aug 2025 09:14:47 -0600
Subject: [PATCH 250/256] CI off

---
 .github/workflows/e2e-tests.yaml | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index f9b5e9cf3b89d..99f34871141d7 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -24,7 +24,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       KUBECONFIG: "/tmp/kubeconfig"
-      CI: "true"
       FAIL_FAST: "true"
       INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
       POD_SECURITY_POLICIES: "false"
@@ -42,8 +41,6 @@ jobs:
       GATHER_QA_TEST_DEBUG_LOGS: "false"
       QA_TEST_DEBUG_LOGS: ""
       SHARED_DIR: "/tmp"
-      SENSOR_HELM_DEPLOY: "false"
-      ROX_SCANNER_V4: "false"
       #ORCHESTRATOR_FLAVOR: "openshift"
     steps:
       - uses: actions/checkout@v4
@@ -69,7 +66,6 @@ jobs:
           mkdir -p "$ARTIFACT_DIR"
 
           cat >> "$GITHUB_ENV" <<EOF
-          CI=true
           SHARED_DIR=${SHARED_DIR:-/tmp}
           ARTIFACT_DIR=${ARTIFACT_DIR}
 
@@ -142,12 +138,13 @@ jobs:
           DEPLOY_STACKROX_VIA_OPERATOR: "false"
           # LOAD_BALANCER: "lb"
           # SCANNER_SUPPORT: "true"
+          SENSOR_HELM_DEPLOY: "true"
+          ROX_SCANNER_V4: "false"
           MONITORING_SUPPORT: "false"
           LOCAL_DEPLOYMENT: "true"
           ROX_BASELINE_GENERATION_DURATION: "1m"
           ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
         run: |
-          set +e
           echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
           kubectl get namespace stackrox \
             || { 
@@ -158,7 +155,7 @@ jobs:
                       kubectl create -f "qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml";
                     }
               fi
-              CI=true ./deploy/deploy-local.sh;
+              CI= ./deploy/deploy-local.sh;
               find deploy -name password;
               echo "ROX_USERNAME=admin" >> "$GITHUB_ENV";
               echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";

From 3e8bd562f2e9288acf51181cda7669f806d180e3 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 20 Aug 2025 09:30:45 -0600
Subject: [PATCH 251/256] default remote clustername

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 99f34871141d7..495dd1b802f1e 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -145,7 +145,6 @@ jobs:
           ROX_BASELINE_GENERATION_DURATION: "1m"
           ROX_NETWORK_BASELINE_OBSERVATION_PERIOD: "2m"
         run: |
-          echo "ROX_SCANNER_V4:${ROX_SCANNER_V4:-}"
           kubectl get namespace stackrox \
             || { 
               if [[ "${CLUSTER,,}" == "openshift" ]]; then
@@ -212,6 +211,7 @@ jobs:
 
       # Only needed for Groovy
       - name: Get secured cluster name
+        continue-on-error: true
         if: ${{ steps.stackrox-api.outcome != 'failure' }}
         run: |
           echo "Groovy tests expect a secured cluster. Default name is 'remote' but the Infra demo clusters set names like 'production'. Look up what secured clusters exist:"

From 64b0bc40809de9f5b108bc661e7d6752b0f74f72 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 20 Aug 2025 12:43:01 -0600
Subject: [PATCH 252/256] continue on err

---
 .github/workflows/e2e-tests.yaml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 495dd1b802f1e..c2812becbf985 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -154,6 +154,7 @@ jobs:
                       kubectl create -f "qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml";
                     }
               fi
+              tests/e2e/run-e2e-tests.sh --config-only qa && exit 1
               CI= ./deploy/deploy-local.sh;
               find deploy -name password;
               echo "ROX_USERNAME=admin" >> "$GITHUB_ENV";
@@ -201,9 +202,10 @@ jobs:
 
       - name: Stackrox API test
         if: ${{ steps.stackrox-api.outcome != 'failure' }}
+        continue-on-error: true
         run: |
-          if [[ "${API_HOSTNAME:-}" == 'localhost' && -e /tmp/forward.pid ]]; then 
-            cat /tmp/forward.pid
+          if [[ "${API_HOSTNAME:-}" == 'localhost' || -e /tmp/forward.pid ]]; then 
+            cat /tmp/forward.pid || true
             ps -ef | grep kube || true
           fi
 
@@ -266,11 +268,13 @@ jobs:
           make -C qa-tests-backend compile
 
       - timeout-minutes: 35
+        continue-on-error: true
         run: |
           export TEST_TARGET="smoke-test"
           cd qa-tests-backend && ./gradlew testSMOKE --fail-fast
 
       - timeout-minutes: 35
+        continue-on-error: true
         run: |
           export TEST_TARGET="bat-test"
           cd qa-tests-backend && ./gradlew testParallelBAT testBAT --fail-fast --warn

From 5239b867c95dc0e64066604d8b516b3888efc058 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 25 Aug 2025 13:46:18 -0600
Subject: [PATCH 253/256] set cluster for k8s

---
 .github/workflows/e2e-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index c2812becbf985..7caaf820dac12 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -100,6 +100,8 @@ jobs:
           set -x
           if kubectl get scc; then
             echo "CLUSTER=OPENSHIFT" | tee -a "$GITHUB_ENV"
+          else
+            echo "CLUSTER=K8S" | tee -a "$GITHUB_ENV"
           fi 
           exit 0
           # groovy tests require CLUSTER='OPENSHIFT' for ocp or ='K8S' for GKE.

From ab9611a657e9f5d864fabdf901617cab6c2643dc Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 25 Aug 2025 14:04:28 -0600
Subject: [PATCH 254/256] only install with qa wrapper

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index 7caaf820dac12..de61c31405d28 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -157,7 +157,7 @@ jobs:
                     }
               fi
               tests/e2e/run-e2e-tests.sh --config-only qa && exit 1
-              CI= ./deploy/deploy-local.sh;
+              # CI= ./deploy/deploy-local.sh;
               find deploy -name password;
               echo "ROX_USERNAME=admin" >> "$GITHUB_ENV";
               echo "ROX_ADMIN_PASSWORD=$(cat deploy/*/central-deploy/password)" >> "$GITHUB_ENV";

From 6391db825858135926f1a3c666a9a572c6e3ed74 Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Mon, 25 Aug 2025 14:40:25 -0600
Subject: [PATCH 255/256] specify image to deploy in config

---
 .github/workflows/e2e-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
index de61c31405d28..61884750bcf68 100644
--- a/.github/workflows/e2e-tests.yaml
+++ b/.github/workflows/e2e-tests.yaml
@@ -156,7 +156,7 @@ jobs:
                       kubectl create -f "qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml";
                     }
               fi
-              tests/e2e/run-e2e-tests.sh --config-only qa && exit 1
+              tests/e2e/run-e2e-tests.sh -t "${MAIN_IMAGE_TAG}" --config-only qa && exit 1
               # CI= ./deploy/deploy-local.sh;
               find deploy -name password;
               echo "ROX_USERNAME=admin" >> "$GITHUB_ENV";

From 2e5dfa39b591d11e45e3b9e8d5a9e8c600ff0a0e Mon Sep 17 00:00:00 2001
From: davdhacs <105243888+davdhacs@users.noreply.github.com>
Date: Wed, 10 Sep 2025 17:54:07 -0600
Subject: [PATCH 256/256] restore master scc anyuid

---
 qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml b/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml
index 95094f6e52fe7..c28290f809c30 100644
--- a/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml
+++ b/qa-tests-backend/src/k8s/scc-qatest-anyuid.yaml
@@ -19,15 +19,14 @@ metadata:
       harness augments qatest-anyuid capabilities in the test environment setup
       stage.
     release.openshift.io/create-only: "false"
-allowHostDirVolumePlugin: true
+allowHostDirVolumePlugin: false
 allowHostIPC: false
-allowHostNetwork: true
+allowHostNetwork: false
 allowHostPID: false
-allowHostPorts: true
+allowHostPorts: false
 allowPrivilegeEscalation: true
-allowPrivilegedContainer: true
-allowedCapabilities: ["*"]
-allowedUnsafeSysctls: ["*"]
+allowPrivilegedContainer: false
+allowedCapabilities: null
 defaultAddCapabilities: null
 fsGroup:
   type: RunAsAny
@@ -35,14 +34,15 @@ groups:
 - system:cluster-admins
 priority: 10
 readOnlyRootFilesystem: false
-requiredDropCapabilities: null
+requiredDropCapabilities:
+- MKNOD
 runAsUser:
   type: RunAsAny
 seLinuxContext:
   type: MustRunAs
 supplementalGroups:
   type: RunAsAny
-users: [system:serviceaccount:stackrox:default]
+users: []
 volumes:
 - configMap
 - downwardAPI