diff --git a/.github/workflows/check-crd-compatibility.yaml b/.github/workflows/check-crd-compatibility.yaml index 95ba18adc4a75..f0a0576396ee1 100644 --- a/.github/workflows/check-crd-compatibility.yaml +++ b/.github/workflows/check-crd-compatibility.yaml @@ -11,6 +11,8 @@ on: - opened - reopened - synchronize + - labeled + - unlabeled concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -20,15 +22,31 @@ jobs: detect-changes: uses: ./.github/workflows/detect-changes.yml + plan: + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/dispatcher-plan.yaml + check-crd-compatibility: - needs: detect-changes - if: ${{ !needs.detect-changes.outputs.ui_only }} + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && !needs.detect-changes.outputs.ui_only + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['check-crd-compatibility'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest container: image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11@sha256:df3459cf038e73775314b3a5d36d14dda41e6df438a0273c9efa8d23e0f2358a # ratchet:quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11 outputs: released_version: ${{ steps.get_previous_released_version.outputs.released_version }} steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout stackrox/stackrox uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: diff --git a/.github/workflows/dispatcher-plan.yaml b/.github/workflows/dispatcher-plan.yaml new file mode 100644 index 0000000000000..7cf5ce4d9481c --- /dev/null +++ b/.github/workflows/dispatcher-plan.yaml @@ -0,0 +1,86 @@ +name: Dispatcher plan + +# Called by the test workflows. Prints the plan and returns, for each +# GitHub Actions job, whether the dispatcher would start it. +# The label ci-dispatcher-enforce is what makes a caller obey that answer. +# Without the label the caller keeps starting jobs the way it does today. + +on: + workflow_call: + outputs: + enforce: + description: true when the pull request has ci-dispatcher-enforce + value: ${{ jobs.plan.outputs.enforce }} + jobs: + description: JSON object of job id to whether the dispatcher would start it + value: ${{ jobs.plan.outputs.jobs }} + +permissions: + contents: read + pull-requests: read + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + enforce: ${{ steps.out.outputs.enforce }} + jobs: ${{ steps.out.outputs.jobs }} + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 + + - name: Install PyYAML + run: | + python3 -m venv "${RUNNER_TEMP}/selector" + "${RUNNER_TEMP}/selector/bin/pip" install 'PyYAML==6.0.3' + + - name: Plan GitHub Actions jobs + id: out + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} + DRAFT: ${{ github.event.pull_request.draft }} + FORK: ${{ github.event.pull_request.head.repo.fork }} + ENFORCE: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') }} + run: | + set -euo pipefail + py="${RUNNER_TEMP}/selector/bin/python" + flags=() + if [[ "${DRAFT}" == "true" ]]; then flags+=(--draft); fi + if [[ "${FORK}" == "true" ]]; then flags+=(--fork); fi + if [[ "${ENFORCE}" == "true" ]]; then flags+=(--enforce); fi + + files="$(mktemp)" + if ! gh api "repos/${GH_REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename' > "${files}"; then + echo "Could not list changed files. Showing the run-everything result." + : > "${files}" + elif [[ "$(wc -l < "${files}")" -ge 3000 ]]; then + echo "3000 or more changed files. Showing the run-everything result." + : > "${files}" + fi + + common=( + "${py}" ci/exploration/dispatcher.py + --mapping ci/test-domains.toml + --defaults ci/exploration/gha_defaults.yaml + --labels "${LABELS}" + "${flags[@]}" + ) + + "${common[@]}" --title "This pull request" - < "${files}" + "${common[@]}" --format json - < "${files}" > "${RUNNER_TEMP}/plan.json" + + python3 - << 'PY' + import json + import os + + plan = json.load(open(os.environ["RUNNER_TEMP"] + "/plan.json")) + selected = {item["job"]: bool(item["would_run"]) for item in plan["jobs"]} + with open(os.environ["GITHUB_OUTPUT"], "a") as handle: + handle.write(f"enforce={os.environ['ENFORCE']}\n") + handle.write("jobs<- + always() && !cancelled() && !github.event.pull_request.head.repo.fork && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['e2e-test-on-kind'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) permissions: id-token: write contents: read steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Login to Quay.io uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # ratchet:docker/login-action@v4 with: diff --git a/.github/workflows/scanner-db-integration-tests.yaml b/.github/workflows/scanner-db-integration-tests.yaml index 785f47924b715..3b75673e3105f 100644 --- a/.github/workflows/scanner-db-integration-tests.yaml +++ b/.github/workflows/scanner-db-integration-tests.yaml @@ -12,18 +12,36 @@ on: - opened - reopened - synchronize + - labeled + - unlabeled jobs: detect-changes: uses: ./.github/workflows/detect-changes.yml + plan: + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/dispatcher-plan.yaml + db-integration-tests: - needs: detect-changes - if: ${{ !needs.detect-changes.outputs.ui_only }} + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && !needs.detect-changes.outputs.ui_only + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['db-integration-tests'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest container: image: quay.io/stackrox-io/apollo-ci:scanner-test-0.5.11@sha256:cea78f2b3a5a38c9ccda678712daaed90b7c372d6b5df963feb758b32dccebe1 # ratchet:quay.io/stackrox-io/apollo-ci:scanner-test-0.5.11 steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: diff --git a/.github/workflows/scanner-e2e-test.yaml b/.github/workflows/scanner-e2e-test.yaml index e4ee1333e3964..e52d3d699d12b 100644 --- a/.github/workflows/scanner-e2e-test.yaml +++ b/.github/workflows/scanner-e2e-test.yaml @@ -7,15 +7,35 @@ on: - opened - reopened - synchronize + - labeled + - unlabeled jobs: + plan: + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/dispatcher-plan.yaml + scanner-ft-images: name: Wait for images - if: > - github.event_name != 'pull_request' - || contains(github.event.pull_request.labels.*.name, 'scanner-functional-tests') + needs: plan + if: >- + always() && !cancelled() && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && ( + github.event_name != 'pull_request' + || contains(github.event.pull_request.labels.*.name, 'scanner-functional-tests') + ) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['scanner-ft-images'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout repo uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: @@ -36,6 +56,15 @@ jobs: name: Deploy and run needs: - scanner-ft-images + - plan + if: >- + always() && !cancelled() && ( + needs.scanner-ft-images.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['scanner-ft-run'] + ) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest env: SCANNER_E2E_QUAY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} @@ -44,6 +73,10 @@ jobs: SCANNER_E2E_REDHAT_USERNAME: ${{ secrets.REDHAT_USERNAME }} SCANNER_E2E_REDHAT_PASSWORD: ${{ secrets.REDHAT_PASSWORD }} steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml index 0cfe9f54c7978..a211281dfa16b 100644 --- a/.github/workflows/style.yaml +++ b/.github/workflows/style.yaml @@ -11,6 +11,8 @@ on: - opened - reopened - synchronize + - labeled + - unlabeled env: ROX_PRODUCT_BRANDING: RHACS_BRANDING @@ -22,13 +24,31 @@ jobs: detect-changes: uses: ./.github/workflows/detect-changes.yml + # The label ci-dispatcher-enforce makes the jobs below follow the plan. + # Without the label, each job keeps its own start rule. + plan: + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/dispatcher-plan.yaml + check-generated-files: - needs: detect-changes - if: ${{ !needs.detect-changes.outputs.ui_only }} + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && !needs.detect-changes.outputs.ui_only + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['check-generated-files'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) env: ARTIFACT_DIR: junit-reports/ runs-on: ubuntu-latest steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: @@ -52,10 +72,21 @@ jobs: run: scripts/ci/jobs/check-generated.sh misc-checks: + needs: plan + if: >- + always() && !cancelled() && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['misc-checks'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) env: ARTIFACT_DIR: junit-reports/ runs-on: ubuntu-latest steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: @@ -97,8 +128,19 @@ jobs: run: if [ -s scripts/ci/jobs/check-dependabot-gomod.sh ]; then scripts/ci/jobs/check-dependabot-gomod.sh; fi style-check: + needs: plan + if: >- + always() && !cancelled() && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['style-check'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: @@ -156,11 +198,20 @@ jobs: run: make style-slim golangci-lint: - needs: detect-changes - if: ${{ !needs.detect-changes.outputs.ui_only }} + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && !needs.detect-changes.outputs.ui_only + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['golangci-lint'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) timeout-minutes: 240 runs-on: ubuntu-latest steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 @@ -203,13 +254,22 @@ jobs: run: make golangci-lint-cache-status check-dependent-images-exist: - needs: detect-changes - if: ${{ !needs.detect-changes.outputs.ui_only }} + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && !needs.detect-changes.outputs.ui_only + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['check-dependent-images-exist'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) # This job ensures that COLLECTOR_VERSION or FACT_VERSION files cannot be updated to a version # for which the image was not successfully built on Konflux (suffix "-fast"). It also verifies that GHA-built image # is there (no suffix) so that the failure also happens in this job. runs-on: ubuntu-latest steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout repo uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 @@ -249,38 +309,64 @@ jobs: rhacs-eng/release-fact:${{ env.fact_tag }}-fast github-actions-lint: - needs: detect-changes + needs: [detect-changes, plan] if: >- - !needs.detect-changes.outputs.files || - contains(needs.detect-changes.outputs.files, '^.github/') + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && ( + !needs.detect-changes.outputs.files || + contains(needs.detect-changes.outputs.files, '^.github/') + ) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['github-actions-lint'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Check workflow files run: make github-actions-lint github-actions-shellcheck: - needs: detect-changes + needs: [detect-changes, plan] if: >- - !needs.detect-changes.outputs.files || - contains(needs.detect-changes.outputs.files, '^.github/') + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && ( + !needs.detect-changes.outputs.files || + contains(needs.detect-changes.outputs.files, '^.github/') + ) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['github-actions-shellcheck'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 - uses: ./.github/actions/job-preamble with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Check scripts with shellcheck run: shellcheck -P SCRIPTDIR -x ./.github/workflows/scripts/*.sh openshift-ci-lint: - needs: detect-changes + needs: [detect-changes, plan] if: >- - !needs.detect-changes.outputs.files || - contains(needs.detect-changes.outputs.files, '^.openshift-ci/') + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && ( + !needs.detect-changes.outputs.files || + contains(needs.detect-changes.outputs.files, '^.openshift-ci/') + ) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['openshift-ci-lint'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest steps: - name: Checkout @@ -293,6 +379,10 @@ jobs: with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Install Python linters run: python3 -m pip install -r .openshift-ci/dev-requirements.txt diff --git a/.github/workflows/test-selection-shadow.yaml b/.github/workflows/test-selection-shadow.yaml new file mode 100644 index 0000000000000..d911e23bad36f --- /dev/null +++ b/.github/workflows/test-selection-shadow.yaml @@ -0,0 +1,31 @@ +name: Test selection (shadow) + +# Prints the plan for this pull request. +# The jobs that actually run live in their own workflows. The label +# ci-dispatcher-enforce is what makes those workflows obey the plan. +# Without the label they keep their own pull_request triggers. + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + - labeled + - unlabeled + +permissions: + contents: read + pull-requests: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +jobs: + plan: + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/dispatcher-plan.yaml diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index d5ca5b28812be..2c20021182cdc 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -11,6 +11,8 @@ on: - opened - reopened - synchronize + - labeled + - unlabeled concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -33,8 +35,23 @@ jobs: ^.github/workflows/unit-tests.yaml$ ^.github/actions/job-preamble/ + # The label ci-dispatcher-enforce makes the jobs below follow the plan. + # Without the label, each job keeps its own start rule. + plan: + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/dispatcher-plan.yaml + go: - needs: detect-changes + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['go'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) strategy: fail-fast: false matrix: @@ -45,6 +62,10 @@ jobs: outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 @@ -107,7 +128,13 @@ jobs: directory: 'junit-reports' go-postgres: - needs: detect-changes + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['go-postgres'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) strategy: fail-fast: false matrix: @@ -122,6 +149,10 @@ jobs: env: POSTGRES_PASSWORD: testpassword steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 @@ -203,10 +234,19 @@ jobs: # they compile and execute without errors. A single postgres version is # sufficient since go-postgres already tests against multiple versions. go-bench: - needs: detect-changes - if: ${{ !needs.detect-changes.outputs.ui_only }} + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && !needs.detect-changes.outputs.ui_only + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['go-bench'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 @@ -259,11 +299,17 @@ jobs: # https://github.com/jstemmer/go-junit-report/issues/174 ui: - needs: detect-changes + needs: [detect-changes, plan] if: >- - !needs.detect-changes.outputs.files || - contains(needs.detect-changes.outputs.files, '^ui/') || - contains(needs.detect-changes.outputs.files, '^.github/actions/cache-ui-dependencies/') + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && ( + !needs.detect-changes.outputs.files || + contains(needs.detect-changes.outputs.files, '^ui/') || + contains(needs.detect-changes.outputs.files, '^.github/actions/cache-ui-dependencies/') + ) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['ui'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} @@ -275,6 +321,10 @@ jobs: with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Set up Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # ratchet:actions/setup-node@v6 with: @@ -313,11 +363,17 @@ jobs: directory: 'ui/apps/platform/test-results/reports' ui-component: - needs: detect-changes + needs: [detect-changes, plan] if: >- - !needs.detect-changes.outputs.files || - contains(needs.detect-changes.outputs.files, '^ui/') || - contains(needs.detect-changes.outputs.files, '^.github/actions/cache-ui-dependencies/') + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && ( + !needs.detect-changes.outputs.files || + contains(needs.detect-changes.outputs.files, '^ui/') || + contains(needs.detect-changes.outputs.files, '^.github/actions/cache-ui-dependencies/') + ) + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['ui-component'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} @@ -329,6 +385,10 @@ jobs: with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Set up Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # ratchet:actions/setup-node@v6 with: @@ -378,8 +438,13 @@ jobs: directory: 'ui/apps/platform/cypress/test-results/reports' local-roxctl-tests: - needs: detect-changes - if: ${{ !needs.detect-changes.outputs.ui_only }} + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && !needs.detect-changes.outputs.ui_only + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['local-roxctl-tests'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} @@ -389,6 +454,10 @@ jobs: - /usr:/mnt/usr - /opt:/mnt/opt steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 with: @@ -424,8 +493,13 @@ jobs: directory: 'roxctl-test-output' shell-unit-tests: - needs: detect-changes - if: ${{ !needs.detect-changes.outputs.ui_only }} + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && !needs.detect-changes.outputs.ui_only + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['shell-unit-tests'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} @@ -441,6 +515,10 @@ jobs: with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Shell Unit Tests run: make shell-unit-tests @@ -462,8 +540,13 @@ jobs: directory: 'shell-test-output' openshift-ci-unit-tests: - needs: detect-changes - if: ${{ !needs.detect-changes.outputs.ui_only }} + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && !needs.detect-changes.outputs.ui_only + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['openshift-ci-unit-tests'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) runs-on: ubuntu-latest container: image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11@sha256:df3459cf038e73775314b3a5d36d14dda41e6df438a0273c9efa8d23e0f2358a # ratchet:quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11 @@ -478,18 +561,31 @@ jobs: with: gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }} + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: OpenShift CI Wrapper Unit Tests run: make -C .openshift-ci test sensor-integration-tests: - needs: detect-changes - if: ${{ !needs.detect-changes.outputs.ui_only }} + needs: [detect-changes, plan] + if: >- + always() && !cancelled() && needs.detect-changes.result == 'success' && ( + (github.event_name != 'pull_request' || !contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce')) && !needs.detect-changes.outputs.ui_only + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result == 'success' && fromJSON(needs.plan.outputs.jobs)['sensor-integration-tests'] + || github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + ) env: KUBECONFIG: "/tmp/kubeconfig" runs-on: ubuntu-latest outputs: new-jiras: ${{ steps.junit2jira.outputs.new-jiras }} steps: + - name: Fail closed when the dispatcher plan did not succeed + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci-dispatcher-enforce') && needs.plan.result != 'success' + run: echo "The dispatcher plan did not succeed, so this job is not running its tests." && exit 1 + - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v7.0.0 diff --git a/.gitignore b/.gitignore index e51e9ea58251c..a2e87871e6348 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ monitoring/grafana/rawcaptures # Test output /junit-reports/ *.test +__pycache__/ +*.pyc # Makefile cruft deps diff --git a/.openshift-ci/dispatch.sh b/.openshift-ci/dispatch.sh index bd8e4ce1a1a27..2538026d5304f 100755 --- a/.openshift-ci/dispatch.sh +++ b/.openshift-ci/dispatch.sh @@ -29,6 +29,30 @@ case "$ci_job" in ;; esac +# The resolver is the only start/skip decision. Exit 0 on skip, before +# credentials and cluster setup, so the Prow context is a success. A resolver +# error runs the job: a broken gate must not turn a required job green. +if is_in_PR_context; then + resolver_status=0 + if [[ -z "${PULL_BASE_SHA:-}" ]]; then + info "Resolver gate: no base SHA; running ${ci_job}" + elif ! changed="$(git -C "$ROOT" diff --name-only "${PULL_BASE_SHA}...HEAD" 2>/dev/null)"; then + info "Resolver gate: diff failed; running ${ci_job}" + else + printf '%s\n' "$changed" | python3 "$ROOT/ci/exploration/resolver.py" \ + --mapping "$ROOT/ci/test-domains.toml" \ + --job "$ci_job" \ + --enforce \ + - || resolver_status=$? + if [[ "$resolver_status" -eq 10 ]]; then + exit 0 + fi + if [[ "$resolver_status" -ne 0 ]]; then + info "Resolver gate failed (${resolver_status}); running ${ci_job}" + fi + fi +fi + if [[ -f "${SHARED_DIR:-}/shared_env" ]]; then # shellcheck disable=SC1091 source "${SHARED_DIR:-}/shared_env" diff --git a/central/policy/test-selection-shadow.txt b/central/policy/test-selection-shadow.txt new file mode 100644 index 0000000000000..7b933cd1a89ce --- /dev/null +++ b/central/policy/test-selection-shadow.txt @@ -0,0 +1,2 @@ +The shadow test-selection map treats every file under central/policy/ as a Central policy change. +This file is only here so that rule has a changed path to report. diff --git a/ci/exploration/dispatcher.py b/ci/exploration/dispatcher.py new file mode 100644 index 0000000000000..9390bd6440360 --- /dev/null +++ b/ci/exploration/dispatcher.py @@ -0,0 +1,633 @@ +"""Turn resolver opinions into a run, skip, or unsure plan. + +Only opinion run starts a job. The plan includes Prow jobs. A default line +is the GitHub Actions trigger recorded for that job. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +import yaml + +from resolver import Mapping, Selection, load_mapping, resolve + +ENFORCE_LABEL = "ci-dispatcher-enforce" + + +@dataclass(frozen=True) +class PullRequest: + draft: bool + fork: bool + labels: frozenset[str] + files: tuple[str, ...] + + +@dataclass(frozen=True) +class JobPlan: + job: str + opinion: str + default_starts: bool + action: str + # False for a Prow job: gha_defaults.yaml has no trigger to compare. + has_default: bool = True + + @property + def would_run(self) -> bool: + # Unsure does not run. Only an explicit run starts the job. + return self.action == "start" + + +def load_defaults(path: str | Path) -> dict[str, dict[str, str]]: + data = yaml.safe_load(Path(path).read_text()) + jobs = data.get("jobs") + if not isinstance(jobs, dict) or not jobs: + raise ValueError("jobs must be a non-empty mapping") + for name, rule in jobs.items(): + if not isinstance(rule, dict) or "when" not in rule: + raise ValueError(f"{name} needs a when field") + return jobs + + +def dispatch( + selection: Selection, + defaults: dict[str, dict[str, str]], + pull_request: PullRequest, +) -> tuple[JobPlan, ...]: + # Defaults cover GitHub Actions. The resolver also decides Prow jobs. + names = set(defaults) | selection.run | selection.skip | selection.unsure + plans = [] + for job in sorted(names): + opinion = _opinion(selection, job) + if opinion == "run": + action = "start" + elif opinion == "skip": + action = "stop" + else: + action = "default" + known = job in defaults + starts = default_would_start(defaults[job], pull_request) if known else False + plans.append(JobPlan(job, opinion, starts, action, has_default=known)) + return tuple(plans) + + +def default_would_start(rule: dict[str, str], pull_request: PullRequest) -> bool: + """default_would_start reports whether the recorded trigger would start this job.""" + if pull_request.fork and rule.get("skip_fork"): + return False + when = rule["when"] + if when == "always": + return True + if when == "not-ui": + return not _ui_only(pull_request.files) + if when == "ready": + return _ready(pull_request) + if when == "label": + return rule["label"] in pull_request.labels and not pull_request.fork + if when == "path": + return _path_matches(rule["path"], pull_request.files) + if when == "konflux": + if "e2e-qa-tests-gke-konflux" in pull_request.labels and not pull_request.fork: + return True + return "konflux-build" in pull_request.labels and _ready(pull_request) + raise ValueError(f"unknown when: {when}") + + +def default_sentence(rule: dict[str, str], pull_request: PullRequest) -> str: + """default_sentence states why default_would_start returned its answer.""" + if pull_request.fork and rule.get("skip_fork"): + return "stays off because this pull request is from a fork" + when = rule["when"] + if when == "always": + return "starts on every pull request" + if when == "not-ui": + return _not_ui_sentence(pull_request) + if when == "ready": + return _ready_sentence(pull_request) + if when == "label": + return _label_sentence(rule["label"], pull_request) + if when == "path": + return _path_sentence(rule["path"], pull_request) + if when == "konflux": + return _konflux_sentence(pull_request) + raise ValueError(f"unknown when: {when}") + + +@dataclass(frozen=True) +class _Section: + title: str + body: tuple[str, ...] + collapse: bool = False + + +def format_plan( + plans: tuple[JobPlan, ...], + *, + shadow: bool, + title: str, + selection: Selection, + defaults: dict[str, dict[str, str]], + pull_request: PullRequest, + mapping: Mapping, +) -> str: + """format_plan is the log text. Long sections fold with GitHub log groups.""" + sections = _sections(plans, selection, defaults, pull_request, mapping) + return _render_text(shadow, title, sections) + + +def format_summary( + plans: tuple[JobPlan, ...], + *, + shadow: bool, + title: str, + selection: Selection, + defaults: dict[str, dict[str, str]], + pull_request: PullRequest, + mapping: Mapping, +) -> str: + """format_summary is the same plan as Markdown for the job summary.""" + sections = _sections(plans, selection, defaults, pull_request, mapping) + return _render_markdown(shadow, title, sections) + + +def github_commands( + plans: tuple[JobPlan, ...], + conflicts: tuple = (), + mapping: Mapping | None = None, +) -> tuple[str, ...]: + """github_commands is one notice, plus a warning per rule conflict and per dropped default.""" + start = sum(plan.would_run for plan in plans) + skip = sum(plan.action == "stop" for plan in plans) + unsure = sum(plan.action == "default" for plan in plans) + dropped = sum(_default_dropped(plan) for plan in plans) + added = sum(_default_added(plan) for plan in plans) + notice = f"Would start {start}, skip {skip}, unsure {unsure}." + if added: + notice += f" This plan would start {_jobs(added)} the default would leave off." + commands = [f"::notice title=Dispatcher plan::{notice}"] + if mapping is not None: + for conflict in conflicts: + commands.append( + "::warning title=Dispatcher plan::" + _conflict_text(conflict, mapping) + ) + if dropped: + commands.append( + "::warning title=Dispatcher plan::" + f"This plan would not start {_jobs(dropped)} the default would start." + ) + return tuple(commands) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Plan GitHub Actions jobs for a pull request.") + parser.add_argument("--mapping", required=True) + parser.add_argument("--defaults", required=True) + parser.add_argument("--labels", default="") + parser.add_argument("--draft", action="store_true") + parser.add_argument("--fork", action="store_true") + parser.add_argument("--enforce", action="store_true") + parser.add_argument("--title", default="This pull request") + parser.add_argument("--format", choices=("human", "json"), default="human") + parser.add_argument("files", nargs="*") + args = parser.parse_args(argv) + + files = list(args.files) + if files == ["-"]: + files = [line.strip() for line in sys.stdin if line.strip()] + labels = frozenset(label for label in args.labels.split(",") if label) + pull_request = PullRequest(args.draft, args.fork, labels, tuple(files)) + mapping = load_mapping(args.mapping) + selection = resolve(files, mapping, list(labels), shadow=not args.enforce) + defaults = load_defaults(args.defaults) + plans = dispatch(selection, defaults, pull_request) + if args.format == "json": + json.dump(_json(plans), sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + report = { + "shadow": not args.enforce, + "title": args.title, + "selection": selection, + "defaults": defaults, + "pull_request": pull_request, + "mapping": mapping, + } + sys.stdout.write(format_plan(plans, **report)) + for command in github_commands(plans, selection.conflicts, mapping): + print(command) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as handle: + handle.write(format_summary(plans, **report)) + return 0 + + +def _json(plans: tuple[JobPlan, ...]) -> dict[str, object]: + return { + "jobs": [ + { + "job": item.job, + "opinion": item.opinion, + "default_starts": item.default_starts, + "has_default": item.has_default, + "action": item.action, + "would_run": item.would_run, + } + for item in plans + ] + } + + +def _opinion(selection: Selection, job: str) -> str: + if job in selection.run: + return "run" + if job in selection.skip: + return "skip" + return "unsure" + + +def _ready(pull_request: PullRequest) -> bool: + return not pull_request.draft and not pull_request.fork and not _ui_only(pull_request.files) + + +def _ui_only(files: tuple[str, ...]) -> bool: + return bool(files) and all(path.startswith("ui/") for path in files) + + +def _path_matches(pattern: str, files: tuple[str, ...]) -> bool: + if not files: + return True + compiled = re.compile(pattern) + return any(compiled.search(path) for path in files) + + +def _sections( + plans: tuple[JobPlan, ...], + selection: Selection, + defaults: dict[str, dict[str, str]], + pull_request: PullRequest, + mapping: Mapping, +) -> tuple[_Section, ...]: + by_action = {"start": [], "stop": [], "default": []} + for plan in plans: + by_action[plan.action].append(plan) + sections = [ + _Section("Changed files", _changed_files(selection)), + _rules_section(selection, mapping), + _conflicts_section(selection, mapping), + _job_section( + "Will run", + by_action["start"], + _run_reasons, + selection, + defaults, + pull_request, + mapping, + ), + _job_section( + "Will skip", + by_action["stop"], + _skip_reasons, + selection, + defaults, + pull_request, + mapping, + ), + _unsure_section(by_action["default"], selection, defaults, pull_request), + ] + sections.extend(_default_sections(plans)) + return tuple(sections) + + +def _job_section(title, plans, reasons, selection, defaults, pull_request, mapping): + entries = [ + ( + plan.job, + reasons(plan.job, selection, mapping), + _default_clause(plan, defaults, pull_request), + ) + for plan in plans + ] + body = _render_jobs(entries) or [" none"] + return _Section( + f"{title} ({len(entries)})", + tuple(body), + collapse=_collapses(title, len(entries)), + ) + + +def _unsure_section(plans, _selection, defaults, pull_request): + reason = ("no rule decided these jobs",) + entries = [ + (plan.job, reason, _default_clause(plan, defaults, pull_request)) + for plan in plans + ] + body = _render_jobs(entries) or [" none"] + return _Section( + f"Unsure ({len(plans)})", + tuple(body), + collapse=_collapses("Unsure", len(plans)), + ) + + +def _default_clause(plan: JobPlan, defaults, pull_request: PullRequest) -> str: + if not plan.has_default: + return "" + return default_sentence(defaults[plan.job], pull_request) + + +def _default_dropped(plan: JobPlan) -> bool: + return plan.has_default and plan.default_starts and not plan.would_run + + +def _default_added(plan: JobPlan) -> bool: + return plan.has_default and plan.would_run and not plan.default_starts + + +def _default_sections(plans: tuple[JobPlan, ...]) -> list[_Section]: + dropped = [plan.job for plan in plans if _default_dropped(plan)] + added = [plan.job for plan in plans if _default_added(plan)] + sections = [] + if dropped: + sections.append( + _Section( + f"Default would start these, and this plan would not ({len(dropped)})", + tuple(f" {job}" for job in dropped), + collapse=_collapses("Default would start", len(dropped)), + ) + ) + if added: + sections.append( + _Section( + f"This plan would start these, and the default would not ({len(added)})", + tuple(f" {job}" for job in added), + collapse=_collapses("This plan would start", len(added)), + ) + ) + if not sections: + sections.append(_Section("Default matches this plan.", ())) + return sections + + +def _collapses(title: str, count: int) -> bool: + if count <= 3: + return False + return title.startswith( + ("Unsure", "Will skip", "Conflicts", "Default would start", "This plan would start") + ) + + +def _render_jobs(entries: list[tuple[str, tuple[str, ...], str]]) -> list[str]: + if not entries: + return [] + grouped: dict[tuple[str, ...], list[tuple[str, str]]] = {} + for job, reasons, clause in entries: + grouped.setdefault(reasons, []).append((job, clause)) + lines: list[str] = [] + for reasons, members in grouped.items(): + recorded = [(job, clause) for job, clause in members if clause] + without_default = [job for job, clause in members if not clause] + if len(members) == 1 and recorded: + job, clause = recorded[0] + lines.append(f" {job}") + lines.extend(f" {reason}" for reason in reasons) + lines.append(f" default: {clause}") + continue + if len(members) == 1: + lines.append(f" {without_default[0]}") + lines.extend(f" {reason}" for reason in reasons) + continue + lines.extend(f" {reason}" for reason in reasons) + by_clause: dict[str, list[str]] = {} + for job, clause in recorded: + by_clause.setdefault(clause, []).append(job) + for clause, jobs in by_clause.items(): + lines.append(f" default: {clause}") + lines.extend(f" {job}" for job in jobs) + if without_default: + lines.append(" no GitHub Actions default") + lines.extend(f" {job}" for job in without_default) + return lines + + +def _rules_section(selection: Selection, mapping: Mapping) -> _Section: + lines: list[str] = [] + for number in selection.matched_rules: + rule = mapping.by_number[number] + lines.append(f" {number} {rule.name}") + lines.append(f" {rule.condition()}") + if not lines: + lines.append(" none") + return _Section(f"Rules ({len(selection.matched_rules)})", tuple(lines)) + + +def _conflicts_section(selection: Selection, mapping: Mapping) -> _Section: + if not selection.conflicts: + return _Section("Conflicts (0)", (" none",)) + lines: list[str] = [] + for conflict in selection.conflicts: + lines.append(f" {conflict.job}") + lines.append(f" {_conflict_text(conflict, mapping)}") + return _Section( + f"Conflicts ({len(selection.conflicts)})", + tuple(lines), + collapse=_collapses("Conflicts", len(selection.conflicts)), + ) + + +def _conflict_text(conflict, mapping: Mapping) -> str: + ran = _rule_phrase(conflict.run_rules, mapping) + skipped = _rule_phrase(conflict.skip_rules, mapping) + return f"{ran} runs {conflict.job} and {skipped} skips it. Run wins." + + +def _rule_phrase(numbers: tuple[int, ...], mapping: Mapping) -> str: + labels = [f"rule {number} {mapping.by_number[number].name}" for number in numbers] + return _join_clauses(labels) + + +def _run_reasons(job: str, selection: Selection, mapping: Mapping) -> tuple[str, ...]: + requirers = _requirers(job, selection, mapping) + if job in selection.required_runs and requirers: + return (f"{_with_verb(requirers, 'requires', 'require')} this job",) + decision = selection.decision_for(job) + if decision is None: + return ("a rule says run",) + lines = [_rule_phrase(decision.rules, mapping)] + conflict = selection.conflict_for(job) + if conflict is not None: + lines.append(f"{_rule_phrase(conflict.skip_rules, mapping)} skips it") + lines.append("run wins") + return tuple(lines) + + +def _requirers(job: str, selection: Selection, mapping: Mapping) -> list[str]: + return sorted( + other + for other, needs in mapping.requires.items() + if job in needs and other in selection.run + ) + + +def _skip_reasons(job: str, selection: Selection, mapping: Mapping) -> tuple[str, ...]: + decision = selection.decision_for(job) + if decision is None: + return ("no rule decided this job",) + return (_rule_phrase(decision.rules, mapping),) + + +def _changed_files(selection: Selection) -> tuple[str, ...]: + if not selection.files: + return (" (no file list)",) + return tuple(f" {trace.path}" for trace in selection.files) + + +def _header(shadow: bool) -> tuple[str, ...]: + if shadow: + return ( + "Shadow. Other GitHub workflows still start themselves.", + f"The switch is the pull request label {ENFORCE_LABEL}.", + *_plan_scope(), + ) + return ( + "Enforce. This dispatcher is the starter for these GitHub Actions jobs.", + *_plan_scope(), + ) + + +def _plan_scope() -> tuple[str, ...]: + return ( + "Run, skip, and unsure include Prow. Default lines are GitHub Actions triggers.", + "dispatch.sh applies this plan to Prow before the cluster is created.", + "Only a job whose opinion is run is started. Unsure does not run.", + ) + + +def _render_text(shadow: bool, title: str, sections: tuple[_Section, ...]) -> str: + chunks = [*_header(shadow), "", title, ""] + for section in sections: + chunks.extend(_text_section(section)) + chunks.append("") + return "\n".join(chunks).rstrip() + "\n" + + +def _text_section(section: _Section) -> list[str]: + if section.collapse: + return [f"::group::{section.title}", *section.body, "::endgroup::"] + return [section.title, *section.body] + + +def _render_markdown(shadow: bool, title: str, sections: tuple[_Section, ...]) -> str: + parts = ["\n".join(_header(shadow)), "", f"## {title}", ""] + for section in sections: + parts.extend(_markdown_section(section)) + return "\n".join(parts).rstrip() + "\n" + + +def _markdown_section(section: _Section) -> list[str]: + if not section.body: + return [f"## {section.title}", ""] + block = _fence(section.body) + if section.collapse: + return ["
", f"{section.title}", "", *block, "
", ""] + return [f"## {section.title}", "", *block, ""] + + +def _fence(body: tuple[str, ...]) -> list[str]: + return ["```", *body, "```"] + + +def _not_ui_sentence(pull_request: PullRequest) -> str: + if _ui_only(pull_request.files): + return "stays off because every changed file is under ui/" + outside = [path for path in pull_request.files if not path.startswith("ui/")] + if not outside: + return "starts because the pull request is not ui-only" + # A long diff does not explain this rule. One to three paths do. + if len(outside) > 3: + return "starts because a changed file is outside ui/" + return f"starts because {_with_verb(outside, 'is outside ui/', 'are outside ui/')}" + + +def _ready_sentence(pull_request: PullRequest) -> str: + if pull_request.draft: + return "stays off because this pull request is a draft" + if pull_request.fork: + return "stays off because this pull request is from a fork" + if _ui_only(pull_request.files): + return "stays off because every changed file is under ui/" + return "starts because this pull request is ready" + + +def _label_sentence(label: str, pull_request: PullRequest) -> str: + if pull_request.fork: + return "stays off because this pull request is from a fork" + if label in pull_request.labels: + return f"starts because this pull request has label {label}" + return f"stays off because label {label} is absent" + + +def _path_sentence(pattern: str, pull_request: PullRequest) -> str: + if not pull_request.files: + return "starts because there is no file list, so the path rule matches" + matched = [path for path in pull_request.files if re.search(pattern, path)] + if matched: + # The pattern stays out of this sentence. A long one hides the file that matched. + return f"starts because {_with_verb(matched, 'matches', 'match')} the path rule" + return f"stays off because no changed file matches {pattern}" + + +def _konflux_sentence(pull_request: PullRequest) -> str: + if "e2e-qa-tests-gke-konflux" in pull_request.labels and not pull_request.fork: + return "starts because this pull request has label e2e-qa-tests-gke-konflux" + if pull_request.fork: + return "stays off because this pull request is from a fork" + if "konflux-build" not in pull_request.labels: + return ( + "stays off because neither label e2e-qa-tests-gke-konflux nor konflux-build is set" + ) + if pull_request.draft: + return "stays off because this pull request is a draft" + if _ui_only(pull_request.files): + return "stays off because every changed file is under ui/" + return "starts because this pull request has label konflux-build and is ready" + + +def _with_verb(paths: list[str], singular: str, plural: str) -> str: + verb = singular if len(paths) == 1 else plural + return f"{_join_files(paths)} {verb}" + + +def _join_files(paths: list[str]) -> str: + if len(paths) == 1: + return paths[0] + if len(paths) == 2: + return f"{paths[0]} and {paths[1]}" + if len(paths) == 3: + return f"{paths[0]}, {paths[1]}, and {paths[2]}" + head = ", ".join(paths[:3]) + return f"{head}, and {len(paths) - 3} more" + + +def _join_clauses(parts: list[str]) -> str: + if len(parts) == 1: + return parts[0] + if len(parts) == 2: + return f"{parts[0]} and {parts[1]}" + return f"{', '.join(parts[:-1])}, and {parts[-1]}" + + +def _jobs(count: int) -> str: + if count == 1: + return "1 job" + return f"{count} jobs" + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/exploration/fixture.toml b/ci/exploration/fixture.toml new file mode 100644 index 0000000000000..27c637baab9ce --- /dev/null +++ b/ci/exploration/fixture.toml @@ -0,0 +1,64 @@ +# Small rule list for the resolver tests. + +version = 1 + +jobs = [ + "style", + "go", + "sensor-integration-tests", + "go-postgres", + "wait-for-images", + "should-dispatch", + "other", +] + +[[rules]] +name = "run-all-label" +when = "label-exists" +label = "ci-run-all-tests" +run = ["*"] + +[[rules]] +name = "no-files" +when = "no-file-changed" +run = ["*"] + +[[rules]] +name = "docs-only" +when = "all-files-changed" +paths = ['\.md$', '^docs/'] +run = ["style"] +skip = ["go"] + +[[rules]] +name = "go-sources" +when = "any-file-changed" +paths = ['\.go$'] +run = ["go"] + +[[rules]] +name = "sensor" +when = "any-file-changed" +paths = ['^sensor/'] +run = ["sensor-integration-tests"] +skip = ["go-postgres"] + +[[rules]] +name = "central-policy" +when = "any-file-changed" +paths = ['^central/policy/'] +run = ["go-postgres"] +skip = ["sensor-integration-tests"] + +[[rules]] +name = "image-wait" +when = "always" +run = ["wait-for-images"] + +[[rules]] +name = "remaining" +when = "remaining" +skip = ["*"] + +[requires] +wait-for-images = ["should-dispatch"] diff --git a/ci/exploration/gha_defaults.yaml b/ci/exploration/gha_defaults.yaml new file mode 100644 index 0000000000000..1b2a6219f6a8c --- /dev/null +++ b/ci/exploration/gha_defaults.yaml @@ -0,0 +1,88 @@ +# Whether each GitHub Actions job would start on a pull request today, +# before this dispatcher is the only starter. +# +# when: +# always - every pull request +# not-ui - every pull request that is not only ui/ +# ready - not a draft, not a fork, and not only ui/ +# label - the pull request has this label, and it is not a fork +# path - a changed file matches path; an empty file list counts as a match +# konflux - label e2e-qa-tests-gke-konflux, or konflux-build on a ready pull request + +jobs: + style-check: + when: always + misc-checks: + when: always + check-generated-files: + when: not-ui + golangci-lint: + when: not-ui + check-dependent-images-exist: + when: not-ui + github-actions-lint: + when: path + path: '^\.github/' + github-actions-shellcheck: + when: path + path: '^\.github/' + openshift-ci-lint: + when: path + path: '^\.openshift-ci/' + check-crd-compatibility: + when: not-ui + go: + when: always + go-postgres: + when: always + go-bench: + when: not-ui + ui: + when: path + path: '^ui/|^\.github/actions/cache-ui-dependencies/' + ui-component: + when: path + path: '^ui/|^\.github/actions/cache-ui-dependencies/' + local-roxctl-tests: + when: not-ui + shell-unit-tests: + when: not-ui + openshift-ci-unit-tests: + when: not-ui + sensor-integration-tests: + when: not-ui + db-integration-tests: + when: not-ui + scanner-ft-images: + when: label + label: scanner-functional-tests + scanner-ft-run: + when: label + label: scanner-functional-tests + e2e-byodb-tests: + when: ready + e2e-db-backup-restore-tests: + when: ready + e2e-gke-upgrade-tests: + when: ready + e2e-nongroovy-tests: + when: ready + e2e-qa-tests-gke: + when: label + label: e2e-qa-tests-gke + e2e-qa-tests-gke-konflux: + when: konflux + e2e-qa-tests-ocp-4-22: + when: label + label: e2e-qa-tests-ocp-4-22 + e2e-test-on-kind: + when: path + path: '^central/notifiers/email/|^central/notifiers/acscsemail/|^central/auth/m2m/|^\.github/workflows/emailsender-central-compatibility\.yaml$' + skip_fork: true + should-dispatch: + when: always + wait-for-images: + when: always + wait-for-konflux-images: + when: label + label: konflux-build diff --git a/ci/exploration/resolver.py b/ci/exploration/resolver.py new file mode 100644 index 0000000000000..ad6c48464bd71 --- /dev/null +++ b/ci/exploration/resolver.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python3 +"""Classify CI jobs for one diff by walking an ordered rule list. + +Rules are data in a TOML file. The list is walked from top to bottom. +A later run replaces an earlier skip. A later skip does not replace an +earlier run: that is a conflict, the plan warns, and the job runs. +A job no rule has mentioned is unsure until the last rule, which must +skip every remaining job. Shadow mode still executes every known job. +Pass shadow=False (CLI: --enforce) to execute only run. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +import tomllib +from dataclasses import dataclass, field +from pathlib import Path + +_WHEN = frozenset( + { + "all-files-changed", + "any-file-changed", + "always", + "label-exists", + "no-file-changed", + "remaining", + } +) +_PATH_WHEN = frozenset({"all-files-changed", "any-file-changed"}) + + +@dataclass(frozen=True) +class Rule: + number: int + name: str + when: str + paths: tuple[re.Pattern[str], ...] + path_patterns: tuple[str, ...] + label: str + run: frozenset[str] + skip: frozenset[str] + run_all: bool + skip_all: bool + + def condition(self) -> str: + if self.when == "always": + return "every pull request" + if self.when == "no-file-changed": + return "the diff has no files" + if self.when == "label-exists": + return f"label {self.label} is set" + if self.when == "all-files-changed": + return "every changed file matches" + if self.when == "any-file-changed": + return "any changed file matches" + return "every job no earlier rule decided" + + +@dataclass(frozen=True) +class Conflict: + job: str + run_rules: tuple[int, ...] + skip_rules: tuple[int, ...] + + +@dataclass(frozen=True) +class Decision: + job: str + opinion: str + rules: tuple[int, ...] + + +@dataclass +class _Votes: + run_rules: list[int] = field(default_factory=list) + skip_rules: list[int] = field(default_factory=list) + + +@dataclass(frozen=True) +class Mapping: + jobs: frozenset[str] + rules: tuple[Rule, ...] + by_number: dict[int, Rule] + # Key requires every job in the tuple. A run of the key is a run of each. + requires: dict[str, tuple[str, ...]] + + +@dataclass(frozen=True) +class FileTrace: + """One changed path. The rule list, not the file, decides the jobs.""" + + path: str + kind: str = "file" + domain: str = "" + explicit_runs: tuple[str, ...] = () + explicit_skips: tuple[str, ...] = () + + +@dataclass(frozen=True) +class Selection: + run: frozenset[str] + skip: frozenset[str] + unsure: frozenset[str] + execute: frozenset[str] + reason: str + shadow: bool + files: tuple[FileTrace, ...] = () + required_runs: frozenset[str] = frozenset() + matched_rules: tuple[int, ...] = () + decisions: tuple[Decision, ...] = () + conflicts: tuple[Conflict, ...] = () + + def decision_for(self, job: str) -> Decision | None: + for decision in self.decisions: + if decision.job == job: + return decision + return None + + def conflict_for(self, job: str) -> Conflict | None: + for conflict in self.conflicts: + if conflict.job == job: + return conflict + return None + + def to_dict(self) -> dict[str, object]: + return { + "shadow": self.shadow, + "reason": self.reason, + "run": sorted(self.run), + "skip": sorted(self.skip), + "unsure": sorted(self.unsure), + "execute": sorted(self.execute), + "matched_rules": list(self.matched_rules), + "required_runs": sorted(self.required_runs), + "conflicts": [ + { + "job": conflict.job, + "run_rules": list(conflict.run_rules), + "skip_rules": list(conflict.skip_rules), + } + for conflict in self.conflicts + ], + "decisions": [ + { + "job": decision.job, + "opinion": decision.opinion, + "rules": list(decision.rules), + } + for decision in self.decisions + ], + "files": [trace.path for trace in self.files], + } + + +def load_mapping(path: str | Path) -> Mapping: + text = Path(path).read_text(encoding="utf-8") + if not str(path).endswith(".toml"): + raise ValueError("mapping must be a TOML file") + data = tomllib.loads(text) + if not isinstance(data, dict): + raise ValueError("mapping must be a TOML table") + return parse_mapping(data) + + +def parse_mapping(data: dict) -> Mapping: + if data.get("version") != 1: + raise ValueError(f"unsupported mapping version: {data.get('version')!r}") + + jobs = _string_tuple(data, "jobs") + if len(jobs) != len(set(jobs)): + raise ValueError("duplicate job name") + job_set = frozenset(jobs) + rules = _parse_rules(data.get("rules"), job_set) + return Mapping( + jobs=job_set, + rules=rules, + by_number={rule.number: rule for rule in rules}, + requires=_parse_requires(data, job_set), + ) + + +def resolve( + changed_files: list[str] | None, + mapping: Mapping, + labels: list[str] | None = None, + *, + shadow: bool = True, +) -> Selection: + """Classify mapping.jobs by walking mapping.rules from top to bottom.""" + files = _dedupe(changed_files) if changed_files else [] + label_set = set(labels or []) + votes = {job: _Votes() for job in mapping.jobs} + matched: list[int] = [] + + for rule in mapping.rules: + if rule.when == "remaining": + pending = [ + job + for job, vote in votes.items() + if not vote.run_rules and not vote.skip_rules + ] + if not pending: + continue + matched.append(rule.number) + _record(votes, pending, "skip", rule.number) + continue + if not _rule_matches(rule, files, label_set): + continue + matched.append(rule.number) + if rule.run_all or rule.run: + _record(votes, _targets(rule, "run", mapping.jobs), "run", rule.number) + if rule.skip_all or rule.skip: + _record(votes, _targets(rule, "skip", mapping.jobs), "skip", rule.number) + + run: set[str] = set() + skip: set[str] = set() + unsure: set[str] = set() + decisions: list[Decision] = [] + conflicts: list[Conflict] = [] + for job in sorted(mapping.jobs): + vote = votes[job] + if vote.run_rules and vote.skip_rules: + conflicts.append( + Conflict(job, tuple(vote.run_rules), tuple(vote.skip_rules)) + ) + run.add(job) + decisions.append(Decision(job, "run", tuple(vote.run_rules))) + elif vote.run_rules: + run.add(job) + decisions.append(Decision(job, "run", tuple(vote.run_rules))) + elif vote.skip_rules: + skip.add(job) + decisions.append(Decision(job, "skip", tuple(vote.skip_rules))) + else: + unsure.add(job) + + run_set, skip_set, unsure_set, required = _apply_requires( + frozenset(run), frozenset(skip), frozenset(unsure), mapping.requires + ) + execute = mapping.jobs if shadow else run_set + return Selection( + run=run_set, + skip=skip_set, + unsure=unsure_set, + execute=execute, + reason="rules", + shadow=shadow, + files=tuple(FileTrace(path) for path in files), + required_runs=required, + matched_rules=tuple(matched), + decisions=tuple(decisions), + conflicts=tuple(conflicts), + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Classify CI jobs for a diff.") + parser.add_argument("--mapping", required=True, help="Path to the rule list TOML") + parser.add_argument("--labels", default="", help="Comma-separated PR labels") + parser.add_argument("--format", choices=("json", "human"), default="json") + parser.add_argument( + "--enforce", + action="store_true", + help="Execute run only, and drop skip. The default is shadow mode.", + ) + parser.add_argument( + "--job", + default="", + help="Decide this job only. Exit 10 when the decision is skip.", + ) + parser.add_argument("files", nargs="*", help="Changed paths. '-' reads stdin.") + args = parser.parse_args(argv) + + files = list(args.files) + if files == ["-"]: + files = [line.strip() for line in sys.stdin if line.strip()] + labels = [label for label in args.labels.split(",") if label] + mapping = load_mapping(args.mapping) + selection = resolve(files, mapping, labels, shadow=not args.enforce) + if args.job: + return _decide_job(selection, mapping, args.job) + if args.format == "json": + json.dump(selection.to_dict(), sys.stdout, indent=2) + sys.stdout.write("\n") + else: + _print_human(selection, mapping) + return 0 + + +def _decide_job(selection: Selection, mapping: Mapping, job: str) -> int: + """_decide_job reports one job. Exit 10 means skip; every other code means run.""" + if job not in mapping.jobs: + print(f"resolver: {job} run (not in the rule list)", file=sys.stderr) + return 0 + if job in selection.run: + if job in selection.required_runs: + detail = "required by a job the rules run" + else: + decision = selection.decision_for(job) + detail = _cite(decision, mapping) if decision else "no rule" + print(f"resolver: {job} run ({detail})", file=sys.stderr) + return 0 + if job in selection.skip: + decision = selection.decision_for(job) + detail = _cite(decision, mapping) if decision else "no rule" + print(f"resolver: {job} skip ({detail})", file=sys.stderr) + return 10 + print(f"resolver: {job} run (no rule decided it)", file=sys.stderr) + return 0 + + +def _cite(decision: Decision, mapping: Mapping) -> str: + names = [f"rule {number} {mapping.by_number[number].name}" for number in decision.rules] + return " and ".join(names) + + +def _record( + votes: dict[str, _Votes], + jobs: frozenset[str] | list[str], + opinion: str, + number: int, +) -> None: + for job in jobs: + vote = votes[job] + bucket = vote.run_rules if opinion == "run" else vote.skip_rules + if number not in bucket: + bucket.append(number) + + +def _targets(rule: Rule, opinion: str, jobs: frozenset[str]) -> frozenset[str]: + if opinion == "run": + return jobs if rule.run_all else rule.run + return jobs if rule.skip_all else rule.skip + + +def _rule_matches(rule: Rule, files: list[str], labels: set[str]) -> bool: + if rule.when == "always": + return True + if rule.when == "no-file-changed": + return not files + if rule.when == "label-exists": + return rule.label in labels + if rule.when == "any-file-changed": + return any(_matches(path, rule.paths) for path in files) + if rule.when == "all-files-changed": + return bool(files) and all(_matches(path, rule.paths) for path in files) + return False + + +def _matches(path: str, patterns: tuple[re.Pattern[str], ...]) -> bool: + return any(pattern.search(path) for pattern in patterns) + + +def _apply_requires( + run: frozenset[str], + skip: frozenset[str], + unsure: frozenset[str], + requires: dict[str, tuple[str, ...]], +) -> tuple[frozenset[str], frozenset[str], frozenset[str], frozenset[str]]: + """A running job pulls in the jobs it requires. A skip does not.""" + running = set(run) + skipped = set(skip) + unknown = set(unsure) + pulled: set[str] = set() + changed = True + while changed: + changed = False + for job in list(running): + for needed in requires.get(job, ()): + if needed in running: + continue + running.add(needed) + skipped.discard(needed) + unknown.discard(needed) + pulled.add(needed) + changed = True + return frozenset(running), frozenset(skipped), frozenset(unknown), frozenset(pulled) + + +def _parse_rules(raw: object, jobs: frozenset[str]) -> tuple[Rule, ...]: + if not isinstance(raw, list) or not raw: + raise ValueError("rules must be a non-empty list") + rules: list[Rule] = [] + names: set[str] = set() + for index, body in enumerate(raw, start=1): + if not isinstance(body, dict): + raise ValueError(f"rule {index} must be a table") + rule = _parse_rule(body, index, jobs) + if rule.name in names: + raise ValueError(f"duplicate rule name {rule.name}") + names.add(rule.name) + rules.append(rule) + last = rules[-1] + if last.when != "remaining" or not last.skip_all or last.run or last.run_all: + raise ValueError("the last rule must skip every job no earlier rule decided") + return tuple(rules) + + +def _parse_rule(body: dict, index: int, jobs: frozenset[str]) -> Rule: + name = body.get("name") + if not isinstance(name, str) or not name: + raise ValueError(f"rule {index} needs a name") + if "number" in body: + raise ValueError(f"rule {index} {name} does not take number") + number = index + when = body.get("when") + if when not in _WHEN: + raise ValueError(f"rule {number} {name} has unknown when: {when!r}") + if "unsure" in body: + raise ValueError(f"rule {number} {name} cannot say unsure") + patterns = _optional_strings(body, "paths", number, name) + label = body.get("label", "") + if not isinstance(label, str): + raise ValueError(f"rule {number} {name} label must be a string") + _check_when_fields(when, patterns, label, number, name) + run, run_all = _job_list(body, "run", number, name, jobs) + skip, skip_all = _job_list(body, "skip", number, name, jobs) + if not run and not run_all and not skip and not skip_all: + raise ValueError(f"rule {number} {name} must run or skip a job") + overlap = run & skip + if overlap or (run_all and skip_all): + raise ValueError(f"rule {number} {name} both runs and skips a job") + if run_all and skip or skip_all and run: + raise ValueError(f"rule {number} {name} both runs and skips a job") + return Rule( + number=number, + name=name, + when=when, + paths=tuple(_compile(pattern, f"rule {number} {name}") for pattern in patterns), + path_patterns=patterns, + label=label, + run=run, + skip=skip, + run_all=run_all, + skip_all=skip_all, + ) + + +def _check_when_fields( + when: str, + patterns: tuple[str, ...], + label: str, + number: int, + name: str, +) -> None: + where = f"rule {number} {name}" + if when in _PATH_WHEN and not patterns: + raise ValueError(f"{where} needs paths") + if when == "label-exists" and not label: + raise ValueError(f"{where} needs a label") + if when not in _PATH_WHEN and patterns: + raise ValueError(f"{where} does not take paths") + if when != "label-exists" and label: + raise ValueError(f"{where} does not take a label") + + +def _job_list( + body: dict, key: str, number: int, name: str, jobs: frozenset[str] +) -> tuple[frozenset[str], bool]: + raw = body.get(key, []) + if raw is None: + raw = [] + if not isinstance(raw, list) or not all(isinstance(item, str) for item in raw): + raise ValueError(f"rule {number} {name} {key} must be a list of jobs") + if not raw: + return frozenset(), False + if "*" in raw: + if raw != ["*"]: + raise ValueError(f"rule {number} {name} {key} mixes * with job names") + return frozenset(), True + chosen = frozenset(raw) + unknown = chosen - jobs + if unknown: + listed = ", ".join(sorted(unknown)) + raise ValueError(f"unknown job {listed} in rule {number} {name}") + if len(raw) != len(chosen): + raise ValueError(f"rule {number} {name} repeats a job in {key}") + return chosen, False + + +def _parse_requires(data: dict, jobs: frozenset[str]) -> dict[str, tuple[str, ...]]: + raw = data.get("requires", {}) + if raw is None: + raw = {} + if not isinstance(raw, dict): + raise ValueError("requires must be a table") + requires: dict[str, tuple[str, ...]] = {} + for job, needed in raw.items(): + if job not in jobs: + raise ValueError(f"unknown job {job} in requires") + if not isinstance(needed, list) or not needed: + raise ValueError(f"{job} requires a non-empty list") + names = tuple(str(item) for item in needed) + if len(names) != len(set(names)): + raise ValueError(f"duplicate requirement for {job}") + for name in names: + if name not in jobs: + raise ValueError(f"unknown job {name} required by {job}") + if name == job: + raise ValueError(f"{job} requires itself") + requires[str(job)] = names + _detect_require_cycles(requires) + return requires + + +def _detect_require_cycles(requires: dict[str, tuple[str, ...]]) -> None: + visiting: set[str] = set() + visited: set[str] = set() + + def walk(name: str) -> None: + if name in visited: + return + if name in visiting: + raise ValueError(f"requires cycle at {name}") + visiting.add(name) + for needed in requires.get(name, ()): + walk(needed) + visiting.remove(name) + visited.add(name) + + for name in requires: + walk(name) + + +def _optional_strings(body: dict, key: str, number: int, name: str) -> tuple[str, ...]: + raw = body.get(key, []) + if raw is None: + raw = [] + if not isinstance(raw, list) or not all(isinstance(item, str) for item in raw): + raise ValueError(f"rule {number} {name} {key} must be a list of strings") + return tuple(raw) + + +def _string_tuple(data: dict, key: str) -> tuple[str, ...]: + value = data.get(key) + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{key} must be a list of strings") + return tuple(value) + + +def _compile(pattern: str, where: str) -> re.Pattern[str]: + try: + return re.compile(pattern) + except re.error as exc: + raise ValueError(f"invalid pattern {pattern!r} in {where}") from exc + + +def _dedupe(files: list[str]) -> list[str]: + seen: set[str] = set() + unique: list[str] = [] + for name in files: + if name in seen: + continue + seen.add(name) + unique.append(name) + return unique + + +def _print_human(selection: Selection, mapping: Mapping) -> None: + print("Rules:") + if not selection.matched_rules: + print(" none") + for number in selection.matched_rules: + rule = mapping.by_number[number] + print(f" {number} {rule.name}") + print(f" {rule.condition()}") + print("Conflicts:") + if not selection.conflicts: + print(" none") + for conflict in selection.conflicts: + print(f" {conflict.job}") + print(" run wins") + print(f"Run ({len(selection.run)}):") + _print_jobs(selection.run, selection, mapping) + print(f"Skip ({len(selection.skip)}):") + _print_jobs(selection.skip, selection, mapping) + + +def _print_jobs(jobs: frozenset[str], selection: Selection, mapping: Mapping) -> None: + if not jobs: + print(" -") + return + for job in sorted(jobs): + decision = selection.decision_for(job) + if job in selection.required_runs: + note = "required by a running job" + elif decision is None: + note = "no rule" + else: + note = ", ".join( + f"{number} {mapping.by_number[number].name}" for number in decision.rules + ) + print(f" {job} ({note})") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/exploration/samples/docs-note.md b/ci/exploration/samples/docs-note.md new file mode 100644 index 0000000000000..27f479a06b7ba --- /dev/null +++ b/ci/exploration/samples/docs-note.md @@ -0,0 +1,2 @@ +The shadow test-selection map treats a markdown file as documentation. +This file is only here so that rule has a changed path to report. diff --git a/ci/exploration/test_dispatcher.py b/ci/exploration/test_dispatcher.py new file mode 100644 index 0000000000000..860ffc49b2d27 --- /dev/null +++ b/ci/exploration/test_dispatcher.py @@ -0,0 +1,459 @@ +"""The GitHub Actions dispatcher turns resolver opinions into start, stop, or default.""" + +import json +from pathlib import Path + +from dispatcher import ( + JobPlan, + PullRequest, + default_sentence, + default_would_start, + dispatch, + format_plan, + format_summary, + github_commands, + load_defaults, + main, +) +from resolver import FileTrace, Selection, load_mapping, resolve + +DEFAULTS = Path(__file__).with_name("gha_defaults.yaml") +MAPPING = Path(__file__).resolve().parents[1] / "test-domains.toml" + +READY = PullRequest(draft=False, fork=False, labels=frozenset(), files=()) + + +def test_run_starts_whether_or_not_the_old_trigger_would(): + defaults = { + "style-check": {"when": "always"}, + "e2e-qa-tests-gke": {"when": "label", "label": "e2e-qa-tests-gke"}, + } + selection = resolve( + ["sensor/common/foo.go"], + load_mapping(MAPPING), + ) + plan = dispatch(selection, defaults, READY) + by_job = {item.job: item for item in plan} + assert by_job["style-check"].action == "start" + assert by_job["style-check"].would_run is True + assert by_job["e2e-qa-tests-gke"].opinion == "skip" + assert by_job["e2e-qa-tests-gke"].would_run is False + + +def test_run_starts_a_job_the_old_trigger_would_leave_off(): + defaults = {"e2e-qa-tests-gke": {"when": "label", "label": "e2e-qa-tests-gke"}} + selection = Selection( + run=frozenset({"e2e-qa-tests-gke"}), + skip=frozenset(), + unsure=frozenset(), + execute=frozenset({"e2e-qa-tests-gke"}), + reason="rules", + shadow=True, + ) + plan = dispatch(selection, defaults, READY) + job = plan[0] + assert job.default_starts is False + assert job.action == "start" + assert job.would_run is True + + +def test_skip_stops_a_job_the_old_trigger_would_start(): + defaults = load_defaults(DEFAULTS) + selection = resolve(["sensor/common/foo.go"], load_mapping(MAPPING)) + plan = dispatch(selection, defaults, READY) + go_postgres = next(item for item in plan if item.job == "go-postgres") + assert go_postgres.opinion == "skip" + assert go_postgres.default_starts is True + assert go_postgres.action == "stop" + assert go_postgres.would_run is False + + +def test_skip_does_not_run(): + defaults = load_defaults(DEFAULTS) + selection = resolve(["sensor/common/foo.go"], load_mapping(MAPPING)) + draft = PullRequest(draft=True, fork=False, labels=frozenset(), files=("sensor/common/foo.go",)) + plan = dispatch(selection, defaults, draft) + by_job = {item.job: item for item in plan} + assert by_job["go"].action == "start" + assert by_job["go"].would_run is True + assert by_job["go-postgres"].action == "stop" + assert by_job["go-postgres"].default_starts is True + assert by_job["go-postgres"].would_run is False + assert by_job["e2e-byodb-tests"].action == "stop" + assert by_job["e2e-byodb-tests"].would_run is False + + +def test_label_is_the_old_trigger_for_optional_e2e(): + defaults = load_defaults(DEFAULTS) + selection = resolve(["README.md"], load_mapping(MAPPING)) + labeled = PullRequest( + draft=True, + fork=False, + labels=frozenset({"e2e-qa-tests-gke"}), + files=("README.md",), + ) + plan = dispatch(selection, defaults, labeled) + job = next(item for item in plan if item.job == "e2e-qa-tests-gke") + assert job.default_starts is True + + +def test_defaults_cover_the_github_actions_jobs(): + mapping = load_mapping(MAPPING) + defaults = load_defaults(DEFAULTS) + prow = { + "aks-qa-e2e-tests", + "aro-qa-e2e-tests", + "eks-qa-e2e-tests", + "gke-external-pg-17-qa-e2e-tests", + "gke-nongroovy-compatibility-tests", + "gke-nongroovy-e2e-tests", + "gke-operator-e2e-tests", + "gke-qa-e2e-tests", + "gke-race-condition-qa-e2e-tests", + "gke-scale-tests", + "gke-scanner-v4-install-tests", + "gke-ui-e2e-tests", + "gke-version-compatibility-tests", + "ibmcloudz-qa-e2e-tests", + "ocp-compliance-e2e-tests", + "ocp-nongroovy-e2e-tests", + "ocp-operator-e2e-tests", + "ocp-perf-scale-tests", + "ocp-qa-e2e-tests", + "ocp-scanner-v4-install-tests", + "ocp-ui-e2e-tests", + "ocp-vm-scanning-e2e-tests", + "osd-aws-qa-e2e-tests", + "osd-gcp-qa-e2e-tests", + "powervs-qa-e2e-tests", + "rosa-qa-e2e-tests", + "ui-component-tests", + } + assert set(defaults) == mapping.jobs - prow + + +def test_not_ui_sentence_names_a_short_diff_and_not_a_long_one(): + one = PullRequest(False, False, frozenset(), ("sensor/common/foo.go",)) + assert default_sentence({"when": "not-ui"}, one) == ( + "starts because sensor/common/foo.go is outside ui/" + ) + many = PullRequest(False, False, frozenset(), tuple(f"pkg/{i}.go" for i in range(6))) + assert default_sentence({"when": "not-ui"}, many) == ( + "starts because a changed file is outside ui/" + ) + + +def test_default_sentence_agrees_with_whether_the_default_starts(): + defaults = load_defaults(DEFAULTS) + pull_requests = [ + PullRequest(False, False, frozenset(), ("sensor/common/foo.go",)), + PullRequest(True, False, frozenset(), ("sensor/common/foo.go",)), + PullRequest(False, True, frozenset(), ("sensor/common/foo.go",)), + PullRequest(False, False, frozenset(), ("ui/src/App.tsx",)), + PullRequest(False, False, frozenset(), ()), + PullRequest(False, False, frozenset({"e2e-qa-tests-gke"}), ("README.md",)), + PullRequest(False, True, frozenset({"e2e-qa-tests-gke"}), ("README.md",)), + PullRequest(False, False, frozenset(), (".github/workflows/style.yaml",)), + PullRequest( + False, False, frozenset({"e2e-qa-tests-gke-konflux"}), ("sensor/a.go",) + ), + PullRequest( + False, True, frozenset({"e2e-qa-tests-gke-konflux"}), ("sensor/a.go",) + ), + PullRequest(False, False, frozenset({"konflux-build"}), ("sensor/a.go",)), + PullRequest(True, False, frozenset({"konflux-build"}), ("sensor/a.go",)), + PullRequest(False, False, frozenset({"konflux-build"}), ("ui/App.tsx",)), + ] + for pull_request in pull_requests: + for rule in defaults.values(): + sentence = default_sentence(rule, pull_request) + if default_would_start(rule, pull_request): + assert sentence.startswith("starts"), sentence + assert "stays off" not in sentence + else: + assert sentence.startswith("stays off"), sentence + + +def test_plan_text_for_a_sensor_change(): + text, summary = _report(("sensor/common/foo.go",)) + assert text.index("Changed files") < text.index("Rules (") + assert text.index("Rules (") < text.index("Conflicts (") + assert text.index("Conflicts (") < text.index("Will run (") + assert text.index("Will run (") < text.index("Will skip (") + assert text.index("Will skip (") < text.index("Unsure (") + assert text.index("Unsure (") < text.index("Default would start these") + assert "today" not in text.lower() + assert "Shadow." in text + assert "ci-dispatcher-enforce" in text + assert "::group::Unsure (" not in text + assert "::group::Will run (" not in text + + assert "sensor/common/foo.go" in text + run = _between(text, "Will run (", "Will skip (") + assert "rule 5 go-sources" in run + assert "rule 7 sensor" in run + assert "rule 6 style" in run + assert "rule 9 image-wait" in run + assert "wait-for-images requires this job" in run + assert "Conflicts (0)" in text + + skip = _between(text, "Will skip (", "Unsure (") + assert "rule 7 sensor" in skip + assert "go-postgres" in skip + assert "rule 12 remaining" in skip + + assert "Unsure (0)" in text + dropped = _between(text, "Default would start these", "\n\n") + assert "\n go-postgres\n" in dropped + assert "\n go\n" not in dropped + + assert "## Will run (" in summary + assert "Will skip (" in summary + assert "::group::" not in summary + + +def test_plan_text_includes_prow_jobs_when_dispatch_changes(): + text, summary = _report((".openshift-ci/dispatch.sh",)) + run = _between(text, "Will run (", "Will skip (") + skip = _between(text, "Will skip (", "Unsure (") + assert "gke-nongroovy-e2e-tests" in run + assert "ocp-vm-scanning-e2e-tests" in run + assert "rule 10 prow-demo-run" in run + assert "gke-nongroovy-e2e-tests\n rule 10 prow-demo-run\n default:" not in text + assert "gke-ui-e2e-tests" in skip + assert "gke-qa-e2e-tests" in skip + assert "rule 11 prow-demo-skip" in skip + assert "no GitHub Actions default" in skip + dropped = _between(text, "Default would start these", "\n\n") + assert "gke-ui-e2e-tests" not in dropped + assert "gke-nongroovy-e2e-tests" not in dropped + assert "gke-nongroovy-e2e-tests" in summary + assert "gke-ui-e2e-tests" in summary + + mapping = load_mapping(MAPPING) + selection = resolve([".openshift-ci/dispatch.sh"], mapping) + plans = dispatch(selection, load_defaults(DEFAULTS), READY) + notice = github_commands(plans, selection.conflicts, mapping)[0] + start = sum(plan.would_run for plan in plans) + skipped = sum(plan.action == "stop" for plan in plans) + unsure = sum(plan.action == "default" for plan in plans) + assert f"Would start {start}, skip {skipped}, unsure {unsure}." in notice + assert f"Will run ({start})" in text + assert f"Will skip ({skipped})" in text + + +def test_plan_text_shows_a_run_winning_over_a_skip(): + files_for_conflict = ( + "sensor/test-selection-shadow.txt", + "central/policy/test-selection-shadow.txt", + ".github/workflows/style.yaml", + ) + text, _summary = _report(files_for_conflict) + assert ".github/workflows/style.yaml" in text + conflicts = _between(text, "Conflicts (", "Will run (") + assert "go-postgres" in conflicts + assert "sensor-integration-tests" in conflicts + assert ( + "rule 8 central-policy runs go-postgres and rule 7 sensor skips it. Run wins." + in conflicts + ) + assert ( + "rule 7 sensor runs sensor-integration-tests and rule 8 central-policy skips it. Run wins." + in conflicts + ) + run = _between(text, "Will run (", "Will skip (") + assert "run wins" in run + assert "rule 8 central-policy" in run + assert "rule 7 sensor" in run + mapping = load_mapping(MAPPING) + selection = resolve(list(files_for_conflict), mapping) + plans = dispatch(selection, load_defaults(DEFAULTS), READY) + warnings = [ + command + for command in github_commands(plans, selection.conflicts, mapping) + if command.startswith("::warning") + ] + assert any("go-postgres" in command and "Run wins." in command for command in warnings) + + +def test_plan_text_keeps_a_skip_when_another_file_matches_no_rule(): + text, _summary = _report( + ("sensor/common/foo.go", ".github/workflows/style.yaml") + ) + skip = _between(text, "Will skip (", "Unsure (") + assert "go-postgres" in skip + assert "rule 7 sensor" in skip + assert "Conflicts (0)" in text + + +def test_plan_text_for_docs_only(): + text, summary = _report(("README.md",)) + assert "ignored because other files" not in text + run = _between(text, "Will run (", "Will skip (") + assert "style-check" in run + assert "rule 4 docs-only" in run + assert "rule 6 style" in run + assert "rule 9 image-wait" in run + assert "wait-for-images requires this job" in run + skip = _between(text, "Will skip (", "Unsure (") + assert "go" in skip + assert "rule 4 docs-only" in skip + assert "::group::Will skip (" in text + assert "Unsure (0)" in text + assert "::group::Unsure" not in text + assert "Will skip (" in summary + + +def test_plan_text_for_files_that_match_nothing(): + text, _summary = _report((".github/workflows/style.yaml",)) + assert "Will run (3)" in text + assert "rule 6 style" in text + assert "rule 9 image-wait" in text + assert "wait-for-images requires this job" in text + assert "Will skip (" in text + skip = _between(text, "Will skip (", "Unsure (") + assert "github-actions-lint" in skip + assert "github-actions-shellcheck" in skip + assert "rule 12 remaining" in skip + assert ".github/workflows/style.yaml matches the path rule" in skip + + +def test_plan_text_names_a_job_the_plan_starts_and_the_default_leaves_off(): + defaults = { + "e2e-qa-tests-gke": {"when": "label", "label": "e2e-qa-tests-gke"} + } + selection = Selection( + run=frozenset({"e2e-qa-tests-gke"}), + skip=frozenset(), + unsure=frozenset(), + execute=frozenset({"e2e-qa-tests-gke"}), + reason="rules", + shadow=True, + files=( + FileTrace( + "sensor/x.go", + "domain", + "sensor", + ("e2e-qa-tests-gke",), + ), + ), + ) + text, _summary = _report(("sensor/x.go",), defaults=defaults, selection=selection) + assert "a rule says run" in text + assert "default: stays off because label e2e-qa-tests-gke is absent" in text + assert "This plan would start these, and the default would not" in text + assert "Default would start these" not in text + assert "Default matches this plan." not in text + + +def test_plan_text_says_when_the_plan_and_the_default_agree(): + defaults = {"style-check": {"when": "always"}} + text, _summary = _report(("central/policy/service.go",), defaults=defaults) + assert "rule 6 style" in text + assert "Default matches this plan." in text + assert "Default would start these" not in text + + +def test_enforce_header_replaces_the_shadow_header(): + text, _summary = _report(("sensor/common/foo.go",), shadow=False) + assert text.startswith("Enforce.") + assert "Shadow." not in text + + +def test_github_commands_warn_when_the_default_would_start_a_stopped_job(): + plans = ( + JobPlan("style-check", "run", True, "start"), + JobPlan("go-postgres", "skip", True, "stop"), + JobPlan("e2e-qa-tests-gke", "unsure", False, "default"), + ) + commands = github_commands(plans) + assert len(commands) == 2 + assert commands[0].startswith("::notice title=Dispatcher plan::") + assert "Would start 1, skip 1, unsure 1." in commands[0] + assert commands[1].startswith("::warning title=Dispatcher plan::") + assert "1 job" in commands[1] + + +def test_github_commands_skip_the_warning_when_nothing_is_dropped(): + plans = (JobPlan("e2e-qa-tests-gke", "run", False, "start"),) + commands = github_commands(plans) + assert len(commands) == 1 + assert commands[0].startswith("::notice title=Dispatcher plan::") + assert "1 job the default would leave off" in commands[0] + assert not any(command.startswith("::warning") for command in commands) + + +def test_cli_prints_the_plan_and_writes_the_summary(capsys, tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + exit_code = main( + [ + "--mapping", + str(MAPPING), + "--defaults", + str(DEFAULTS), + "--title", + "Example", + "sensor/common/foo.go", + ] + ) + assert exit_code == 0 + out = capsys.readouterr().out + assert "Will run (" in out + assert "::notice title=Dispatcher plan::" in out + written = summary.read_text() + assert "Will skip (" in written + assert "::group::" not in written + + +def test_cli_json_stays_json(capsys): + exit_code = main( + [ + "--mapping", + str(MAPPING), + "--defaults", + str(DEFAULTS), + "--format", + "json", + "sensor/common/foo.go", + ] + ) + assert exit_code == 0 + payload = json.loads(capsys.readouterr().out) + assert "jobs" in payload + + +def _report( + files, + *, + draft=False, + fork=False, + labels=(), + shadow=True, + title="Example", + defaults=None, + selection=None, +): + pull_request = PullRequest(draft, fork, frozenset(labels), tuple(files)) + mapping = load_mapping(MAPPING) + if defaults is None: + defaults = load_defaults(DEFAULTS) + if selection is None: + selection = resolve(list(files), mapping, list(pull_request.labels)) + plans = dispatch(selection, defaults, pull_request) + kwargs = { + "shadow": shadow, + "title": title, + "selection": selection, + "defaults": defaults, + "pull_request": pull_request, + "mapping": mapping, + } + return format_plan(plans, **kwargs), format_summary(plans, **kwargs) + + +def _between(text, start, end): + begin = text.index(start) + if end and end in text[begin + len(start) :]: + return text[begin : text.index(end, begin + len(start))] + return text[begin:] diff --git a/ci/exploration/test_production_map.py b/ci/exploration/test_production_map.py new file mode 100644 index 0000000000000..3b33afc83073d --- /dev/null +++ b/ci/exploration/test_production_map.py @@ -0,0 +1,118 @@ +"""The production rule list in ci/test-domains.toml.""" + +from pathlib import Path + +from resolver import load_mapping, main, resolve + +PRODUCTION = Path(__file__).resolve().parents[1] / "test-domains.toml" + +IMAGE_WAIT = frozenset({"wait-for-images", "should-dispatch"}) + + +def decide(files, labels=None): + mapping = load_mapping(PRODUCTION) + return mapping, resolve(files, mapping, labels) + + +def test_docs_and_changelog_skip_go(): + mapping, result = decide(["README.md", "CHANGELOG.md", "docs/guide.md"]) + assert result.unsure == frozenset() + assert "style-check" in result.run + assert "go" in result.skip + assert result.decision_for("go").rules == (4,) + assert result.conflicts == () + assert IMAGE_WAIT <= result.run + assert result.required_runs == frozenset({"should-dispatch"}) + assert result.skip == mapping.jobs - result.run + + +def test_go_source_runs_go_unit_tests(): + _mapping, result = decide(["sensor/common/foo.go"]) + assert "go" in result.run + assert result.decision_for("go").rules == (5,) + assert "sensor-integration-tests" in result.run + assert "go-postgres" in result.skip + assert result.conflict_for("go") is None + + +def test_sensor_and_central_conflict_and_run_wins(): + _mapping, result = decide( + ["sensor/test-selection-shadow.txt", "central/policy/test-selection-shadow.txt"] + ) + assert "go" in result.skip + assert "sensor-integration-tests" in result.run + assert "go-postgres" in result.run + assert result.conflict_for("go-postgres") is not None + assert result.conflict_for("sensor-integration-tests") is not None + + +def test_dispatch_change_runs_one_prow_suite_and_skips_the_others(): + _mapping, result = decide([".openshift-ci/dispatch.sh"]) + assert "gke-nongroovy-e2e-tests" in result.run + assert result.decision_for("gke-nongroovy-e2e-tests").rules == (10,) + assert "ocp-vm-scanning-e2e-tests" in result.run + assert result.decision_for("ocp-vm-scanning-e2e-tests").rules == (10,) + assert "gke-ui-e2e-tests" in result.skip + assert result.decision_for("gke-ui-e2e-tests").rules == (11,) + assert "gke-qa-e2e-tests" in result.skip + assert result.decision_for("gke-qa-e2e-tests").rules == (11,) + + +def test_job_flag_exits_for_the_prow_demonstration(capsys): + skip = main( + [ + "--mapping", + str(PRODUCTION), + "--job", + "gke-ui-e2e-tests", + "--enforce", + ".openshift-ci/dispatch.sh", + ] + ) + assert skip == 10 + assert "skip" in capsys.readouterr().err + run = main( + [ + "--mapping", + str(PRODUCTION), + "--job", + "gke-nongroovy-e2e-tests", + "--enforce", + ".openshift-ci/dispatch.sh", + ] + ) + assert run == 0 + assert "prow-demo-run" in capsys.readouterr().err + vm = main( + [ + "--mapping", + str(PRODUCTION), + "--job", + "ocp-vm-scanning-e2e-tests", + "--enforce", + ".openshift-ci/dispatch.sh", + ] + ) + assert vm == 0 + assert "ocp-vm-scanning-e2e-tests run" in capsys.readouterr().err + outside = main( + [ + "--mapping", + str(PRODUCTION), + "--job", + "test-binary-build-commands", + "--enforce", + ".openshift-ci/dispatch.sh", + ] + ) + assert outside == 0 + + +def test_ci_only_change_skips_go_and_runs_style_and_the_image_wait(): + mapping, result = decide(["ci/test-domains.toml", ".github/workflows/style.yaml"]) + assert "go" in result.skip + assert result.decision_for("go").rules == (12,) + assert "style-check" in result.run + assert IMAGE_WAIT <= result.run + assert result.unsure == frozenset() + assert result.skip == mapping.jobs - result.run diff --git a/ci/exploration/test_resolver.py b/ci/exploration/test_resolver.py new file mode 100644 index 0000000000000..c5eaec7515a4f --- /dev/null +++ b/ci/exploration/test_resolver.py @@ -0,0 +1,163 @@ +"""Behavior of the ordered rule list.""" + +import json +from pathlib import Path + +import pytest + +from resolver import load_mapping, main, parse_mapping, resolve + +FIXTURE = Path(__file__).with_name("fixture.toml") + +JOBS = frozenset( + { + "style", + "go", + "sensor-integration-tests", + "go-postgres", + "wait-for-images", + "should-dispatch", + "other", + } +) + + +@pytest.fixture +def mapping(): + return load_mapping(FIXTURE) + + +def decide(mapping, files, labels=None, shadow=True): + return resolve(files, mapping, labels, shadow=shadow) + + +def test_docs_only_skips_go_and_runs_style(mapping): + result = decide(mapping, ["README.md", "docs/guide.md"]) + assert result.reason == "rules" + assert result.unsure == frozenset() + assert "style" in result.run + assert "go" in result.skip + assert result.decision_for("go").rules == (3,) + assert result.conflicts == () + assert "wait-for-images" in result.run + assert "should-dispatch" in result.required_runs + + +def test_go_file_runs_go_even_when_a_directory_rule_also_matches(mapping): + result = decide(mapping, ["sensor/common/foo.go"]) + assert "go" in result.run + assert result.decision_for("go").rules == (4,) + assert "sensor-integration-tests" in result.run + assert "go-postgres" in result.skip + assert result.decision_for("go-postgres").rules == (5,) + assert result.conflicts == () + + +def test_opposite_votes_warn_and_run_wins(mapping): + result = decide(mapping, ["sensor/common/foo.go", "central/policy/service.go"]) + assert "go-postgres" in result.run + assert "sensor-integration-tests" in result.run + by_job = {conflict.job: conflict for conflict in result.conflicts} + assert by_job["go-postgres"].run_rules == (6,) + assert by_job["go-postgres"].skip_rules == (5,) + assert by_job["sensor-integration-tests"].run_rules == (5,) + assert by_job["sensor-integration-tests"].skip_rules == (6,) + + +def test_run_all_label_beats_a_later_skip(mapping): + result = decide(mapping, ["README.md"], labels=["ci-run-all-tests"]) + assert result.run == JOBS + assert result.skip == frozenset() + go = result.conflict_for("go") + assert go is not None + assert go.run_rules == (1,) + assert go.skip_rules == (3,) + + +def test_missing_diff_runs_every_job_and_docs_do_not_match(mapping): + result = decide(mapping, []) + assert result.run == JOBS + assert result.conflicts == () + assert 3 not in result.matched_rules + + +def test_remaining_does_not_conflict_with_an_earlier_run(mapping): + result = decide(mapping, ["README.md"]) + assert result.conflict_for("style") is None + assert result.conflict_for("wait-for-images") is None + assert result.decision_for("other").rules == (8,) + + +def test_enforce_executes_only_run(mapping): + result = decide(mapping, ["sensor/common/foo.go"], shadow=False) + assert result.execute == result.run + assert result.execute.isdisjoint(result.skip) + + +def test_shadow_executes_every_job(mapping): + result = decide(mapping, ["sensor/common/foo.go"]) + assert result.shadow is True + assert result.execute == JOBS + + +def test_last_rule_must_skip_what_remains(): + data = _rules() + data["rules"][-1]["skip"] = ["other"] + with pytest.raises(ValueError, match="last rule"): + parse_mapping(data) + + +def test_a_rule_does_not_take_number(): + data = _rules() + data["rules"][0]["number"] = 1 + with pytest.raises(ValueError, match="number"): + parse_mapping(data) + + +def test_a_rule_cannot_say_unsure(): + data = _rules() + data["rules"][0]["unsure"] = ["other"] + with pytest.raises(ValueError, match="unsure"): + parse_mapping(data) + + +def test_human_report_lists_rules_then_conflicts(capsys): + exit_code = main( + [ + "--mapping", + str(FIXTURE), + "--format", + "human", + "sensor/common/foo.go", + "central/policy/service.go", + ] + ) + assert exit_code == 0 + report = capsys.readouterr().out + assert report.index("Rules:") < report.index("Conflicts:") + assert report.index("Conflicts:") < report.index("Run (") + assert "go-postgres" in report + assert "run wins" in report + + +def test_cli_json_lists_conflicts(capsys): + exit_code = main( + ["--mapping", str(FIXTURE), "--format", "json", "sensor/a.go", "central/policy/b.go"] + ) + assert exit_code == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["reason"] == "rules" + assert payload["unsure"] == [] + jobs = {item["job"] for item in payload["conflicts"]} + assert jobs == {"go-postgres", "sensor-integration-tests"} + + +def _rules(): + return { + "version": 1, + "jobs": ["style", "other"], + "rules": [ + {"name": "style", "when": "always", "run": ["style"]}, + {"name": "remaining", "when": "remaining", "skip": ["*"]}, + ], + } diff --git a/ci/test-domains.toml b/ci/test-domains.toml new file mode 100644 index 0000000000000..a96aa2d7a0fc1 --- /dev/null +++ b/ci/test-domains.toml @@ -0,0 +1,168 @@ +# Rules are walked from top to bottom. A later run overrides an earlier skip. +# A later skip does not override an earlier run: the plan warns, and the +# job runs. A job no rule has mentioned stays unsure until the last rule, +# which skips whatever is left. Rules do not say unsure. +# +# The jobs list is every target a pull request can hit: GitHub Actions +# job ids, then Prow job names. A prerequisite is a target too. + +version = 1 + +jobs = [ + "style-check", + "misc-checks", + "golangci-lint", + "check-generated-files", + "github-actions-lint", + "github-actions-shellcheck", + "openshift-ci-lint", + "check-dependent-images-exist", + "check-crd-compatibility", + "go", + "go-postgres", + "go-bench", + "ui", + "ui-component", + "local-roxctl-tests", + "shell-unit-tests", + "openshift-ci-unit-tests", + "sensor-integration-tests", + "db-integration-tests", + "scanner-ft-images", + "scanner-ft-run", + "e2e-byodb-tests", + "e2e-db-backup-restore-tests", + "e2e-gke-upgrade-tests", + "e2e-nongroovy-tests", + "e2e-qa-tests-gke", + "e2e-qa-tests-gke-konflux", + "e2e-qa-tests-ocp-4-22", + "e2e-test-on-kind", + "should-dispatch", + "wait-for-images", + "wait-for-konflux-images", + "aks-qa-e2e-tests", + "aro-qa-e2e-tests", + "eks-qa-e2e-tests", + "gke-external-pg-17-qa-e2e-tests", + "gke-nongroovy-compatibility-tests", + "gke-nongroovy-e2e-tests", + "gke-operator-e2e-tests", + "gke-qa-e2e-tests", + "gke-race-condition-qa-e2e-tests", + "gke-scale-tests", + "gke-scanner-v4-install-tests", + "gke-ui-e2e-tests", + "gke-version-compatibility-tests", + "ibmcloudz-qa-e2e-tests", + "ocp-compliance-e2e-tests", + "ocp-nongroovy-e2e-tests", + "ocp-operator-e2e-tests", + "ocp-perf-scale-tests", + "ocp-qa-e2e-tests", + "ocp-scanner-v4-install-tests", + "ocp-ui-e2e-tests", + "ocp-vm-scanning-e2e-tests", + "osd-aws-qa-e2e-tests", + "osd-gcp-qa-e2e-tests", + "powervs-qa-e2e-tests", + "rosa-qa-e2e-tests", + "ui-component-tests", +] + +# The run-all label starts every job. A later skip does not undo that. +[[rules]] +name = "run-all-label" +when = "label-exists" +label = "ci-run-all-tests" +run = ["*"] + +# A missing diff may be the whole tree. +[[rules]] +name = "no-files" +when = "no-file-changed" +run = ["*"] + +# One of these files means the whole tree may have changed. +[[rules]] +name = "tree-may-have-changed" +when = "any-file-changed" +paths = ['^go\.mod$', '^go\.sum$', '^proto/', '^generated/'] +run = ["*"] + +# Every changed file is documentation. Go tests do not run. +[[rules]] +name = "docs-only" +when = "all-files-changed" +paths = ['\.md$', '^docs/', '^CHANGELOG'] +run = ["style-check"] +skip = ["go"] + +# Any Go source runs the Go unit tests. A directory rule does not hide this. +[[rules]] +name = "go-sources" +when = "any-file-changed" +paths = ['\.go$'] +run = ["go"] + +# Any diff that has a file runs style. +[[rules]] +name = "style" +when = "any-file-changed" +paths = ['.*'] +run = ["style-check"] + +# Sensor changes run the Sensor integration job and skip Central Postgres. +[[rules]] +name = "sensor" +when = "any-file-changed" +paths = ['^sensor/'] +run = ["sensor-integration-tests"] +skip = ["go-postgres"] + +# Policy package changes run Central Postgres and skip Sensor integration. +[[rules]] +name = "central-policy" +when = "any-file-changed" +paths = ['^central/policy/'] +run = ["go-postgres"] +skip = ["sensor-integration-tests"] + +# The image wait has to pass before an image-backed test can start. +[[rules]] +name = "image-wait" +when = "always" +run = ["wait-for-images"] + +# A change to the Prow entrypoint keeps two suites: one Prow starts on +# every pull request, and vm-scanning, which stays off unless someone asks. +[[rules]] +name = "prow-demo-run" +when = "any-file-changed" +paths = ['^\.openshift-ci/dispatch\.sh$'] +run = ["gke-nongroovy-e2e-tests", "ocp-vm-scanning-e2e-tests"] + +# That same change skips the other suites Prow starts on every pull request. +[[rules]] +name = "prow-demo-skip" +when = "any-file-changed" +paths = ['^\.openshift-ci/dispatch\.sh$'] +skip = ["gke-ui-e2e-tests", "gke-qa-e2e-tests"] + +# Mandatory. Skips every job that is still unsure. +[[rules]] +name = "remaining" +when = "remaining" +skip = ["*"] + +[requires] +wait-for-images = ["should-dispatch"] +wait-for-konflux-images = ["should-dispatch"] +e2e-byodb-tests = ["wait-for-images"] +e2e-db-backup-restore-tests = ["wait-for-images"] +e2e-gke-upgrade-tests = ["wait-for-images"] +e2e-nongroovy-tests = ["wait-for-images"] +e2e-qa-tests-gke = ["wait-for-images"] +e2e-qa-tests-ocp-4-22 = ["wait-for-images"] +e2e-qa-tests-gke-konflux = ["wait-for-konflux-images"] +scanner-ft-run = ["scanner-ft-images"] diff --git a/sensor/test-selection-shadow.txt b/sensor/test-selection-shadow.txt new file mode 100644 index 0000000000000..58b1dad586d32 --- /dev/null +++ b/sensor/test-selection-shadow.txt @@ -0,0 +1,2 @@ +The shadow test-selection map treats every file under sensor/ as a Sensor change. +This file is only here so that rule has a changed path to report.