name: Pull Request

env:

# TODO: unhardcode this

REGISTRY: iad.ocir.io/idyksih5sir9/cozystack

on:

pull_request:

# `labeled` makes the promote PR's release-candidate verification and manual

# full-e2e opt-in actionable after the bot opens it. The plan/resolve guards

# below discard every other label event before it can launch work.

types: [opened, synchronize, reopened, labeled]

# Cancel in‑flight runs for the same PR when a new push arrives.

#

# A `labeled` event that `plan` discards gets its OWN group, so labelling an

# open PR cannot cancel a live opened/synchronize run. Any label other than

# full-e2e/release makes `plan` skip, and `e2e-report` skips with it, so without the

# split the pair "cancel the real run, then skip" left the head SHA with no

# "E2E Tests" status at all — and an unposted required status stays "Expected",

# blocking the merge until someone pushes again or re-runs the cancelled run.

# Observed on live CI: run 30614258694 (cancelled) followed by 30614280603

# (skipped) on the same SHA, 24s apart, from a maintainer adding kind/* labels

# just after a push. (`pr-labeler` cannot cause this — it labels with the

# default GITHUB_TOKEN, which by design triggers no further workflow runs.)

#

# The condition is the exact complement of `plan`'s guard, so only a run that

# will publish NOTHING is moved aside. A `labeled` run carrying full-e2e or

# release does publish, so it stays in the main group and supersedes the head

# run the way a push does. That keeps exactly one publisher of the `E2E Tests`

# context alive per head SHA, which the status depends on: two live publishers are latest-wins

# per context, and the full suite is the superset, so a full-e2e run that fails

# early — a build flake, a suite failing at minute twenty — would finish before

# the narrower head run it was meant to widen, and that run's later green would

# erase the failure. Cancelling also means opting into the full suite costs one

# suite rather than two, and a second label landing afterwards cannot cancel the

# full-e2e run, because it is no longer in the same group as it.

#

# `release` publishes on the SAME-REPO lane only, hence the fork term. A promote

# PR is opened same-repo by the release bot, so the label on a fork PR is always

# a mislabel — and on that lane it would be a destructive one. e2e-fork.yaml's

# `resolve` decides "this run built nothing, so it is not a verdict" by asking

# whether `Plan build` skipped; a fork run that gets past `plan` then fails

# closed on the pr-patch artifact its own `release` label gated off, posting red

# on a suite that was never going to run, with no `unlabeled` trigger to undo

# it. Excluding forks keeps the label as inert on that lane as it was before

# promotion started using it. The term appears in both this key and `plan`'s

# guard because the two must stay exact complements: NOT(A OR B OR (C AND NOT

# F)) is (NOT A) AND (NOT B) AND (NOT C OR F), and for a same-repo PR F is false

# and both sides reduce to exactly what they were.

concurrency:

group: pr-${{ github.workflow }}-${{ github.event.pull_request.number }}${{ (github.event.action == 'labeled' && github.event.label.name != 'full-e2e' && (github.event.label.name != 'release' || github.event.pull_request.head.repo.fork)) && '-label' || '' }}

cancel-in-progress: true

# Top-level token defaults to read-only. A job-level block replaces this one

# rather than adding to it, so each job restates every scope it needs.

permissions:

contents: read

jobs:

# Compute the docs-only gate and the per-package build matrix from the PR

# diff. hack/build-matrix.sh parses the root Makefile `build:` target (the

# single source of truth) and emits only the units whose package dir changed,

# escalating to the full set on a shared-dependency change. talos + installer

# are handled by dedicated jobs (see build-matrix.sh for why).

plan:

name: Plan build

runs-on: ubuntu-latest

# Exact complement of the concurrency key above; see the reasoning there,

# including why `release` is admitted for same-repo PRs only.

if: github.event.action != 'labeled' || github.event.label.name == 'full-e2e' || (github.event.label.name == 'release' && !github.event.pull_request.head.repo.fork)

permissions:

contents: read

# For the opening `E2E Tests` status below; e2e-report writes the terminal

# one with the same scope.

statuses: write

outputs:

code: ${{ steps.p.outputs.code }}

matrix: ${{ steps.p.outputs.matrix }}

any: ${{ steps.p.outputs.any }}

touched: ${{ steps.p.outputs.touched }}

talos: ${{ steps.p.outputs.talos }}

steps:

# Claim the required context for THIS run before anything can be merged on

# the strength of the last one. Commit statuses are latest-wins per (sha,

# context) and nothing expires them, so a head SHA that already went green

# keeps reading green while a newly started run works — re-run all jobs, a

# full-e2e label, a reopen — and the PR stays mergeable through a suite

# that may be about to fail. While the gate was a job, GitHub created a

# fresh non-success check run each time and that window did not exist.

# Overwriting with `pending` here restores it: the run that owns the

# status says so at the start and concludes it in e2e-report.

#

# Same-repo only. A fork's `pull_request` token is read-only, so this call

# would 403 and fail `plan` on every fork PR, and that read-only token is

# exactly what makes the whole commit-status design safe (§10). Fork PRs

# get their pending from e2e-fork.yaml's `resolve` instead, which is

# privileged.

#

# Note what that costs a fork. `resolve` cannot start until this entire

# run has finished, so the window this step shuts in seconds stays open

# for the length of the build on the fork lane. Structural rather than an

# oversight, since a read-only token cannot post a status at any point in

# the run, and bounded by the build. §10 records it.

- name: Open the "E2E Tests" status for this run

if: ${{ !github.event.pull_request.head.repo.fork }}

# A transient status-API failure here must not take the run with it.

# This step is the first in `plan`, and `plan` is what `checks`,

# `build`, `build-talos`, `finalize` and `e2e` all hang off, so a

# failure costs a full rebuild for nothing. Skipping the opener only

# degrades to the behaviour that preceded it — the terminal status

# still gets posted by `e2e-report`, and a run that dies before that

# leaves the merge blocked either way.

continue-on-error: true

uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7

env:

HEAD_SHA: ${{ github.event.pull_request.head.sha }}

RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

with:

script: |

const { HEAD_SHA, RUN_URL } = process.env;

await github.rest.repos.createCommitStatus({

owner: context.repo.owner,

repo: context.repo.repo,

sha: HEAD_SHA,

state: 'pending',

context: 'E2E Tests',

description: 'e2e in progress — this run has not reported yet',

target_url: RUN_URL,

});

- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

with:

fetch-depth: 0

# e2e-fork.yaml pushes a fork's images only under names the base tree

# already exports, so an image the PR adds is refused there after the

# whole build. Refuse it here instead, before anything builds. The push

# check stays the boundary; a fork can edit this step away.

- name: Refuse images the base branch does not build (fork PR)

if: ${{ github.event.pull_request.head.repo.fork }}

run: |

bt="$(mktemp -d)/basetree"

git worktree add --quiet --detach "$bt" HEAD^1

hack/fork-new-images.sh "$bt" .

- name: Compute changed files and build matrix

id: p

env:

BASE_REF: ${{ github.base_ref }}

run: |

git diff --name-only "origin/${BASE_REF}...HEAD" > /tmp/changed.txt

# The docs-only decision reads a RENAME-BLIND list. `git diff

# --name-only` applies rename detection and reports a rename as its

# new path alone, so moving a code file into docs/ reads as a

# docs-only change: nothing builds, no e2e runs, and the removal of

# that code path ships untested. It also deadlocks the fork lane,

# which answers the same question from `pulls.listFiles` and does see

# the old path through `previous_filename`. That lane concludes e2e is

# required, then fails closed on a pr-patch artifact this lane never

# produced — an unfixable red on a legitimate contribution, with no

# label override and no push that clears it. Both lanes have to answer

# "is this docs-only" the same way, and the only answer that is also

# correct keeps both sides of a rename.

#

# The build matrix still reads the rename-detected list, which is not

# a claim that doing so is right. Moving a file between packages

# leaves the source package changed and unselected, so the matrix

# under-builds on a cross-package rename. That behaviour predates this

# line and is not what the docs-only decision is for, so it is left

# alone here rather than widened by a side effect.

git diff --name-only --no-renames "origin/${BASE_REF}...HEAD" > /tmp/changed_norenames.txt

echo "Changed files:"; sed 's/^/ /' /tmp/changed_norenames.txt

if grep -qvE '^docs/' /tmp/changed_norenames.txt; then

echo "code=true" >> "$GITHUB_OUTPUT"

else

echo "code=false" >> "$GITHUB_OUTPUT"

fi

matrix="$(hack/build-matrix.sh /tmp/changed.txt)"

echo "Build matrix: $matrix"

echo "matrix=$matrix" >> "$GITHUB_OUTPUT"

talos="$(hack/build-talos-needed.sh /tmp/changed_norenames.txt)"

echo "Talos build needed: $talos"

echo "talos=$talos" >> "$GITHUB_OUTPUT"

# Package dirs the PR edited (packages/<area>/<name>). Their committed

# refs are the PR's intent and must survive the current-main overlay —

# otherwise an upstream image bump in a non-build-unit package would be

# silently reverted in e2e/the installer artifact (see

# hack/overlay-main-images.sh). Whitespace-separated, no trailing slash.

touched="$(sed -nE 's#^(packages/[^/]+/[^/]+)/.*#\1#p' /tmp/changed.txt | sort -u | tr '\n' ' ')"

echo "Touched packages: $touched"

echo "touched=$touched" >> "$GITHUB_OUTPUT"

if [ "$matrix" = "[]" ]; then

echo "any=false" >> "$GITHUB_OUTPUT"

else

echo "any=true" >> "$GITHUB_OUTPUT"

fi

# Promotion publishes the rewritten packages artifact before opening its PR.

# Verify GitHub's prospective merge tree here, so a `packages/` edit pushed to

# this PR fails the required E2E Tests status before it can merge.

#

# Checking the MERGE tree rather than the head is what makes base drift

# visible when this job runs — but nothing re-runs it when the base moves:

# GitHub does not fire `pull_request` on the base branch advancing, it only

# recomputes refs/pull/N/merge. So a `packages/` change landing on the base

# after this went green is never re-verified here, and the PR stays mergeable

# on the stale result. Finalize repeats the check against the real merge

# commit before creating the write-once stable tag, which is where that case

# is actually caught — loudly, and before any stable name exists.

verify-release-candidate:

name: Verify release packages candidate

runs-on: ubuntu-latest

# Matching `checks` and `finalize`. Without a ceiling the job inherits the

# 6-hour default, and a stalled `flux pull artifact` would hold the required

# `E2E Tests` status pending for all of it, since e2e-report needs this job.

timeout-minutes: 30

if: |

contains(github.event.pull_request.labels.*.name, 'release')

&& github.event.pull_request.user.login == 'cozystack-ci[bot]'

&& (github.event.action != 'labeled' || github.event.label.name == 'full-e2e' || github.event.label.name == 'release')

permissions:

contents: read

packages: read

steps:

# The default pull_request checkout is GitHub's prospective merge commit,

# not just the head branch. That makes base-branch package drift visible.

- name: Checkout prospective merge

uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

with:

fetch-depth: 0

# Execute the verifier from the trusted base SHA. An older rc-derived head

# may not contain it, and a PR must not be able to weaken its own guard.

- name: Checkout release tooling from base

uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

with:

ref: ${{ github.event.pull_request.base.sha }}

path: .release-tooling

sparse-checkout: hack

persist-credentials: false

# Both tools are version-pinned. This job decides whether a stable release

# may be created, so what it runs has to be a fixed input: `latest/download`

# resolves to whatever mikefarah shipped this morning, and a yq that changed

# how it emits or compares a value turns a release gate into a moving one.

# The version tracks packages/core/testing/images/e2e-sandbox/Dockerfile.

# yq is additionally pinned by content; flux comes from the commit-pinned

# upstream setup action, which checksums the release tarball.

# promote-rc.yaml's copy of this step carries the full reasoning for both.

- name: Set up flux

uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6

with:

version: "2.8.6"

- name: Set up promotion toolchain (yq)

env:

YQ_VERSION: "4.53.3"

YQ_SHA256: "fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4"

run: |

if ! yq --version 2>/dev/null | grep -q "mikefarah.* version v${YQ_VERSION}\$"; then

yq_bin="$(mktemp)"

curl -fsSL -o "$yq_bin" \

"https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64"

printf '%s %s\n' "$YQ_SHA256" "$yq_bin" | sha256sum -c -

sudo install -m 0755 "$yq_bin" /usr/local/bin/yq

rm -f "$yq_bin"

fi

- name: Login to registry (GHCR)

uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3

with:

username: ${{ github.repository_owner }}

password: ${{ secrets.GITHUB_TOKEN }}

registry: ghcr.io

env:

DOCKER_CONFIG: ${{ runner.temp }}/.docker

- name: Verify stable packages candidate

env:

DOCKER_CONFIG: ${{ runner.temp }}/.docker

HEAD_BRANCH: ${{ github.head_ref }}

run: |

STABLE_VERSION="${HEAD_BRANCH#release-}"

.release-tooling/hack/verify-promoted-packages.sh "$STABLE_VERSION"

# Helm-unittest + Go controller tests. Decoupled from the image build and from

# e2e so they give fast, independent signal; they gate the PR as their own

# required check rather than blocking e2e from starting.

checks:

name: Unit & controller tests

runs-on: oracle-vm-4cpu-16gb-x86-64

timeout-minutes: 30

env:

# The pipe falls back to direct on any proxy error (hack/common-envs.mk).

GOPROXY: "https://proxy.golang.org|direct"

permissions:

contents: read

needs: ["plan"]

# Unit contracts read both code and documentation. Run this job for every

# non-release PR: code changes take the complete unit path, while docs-only

# changes run the Bats lane that owns those cross-tree contracts. Builds and

# E2E remain gated on `code` below.

if: |

!contains(github.event.pull_request.labels.*.name, 'release')

steps:

- name: Checkout code

uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

with:

fetch-depth: 0

# The module cache is restored from the key build-main.yaml writes on

# main, which every PR run can read, so a PR's first run starts warm. The

# download fetches whatever the cache lacks, the whole set on a miss. The

# Go steps are for code changes only: a docs-only run is the Bats lane alone.

- name: Locate Go module cache

if: needs.plan.outputs.code == 'true'

id: gomodcache

run: echo "dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"

- name: Cache Go modules

if: needs.plan.outputs.code == 'true'

uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0

with:

path: ${{ steps.gomodcache.outputs.dir }}

key: go-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }}

restore-keys: go-mod-${{ runner.os }}-

- name: Download Go modules

if: needs.plan.outputs.code == 'true'

run: |

for mod in $(git ls-files go.mod '*/go.mod' | xargs -n1 dirname); do

(cd "$mod" && "$GITHUB_WORKSPACE/hack/go-mod-download.sh")

done

# Ephemeral runners don't carry the helm-unittest plugin the persistent

# self-hosted runner accreted. The plugin embeds its own Helm, so its

# version is pinned: a plugin release must not change test results. Bats is

# installed on every run, docs-only ones included.

- name: Set up test toolchain

env:

CODE_CHANGED: ${{ needs.plan.outputs.code }}

HELM_UNITTEST_VERSION: "1.1.2"

BATS_VERSION: "1.14.0"

BATS_COMMIT: eb7f42f8d608ac693d7a4b67474f6714ea68cfc5

run: |

set -euo pipefail

if [ "$CODE_CHANGED" = true ]; then

installed="$(helm plugin list 2>/dev/null | awk '$1 == "unittest" { print $2 }' || true)"

if [ "$installed" != "$HELM_UNITTEST_VERSION" ]; then

[ -z "$installed" ] || helm plugin uninstall unittest

helm plugin install https://github.com/helm-unittest/helm-unittest --version "v${HELM_UNITTEST_VERSION}"

fi

fi

# Source archive bytes can change; verify the tag's commit before install.

if ! bats --version 2>/dev/null | grep -qx "Bats $BATS_VERSION"; then

bats_parent=$(mktemp -d "$RUNNER_TEMP/bats-core.XXXXXX")

bats_source="$bats_parent/source"

git clone --quiet --depth=1 --branch "v$BATS_VERSION" \

https://github.com/bats-core/bats-core.git "$bats_source"

test "$(git -C "$bats_source" rev-parse HEAD)" = "$BATS_COMMIT"

sudo "$bats_source/install.sh" /usr/local

fi

bats --version | grep -qx "Bats $BATS_VERSION"

# setup-envtest and the kube-apiserver and etcd it fetches, so a pull

# request downloads them only when the versions pinned in the Makefile

# change, and an outage at their sources cannot fail unrelated changes.

# The envtest target runs the controller suites a fake client cannot

# stand in for.

- name: Cache envtest binaries

if: needs.plan.outputs.code == 'true'

uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0

with:

path: _out/envtest

key: envtest-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }}

# -k is required, not cosmetic: unit-tests and test-controllers are two

# independent goals on the same command line, and without -k make stops

# handing out new work the moment any unit-tests prerequisite fails —

# test-controllers then never starts and reports nothing, so a red BATS

# file hides the entire Go controller suite. -k lets already-schedulable

# goals keep running; the job still fails overall on any red target.

- name: Run unit and controller tests

if: needs.plan.outputs.code == 'true'

run: make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target

- name: Run Bats unit tests for docs-only changes

if: needs.plan.outputs.code == 'false'

run: make bats-unit-tests

- name: Upload unit test report

if: always()

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4

with:

name: bats-unit-report

path: _out/test-reports/

if-no-files-found: ignore

# One image (package) per job, fanned out across the ephemeral pool. Each job

# gets its own buildx (docker-container driver) so the shared-buildkit

# contention that wedged the old single Build job (#2937) is severed by

# construction. Each job patches only its own package's values.yaml/.tag files

# and uploads that diff as a patch fragment; `finalize` reassembles them.

build:

name: Build ${{ matrix.pkg }}

needs: ["plan"]

if: |

needs.plan.outputs.code == 'true'

&& needs.plan.outputs.any == 'true'

&& !contains(github.event.pull_request.labels.*.name, 'release')

runs-on: oracle-vm-4cpu-16gb-x86-64

timeout-minutes: 30

permissions:

contents: read

strategy:

fail-fast: false

matrix:

pkg: ${{ fromJSON(needs.plan.outputs.matrix) }}

steps:

- name: Checkout code

uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

with:

fetch-depth: 0

fetch-tags: true

# installer is built in finalize, but some package image targets shell out

# to flux.

- name: Set up flux

uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6

with:

version: "2.8.6"

- name: Set up Docker config

run: |

if [ -d ~/.docker ]; then

cp -r ~/.docker "${{ runner.temp }}/.docker"

fi

- name: Login to GitHub Container Registry

if: ${{ !github.event.pull_request.head.repo.fork }}

uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3

with:

username: ${{ secrets.OCIR_USER }}

password: ${{ secrets.OCIR_TOKEN }}

registry: iad.ocir.io

env:

DOCKER_CONFIG: ${{ runner.temp }}/.docker

# docker-container driver: isolated buildkit per job (no shared-buildkit

# contention) and a prerequisite for the mode=max cache reads (#2938).

# Its buildkitd also routes docker.io base-image pulls through

# mirror.gcr.io (hack/buildkitd.toml): the Build step authenticates only

# to the push registry (OCIR), so base images (golang/ubuntu/nginx/...)

# would otherwise be pulled from Docker Hub anonymously and intermittently

# trip its anonymous-pull rate limit; the mirror sidesteps that and falls

# back to Docker Hub on a miss/outage. Create the builder under the SAME

# DOCKER_CONFIG the Build step reads (runner.temp/.docker, holding the

# OCIR auth) so buildx forwards push/cache auth to the container builder.

- name: Set up Buildx (docker-container driver, docker.io mirror)

id: buildx

uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0

with:

driver: docker-container

buildkitd-config: hack/buildkitd.toml

env:

DOCKER_CONFIG: ${{ runner.temp }}/.docker

- name: Build ${{ matrix.pkg }}

run: |

mkdir -p _out/oci

make -C "$PKG" image

env:

PKG: ${{ matrix.pkg }}

DOCKER_CONFIG: ${{ runner.temp }}/.docker

# Build on the docker-container builder created above so base-image

# pulls go through the docker.io mirror (BUILDX_ARGS adds

# --builder=$(BUILDER)).

BUILDER: ${{ steps.buildx.outputs.name }}

# common-envs.mk defaults to amd64+arm64. Nothing here sets up QEMU,

# and on this x86-64 runner the arm64 half would be emulated anyway;

# CI builds amd64 until arm64 has a native leg of its own.

PLATFORM: linux/amd64

# Unique per (PR, commit) so concurrent PR builds never push the same

# tag. PR images are consumed by digest; :latest belongs to releases.

IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}

PUBLISH_VERSIONED: '0'

PUBLISH_FLOATING: '0'

# Fork PRs get no registry credentials, so instead of pushing, build

# each image to <workspace>/_out/oci/<name>.oci.tar and upload it below;

# the privileged e2e-fork.yaml workflow_run pushes them by digest. Empty

# on same-repo PRs → normal push path. Setting it also forces PUSH/LOAD

# off in common-envs.mk (#3257). Must be ABSOLUTE (workspace-rooted):

# the build runs `make -C <pkg>`, so buildx resolves a relative dest

# against the package dir — the archive would land in <pkg>/_out/oci and

# miss the repo-root _out/oci the upload reads.

OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}

- name: Capture digest patch fragment

id: frag

env:

PKG: ${{ matrix.pkg }}

run: |

safe="$(echo "$PKG" | tr '/' '-')"

mkdir -p _out/fragments

# --binary so vendored binary provider bundles (components*.gz),

# which some package image targets regenerate, survive reassembly

# in finalize (a plain "Binary files differ" stub is unappliable).

git diff --binary HEAD > "_out/fragments/${safe}.patch"

echo "safe=${safe}" >> "$GITHUB_OUTPUT"

echo "Fragment for ${PKG}:"; git --no-pager diff --stat HEAD

- name: Upload patch fragment

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4

with:

name: pr-patch-fragment-${{ steps.frag.outputs.safe }}

path: _out/fragments/${{ steps.frag.outputs.safe }}.patch

if-no-files-found: error

# Fork PR only: the images were exported to OCI archives instead of pushed

# (OCI_EXPORT_DIR above). Upload them for e2e-fork.yaml to push by digest.

- name: Upload OCI image archives (fork PR)

if: ${{ github.event.pull_request.head.repo.fork }}

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4

with:

name: pr-oci-${{ steps.frag.outputs.safe }}

path: _out/oci/*.oci.tar

if-no-files-found: error

# Talos is its own leg: the installer tarball is heavy and the Talos image is

# built from it, so both live in one job. No nocloud disk is built here any

# more, and e2e no longer downloads one -- the container lane boots the

# upstream Talos container image. `images/talos/profiles/nocloud.yaml` is

# therefore compiled by nightly.yaml and by the tag build's `make assets`.

#

# For same-repo PRs, the plan job classifies the rename-blind changed-path

# list with build-talos-needed.sh. That list includes direct Talos sources and

# the shared Makefile/Buildx/workflow inputs that determine this job's output;

# line-based classification avoids depending on the formatting of `touched`.

# A successful run stamps packages/extra/bootbox/images/matchbox.tag, and

# finalize preserves that generated ref when it overlays base-branch images.

#

# Fork PRs keep the previous unconditional behavior as a conservative scope

# boundary. Their build/export and privileged publication path spans a second

# workflow and is not optimized here. The installer job also exports an OCI

# archive, so archive availability is not a reason to require Talos itself.

build-talos:

name: Build Talos

needs: ["plan"]

if: |

needs.plan.outputs.code == 'true'

&& !contains(github.event.pull_request.labels.*.name, 'release')

&& (

github.event.pull_request.head.repo.fork

|| needs.plan.outputs.talos == 'true'

)

runs-on: oracle-vm-4cpu-16gb-x86-64

timeout-minutes: 40

permissions:

contents: read

steps:

- name: Checkout code

uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

with:

fetch-depth: 0

fetch-tags: true

- name: Set up Docker config

run: |

if [ -d ~/.docker ]; then

cp -r ~/.docker "${{ runner.temp }}/.docker"

fi

- name: Login to GitHub Container Registry

if: ${{ !github.event.pull_request.head.repo.fork }}

uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3

with:

username: ${{ secrets.OCIR_USER }}

password: ${{ secrets.OCIR_TOKEN }}

registry: iad.ocir.io

env:

DOCKER_CONFIG: ${{ runner.temp }}/.docker

# buildkitd-config routes docker.io base-image pulls through mirror.gcr.io

# (hack/buildkitd.toml) so the talos image build dodges Docker Hub's

# anonymous-pull rate limit; see the build job for the full rationale.

- name: Set up Buildx (docker-container driver, docker.io mirror)

id: buildx

uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0

with:

driver: docker-container

buildkitd-config: hack/buildkitd.toml

env:

DOCKER_CONFIG: ${{ runner.temp }}/.docker

- name: Build Talos image (matchbox + installer tarball)

run: |

mkdir -p _out/oci

make -C packages/core/talos image

env:

DOCKER_CONFIG: ${{ runner.temp }}/.docker

BUILDER: ${{ steps.buildx.outputs.name }}

# amd64 only, as in the build job. The arm64 installer needs no QEMU,

# but nothing a PR runs boots it. matchbox still gets the arm64 kernel

# and initramfs.

PLATFORM: linux/amd64

IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}

PUBLISH_VERSIONED: '0'

PUBLISH_FLOATING: '0'

# Fork PR: export to OCI archive instead of pushing (see build job).

OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}

- name: Capture Talos patch fragment

run: |

mkdir -p _out/fragments

git diff --binary HEAD > _out/fragments/talos.patch

echo "Talos fragment:"; git --no-pager diff --stat HEAD

- name: Upload Talos patch fragment

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4

with:

name: pr-patch-fragment-talos

path: _out/fragments/talos.patch

if-no-files-found: error

# Fork PR only: upload the Talos image OCI archive for e2e-fork.yaml.

- name: Upload Talos OCI image archive (fork PR)

if: ${{ github.event.pull_request.head.repo.fork }}

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4

with:

name: pr-oci-talos

path: _out/oci/*.oci.tar

if-no-files-found: error

# Reassemble the unified pr.patch from every fragment, then build the installer

# LAST: `make -C packages/core/installer image` runs `flux push artifact

# --path=packages`, which bundles the WHOLE digest-patched tree into the OCI

# artifact the operator pulls — so every other package's digest edit must

# already be applied here. e2e consumes the resulting pr.patch unchanged.

finalize:

name: Finalize (merge patches + installer)

# Depends on `checks` too: gating finalize on unit/controller tests cascades

# to e2e (which needs finalize), so a red unit test skips the expensive

# install+e2e path instead of wasting ~1h on it. Restores the gating the old

# in-build-job tests gave, without re-coupling the cheap parallel builds.

needs: ["plan", "checks", "build", "build-talos"]

if: |

!cancelled()

&& needs.plan.outputs.code == 'true'

&& !contains(github.event.pull_request.labels.*.name, 'release')

&& needs.checks.result == 'success'

&& (needs.build.result == 'success' || needs.build.result == 'skipped')

&& (needs.build-talos.result == 'success' || needs.build-talos.result == 'skipped')

runs-on: oracle-vm-4cpu-16gb-x86-64

timeout-minutes: 30

permissions:

contents: read

steps:

- name: Checkout code

uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

with:

fetch-depth: 0

fetch-tags: true

- name: Set up flux

uses: fluxcd/flux2/action@04acaec6161ac4fb1a82ffafa88901c03271d34f # v2.8.6

with:

version: "2.8.6"

- name: Download patch fragments

uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4

with:

pattern: pr-patch-fragment-*

merge-multiple: true

path: _out/fragments

- name: Apply digest fragments

run: |

applied=0

# Each fragment patches only its own package's files; --3way lets two

# fragments that touch adjacent lines of a shared file (e.g. a package

# Makefile that writes a sibling package's values.yaml) merge cleanly

# instead of the second apply aborting the whole finalize.

for f in _out/fragments/*.patch; do

[ -s "$f" ] || continue

echo "Applying $f"

git apply --3way "$f"

applied=$((applied + 1))

done

echo "Applied $applied fragment(s)"

git --no-pager diff --stat HEAD

- name: Set up Docker config

run: |

if [ -d ~/.docker ]; then

cp -r ~/.docker "${{ runner.temp }}/.docker"

fi

- name: Login to GitHub Container Registry

if: ${{ !github.event.pull_request.head.repo.fork }}

uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3

with:

username: ${{ secrets.OCIR_USER }}

password: ${{ secrets.OCIR_TOKEN }}

registry: iad.ocir.io

env:

DOCKER_CONFIG: ${{ runner.temp }}/.docker

# Overlay current-main image refs onto every package this PR did NOT

# rebuild. Without it, unbuilt packages fall back to the last-release refs

# committed in the tree (e.g. v1.5.0) even when main is hundreds of commits

# ahead — so e2e and the installer's packages artifact would ship stale

# images for everything outside this PR's build matrix. Source of truth is

# the cozystack-packages:main OCI artifact build-main.yaml already

# publishes (the whole current-main packages tree, refs digest-pinned to

# main's images). Runs after login (creds available) and before the

# installer build below so the flux-pushed packages tree — and the

# pr.patch e2e applies — already reflect the overlay. A missing artifact

# (build-main not yet green) is a no-op: packages keep their committed

# refs (prior behaviour). See hack/overlay-main-images.sh.

#

# Read the artifact for the PR's OWN base branch, not always main. Overlaying

# main's images onto a release-line PR mixes generations: main's binaries

# against that line's charts. #3437 (a one-line change on release-1.6) failed

# install deterministically that way, because main's cozystack-controller

# served an aggregated OpenAPI release-1.6's charts could not validate

# against (`unknown model in reference: …v1alpha1.OptionSpec`) — a broken

# lane reading as a broken PR. build-release.yaml publishes

# cozystack-packages:<line> for every maintained release-<major>.<minor>

# branch, so each base branch has its own artifact to read.

#

# A base that is neither main nor a line (a stacked PR's feature branch) has

# no artifact, and a `/` in it is not even a valid OCI tag. It reads the

# artifact of whichever of main and the lines it has the fewest commits on

# top of. A tie goes to main. A line with no commit of its own yet is, in

# the graph, main at the cut commit, so a stack on it ties and reads the

# artifact built from main's tip. That artifact carries newer images than

# the line once main moves past the cut. The stack keeps reading it until

# it contains a commit of the line's own. Breaking the tie towards the

# line instead would misroute the far more common branch forked from main

# before the cut, which ties the same way.

#

# A missing artifact stays a no-op (packages keep their committed refs), which

# also covers a line whose first build-release run has not landed yet.

- name: Pull base-branch packages tree

run: |

ARTIFACT_REF="$BASE_REF"

if [ "$BASE_REF" != "main" ] \

&& ! printf '%s' "$BASE_REF" | grep -qE '^release-[0-9]+\.[0-9]+$'; then

ARTIFACT_REF=main

# A re-run can outlive its base: the payload keeps a branch deleted since.

if git rev-parse --verify --quiet "origin/${BASE_REF}" >/dev/null; then

best="$(git rev-list --count "origin/main..origin/${BASE_REF}")"

for line in $(git for-each-ref --format='%(refname:lstrip=3)' 'refs/remotes/origin/release-*' \

| grep -E '^release-[0-9]+\.[0-9]+$'); do

n="$(git rev-list --count "origin/${line}..origin/${BASE_REF}")"

if [ "$n" -lt "$best" ]; then best="$n"; ARTIFACT_REF="$line"; fi

done

fi

echo "::notice::Base ${BASE_REF} publishes no packages artifact; overlaying cozystack-packages:${ARTIFACT_REF}."

fi

rm -rf _out/mainpkgs

mkdir -p _out/mainpkgs

if ! flux pull artifact "oci://${REGISTRY}/cozystack-packages:${ARTIFACT_REF}" \

--output _out/mainpkgs; then

rm -rf _out/mainpkgs

# Degrading to committed refs is safe, but SAY so: on a release line a

# missing artifact means build-release.yaml has not published this line

# yet (first build pending, or its branch filter does not match this

# branch), and the PR is then testing the line's last release instead of

# its tip. Silent would look identical to a working overlay.

if [ "${ARTIFACT_REF}" = "main" ]; then

echo "cozystack-packages:${ARTIFACT_REF} pull failed — keeping committed refs"

else

echo "::warning title=No packages artifact for ${ARTIFACT_REF}::Keeping committed refs, so unbuilt packages use this line's last release rather than its tip. Check that build-release.yaml has run for ${ARTIFACT_REF}."

fi

fi

env:

DOCKER_CONFIG: ${{ runner.temp }}/.docker

# Env-passed, not interpolated into the script: base_ref is repo-controlled

# here (a branch name in this repo), but the workflow-injection hygiene

# applies uniformly.

BASE_REF: ${{ github.base_ref }}

- name: Overlay base-branch refs for unbuilt packages

# Best-effort: this only improves WHICH images e2e and the installer use,

# so an unexpected failure must degrade to the committed refs (prior

# behaviour), never block the PR. overlay-main-images.sh already handles

# the expected paths (missing artifact, drift, absent unit) internally —

# including the empty directory the step above leaves when a line has no

# artifact yet.

run: |

rebuilt="$BUILD_MATRIX"

if [ "$TALOS_RESULT" = "success" ]; then

case "$rebuilt" in

'[]') rebuilt='["packages/extra/bootbox"]' ;;

*) rebuilt="${rebuilt%]},\"packages/extra/bootbox\"]" ;;

esac

fi

hack/overlay-main-images.sh _out/mainpkgs "$rebuilt" "$PR_TOUCHED" \

|| echo "::warning::overlay-main-images failed; unbuilt packages keep committed refs"

env:

# The set of packages THIS PR rebuilt; their pr-<N>-<sha> refs are

# authoritative, so overlay-main-images.sh skips them. Computed by the

# plan job from the Makefile (a JSON array of package dirs), not from

# untrusted PR input — passed via env per workflow-injection hygiene.

BUILD_MATRIX: ${{ needs.plan.outputs.matrix }}

# build-talos writes matchbox.tag under extra/bootbox rather than its

# own package. Preserve that generated ref only when the job actually

# succeeded; when it was skipped, the base-branch ref should overlay.

TALOS_RESULT: ${{ needs.build-talos.result }}

# Packages THIS PR edited (changed-file dirs from the plan job's git

# diff). Their committed refs are the PR's intent — an upstream image

# bump in a non-build-unit package would otherwise be silently reverted

# by the overlay — so overlay-main-images.sh preserves them. Also

# env-passed, derived from the diff, not from untrusted PR input.

PR_TOUCHED: ${{ needs.plan.outputs.touched }}

# buildkitd-config routes docker.io base-image pulls through mirror.gcr.io

# (hack/buildkitd.toml) so the installer build dodges Docker Hub's

# anonymous-pull rate limit; see the build job for the full rationale.

- name: Set up Buildx (docker-container driver, docker.io mirror)

id: buildx

uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0

with:

driver: docker-container

buildkitd-config: hack/buildkitd.toml

env:

DOCKER_CONFIG: ${{ runner.temp }}/.docker

# Same-repo PR: build the operator image AND flux-push the now-fully-patched

# packages tree, then patch the installer's platformSourceRef to that

# artifact. Fork PR: no registry creds — build ONLY the operator image to an

# OCI archive (image-packages/chart, which push, are skipped) and upload it;

# e2e-fork.yaml pushes the operator + flux-pushes the packages tree and

# bakes platformSourceRef there. The overlay above already ran, so the

# pr.patch assembled below carries the fully digest-pinned tree either way.

- name: Build installer (operator + packages artifact)

run: |

mkdir -p _out/oci

if [ "$IS_FORK" = "true" ]; then

make -C packages/core/installer pre-checks image-operator

else

make -C packages/core/installer image

fi

env:

DOCKER_CONFIG: ${{ runner.temp }}/.docker

BUILDER: ${{ steps.buildx.outputs.name }}

# amd64 only, as in the build job.

PLATFORM: linux/amd64

IMAGE_TAG: pr-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}

PUBLISH_VERSIONED: '0'

PUBLISH_FLOATING: '0'

IS_FORK: ${{ github.event.pull_request.head.repo.fork }}

OCI_EXPORT_DIR: ${{ github.event.pull_request.head.repo.fork && format('{0}/_out/oci', github.workspace) || '' }}

# Fork PR only: upload the operator image OCI archive for e2e-fork.yaml.

- name: Upload installer OCI image archive (fork PR)

if: ${{ github.event.pull_request.head.repo.fork }}

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4

with:

name: pr-oci-installer

path: _out/oci/*.oci.tar

if-no-files-found: error

- name: Assemble pr.patch

run: |

mkdir -p _out/assets

# --binary so regenerated provider bundles (components*.gz) stay in

# the patch e2e applies; mirrors main's pr.patch binary fix (#e566992).

git diff --binary HEAD > _out/assets/pr.patch

echo "Final pr.patch:"; git --no-pager diff --stat HEAD

- name: Upload git diff patch

uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4

with:

name: pr-patch

path: _out/assets/pr.patch

if-no-files-found: error

resolve_assets:

name: "Resolve assets"

runs-on: ubuntu-latest

# In lockstep with `plan`, fork term included: this job is on `e2e`'s

# release arm, so a label event `plan` runs for and this job skips for

# leaves `e2e` with both arms false while `e2e-report` still runs. On a

# promote PR already carrying full-e2e that reaches the final `else` — the

# "e2e not repeated" branch is bypassed by full-e2e — and posts `E2E Tests`

# = failure over a green one, unclearable without a push because

# `unlabeled` is not a trigger. The fork term keeps a `release` label on a

# fork PR as inert as it was before promotion used the label: without it

# this job runs there and fails resolving a draft release for a branch that

# is not `release-X.Y.Z`. It short-circuits on `action != 'labeled'`, so the

# pre-existing behaviour on a push to such a PR is unchanged.

if: |

contains(github.event.pull_request.labels.*.name, 'release')

&& (github.event.action != 'labeled' || github.event.label.name == 'full-e2e' || (github.event.label.name == 'release' && !github.event.pull_request.head.repo.fork))

permissions:

contents: read

steps:

- name: Generate GitHub App token

id: app-token

uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1

with:

app-id: ${{ secrets.COZYSTACK_CI_APP_ID }}

private-key: ${{ secrets.COZYSTACK_CI_PRIVATE_KEY }}

owner: cozystack

repositories: ${{ github.event.repository.name }}

permission-contents: read

- name: Checkout code

if: contains(github.event.pull_request.labels.*.name, 'release')

uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

with:

fetch-depth: 0

fetch-tags: true

- name: Extract tag from PR branch (release PR)

if: contains(github.event.pull_request.labels.*.name, 'release')

id: get_tag

uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7

with:

script: |

const branch = context.payload.pull_request.head.ref;

const m = branch.match(/^release-(\d+\.\d+\.\d+(?:[-\w\.]+)?)$/);

if (!m) {

core.setFailed(`❌ Branch '${branch}' does not match 'release-X.Y.Z[-suffix]'`);

return;

}

core.setOutput('tag', `v${m[1]}`);

# The draft release has to exist for a promote PR's e2e arm to mean

# anything, and this job is what proves it. It used to also resolve the

# nocloud disk asset and export its id; the container lane boots no disk,

# so nothing in this workflow reads that id any more and resolving it only

# added a way to fail a release PR over an asset it does not use. The

# asset's presence is still checked in e2e-tag.yaml and promote-rc.yaml.

- name: Verify draft release exists (release PR)

if: contains(github.event.pull_request.labels.*.name, 'release')

uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7

with:

github-token: ${{ steps.app-token.outputs.token }}

script: |

const tag = '${{ steps.get_tag.outputs.tag }}';

const releases = await github.rest.repos.listReleases({

owner: context.repo.owner,

repo: context.repo.repo,

per_page: 100

});

const draft = releases.data.find(r => r.tag_name === tag && r.draft);

if (!draft) {

core.setFailed(`Draft release '${tag}' not found`);

return;

}

# In-tree e2e for SAME-REPO PRs, on Talos nodes running as CONTAINERS, like

# every other e2e lane. What containers cannot carry has no lane at all:

# DRBD, and with it the `replicated` StorageClass, its Immediate binding mode

# and the tenant StorageClass propagation that only applies to

# remotely-accessible classes.

#

# Deliberately NOT named "E2E Tests": the

# required "E2E Tests" context is a commit status posted by the `e2e-report`

# job (non-fork) or by e2e-fork.yaml (fork). A required *job* that skips

# counts as PASSED by branch protection, which is exactly how fork PRs used

# to merge with e2e never run (#3257); routing the required context through

# a status removes that skip-to-green path. Fork PRs skip this job

# (they have no pushed images to pull) and run e2e in the privileged

# workflow_run instead.

e2e:

name: "E2E (in-tree)"

# hack/e2e-compose.yaml gives each of the three nodes `cpus: 8` and

# `mem_limit: 24576m`, matching what the QEMU guests were given, so the nodes

# still demand 24 vCPU and 72 GiB between them. The same headroom argument

# therefore still decides the class: on a 24-vCPU runner that is a 1:1 commit

# with nothing left for containerd, the kernel, the host iptables MASQUERADE

# every image pull traverses and the runner agent, and during "Install

# Cozystack" the nodes contend with those and lose cycles. Readiness and

# liveness probes then miss their deadlines, a HelmRelease flips InProgress ->

# Failed and its dependents cascade to DependencyNotReady -- on a different

# component each run, the signature of resource thrash rather than a product

# bug. The 32-vCPU / 128 GiB class leaves 8 vCPU (25%) and 56 GiB of headroom

# and removes that oversubscription.

#

# One thing this class does NOT fix, and it is worth knowing before reading a

# red run: a container node reports the HOST's CPU and memory as its capacity,

# and hack/e2e-container-up.sh counterbalances that with kubelet

# systemReserved so the scheduler sees 8 CPU / 24 GiB. That corrects the

# scheduler and not what a pod reads from /proc, so a workload that sizes

# itself from /proc/cpuinfo or /proc/meminfo is configured for the whole host.

runs-on: oracle-vm-32cpu-128gb-x86-64

# Covers a green full suite. It does not cover a tenant-Kubernetes suite

# running into its node-join deadline, so a red there can end as a

# cancelled job. Held to one figure across every lane that runs the suite.

timeout-minutes: 215

permissions:

contents: read

packages: read

checks: write

needs: ["finalize", "resolve_assets", "verify-release-candidate"]

# Regular PRs still enter through finalize. A release-labelled promote PR

# enters through resolve_assets only when a maintainer explicitly adds

# full-e2e; without that label the immutable rc's already-green suite is not

# repeated. The labeled trigger above makes that post-open opt-in start a run.

#

# Fork PRs never enter here: their build exports OCI archives instead of

# pushing images, so there is no closure for this job to test. e2e-fork.yaml

# runs the suite for them from the privileged workflow_run and concludes the

# required "E2E Tests" status; for same-repo PRs e2e-report concludes it.

if: |

!cancelled()

&& !github.event.pull_request.head.repo.fork

&& (

needs.finalize.result == 'success'

|| (

needs.resolve_assets.result == 'success'

&& needs.verify-release-candidate.result == 'success'

&& contains(github.event.pull_request.labels.*.name, 'release')

&& contains(github.event.pull_request.labels.*.name, 'full-e2e')

)

)

steps:

# ▸ Checkout and prepare the codebase

# fetch-depth: 0 is required by the "Select E2E tests" step so that

# `git diff origin/${BASE_REF}...HEAD` can resolve the base ref.

- name: Checkout code

uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4

with:

fetch-depth: 0

# ▸ Regular PR path – pr.patch from finalize. No Talos disk: the nodes boot

# from ghcr.io/siderolabs/talos directly (hack/e2e-compose.yaml), so this

# lane consumes no nocloud asset and waits on no Talos build.

- name: Download PR patch

if: "!contains(github.event.pull_request.labels.*.name, 'release')"

uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4

with:

name: pr-patch