From aa106512fc32ff7f0104a893f83f7a3a05e2ccdc Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 24 Jul 2025 13:22:49 +0200 Subject: [PATCH 1/4] fix: Update git state before doing important things --- release/create-release-candidate-branch.sh | 8 ++++++++ release/post-release.sh | 12 ++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/release/create-release-candidate-branch.sh b/release/create-release-candidate-branch.sh index 6623cec..5e1a9ae 100755 --- a/release/create-release-candidate-branch.sh +++ b/release/create-release-candidate-branch.sh @@ -97,6 +97,10 @@ check_products() { fi cd "$TEMP_RELEASE_FOLDER/$DOCKER_IMAGES_REPO" + # Need to update here because if we deleted the local state, or someone else continues + # we might be back on main, or on the release branch without having pulled updates from fixes. + git fetch && git switch "$RELEASE_BRANCH" && git pull + # switch to the release branch, which should exist as tagging # is subsequent to creating the branch. # Note, if this needs to check the branch exists locally, then use: @@ -133,6 +137,10 @@ check_operators() { fi cd "$TEMP_RELEASE_FOLDER/${operator}" + + # Need to update here because if we deleted the local state, or someone else continues + # we might be back on main, or on the release branch without having pulled updates from fixes. + git fetch && git switch "$RELEASE_BRANCH" && git pull # Note, if this needs to check the branch exists locally, then use: # "^[ *]*$RELEASE_BRANCH\$" if ! git branch -a | grep -E "$RELEASE_BRANCH\$"; then diff --git a/release/post-release.sh b/release/post-release.sh index 57f22bd..6f7f07f 100755 --- a/release/post-release.sh +++ b/release/post-release.sh @@ -84,10 +84,14 @@ update_operators() { while IFS="" read -r OPERATOR || [ -n "$OPERATOR" ] do cd "$TEMP_RELEASE_FOLDER/$OPERATOR" + + git checkout main + git pull + # New branch that updates the CHANGELOG CHANGELOG_BRANCH="chore/update-changelog-from-release-$RELEASE_TAG" # Branch out from main - git switch -c "$CHANGELOG_BRANCH" main + git switch -c "$CHANGELOG_BRANCH" # Checkout CHANGELOG changes from the release tag git checkout "$RELEASE_TAG" -- CHANGELOG.md # Ensure only the CHANGELOG has been modified and there @@ -146,10 +150,14 @@ check_products() { # the changelog in the release branch. update_products() { cd "$TEMP_RELEASE_FOLDER/$DOCKER_IMAGES_REPO" + + git checkout main + git pull + # New branch that updates the CHANGELOG CHANGELOG_BRANCH="chore/update-changelog-from-release-$RELEASE_TAG" # Branch out from main - git switch -c "$CHANGELOG_BRANCH" main + git switch -c "$CHANGELOG_BRANCH" # Checkout CHANGELOG changes from the release tag git checkout "$RELEASE_TAG" -- CHANGELOG.md # Ensure only the CHANGELOG has been modified and there From c8467cf36c7b7daa337e118dcc7705e0592b2a1a Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 24 Jul 2025 13:56:42 +0200 Subject: [PATCH 2/4] chore(release): Add image check script to ease the process of validating all images have been pushed. --- release/image-checks.sh | 126 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100755 release/image-checks.sh diff --git a/release/image-checks.sh b/release/image-checks.sh new file mode 100755 index 0000000..53df0f8 --- /dev/null +++ b/release/image-checks.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +# Usage: +# 1. Update the product and SDP versions at the end of the scripts +# 2. ./release/image-checks.sh +# 3. Check for missing artifacts in the output (it should be an obvious change in the pattern) +# +# It could be improved in future to assert things exist, and provide a clear error message when they don't. + +# This is a handy utility for checking the existance of images. +# It does currently reqiure manually choosing product versions to check, +# but this list could be automated through conf.py +read -p "Be sure to update the versions in this script before running. Press Enter to continue" + +# prepend a string to each line of stdout +# Usage: +# echo "patch written to: $PATCH" | prepend "\t" +function prepend { + while read -r line; do + echo -e "${1}${line}" + done +} + +if [ -z "$HARBOR_TOKEN" ]; then + read -s -p "Harbor Token (find it in the web UI): " HARBOR_TOKEN +fi + +HARBOR_AUTH_HEADER="Authorization: Bearer $HARBOR_TOKEN" +TAGS_JQ_QUERY='.tags[] | select(. == $tag)' + +# Check that per-arch images and a manifests exist for an operator. +# Usage: +# check_tags_for_operator airflow-operator 25.7.0 +# check_tags_for_operator airflow-operator 25.7.0 sandbox +check_tags_for_operator() { + local operator="$1" + local sdp_version="$2" + local project="${3:-sdp}" + + echo "Checking $operator for SDP $sdp_version" + local tags_url="https://oci.stackable.tech/v2/$project/$operator/tags/list" + + curl -sSL -H "$HARBOR_AUTH_HEADER" "$tags_url" | jq -re "$TAGS_JQ_QUERY" --arg tag "$sdp_version-arm64" | prepend "\t" + curl -sSL -H "$HARBOR_AUTH_HEADER" "$tags_url" | jq -re "$TAGS_JQ_QUERY" --arg tag "$sdp_version-amd64" | prepend "\t" + curl -sSL -H "$HARBOR_AUTH_HEADER" "$tags_url" | jq -re "$TAGS_JQ_QUERY" --arg tag "$sdp_version" | prepend "\t" +} + + +# this could use some work to handle products under stackable instead of sdp +# Check that per-arch images and a manifests exist for a product. +# Usage: +# check_tags_for_product spark-k8s 3.5.6 25.7.0-rc1 +# check_tags_for_product spark-connect-client 3.5.6 25.7.0 stackable +check_tags_for_product() { + local product_name="$1" + local product_version="$2" + local sdp_version="$3" + local project="${4:-sdp}" + + echo "Checking $project/$product_name@$product_version for SDP $sdp_version" + local tags_url="https://oci.stackable.tech/v2/$project/$product_name/tags/list" + + curl -sSL -H "$HARBOR_AUTH_HEADER" "$tags_url" | jq -re "$TAGS_JQ_QUERY" --arg tag "$product_version-stackable$sdp_version-arm64" | prepend "\t" + curl -sSL -H "$HARBOR_AUTH_HEADER" "$tags_url" | jq -re "$TAGS_JQ_QUERY" --arg tag "$product_version-stackable$sdp_version-amd64" | prepend "\t" + curl -sSL -H "$HARBOR_AUTH_HEADER" "$tags_url" | jq -re "$TAGS_JQ_QUERY" --arg tag "$product_version-stackable$sdp_version" | prepend "\t" +} + +# How it was run for the previous release: +SDP_RELEASE=25.7.0 +check_tags_for_operator airflow-operator "$SDP_RELEASE" +check_tags_for_operator commons-operator "$SDP_RELEASE" +check_tags_for_operator druid-operator "$SDP_RELEASE" +check_tags_for_operator hbase-operator "$SDP_RELEASE" +check_tags_for_operator hdfs-operator "$SDP_RELEASE" +check_tags_for_operator hive-operator "$SDP_RELEASE" +check_tags_for_operator kafka-operator "$SDP_RELEASE" +check_tags_for_operator listener-operator "$SDP_RELEASE" +check_tags_for_operator nifi-operator "$SDP_RELEASE" +check_tags_for_operator opa-operator "$SDP_RELEASE" +check_tags_for_operator secret-operator "$SDP_RELEASE" +check_tags_for_operator spark-k8s-operator "$SDP_RELEASE" +check_tags_for_operator superset-operator "$SDP_RELEASE" +check_tags_for_operator trino-operator "$SDP_RELEASE" +check_tags_for_operator zookeeper-operator "$SDP_RELEASE" + +# Be sure to check the product versions for the release you a checking for. +# This list was hand generated. It could be improved by evaluating conf.py. +check_tags_for_product airflow 2.9.3 "$SDP_RELEASE" +check_tags_for_product airflow 2.10.4 "$SDP_RELEASE" +check_tags_for_product airflow 2.10.5 "$SDP_RELEASE" +check_tags_for_product airflow 3.0.1 "$SDP_RELEASE" +check_tags_for_product druid 30.0.1 "$SDP_RELEASE" +check_tags_for_product druid 31.0.1 "$SDP_RELEASE" +check_tags_for_product druid 33.0.0 "$SDP_RELEASE" +check_tags_for_product hadoop 3.3.6 "$SDP_RELEASE" +check_tags_for_product hadoop 3.4.1 "$SDP_RELEASE" +check_tags_for_product hbase 2.6.1 "$SDP_RELEASE" +check_tags_for_product hbase 2.6.2 "$SDP_RELEASE" +check_tags_for_product hive 3.1.3 "$SDP_RELEASE" +check_tags_for_product hive 4.0.0 "$SDP_RELEASE" +check_tags_for_product hive 4.0.1 "$SDP_RELEASE" +check_tags_for_product kafka-testing-tools 1.0.0 "$SDP_RELEASE" +check_tags_for_product kafka 3.7.2 "$SDP_RELEASE" +check_tags_for_product kafka 3.9.0 "$SDP_RELEASE" +check_tags_for_product kafka 3.9.1 "$SDP_RELEASE" +check_tags_for_product kafka 4.0.0 "$SDP_RELEASE" +check_tags_for_product krb5 1.21.1 "$SDP_RELEASE" +check_tags_for_product nifi 1.27.0 "$SDP_RELEASE" +check_tags_for_product nifi 1.28.1 "$SDP_RELEASE" +check_tags_for_product nifi 2.4.0 "$SDP_RELEASE" +check_tags_for_product omid 1.1.2 "$SDP_RELEASE" +check_tags_for_product omid 1.1.3 "$SDP_RELEASE" +check_tags_for_product opa 1.4.2 "$SDP_RELEASE" +check_tags_for_product opa 1.0.1 "$SDP_RELEASE" +check_tags_for_product spark-connect-client 3.5.6 "$SDP_RELEASE" stackable +check_tags_for_product spark-k8s 3.5.5 "$SDP_RELEASE" +check_tags_for_product spark-k8s 3.5.6 "$SDP_RELEASE" +check_tags_for_product superset 4.0.2 "$SDP_RELEASE" +check_tags_for_product superset 4.1.1 "$SDP_RELEASE" +check_tags_for_product superset 4.1.2 "$SDP_RELEASE" +check_tags_for_product tools 1.0.0 "$SDP_RELEASE" +check_tags_for_product trino 451 "$SDP_RELEASE" +check_tags_for_product trino 470 "$SDP_RELEASE" +check_tags_for_product trino 476 "$SDP_RELEASE" +check_tags_for_product vector 0.47.0 "$SDP_RELEASE" +check_tags_for_product zookeeper 3.9.3 "$SDP_RELEASE" From 272438f18c28818c3eb1bedf27ece29d40090323 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 24 Jul 2025 14:33:02 +0200 Subject: [PATCH 3/4] fix(release): Allow pull failure in dry-run mode --- release/tag-release-candidate.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/release/tag-release-candidate.sh b/release/tag-release-candidate.sh index 67d6c59..2952f6d 100755 --- a/release/tag-release-candidate.sh +++ b/release/tag-release-candidate.sh @@ -17,7 +17,13 @@ tag_products() { cd "$TEMP_RELEASE_FOLDER/$DOCKER_IMAGES_REPO" # the PR branch should already exist - git switch "$RELEASE_BRANCH" && git pull + git switch "$RELEASE_BRANCH" + if $PUSH; then + git pull + else + git pull || echo "Dry-run: remote branch doesn't exist yet..." + # NOTE (@NickLarsenNZ): We could add a fake commit, but that would poison the current state. + fi git tag -sm "release $RELEASE_TAG" "$RELEASE_TAG" push_branch } @@ -25,7 +31,13 @@ tag_products() { tag_operators() { while IFS="" read -r operator || [ -n "$operator" ]; do cd "${TEMP_RELEASE_FOLDER}/${operator}" - git switch "$RELEASE_BRANCH" && git pull + git switch "$RELEASE_BRANCH" + if $PUSH; then + git pull + else + git pull || echo "Dry-run: remote branch doesn't exist yet..." + # NOTE (@NickLarsenNZ): We could add a fake commit, but that would poison the current state. + fi git tag -sm "release $RELEASE_TAG" "$RELEASE_TAG" push_branch done < <(yq '... comments="" | .operators[] ' "$INITIAL_DIR"/release/config.yaml) From 60ab3343191d9b3547d2b2ec5841bc5e0d560668 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 24 Jul 2025 15:17:39 +0200 Subject: [PATCH 4/4] docs(release): Add example process from dry-run, through release-candidates, to the actual releae. --- release/EXAMPLE_PROCESS.md | 184 +++++++++++++++++++++++++++++++++++++ release/README.md | 5 +- 2 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 release/EXAMPLE_PROCESS.md diff --git a/release/EXAMPLE_PROCESS.md b/release/EXAMPLE_PROCESS.md new file mode 100644 index 0000000..0c7d890 --- /dev/null +++ b/release/EXAMPLE_PROCESS.md @@ -0,0 +1,184 @@ +# Release script process + +## Dry run testing + +It is a good idea to do a dry-run check locally, and then delete the local state. + +```sh +# Clone to /tmp/stackable-release-YY.M/docker-images and/or enter the directory. +# Create and switch to branch: release-YY.M. +./release/create-release-branch.sh -b YY.M -w products +./release/create-release-branch.sh -b YY.M -w operators +./release/create-release-branch.sh -b YY.M -w demos + +# Clone to /tmp/stackable-release-YY.M/docker-images and/or enter the directory. +# Fetch and switch to branch: release-YY.M. +# NOTE: this would only work for dry run if the previous step was done on the +# same machine and not deleted. Else the release branch won't be found. +# Create and switch to branch: pr-YY.M.X-rc1. +# Update the change log, and commit it (skip pushing). +# Raise a PR (dry-run). +./release/create-release-candidate-branch.sh -t YY.M.X-rc1 -w products +./release/create-release-candidate-branch.sh -t YY.M.X-rc1 -w operators + +# Pretend the PR exists, and exit. +# This doesn't really test much. +./release/merge-release-candidate.sh -t YY.M.X-rc1 -w products +./release/merge-release-candidate.sh -t YY.M.X-rc1 -w operators + +# Clone to /tmp/stackable-release-YY.M/docker-images and/or enter the directory. +# Fetch and switch to branch: release-YY.M. +# NOTE: this would only work for dry run if the first step was done on the +# same machine and not deleted. Else the release branch won't be found. +# Tag the HEAD commit (in this case it would be the same as `main` where this branch was based). +./release/tag-release-candidate.sh -t YY.M.X-rc1 -w products +./release/tag-release-candidate.sh -t YY.M.X-rc1 -w operators + +# TODO: ./release/post-release.sh +``` + +Finally, delete the dirty state: + +```sh +rm -fr /tmp/stackable-release-YY.M +``` + +## Actual release + +### Release Candidates + +This procedure would be done until a viable release-candidate can be selected. + +```sh +# Clone to /tmp/stackable-release-YY.M/docker-images and/or enter the directory. +# Create and switch to branch: release-YY.M. +# Push the branch. +./release/create-release-branch.sh -b YY.M -w products -p +./release/create-release-branch.sh -b YY.M -w operators -p +./release/create-release-branch.sh -b YY.M -w demos -p + +# Clone to /tmp/stackable-release-YY.M/docker-images and/or enter the directory. +# Fetch and switch to branch: release-YY.M. +# Create and switch to branch: pr-YY.M.X-rc1. +# Update the change log, and commit it. +# Push the branch. +# Raise a PR. +./release/create-release-candidate-branch.sh -t YY.M.X-rc1 -w products -p +./release/create-release-candidate-branch.sh -t YY.M.X-rc1 -w operators -p +``` + +Ask people to do approvals. + +> [!TIP] +> It is nice to drop all the links in a Slack message to make it easier. +> You can search the output for `https://github.com/stackabletech/.*/pull/[0-9]+` + +Once all approvals are done, merge the changes and tag the release branch `HEAD`. + +```sh +# Check that PRs are `OPEN`. +# Merge the PR (this does not seem to wait for checks to complete). +./release/merge-release-candidate.sh -t YY.M.X-rc1 -w products -p +./release/merge-release-candidate.sh -t YY.M.X-rc1 -w operators -p +``` + +It is still worth checking that all PRs are merged. + +```sh +# Clone to /tmp/stackable-release-YY.M/docker-images and/or enter the directory. +# Fetch, including tags. +# Ensure tag does not already exist. +# Switch to the release branch. +# Pull. +# Tag HEAD. +# Push tag. +./release/tag-release-candidate.sh -t YY.M.X-rc1 -w products -p +./release/tag-release-candidate.sh -t YY.M.X-rc1 -w operators -p +``` + +> [!CAUTION] +> Wait for images to be built. +> +> To ensure all image artifacts have been pushed, see [image-checks.sh](./image-checks.sh). + +Now do release candidate Testing. + +Repeat this process if this release-candidate is not viable (changing `rc1` to `rc2` for example). + +### Promote RC to Release + +Once the release candidate has been deemed good, do the release proper: + +> [!NOTE] +> `create-release-candidate-branch` is a misnomer, since it is a release branch. +> But this step is necessary to update the version numbers throughout the repository +> to the final version number for the release. + +```sh +# Clone to /tmp/stackable-release-YY.M/docker-images and/or enter the directory. +# Fetch and switch to branch: release-YY.M. +# Create and switch to branch: pr-YY.M.X. +# Update the change log, and commit it. +# Push the branch. +# Raise a PR. +./release/create-release-candidate-branch.sh -t YY.M.X -w products -p +./release/create-release-candidate-branch.sh -t YY.M.X -w operators -p +``` + +Ask people to do approvals. + +> [!TIP] +> It is nice to drop all the links in a Slack message to make it easier. +> You can search the output for `https://github.com/stackabletech/.*/pull/[0-9]+` + +Once all approvals are done, merge the changes and tag the release branch `HEAD`. + +```sh +# Check that PRs are `OPEN`. +# Merge the PR (this does not seem to wait for checks to complete). +./release/merge-release-candidate.sh -t YY.M.X -w products -p +./release/merge-release-candidate.sh -t YY.M.X -w operators -p + +# Clone to /tmp/stackable-release-YY.M/docker-images and/or enter the directory. +# Fetch, including tags. +# Ensure tag does not already exist. +# Switch to the release branch. +# Pull. +# Tag HEAD. +# Push tag. +./release/tag-release-candidate.sh -t YY.M.X -w products -p +./release/tag-release-candidate.sh -t YY.M.X -w operators -p +``` + +> [!CAUTION] +> Wait for images to be built. +> +> To ensure all image artifacts have been pushed, see [image-checks.sh](./image-checks.sh). + +Once everything is looking good, the release can be mentioned in the `main` branch. + +```sh +# Clone to /tmp/stackable-release-YY.M/docker-images and/or enter the directory. +# Fetch, including tags. +# Ensure the release tag exists. +# Checkout the main branch. +# Pull. +# Create and switch to branch: chore/update-changelog-from-release-YY.M.X. +# Checkout the CHANGELOG.md at tag YY.M.X. +# Ensure only the CHANGELOG.md has changed. +# NOTE: It could be possible that new changes have gone into `main`. +# These will need to be fixed on the branch (they should be obvious when +# reviewing the PR). +# Add and commit the CHANGELOG.md from the release. +# Push the branch. +# Raise a PR. +./release/post-release.sh -t YY.M.X -p +``` + +Ask people to do approvals. + +> [!TIP] +> It is nice to drop all the links in a Slack message to make it easier. +> You can search the output for `https://github.com/stackabletech/.*/pull/[0-9]+` + +Once all approvals are done, manually merge the PRs. diff --git a/release/README.md b/release/README.md index f4d2a54..b4f5820 100644 --- a/release/README.md +++ b/release/README.md @@ -1,6 +1,9 @@ # Release workflow -This is the recommended release flow in a nutshell for the release 25.3. +> [!TIP] +> If you are performing a release, you might want to skip over to [EXAMPLE_PROCESS](./EXAMPLE_PROCESS.md) which illustrates the process start to finish. + +This is the recommended release flow in a nutshell for the release 25.7. These steps are repeated for each release tag i.e. for both release-candidates and non-release-candidates: ```shell