diff --git a/.claude/hooks/md-no-hardwrap.py b/.claude/hooks/md-no-hardwrap.py new file mode 100755 index 0000000000..ad8e998191 --- /dev/null +++ b/.claude/hooks/md-no-hardwrap.py @@ -0,0 +1,802 @@ +#!/usr/bin/env python3 +"""PreToolUse hook: keep hardwrapped prose out of markdown and out of GitHub. + +A markdown renderer collapses a soft line break into a space, so a paragraph +wrapped at ~80 columns renders identically to the same paragraph on one line. +The wrap is not free, though: it breaks narrow viewports, it mangles list and +table rendering, and it turns a one-word edit into a diff that reflows the +whole block. Prose therefore stays on a single line per paragraph, and the +renderer decides where it wraps. + +The rule is easy to state and easy to forget, because an ~80-column wrap is +correct in the places a coding agent spends most of its time (Go comments, +commit message bodies), and the reflex leaks from there into markdown files +and GitHub bodies. This hook is the mechanical check that the reflex slips +past. It guards two surfaces: + + Write / Edit / MultiEdit markdown content headed for a file + Bash a `gh` command publishing a body to GitHub + +Two design rules keep it from becoming a nuisance, which matters more than +catching every last wrap -- a guard that blocks legitimate work gets switched +off, and then it guards nothing: + + * Only NEW wraps are refused. The hook compares the file as it stands + against the file as it would land, and reports only what the edit adds. + Editing one word of a legacy document that is wrapped throughout, or + rewriting a vendored chart README verbatim, therefore stays possible. + + * Anything ambiguous is allowed. A body assembled at runtime by the shell, + an unsplittable command line, a path that is not a regular file, a + document the hook cannot read -- all pass. The hook only ever blocks what + it is certain about. + +Line breaks are rejected only inside prose: a paragraph, a list item, or a +blockquote spanning more than one line. Everywhere else a break carries +meaning and is left alone -- code blocks (fenced and indented), tables, +headings, front matter, HTML, link reference definitions, footnotes, badge +stacks, and explicit hard breaks (two trailing spaces, or a trailing +backslash), which render as
and cannot be reflowed without changing the +output. + +Agent hook protocol: a JSON payload on stdin, exit 0 to allow the tool call, +exit 2 to block it with an explanation on stderr. +""" + +from __future__ import annotations + +import difflib +import json +import os +import re +import shlex +import stat +import sys +from typing import NamedTuple + +MARKDOWN_SUFFIXES = (".md", ".markdown", ".mdx") + +# Paths whose prose the repo does not own. Vendored upstream charts are +# regenerated verbatim by `make update` and AGENTS.md forbids editing them; +# published changelogs are historical records; *.md.gotmpl is a Go template, +# not a document. Reflowing any of them is the wrong instruction to give. +OUT_OF_SCOPE = ( + re.compile(r"packages/[^/]+/[^/]+/charts/"), + re.compile(r"/?docs/changelogs/"), + re.compile(r"/vendor/"), +) + +# Refuse to read a body file larger than this. A body is prose; anything of +# this size is not, and a hook must not stall the agent chewing through it. +MAX_BODY_FILE_BYTES = 1 << 20 # 1 MiB + +# Above this, diffing before against after costs more than the check is worth. +# The hook then cannot tell an added wrap from an inherited one, and allows. +MAX_DIFF_CHARS = 400_000 + +# `gh` subcommands that publish a body. `gh pr view`, `gh pr list` and friends +# are absent by design: they read, they do not publish. +GH_PUBLISHING_SUBCOMMANDS = { + ("pr", "create"), + ("pr", "edit"), + ("pr", "comment"), + ("pr", "review"), + ("issue", "create"), + ("issue", "edit"), + ("issue", "comment"), + ("release", "create"), + ("release", "edit"), + ("api",), +} +GH_NOUNS = {"pr", "issue", "release"} +GH_ACTIONS = {"create", "edit", "comment", "review"} + +# Flag meanings are subcommand-dependent: -F is --body-file for `gh pr`, but +# --raw-field for `gh api`. Getting this backwards silently drops the body. +BODY_FLAGS = ("--body", "-b", "--notes", "-n") +BODY_FILE_FLAGS = ("--body-file", "-F", "--notes-file") +API_FIELD_FLAGS = ("-f", "--field", "-F", "--raw-field") + + +class Violation(NamedTuple): + start_line: int # 1-based, first line of the offending block + end_line: int # 1-based, last line of the offending block + text: str # the block, joined with " | " for the message + + +_RE_HEADING = re.compile(r"^#{1,6}\s") +_RE_LIST_MARKER = re.compile(r"^\s*([-*+]|\d+[.)])\s") +_RE_BLOCKQUOTE = re.compile(r"^\s*>") +_RE_TABLE_ROW = re.compile(r"^\s*\|") +# A table delimiter row is dashes, colons, pipes and spaces, with at least one +# pipe. The pipe is what tells it apart from a `---` horizontal rule or setext +# underline, and it is what makes a pipe-less GFM table (`a | b`) detectable. +_RE_TABLE_DELIMITER = re.compile(r"^\s*:?-{1,}:?(\s*\|\s*:?-{1,}:?)+\s*$") +_RE_SETEXT_UNDERLINE = re.compile(r"^\s*(=+|-+)\s*$") +_RE_ADMONITION_OPENER = re.compile(r"^\s*(\?\?\?|!!!)") +# Any line opening with a tag is HTML, whether or not it also carries content: +# `Test output` and `
all green
` are two lines of +# a
block, not a wrapped paragraph, and joining them would be wrong. +_RE_HTML_BLOCK = re.compile(r"^\s*\s*$") +_RE_FENCE = re.compile(r"^\s*(```|~~~)") +_RE_HR = re.compile(r"^\s*([-*_])(\s*\1){2,}\s*$") +_RE_FRONT_MATTER = re.compile(r"^\s*(---|\+\+\+)\s*$") +# `[label]: https://…` and `[^note]: text` -- one definition per line is the +# only way to write these, so a run of them is not a wrapped paragraph. +_RE_LINK_REF_DEF = re.compile(r"^\s*\[[^\]]+\]:\s") +# `: definition` under a term. +_RE_DEFINITION = re.compile(r"^\s*:\s") +# Indented code: four spaces or a tab, outside a list. +_RE_INDENTED_CODE = re.compile(r"^( |\t)") +# A README shield stack is one badge per line by convention; collapsing it +# would be wrong. A paragraph mixing badges with prose is still wrapped prose. +_RE_BADGE_ROW = re.compile(r"^\s*\[!\[") +_RE_IMAGE_ROW = re.compile(r"^\s*!\[") +# Heredoc opener: `< bool: + return bool(_RE_BADGE_ROW.match(line) or _RE_IMAGE_ROW.match(line)) + + +def ends_with_hard_break(line: str) -> bool: + """True iff the line ends in an explicit
: two spaces, or a backslash. + + Such a break renders as a line break. Reflowing it onto one line would + change the output, so it is a deliberate break and not a wrap. + """ + return line.endswith(" ") or line.rstrip(" ").endswith("\\") + + +def is_structure(line: str) -> bool: + """True iff the line is markdown structure rather than prose.""" + return any( + rx.match(line) + for rx in ( + _RE_HEADING, + _RE_BLOCKQUOTE, + _RE_TABLE_ROW, + _RE_TABLE_DELIMITER, + _RE_SETEXT_UNDERLINE, + _RE_ADMONITION_OPENER, + _RE_HTML_BLOCK, + _RE_HR, + _RE_LINK_REF_DEF, + _RE_DEFINITION, + ) + ) + + +def wrapped_groups(block: list[tuple[int, str]]) -> list[list[tuple[int, str]]]: + """Split a block into logical lines, and return those spanning >1 physical line. + + An explicit hard break ends a logical line. Everything up to the next hard + break (or the end of the block) is one logical line, and if it occupies + more than one physical line, the author wrapped it. + """ + groups: list[list[tuple[int, str]]] = [] + current: list[tuple[int, str]] = [] + + for numbered in block: + current.append(numbered) + if ends_with_hard_break(numbered[1]): + groups.append(current) + current = [] + if current: + groups.append(current) + + return [g for g in groups if len(g) > 1] + + +def find_violations(content: str) -> list[Violation]: + """Return every prose block in content that spans more than one line.""" + violations: list[Violation] = [] + lines = content.split("\n") + + def flush(block: list[tuple[int, str]], strip_quote: bool = False) -> None: + if len(block) < 2: + return + # A stack of badges or images is one per line by convention. + if all(is_badge_or_image_row(text) for _, text in block): + return + for group in wrapped_groups(block): + violations.append( + Violation( + start_line=group[0][0], + end_line=group[-1][0], + text=" | ".join( + (t.lstrip("> ").strip() if strip_quote else t.strip()) + for _, t in group + ), + ) + ) + + in_fence = False + fence_marker = "" + in_html_comment = False + # A leading `---` opens front matter only if a matching delimiter closes it. + # Otherwise it is a horizontal rule, and treating it as an unterminated + # front-matter block would swallow the whole document unchecked. + front_matter_marker = ( + lines[0].strip() if lines and _RE_FRONT_MATTER.match(lines[0]) else "" + ) + in_front_matter = bool(front_matter_marker) and any( + line.strip() == front_matter_marker for line in lines[1:] + ) + in_table = False + + block: list[tuple[int, str]] = [] + block_kind: str | None = None # "para" | "list" | "quote" + + for idx, raw in enumerate(lines, start=1): + if in_front_matter: + if idx > 1 and raw.strip() == front_matter_marker: + in_front_matter = False + continue + + if in_html_comment: + if _RE_HTML_COMMENT_CLOSE.search(raw): + in_html_comment = False + continue + if _RE_HTML_COMMENT_OPEN.match(raw): + flush(block, block_kind == "quote") + block, block_kind = [], None + if not _RE_HTML_COMMENT_CLOSE.search(raw): + in_html_comment = True + continue + + fence = _RE_FENCE.match(raw) + if fence: + flush(block, block_kind == "quote") + block, block_kind = [], None + if not in_fence: + in_fence, fence_marker = True, fence.group(1) + elif fence_marker in raw: + in_fence, fence_marker = False, "" + continue + if in_fence: + continue + + if not raw.strip(): + flush(block, block_kind == "quote") + block, block_kind, in_table = [], None, False + continue + + # A delimiter row proves the line above it was a table header, not the + # first line of a paragraph. This is what makes a pipe-less GFM table + # (`a | b` over `--- | ---`) readable as a table. + if _RE_TABLE_DELIMITER.match(raw): + if block_kind == "para" and block: + block.pop() + flush(block, False) + block, block_kind, in_table = [], None, True + continue + if in_table: + if "|" in raw: + continue + in_table = False + + if _RE_BLOCKQUOTE.match(raw): + if block_kind != "quote": + flush(block, block_kind == "quote") + block, block_kind = [], "quote" + # `>` on its own separates paragraphs inside the quote. + if not raw.lstrip("> ").strip(): + flush(block, True) + block = [] + else: + block.append((idx, raw)) + continue + + if _RE_LIST_MARKER.match(raw): + flush(block, block_kind == "quote") + block, block_kind = [(idx, raw)], "list" + continue + + if is_structure(raw): + flush(block, block_kind == "quote") + block, block_kind = [], None + continue + + # An indented line directly under a list item is that item's lazy + # continuation -- the wrapped-bullet case. Outside a list, and with no + # paragraph open, it is an indented code block. + if _RE_INDENTED_CODE.match(raw) and block_kind != "list" and not block: + continue + + if block_kind == "list": + block.append((idx, raw)) # lazy continuation of the item + continue + + if block_kind != "para": + flush(block, block_kind == "quote") + block, block_kind = [], "para" + block.append((idx, raw)) + + flush(block, block_kind == "quote") + return violations + + +def out_of_scope(path: str) -> bool: + """True iff the repo does not own this file's prose.""" + normalised = path.replace(os.sep, "/") + if normalised.endswith(".gotmpl"): + return True + return any(rx.search(normalised) for rx in OUT_OF_SCOPE) + + +def is_markdown(path: str) -> bool: + return path.lower().endswith(MARKDOWN_SUFFIXES) + + +def read_text_file(path: str) -> str | None: + """Read a regular file, or None if it is missing, huge, or not a file. + + A fifo or a character device would block forever, and the agent with it. + """ + try: + expanded = os.path.expanduser(path) + info = os.stat(expanded) + if not stat.S_ISREG(info.st_mode) or info.st_size > MAX_BODY_FILE_BYTES: + return None + with open(expanded, encoding="utf-8") as fh: + return fh.read() + except (OSError, UnicodeDecodeError): + return None + + +def added_violations(before: str | None, after: str) -> list[Violation]: + """Return the violations whose line breaks THIS edit introduced. + + Legacy documents are wrapped throughout, and demanding that an agent reflow + a file it only meant to touch one word of is how a hook gets switched off. + But "is this violation new" survives neither of the obvious tests: comparing + the violations' text refuses a typo fix inside a paragraph that was already + wrapped, and counting them lets a file be rewritten as entirely fresh + hardwrapped prose so long as the total does not rise. + + The question that does hold up is about the break, not the block: a soft + break is this edit's doing only when BOTH lines it joins are lines the edit + wrote. Touching one line of a pair that was already wrapped leaves the break + where it was; writing both lines is what creates it. + """ + found = find_violations(after) + if before is None or not found: + return found + + before_lines = before.split("\n") + after_lines = after.split("\n") + + # Lines of `after` that came through the edit untouched, 1-based. + untouched: set[int] = set() + matcher = difflib.SequenceMatcher(None, before_lines, after_lines, autojunk=False) + for tag, _, _, start_line, end_line in matcher.get_opcodes(): + if tag == "equal": + untouched.update(range(start_line + 1, end_line + 1)) + + def edit_wrote_this_break(violation: Violation) -> bool: + return any( + number not in untouched and number + 1 not in untouched + for number in range(violation.start_line, violation.end_line) + ) + + return [v for v in found if edit_wrote_this_break(v)] + + +def apply_edits(original: str, tool_input: dict) -> str | None: + """Apply an Edit/MultiEdit payload to the file's current text.""" + edits: list[dict] = [] + if isinstance(tool_input.get("edits"), list): + edits = [e for e in tool_input["edits"] if isinstance(e, dict)] + elif isinstance(tool_input.get("new_string"), str): + edits = [tool_input] + + result = original + for edit in edits: + old = edit.get("old_string") + new = edit.get("new_string") + if not isinstance(old, str) or not isinstance(new, str) or not old: + return None # an empty old_string would prepend, not replace + if old not in result: + return None # cannot reproduce the edit; do not guess + if edit.get("replace_all"): + result = result.replace(old, new) + else: + result = result.replace(old, new, 1) + return result + + +def gh_subcommand(args: list[str]) -> tuple[str, ...] | None: + """Return the (noun, action) a `gh` invocation names, or None. + + Position alone cannot find the subcommand: a global flag may precede it, + and `gh -R owner/repo pr comment` would be read as the subcommand + `owner/repo`, letting the body through. Match on the token's value. + """ + noun: str | None = None + for token in args: + if noun is None: + if token == "api": + return ("api",) + if token in GH_NOUNS: + noun = token + elif token in GH_ACTIONS: + return (noun, token) + return None + + +HEREDOC_MARKER = "__COZY_HEREDOC_{}__" +_RE_HEREDOC_MARKER = re.compile(r"__COZY_HEREDOC_(\d+)__") + + +def split_off_heredocs(command: str) -> tuple[str, list[str]]: + """Split a command line into (command with heredocs marked, their bodies). + + A heredoc body is data, and shlex would tokenise it into nonsense. Lift the + bodies out and leave a marker where each opener stood, so that the body can + later be traced back to the command that opened it -- a heredoc feeding + `kubectl apply -f -` is not a GitHub body just because a `gh` call shares + the line. + + Every line is scanned for an opener, not only the first: a long `gh` call is + routinely spread over backslash-continued lines, the heredoc opening on the + last of them. + """ + kept: list[str] = [] + bodies: list[str] = [] + body: list[str] = [] + pending: list[str] = [] + + for line in command.split("\n"): + if pending: + if line.strip() == pending[0]: + bodies.append("\n".join(body)) + body, pending = [], pending[1:] + else: + body.append(line) + continue + + openers = list(_RE_HEREDOC.finditer(line)) + if openers: + marked = line + for offset, opener in enumerate(openers): + index = len(bodies) + len(pending) + offset + marked = marked.replace( + opener.group(0), HEREDOC_MARKER.format(index), 1 + ) + kept.append(marked) + pending.extend(m.group(2) for m in openers) + else: + kept.append(line) + + if body: # unterminated heredoc: take what we have + bodies.append("\n".join(body)) + + return "\n".join(kept), bodies + + +def stdin_redirect(args: list[str]) -> str | None: + """Return the file a `< path` redirect feeds this segment's stdin from.""" + for index, arg in enumerate(args): + if arg == "<" and index + 1 < len(args): + return args[index + 1] + if arg.startswith("<") and len(arg) > 1 and not arg.startswith("<<"): + return arg[1:] + return None + + +def owned_heredocs(args: list[str], bodies: list[str]) -> list[str]: + """Return the heredoc bodies opened by this segment, in order.""" + owned: list[str] = [] + for arg in args: + for match in _RE_HEREDOC_MARKER.finditer(arg): + index = int(match.group(1)) + if index < len(bodies): + owned.append(bodies[index]) + return owned + + +def newlines_to_separators(command: str) -> str: + """Turn command-separating newlines into `;`, leaving quoted ones alone. + + shlex treats a newline as plain whitespace, so a script whose second line + is the `gh` call lexes into one segment beginning with the FIRST line's + command -- and the `gh` invocation is never recognised as one. A newline + inside a quoted string belongs to the body and must survive; a + backslash-continued newline joins one command, so it is not a separator. + """ + out: list[str] = [] + quote: str | None = None + index = 0 + + while index < len(command): + char = command[index] + + if quote: + if char == "\\" and quote == '"' and index + 1 < len(command): + out.append(char) + out.append(command[index + 1]) + index += 2 + continue + if char == quote: + quote = None + out.append(char) + index += 1 + continue + + if char in ("'", '"'): + quote = char + out.append(char) + index += 1 + continue + + if char == "\\" and index + 1 < len(command) and command[index + 1] == "\n": + out.append(" ") # line continuation: one command, not two + index += 2 + continue + + if char == "\n": + out.append(" ; ") + index += 1 + continue + + out.append(char) + index += 1 + + return "".join(out) + + +def shell_assembled(value: str) -> bool: + """True iff the shell, not the agent, decides what this argument says. + + `--body "$(cat < list[str]: + """Return the prose fields of a JSON document (`gh api --input` payload). + + A JSON heredoc is not markdown: its braces and quoted keys are not a + wrapped paragraph. Only the human-facing values inside it are prose. + """ + try: + data = json.loads(text) + except (json.JSONDecodeError, ValueError): + return [] + + found: list[str] = [] + + def walk(node: object) -> None: + if isinstance(node, dict): + for key, value in node.items(): + if key in ("body", "text") and isinstance(value, str): + found.append(value) + else: + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + + walk(data) + return found + + +def segment_bodies(args: list[str], is_api: bool) -> tuple[list[str], bool]: + """Return (bodies, wants_stdin) for one `gh` segment's arguments.""" + bodies: list[str] = [] + wants_stdin = False + + idx = 0 + while idx < len(args): + arg = args[idx] + nxt = args[idx + 1] if idx + 1 < len(args) else None + + if is_api: + # For `gh api`, -F is --raw-field, not --body-file. + if arg in API_FIELD_FLAGS and nxt is not None: + if nxt.startswith("body=") or nxt.startswith("text="): + value = nxt.split("=", 1)[1] + if value.startswith("@"): + content = read_text_file(value[1:]) + if content is not None: + bodies.append(content) + else: + bodies.append(value) + idx += 2 + continue + if arg == "--input" and nxt is not None: + if nxt == "-": + wants_stdin = True + else: + content = read_text_file(nxt) + if content is not None: + bodies.extend(json_bodies(content)) + idx += 2 + continue + idx += 1 + continue + + if arg in BODY_FLAGS and nxt is not None: + if not shell_assembled(nxt): + bodies.append(nxt) + idx += 2 + continue + if arg.startswith(("--body=", "--notes=")): + value = arg.split("=", 1)[1] + if not shell_assembled(value): + bodies.append(value) + idx += 1 + continue + if arg in BODY_FILE_FLAGS and nxt is not None: + if nxt == "-": + wants_stdin = True + elif not out_of_scope(nxt): + content = read_text_file(nxt) + if content is not None: + bodies.append(content) + idx += 2 + continue + if arg.startswith(("--body-file=", "--notes-file=")): + value = arg.split("=", 1)[1] + if value == "-": + wants_stdin = True + elif not out_of_scope(value): + content = read_text_file(value) + if content is not None: + bodies.append(content) + idx += 1 + continue + idx += 1 + + return bodies, wants_stdin + + +def gh_bodies(command: str) -> list[str]: + """Return every body a `gh` command line would publish to GitHub.""" + if "gh" not in command: + return [] + + stripped, heredocs = split_off_heredocs(command) + stripped = newlines_to_separators(stripped) + + # punctuation_chars makes the lexer return `;`, `&&`, `|`, `(` and friends + # as tokens of their own. Without it `git push; gh pr create …` lexes as + # `push;` plus a `gh` that never starts a segment, and the body it + # publishes is never seen. + try: + lexer = shlex.shlex(stripped, posix=True, punctuation_chars=True) + lexer.whitespace_split = True + tokens = list(lexer) + except ValueError: # unbalanced quotes: not ours to judge + return [] + + # A command line may chain several commands; only `gh` segments matter. + separators = (";", "&", "&&", "||", "|", "(", ")", "{", "}", "\n") + segments: list[list[str]] = [[]] + for token in tokens: + if token in separators: + segments.append([]) + else: + segments[-1].append(token) + + bodies: list[str] = [] + for segment in segments: + # An environment prefix (`GH_TOKEN=… gh …`) would otherwise hide the + # `gh` behind a token that is not a command name. + while segment and re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", segment[0]): + segment = segment[1:] + if not segment or os.path.basename(segment[0]) != "gh": + continue + + subcommand = gh_subcommand(segment[1:]) + if subcommand not in GH_PUBLISHING_SUBCOMMANDS: + continue + + is_api = subcommand == ("api",) + found, wants_stdin = segment_bodies(segment[1:], is_api) + bodies.extend(found) + + # A heredoc THIS segment opened is the body it publishes -- as JSON for + # `gh api`, as markdown otherwise. A heredoc opened by another command + # on the same line (a commit message, a YAML manifest) belongs to that + # command, and blocking on it would be a false positive of exactly the + # kind that gets a hook switched off. + owned = owned_heredocs(segment[1:], heredocs) + for heredoc in owned: + bodies.extend(json_bodies(heredoc) if is_api else [heredoc]) + + # `--body-file -` / `--input -` take the body from stdin; a `< file` + # redirect says which file that is. Without following it the body + # publishes unread. + if wants_stdin and not owned: + source = stdin_redirect(segment[1:]) + if source and not out_of_scope(source): + content = read_text_file(source) + if content is not None: + bodies.extend(json_bodies(content) if is_api else [content]) + + return bodies + + +def truncate(text: str, limit: int = 160) -> str: + return text if len(text) <= limit else text[: limit - 1] + "…" + + +def report(subject: str, violations: list[Violation]) -> None: + lines = [ + "BLOCKED: hardwrapped prose.", + "AGENTS.md rule: one continuous line per paragraph — the renderer wraps to viewer width.", + "Reflow each block below onto a single line, then retry.", + "", + f"Subject: {subject}", + ] + for v in violations[:5]: + lines.append(f" - lines {v.start_line}-{v.end_line}: {truncate(v.text)}") + if len(violations) > 5: + lines.append(f" - … and {len(violations) - 5} more") + print("\n".join(lines), file=sys.stderr) + + +def main() -> int: + try: + payload = json.load(sys.stdin) + except (json.JSONDecodeError, UnicodeDecodeError): + return 0 + if not isinstance(payload, dict): + return 0 + + tool_name = payload.get("tool_name", "") + tool_input = payload.get("tool_input") + if not isinstance(tool_input, dict): + return 0 + + violations: list[Violation] = [] + + if tool_name in ("Write", "Edit", "MultiEdit"): + path = tool_input.get("file_path") + if not isinstance(path, str) or not is_markdown(path) or out_of_scope(path): + return 0 + + subject = path + before = read_text_file(path) + + if tool_name == "Write": + after = tool_input.get("content") + if not isinstance(after, str): + return 0 + else: + if before is None: + return 0 # cannot see the file, cannot judge the hunk + after = apply_edits(before, tool_input) + if after is None: + return 0 + + violations = added_violations(before, after) + + elif tool_name == "Bash": + command = tool_input.get("command") + if not isinstance(command, str): + return 0 + subject = "gh command body" + for body in gh_bodies(command): + violations.extend(find_violations(body)) + + else: + return 0 + + if not violations: + return 0 + + report(str(subject), violations) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000000..395314de68 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,16 @@ +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "Write|Edit|MultiEdit|Bash", + "hooks": [ + { + "type": "command", + "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/md-no-hardwrap.py", + "timeout": 10 + } + ] + } + ] + } +} diff --git a/.clomonitor.yml b/.clomonitor.yml new file mode 100644 index 0000000000..26444486e7 --- /dev/null +++ b/.clomonitor.yml @@ -0,0 +1,8 @@ +# CLOMonitor metadata file, read when CLOMonitor lints this repository: +# https://clomonitor.io/projects/cncf/cozystack +# Format: https://github.com/cncf/clomonitor/blob/main/docs/metadata/.clomonitor.yml + +# License scanning results: the CNCF FOSSA project for this repository, kept +# current by the FOSSA GitHub integration on the cozystack organization (#4758). +licenseScanning: + url: https://app.fossa.com/projects/git%2Bgithub.com%2Fcozystack%2Fcozystack diff --git a/.gemini/styleguide.md b/.gemini/styleguide.md index 7185a6257e..4b920837e3 100644 --- a/.gemini/styleguide.md +++ b/.gemini/styleguide.md @@ -107,4 +107,4 @@ When reviewing `values.schema.json`: - Missing comments on generated code. - `go.sum` changes accompanying `go.mod` changes — expected and correct. - Fork relationships for vendored tooling images — intentional (e.g., `cozystack/kilo` fork is expected). -- Absence of unit tests for vendored chart overrides — covered by E2E tests in `hack/e2e-apps/`. +- Absence of unit tests for vendored chart overrides — covered by E2E tests in `hack/e2e-chainsaw/`. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..b114c596bb --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Unified diff context has a marker before Go tabs and blank lines. +packages/system/capi-providers-infraprovider/images/cluster-api-provider-kubevirt/patches/*.diff -whitespace +packages/system/cilium/images/cilium-operator/patches/*.diff -whitespace +packages/system/seaweedfs/images/seaweedfs/patches/*.diff -whitespace diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2482712d0e..1a31f3689a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,86 @@ -* @kvaps @lllamnyp @lexfrei @androndo @IvanHunters @sircthulhu @myasnikovdaniil +# Path-to-owner mapping for this repository. MAINTAINERS.md names the people and +# points here for the areas; CONTRIBUTOR_LADDER.md defines the roles. Security +# reports do NOT come through this file — see SECURITY.md. +# +# Two mechanics shape the file, and both fail silently. +# +# Matching REPLACES, it does not merge. Only the LAST matching pattern applies, +# and it discards every broader one including the * catch-all, so exactly one +# line decides the owners of any given path. There is no syntax for "these +# people may review anything" — repeating the handles on every line is that +# statement. A rule that omits someone strips their ownership of those paths. +# +# A handle without write access to this repository is ignored and no error is +# reported, which leaves the paths it scopes gating on nobody. Check write access +# before adding a handle. +# +# Rules run broad to specific. An area with no rule of its own is owned by the +# tier rule above it — add a person to an existing rule rather than carving a new +# path for them. + +* @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters + +/api/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @sircthulhu +/cmd/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @sircthulhu +/internal/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @sircthulhu +/pkg/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @sircthulhu + +/packages/core/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @sircthulhu @kingdonb +/packages/system/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @sircthulhu @mattia-eleuteri +/packages/extra/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @sircthulhu @mattia-eleuteri +/packages/apps/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @matthieu-robin @scooby87 + +/packages/apps/vm-*/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @scooby87 @mattia-eleuteri +/packages/system/kubevirt*/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @scooby87 @mattia-eleuteri +/packages/system/vm-*/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @scooby87 @mattia-eleuteri +/packages/system/gpu-operator/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @scooby87 @mattia-eleuteri +/packages/system/hami/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @scooby87 @mattia-eleuteri +/dashboards/gpu/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @scooby87 @mattia-eleuteri +/dashboards/kubevirt/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @scooby87 @mattia-eleuteri + +/api/backups/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @androndo @mattia-eleuteri +/cmd/backup*-controller/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @androndo @mattia-eleuteri +/internal/backupcontroller/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @androndo @mattia-eleuteri +/examples/backups/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @androndo @mattia-eleuteri +/packages/system/backup*-controller/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @androndo @mattia-eleuteri +/packages/system/velero/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @androndo @mattia-eleuteri +/packages/system/vsnap-crd/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @androndo @mattia-eleuteri + +/packages/system/keycloak*/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @sircthulhu + +/packages/system/capi-providers-infraprovider-proxmox/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @sircthulhu @mattia-eleuteri @themoriarti +/packages/system/proxmox-*/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @sircthulhu @mattia-eleuteri @themoriarti + +/cmd/flux-*/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @kingdonb +/internal/flux*/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @kingdonb +/packages/core/flux-aio/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @kingdonb +/packages/system/flux-*/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @kingdonb +/dashboards/flux/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @kingdonb + +/.github/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters +/hack/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters +/test/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters +/tools/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters +/Makefile @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters +/.pre-commit-config.yaml @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters + +/docs/ @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 +/ADOPTERS.md @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 +/CODE_OF_CONDUCT.md @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 +/CONTRIBUTING.md @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 +/CONTRIBUTOR_LADDER.md @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 +/GENERAL_TECHNICAL_REVIEW.md @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 +/GOVERNANCE.md @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 +/LICENSE @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 +/MAINTAINERS.md @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 +/README.md @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 +/ROADMAP.md @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 +/SECURITY.md @kvaps @lllamnyp @lexfrei @myasnikovdaniil @IvanHunters @tym83 + +# `make generate` produces the paths below from each app's values.yaml, which is +# already reviewed under the rules above, and the pre-commit run-make-generate / +# run-make-generate-root CI hooks regenerate and fail on any diff. A pattern +# listed with no owner is what waives the rules above for just these paths. +/api/apps/v1alpha1/*/types.go +/api/apps/v1alpha1/*/zz_generated.deepcopy.go +/packages/system/*/cozyrds/ diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index fefdab821f..fcd2b50db6 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,7 +1,7 @@ --- name: Bug report about: Create a report to help us improve -labels: 'kind/bug' +labels: 'kind/bug, triage/needs-triage' assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..737c5e3f25 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,14 @@ +blank_issues_enabled: false +contact_links: + - name: 💬 Question, help, or an early idea + url: https://github.com/cozystack/cozystack/discussions + about: Usage questions, troubleshooting, or a "should we…" idea that isn't a concrete request yet — start a Discussion. The community and maintainers answer here. + - name: 📐 Design proposal / RFC (architectural or cross-cutting) + url: https://github.com/cozystack/community/tree/main/design-proposals + about: Changes that affect multiple components or APIs, or that need a decision before any code is written (a new subsystem, an API change with tradeoffs). Open a design-proposal PR in the community repo. + - name: 🏛️ Governance, process, or community + url: https://github.com/cozystack/community/issues/new + about: Maintainership, decision-making, meetings, working groups, community resources — these live in the community repo. + - name: 💬 Chat (Telegram / CNCF Slack) + url: https://t.me/cozystack + about: Real-time discussion with the Cozystack community. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..c4028198ad --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,42 @@ +--- +name: Feature request +about: Propose a concrete, scoped improvement to Cozystack +labels: 'kind/feature, triage/needs-triage' +assignees: '' + +--- + + +**Problem / motivation** + +What are you trying to do, and what is missing or painful today? + +**Proposed solution** + +What should Cozystack do? Be as concrete as you can — the API field, flag, value, or behavior you expect. + +**Scope check** + +- [ ] This affects a single component / package (if it spans many, consider a [design proposal](https://github.com/cozystack/community/tree/main/design-proposals)) +- [ ] I can describe the expected behavior concretely (if not, consider a [Discussion](https://github.com/cozystack/cozystack/discussions) first) + +**Alternatives considered** + +Other approaches you weighed, and why this one. + +**Additional context** + +Links, related issues, screenshots, prior art. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 044dd6a083..1e368718e1 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -8,7 +8,7 @@ - Breaking changes: append `!` after type/scope (`feat(api)!: ...`) or add a `BREAKING CHANGE:` footer - If it's a work in progress, consider creating this PR as a draft. - Don't hesistate to ask for opinion and review in the community chats, even if it's still a draft. -- Add the label `backport` if it's a bugfix that needs to be backported to a previous version. +- Add the label `kind/backport` if it's a bugfix that needs to be backported to a previous version. --> ## What this PR does @@ -19,6 +19,34 @@ +### Downstream repositories + + + +- [ ] No downstream repository is affected by this change +- [ ] [cozystack/website](https://github.com/cozystack/website) - follow-up: +- [ ] [cozystack/terraform-provider-cozystack](https://github.com/cozystack/terraform-provider-cozystack) - follow-up: +- [ ] [cozystack/ansible-cozystack](https://github.com/cozystack/ansible-cozystack) - follow-up: +- [ ] [cozystack/ccp](https://github.com/cozystack/ccp) - follow-up: +- [ ] [cozystack/talm](https://github.com/cozystack/talm) - follow-up: +- [ ] [cozystack/cozyhr](https://github.com/cozystack/cozyhr) - follow-up: +- [ ] [cozystack/cozy-proxy](https://github.com/cozystack/cozy-proxy) - follow-up: +- [ ] [cozystack/cozystack-telemetry-server](https://github.com/cozystack/cozystack-telemetry-server) - follow-up: +- [ ] [cozystack/external-apps-example](https://github.com/cozystack/external-apps-example) - follow-up: +- [ ] [cozystack/examples](https://github.com/cozystack/examples) - follow-up: +- [ ] [cozystack/community](https://github.com/cozystack/community) - follow-up: + ### Release note + +## Fixes + +* **[platform] Migrate ACME HTTP-01 to ingressClassName API**: `ClusterIssuer` HTTP-01 solvers (`letsencrypt-prod`, `letsencrypt-stage`) were hardcoded to ingress class `nginx`, which does not exist on Cozystack clusters — real classes are named after tenant namespaces (e.g. `tenant-root`). Certificate issuance only worked because every ingress overrode the issuer via the deprecated `acme.cert-manager.io/http01-ingress-class` annotation. Both sides are now migrated to the modern cert-manager API: the `ClusterIssuer` uses `http01.ingress.ingressClassName` (parameterized from `_cluster.expose-ingress`, defaulting to `tenant-root`) and all ingress templates switch to the `acme.cert-manager.io/http01-ingress-ingressclassname` annotation. The migration is atomic — mixing the two APIs triggers cert-manager's validation error "fields ingressClassName and class cannot be set at the same time" ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2436, backport #2439). + +## Documentation + +* **[website] Document tenant namespace layout and parent/child derivation**: Added two new sections to the Tenant System guide describing how tenant workload namespace names are constructed and how downstream integrations should walk the tenant tree without string-parsing namespace names ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#479). + +* **[website] Document the checkbox-then-edit-CR customization pattern**: Added a *Customizing Tenant Services* section documenting how to configure a tenant-level service (etcd, monitoring, ingress, seaweedfs) beyond the per-tenant boolean flag — enable the flag then `kubectl edit` the downstream `apps.cozystack.io/v1alpha1` CR, with callouts for common wrong approaches ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#485). + +* **[website] Fix 14 broken links and stale talm anchor across v1 docs**: Automated pass corrected 14 broken `{{% ref %}}`/`{{% relref %}}` shortcode targets and raw markdown links across 11 files, plus updated a stale `talm` anchor ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#486). + +* **[website] Rewrite external applications guide for ApplicationDefinition API**: Updated the external apps guide to replace outdated `CozystackResourceDefinition` references with the current `ApplicationDefinition` API and conventions ([**@kitsunoff**](https://github.com/kitsunoff) in cozystack/website#488). + +* **[website] Refactor docs versions to major.minor variants**: Restructured the docs versioning from `v0`/`v1` to per-minor-version directories (`v1.0`, `v1.1`, `v1.2`), making the site compatible with the push-based docs update pipeline ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#477). + +* **[website] Update /docs/v1/ redirect to latest v1.2**: Updated the Netlify redirect `/docs/v1/*` to point to `v1.2` (was outdated `v1.0`) and added a Hugo alias for GitHub Pages compatibility ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#492). + +* **[website] Trunk-based versioning with permanent next/ directory**: Replaced the hardcoded draft-directory approach with a permanent `content/en/docs/next/` trunk for upcoming/unreleased Cozystack docs, excluded from production builds and promoted to `vX.Y/` when the upstream release workflow fires ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#495). + +--- + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.1.6...v1.1.7 diff --git a/docs/changelogs/v1.4.3.md b/docs/changelogs/v1.4.3.md new file mode 100644 index 0000000000..a17f2a3e00 --- /dev/null +++ b/docs/changelogs/v1.4.3.md @@ -0,0 +1,67 @@ + + +# v1.4.3 (2026-06-10) + +A patch release shipping five bug fixes for the dashboard, API server, networking, object storage, and managed Kubernetes, along with a `seaweedfs-cosi-driver` update, an `ouroboros` v0.8.0 upgrade, two `ansible-cozystack` improvements, and a batch of new documentation. + +## Features and Improvements + +* **feat(ouroboros): bump to v0.8.0**: Upgrades the ouroboros proxy to v0.8.0, which now logs an explicit reason when its TCP backend readiness check fails — making stuck-proxy situations immediately diagnosable instead of silently `NotReady`. The upgrade also migrates the kubectl sidecar image from Docker Hub to `mirror.gcr.io`, avoiding anonymous pull rate-limits on shared CI/runner IPs ([**@lexfrei**](https://github.com/lexfrei) in #2807, backport #2835). + +* **[ansible-cozystack] feat(prepare): exclude loop and virtual devices from host LVM scanning**: Sets an LVM `global_filter` in `/etc/lvm/lvm.conf` on all prepare playbooks (Ubuntu, RHEL, SUSE) so the host LVM does not scan or activate DRBD, device-mapper, zd-device, or loop-backed volume groups — preventing unintended VG activation of LINSTOR/DRBD volumes or loop-mounted images. The filter is exposed as the `cozystack_lvm_global_filter` inventory variable for clusters whose own PVs live on device-mapper (LVM-on-LUKS, multipath), and its effectiveness is verified via `lvmconfig` immediately after writing ([**@kvaps**](https://github.com/kvaps) in cozystack/ansible-cozystack#49). + +* **[ansible-cozystack] fix(prepare): enable containerd device_ownership_from_security_context for CDI block imports**: Adds a k3s containerd drop-in config enabling `device_ownership_from_security_context` on the CRI plugin across all prepare playbooks. Without this setting k3s ships the option disabled, so the KubeVirt CDI importer fails with "cannot open /dev/cdi-block-volume: Permission denied" when writing VM disk images into raw block volumes, causing `DataVolume` to hang in `ImportInProgress` and VMs to stay `Pending` ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#48). + +## Fixes + +* **fix(dashboard): grant tenant dashboard read on cozy-public PVCs**: The VM disk source-image dropdown in the console was returning 403 and staying empty even when golden images existed in the `cozy-public` namespace. The `cozy:tenant:dashboard` Role only granted read on Flux HelmRepositories and HelmCharts; `get/list/watch` on PersistentVolumeClaims has been added so tenant identities can list the `vm-default-images-*` PVCs ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2843, backport #2858). + +* **fix(api): emit initial-events-end bookmark for core.cozystack.io watches**: The `TenantSecret`, `TenantModule`, and `TenantNamespace` aggregated API resources never sent the `k8s.io/initial-events-end` bookmark required by the WatchList / streaming-list protocol. Client-go informers using `WatchListClient` (on by default since v1.35) never reached `HasSynced` and logged "hasn't received required bookmark event marking the end of initial events stream" every ~10 seconds. The bookmark is now emitted after initial `ADDED` events, matching the behaviour `apps.cozystack.io/Application` already implemented ([**@sunib**](https://github.com/sunib) in #2786, backport #2844). + +* **fix(networking): point host ouroboros proxy at the root-tenant ingress**: When `publishing.proxyProtocol` was enabled, the host-level ouroboros proxy inherited the wrapper chart's default backend (`ingress-nginx-controller.cozy-ingress-nginx`), a FQDN that describes a managed Kubernetes tenant cluster. On the host, ingress-nginx is deployed by `extra/ingress` as `root-ingress-controller` in `tenant-root`, so the composed FQDN never resolved and the proxy never became `Ready`. The host ouroboros `Package` is now emitted with a `proxy.target` override derived from `publishing.ingressName` ([**@lexfrei**](https://github.com/lexfrei) in #2800, backport #2846). + +* **fix(objectstorage-controller): propagate Bucket readiness to BucketClaim**: The vendored COSI controller (v0.2.2) hardcoded `bucketReady=false` after dynamic provisioning and never re-read the `Bucket` to pick up the `true` transition, so `BucketAccess` was never granted and provisioned buckets ended up without credentials. The controller now re-reads the live `Bucket` after create and propagates its readiness, converging `BucketClaim` to ready on the next resync ([**@lexfrei**](https://github.com/lexfrei) in #2792, backport #2828). + +* **fix(kubernetes): stamp application lineage labels on worker node VMs**: Worker-node VMs of a tenant Kubernetes cluster are created by Cluster API and the KubeVirt provider, so their `virt-launcher` pods were never stamped with the `apps.cozystack.io/application.{group,kind,name}` lineage labels, preventing the dashboard from attributing those pods to their owning Kubernetes application. The labels are now applied to the `KubevirtMachineTemplate` worker VM template. A companion fix also quotes `application.name` so a purely-numeric cluster name renders as a YAML string rather than an integer, which would fail label-value validation ([**@kvaps**](https://github.com/kvaps) in #2779, backport #2790). + +## Dependencies + +* **chore(seaweedfs): bump seaweedfs-cosi-driver to v0.3.1**: v0.3.1 ships a stale-socket self-heal: the COSI driver now removes any leftover UNIX socket before binding, so the objectstorage provisioner recovers automatically from `CrashLoopBackOff` after a non-graceful exit (SIGKILL, OOM, or panic) instead of wedging on "bind: address already in use" ([**@lexfrei**](https://github.com/lexfrei) in #2791, backport #2827). + +## Development, Testing, and CI/CD + +* **ci(release): repair orphaned draft tag_name on retag**: When the git tag was deleted and re-created between draft creation and merge, GitHub orphaned the draft release by setting `tag_name` to `"untagged-"` while preserving the human-readable `name`. The finalize step looked up the draft by `tag_name` and threw "Draft release for \ not found", blocking the release. The workflow now detects the orphaned form, falls back to matching by `name`, repairs `tag_name` via `updateRelease`, and then publishes ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2761, backport #2829). + +## Documentation + +* **docs(ingress): explain how ingress works in the platform**: Adds a "How ingress works" overview section to the ingress package README (rendered at `/docs/v1.4/operations/services/ingress/`), covering per-tenant ingress-nginx controllers, the per-namespace `IngressClass` model, cross-tenant sharing via `namespace.cozystack.io/ingress`, TLS via cert-manager issuers, and the `whitelist`/`cloudflareProxy` access-control options ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2770). + +* **[website] docs(networking): publish Kubernetes API endpoint via external-dns with kuberture**: Documents how to expose the managed Kubernetes API endpoint through external-dns using the kuberture system package, including configuration examples ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#539). + +* **[website] docs: exclude loop devices from LVM global_filter**: Adds documentation explaining the LVM `global_filter` requirement for hosts running Cozystack on bare metal, preventing unintended activation of DRBD/loop-backed volume groups ([**@kvaps**](https://github.com/kvaps) in cozystack/website#563). + +* **[website] feat(blog): add Managed Kubernetes how-to post**: Publishes a practical how-to blog post covering how to deploy and use managed Kubernetes clusters within Cozystack ([**@tym83**](https://github.com/tym83) in cozystack/website#565). + +* **[website] feat(blog): add platform-managed backups introduction post**: Publishes a blog post introducing Cozystack's platform-managed backup capabilities for stateful workloads ([**@tym83**](https://github.com/tym83) in cozystack/website#566). + +* **[website] docs(talm): document DRBD sysctl tuning, keepalive toggle, etcd quota**: Adds documentation for talm covering DRBD sysctl performance tuning, the DRBD keepalive toggle, and the etcd quota configuration ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#567). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@sunib**](https://github.com/sunib) +* [**@tym83**](https://github.com/tym83) + +### New Contributors + +We're excited to welcome our first-time contributors: + +* [**@sunib**](https://github.com/sunib) - First contribution! + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.4.2...v1.4.3 diff --git a/docs/changelogs/v1.4.4.md b/docs/changelogs/v1.4.4.md new file mode 100644 index 0000000000..454916b1ea --- /dev/null +++ b/docs/changelogs/v1.4.4.md @@ -0,0 +1,44 @@ + + +# v1.4.4 (2026-06-18) + +A patch release that fixes the dashboard build on the `release-1.4` branch and ships a new `talm` v0.31.0 with stability and usability improvements. + +## Fixes + +* **fix(dashboard): pin UI source branch to `release-1.4`**: Patch builds of Cozystack 1.4.x were cloning `cozystack-ui@main` instead of `cozystack-ui@release-1.4`, which could pull in unreleased UI features and break the patch release gate entirely. `CONSOLE_BRANCH` is now pinned to `release-1.4` in the dashboard Makefile. The PR also adds documentation explaining that cutting a new minor release (vX.Y.0) requires creating the matching `cozystack-ui/release-X.Y` branch and pinning `CONSOLE_BRANCH` in the dashboard Makefile ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2945). + +## Other repositories + +### talm v0.31.0 + +* **[talm] feat(cozystack): add DRBD-oriented sysctl and etcd backend defaults**: Adds a set of production-tested TCP sysctls (`tcp_orphan_retries`, `tcp_fin_timeout`, `netdev_max_backlog`, `netdev_budget`, `netdev_budget_usecs`) to the `cozystack` preset that prevent TCP port exhaustion during DRBD reconnect storms on node reboots or resyncs. An opt-in `tcpKeepaliveTuning` group is also available for deployments that need faster idle-socket failure detection across all long-lived TCP connections. The etcd backend quota is exposed as a configurable `etcd.quotaBackendBytes` value (default 8 GiB) so control planes holding many DRBD-resource CRDs do not trip etcd's 2 GiB NOSPACE alarm ([**@IvanHunters**](https://github.com/IvanHunters) in cozystack/talm#131). + +* **[talm] fix(cli, engine): emit progress on stderr, enrich lookup error chain, retry transient failures**: Fixes `talm template -f X > Y` producing YAML files with a stray progress line on stdout. Lookup failures now surface the resource kind, namespace, id, and dialed endpoints, with a six-class taxonomy of hints (TLS handshake, connection refused, deadline, auth, resource, unknown) so operators receive actionable guidance instead of a raw gRPC error. Transient connectivity classes (refused, deadline) are retried up to three times with exponential backoff; permanent errors fail immediately ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#212). + +* **[talm] fix(engine): require a template before rendering the chart**: Running `talm template` without `--file`/`--template` previously rendered the entire chart before checking that any template was actually requested, which in online mode triggered live node discovery and produced a misleading `connection refused` error instead of the actionable `templates are not set` message. The guard is now evaluated before any chart render or node dial ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#217). + +* **[talm] fix(cozystack): exclude loop devices from LVM global_filter**: Adds `/dev/loop.*` to the LVM `global_filter` in the generated Talos machine config so the host does not scan or activate volume groups inside loop-mounted images, preventing unexpected LVM activation on boot ([**@kvaps**](https://github.com/kvaps) in cozystack/talm#215). + +## Documentation + +* **[website] feat(blog): add Cozystack vs OpenStack comparison post**: Publishes a vendor-neutral blog post comparing Cozystack and OpenStack across seven dimensions — architecture, compute, networking, storage, managed services, operations, and multi-tenancy — for teams evaluating private cloud options ([**@tym83**](https://github.com/tym83) in cozystack/website#580). + +* **[website] fix(layout): keep fixed-header offset after banner removal**: Restores the correct top-offset for the fixed navigation header after the CozySummit banner was removed, preventing content from being obscured behind the navbar ([**@kvaps**](https://github.com/kvaps) in cozystack/website#579). + +* **[website] chore(banner): remove CozySummit Virtual 2026 announcement**: Removes the expired event banner from the website ([**@kvaps**](https://github.com/kvaps) in cozystack/website#576). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@tym83**](https://github.com/tym83) + + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.4.3...v1.4.4 diff --git a/docs/changelogs/v1.4.5.md b/docs/changelogs/v1.4.5.md new file mode 100644 index 0000000000..0effd00073 --- /dev/null +++ b/docs/changelogs/v1.4.5.md @@ -0,0 +1,40 @@ + + +# v1.4.5 (2026-07-03) + +A patch release with fixes for tenant control-plane deletion, VM node-group sizing, Harbor storage-class drift, CSI hot-plug volume handling, and a new `talm` v0.32.0 with encrypted user values. + +## Fixes + +* **fix(kamaji): backport datastore unused-deletion deadlock fix (clastix/kamaji#1122)**: Deleting a tenant whose etcd `DataStore` had zero referencing `TenantControlPlane`s left the `DataStore` stuck in `Terminating` (blocked by the `kamaji.clastix.io/TenantControlPlane` finalizer), which hung the etcd Helm uninstall and wedged the whole tenant namespace. Tenant deletion involving an otherwise-idle `DataStore` now completes cleanly instead of hanging indefinitely ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3146, backport #3188). +* **fix(kubernetes): gate vmop cert-manager-crds dependency on certManager.enabled**: The `victoria-metrics-operator` `HelmRelease` unconditionally depended on `cert-manager-crds`, even though that `HelmRelease` is only created when `certManager.enabled` is true. Clusters with `monitoringAgents.enabled=true` and `certManager.enabled=false` therefore failed with a "not found" error on `vmop`, which cascaded into `monitoring-agents` and `vertical-pod-autoscaler` never installing. This previously-broken combination now installs correctly ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3130, backport #3186). +* **fix(kubernetes): make node group resources optional and prefer them over instanceType**: Node groups sized purely by `instanceType` (with `resources` omitted) failed `values.schema.json` validation, even though sizing was fully specified. `resources` is now optional, and when both `instanceType` and explicit `resources` (both `cpu` and `memory`) are set, the explicit resources take precedence over the instance type. A partial spec with only one of `cpu`/`memory` alongside an `instanceType` now fails fast at render time with a clear error instead of producing an invalid VM spec ([**@lexfrei**](https://github.com/lexfrei) in #3121, backport #3157). +* **fix(harbor): preserve jobservice and trivy storageClass on upgrade**: The `cozy-harbor` wrapper never forwarded `storageClass` to the `jobservice` job-log PVC, so it was created without a `storageClassName`, and the default-`StorageClass` admission then stamped the cluster default onto it. Because `storageClassName` is immutable, this caused permanent drift between the rendered and live state on every upgrade. The same latent drift existed for Trivy's `volumeClaimTemplate`. Both now preserve an existing PVC/VCT's storage class via lookup, falling back to the configured `storageClass` only for fresh installs, so upgrades no longer show spurious storage-class drift ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2930, backport #3151). +* **fix(kubernetes): add spec.timeout to tenant CSI HelmRelease**: The tenant `kubevirt-csi-node` `HelmRelease` was the only tenant addon without an explicit `timeout: 10m` (all others already had it), so it fell back to Helm's default 5-minute timeout. On slower tenant control-plane bring-up, the CSI DaemonSet could still be `InProgress` past 5 minutes, causing helm-controller to trigger a remediation uninstall/reinstall cycle mid-flight and wedge any in-progress volume mount. Aligning the timeout with the other tenant HelmReleases removes this source of intermittent CSI install/remediation flapping ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2727, backport #2929). +* **fix(csi): detach orphan hot-plug volumes from VMI when parent VM is gone**: Upstream treated a missing parent VM as detach success, so when a VMI outlived its VM during node reprovisioning, the hot-plug pod kept the infra device exclusively attached to the source host and subsequent attach attempts failed. The CSI driver now re-checks VMI volume status after the upstream unpublish call and detaches directly from the VMI when needed, and `kubevirt-csi-driver` is bumped to the latest upstream release carrying this fix, resolving stuck volume attachments after node reprovisioning ([**@kvaps**](https://github.com/kvaps) in #2866, backport #2965). + +## Other repositories + +### talm v0.32.0 + +* **[talm] feat: first-class encrypted user values, honored at template and apply**: Adds support for arbitrary encrypted user secret values (registry passwords, KMS plugin secret IDs, etc.) consumed by chart templates. Value files named `*.encrypted.yaml` are age-decrypted in memory via the existing `talm.key` at both `template` and `apply` time — plaintext never touches disk. `talm init --encrypt`/`--decrypt` now covers `values-secret.yaml` ↔ `values-secret.encrypted.yaml`. Secret-bearing fields are omitted from committed `nodes/*.yaml` files and redacted by default in `talm template` stdout and `talm apply`'s drift preview (`--show-secrets`/`--show-secrets-in-drift` reveal them on demand). `talm apply` also now honors the full set of value-source flags and `templateOptions` from `Chart.yaml`, so a value supplied at template time is no longer silently dropped at apply ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#222). + +## Documentation + +* **[website] docs(operations): add containerized GPU workloads guide**: Adds a guide covering running GPU workloads in containers on Cozystack, helping operators evaluate and configure GPU-accelerated container deployments ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#555). +* **[website] docs(talm): document chart drift detection and strict mode**: Documents `talm`'s chart drift detection and strict mode, explaining how operators can catch and control configuration drift between rendered and applied charts ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#577). +* **[website] docs(talos): warn against changing op: on the machine files block**: Adds a warning against changing the `op:` field on the Talos machine `files` block, helping operators avoid a known misconfiguration pitfall ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#575). +* **[website] docs(platform): document publishing.certificates wildcard options**: Documents the `publishing.certificates` wildcard options for the platform chart, clarifying how operators can configure wildcard certificates for published services ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#588). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) + + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.4.4...v1.4.5 diff --git a/docs/changelogs/v1.5.0.md b/docs/changelogs/v1.5.0.md new file mode 100644 index 0000000000..f2edb8d8d5 --- /dev/null +++ b/docs/changelogs/v1.5.0.md @@ -0,0 +1,378 @@ + + +# Cozystack v1.5.0 + +Cozystack v1.5.0 brings **Gateway API support via Cilium** as an opt-in ingress layer alongside ingress-nginx, **TLS for managed databases and messaging** (Kafka, NATS, Qdrant, and PostgreSQL external endpoints), **backups that work out of the box** with a platform-managed default `BackupClass`, a shared backups bucket, a new etcd backup strategy and a generic Job strategy, the **Flux v2.8 upgrade** with strict server-side apply and kstatus health checking, a new **flux-shard-operator** that spreads tenant HelmReleases across helm-controller shards so one noisy tenant can no longer stall the others, **operator-provided wildcard certificates** for platform and root-tenant ingress, **GPU passthrough that works without manual KubeVirt patching**, a **deletion-protection guardrail** for critical platform objects, and **runtime-populated dashboard dropdowns** via a new `Option` API. The release also rolls up every fix from v1.4.1 through v1.4.4. + +Platform components bumped in this release: **Flux v2.7.3 → v2.8.0** (flux-operator/flux-instance charts v0.33.0 → v0.50.0), **MetalLB v0.15.2 → v0.16.1** (FRR-K8s is now the default BGP backend), **SeaweedFS 4.05 → 4.31**, **etcd-operator v0.4.3 → v0.4.5**, **ouroboros v0.7.2 → v0.8.0**, **seaweedfs-cosi-driver v0.3.1**, and the new **kuberture** system package. + +> **Note:** Items marked *(backported to v1.4.x)* were also shipped in the v1.4.1, v1.4.2, v1.4.3, or v1.4.4 patch releases. + +## Feature Highlights + +### Gateway API Support via Cilium + +Cozystack-native services can now be exposed through the **Gateway API** backed by Cilium, as an opt-in alternative to the existing per-tenant ingress-nginx controllers. The feature is materialized per tenant through a new `gateway.cozystack.io/v1alpha1 TenantGateway` CRD reconciled by `cozystack-controller`. + +Enable it at the platform level with `publishing.gateway.enabled=true`, then either give a tenant its own Gateway, LoadBalancer IP, and certificate with `tenant.spec.gateway=true`, or leave it unset and let the tenant inherit the nearest ancestor's Gateway through the same label-based selector model that already drives ingress inheritance. Two certificate solver modes are supported: HTTP-01 (the default — a per-app certificate with zero platform configuration for new apps) and DNS-01 (opt-in — a single wildcard certificate covering an apex, with cloudflare, route53, digitalocean, and rfc2136 providers). + +Defaults stay on ingress-nginx, so existing clusters are unchanged. Two things to be aware of: Cilium Envoy / Gateway API is now always enabled (an extra `cilium-envoy` DaemonSet, roughly 100 MB RAM per node at idle), and `cozystack-api` now invokes admission (`createValidation` / `deleteValidation`) on Create and Delete for `apps.cozystack.io/*` — so any custom ValidatingAdmissionPolicies or webhooks on those kinds will now fire on all three verbs. See the [Gateway API guide](https://cozystack.io/docs/v1.5/operations/networking/gateway-api/) ([**@lexfrei**](https://github.com/lexfrei) in #2470). + +### TLS for Managed Databases and Messaging + +Four managed-app charts gain TLS support driven by a single `tls.enabled` value with consistent tri-state semantics: when unset it inherits `external` (TLS auto-on when the service is published externally, off when cluster-internal), and an explicit `true`/`false` always wins. In every case the trust anchor is a chart- or operator-managed self-signed CA that clients retrieve and pin — there is no publicly trusted CA. The one upgrade-time behaviour change to plan for: existing instances with `external: true` flip to TLS-on after upgrade; cluster-internal instances are unaffected. + +* **Kafka** serves TLS on its external LoadBalancer listener (port 9094), with certificates managed end-to-end by the Strimzi operator; clients trust via the operator-published `-cluster-ca-cert` / `-clients-ca-cert` secrets. The external listener is now gated only on `external: true`, decoupled from `tls.enabled` ([**@Arsolitt**](https://github.com/Arsolitt) in #2681). +* **NATS** and **Qdrant** gain TLS via a self-contained cert-manager chain (self-signed Issuer → CA → leaf) rendered in the tenant namespace; NATS covers both client connections and cluster routes, Qdrant covers REST and gRPC. Clients trust the `-ca` secret ([**@Arsolitt**](https://github.com/Arsolitt) in #2684, #2685). +* **PostgreSQL** — CNPG already serves TLS unconditionally, so `tls.enabled` here injects the external hostname into the operator-managed server certificate's SANs (when `external: true`), so `sslmode=verify-full` works against the external endpoint. Clients retrieve `ca.crt` from the `-credentials` secret ([**@Arsolitt**](https://github.com/Arsolitt) in #2686). + +### Backups That Work Out of the Box + +This release closes the gap between "backup machinery is installed" and "backups actually work without per-app S3 configuration." + +A platform-managed default **`BackupClass` (`cozy-default`)** is now shipped, backed by a system bucket (`cozy-backups`). Apps opt in with a `useSystemBucket` flag, after which the platform projects shared backup credentials into the tenant namespace (with RBAC isolation and projection metrics) and skips per-release credential Secrets. Default strategies are provided for every backup-capable app — Velero for VMDisk/VMInstance, CNPG for PostgreSQL, MariaDB, Altinity for ClickHouse, FoundationDB, and etcd — and a Velero `BackupStorageLocation` is wired to the system bucket. The legacy per-tenant S3 fields on Postgres and ClickHouse are deprecated in favour of this default flow ([**@androndo**](https://github.com/androndo) in #2716). + +To make that default flow reliable, **Velero is now a default system package** rather than optional. This fixes a deterministic failure where the default `backupstrategy-controller` (which hard-depends on Velero) sat in `DependenciesNotReady` and kept the platform HelmRelease from ever reaching Ready. Existing clusters get Velero in the `cozy-velero` namespace on upgrade; opt out via `bundles.disabledPackages` if you do not back up VMs ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2833). + +Two new backup strategies join the catalog: an **etcd** strategy (cluster-scoped `strategy.backups.cozystack.io Etcd` CRD, S3-only, with snapshot BackupJob and a destructive in-place RestoreJob), and a generic application-agnostic **Job** strategy where the operator supplies a Kubernetes Job template that Cozystack renders and runs as a one-shot backup, then re-renders with `.Mode == "restore"` for recovery — the generic counterpart of the app-specific drivers ([**@androndo**](https://github.com/androndo) in #2641, [**@lllamnyp**](https://github.com/lllamnyp) in #1721). + +### Flux v2.8 Upgrade with Strict Server-Side Apply + +Flux is upgraded from v2.7.3 to v2.8.0 across both the embedded management-cluster Flux and the optional tenant Flux addon (flux-operator / flux-instance charts move v0.33.0 → v0.50.0). Flux v2.8's helm-controller v1.5 ships Server-Side Apply with `--force-conflicts` and kstatus-based health checking by default — so misplaced chart fields that v2.7 silently dropped are now hard errors (fixed here for foundationdb, kafka, kubevirt-instancetypes, vm-instance, and the platform chart), and parent HelmReleases now wait for every child resource to be Ready before reporting Ready themselves. + +**Action required on upgrade:** Kubernetes 1.33+ is now required for the management cluster (and for any tenant cluster enabling the Flux addon). The `upgrade.force: true` knob is removed, so immutable-field changes (for example StatefulSet `volumeClaimTemplates` / `serviceName`) no longer self-heal and must be recreated manually (`kubectl delete sts --cascade=orphan`). Persistent TPM/EFI is re-enabled for Windows KubeVirt preferences (each affected VM provisions one extra RWO PVC), and FoundationDB `imageType` is pinned to `split` to keep upgrades non-disruptive ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2602). + +### flux-shard-operator: Tenant helm-controller Sharding + +A new **flux-shard-operator** spreads tenant HelmReleases across multiple helm-controller shards, so one noisy tenant — for example a HelmRelease stuck in infinite remediation — can no longer degrade reconciliation for everyone else. Placement is per-tenant (all of a tenant's HelmReleases share one shard), assigned greedily by least load, with a CREATE-time mutating webhook stamping the shard label on each HelmRelease. + +It ships with `shardCount: auto` by default, which sizes shards from the tenant HelmRelease count: small clusters stay at a single shard (today's behaviour) while large fleets shard out automatically, and an integer pins the count explicitly. The legacy hand-rolled `flux-tenants` deployment is drained and retired automatically by migration 44 ([**@kvaps**](https://github.com/kvaps) in #2821). + +### Operator-Provided Wildcard Certificates + +Operators can now serve platform services and the root tenant's ingress under a pre-existing wildcard TLS certificate instead of minting per-host ACME certificates. Set `publishing.certificates.wildcardSecretName` to the name of a TLS Secret already created in the publishing namespace (`tenant-root` by default) — only the Secret name travels over the cozystack-values channel, never the key material. + +It works on both ingress paths: with ingress-nginx the controller serves it as `--default-ssl-certificate` and platform Ingresses drop their cert-manager annotations; with Gateway API a new `existingSecret` TenantGateway cert mode references the Secret directly and provisions no Issuer or Certificate. Scope is the root tenant only for now; extending wildcard mode to child tenants is a follow-up ([**@lexfrei**](https://github.com/lexfrei) in #2819). + +### GPU Passthrough Out of the Box + +GPU enablement is wired up across all three paths a GPU can reach a workload, each of which previously needed manual reconciliation. + +For **tenant Kubernetes**, node-groups declaring `gpus` automatically get the `gpu=on` kubelet label (so HAMi's device plugin schedules and advertises `nvidia.com/gpu`), and the tenant gpu-operator loads the driver with `NVreg_NvLinkDisable=1`, fixing single-SXM-GPU passthrough that previously hung at "Fabric State: In Progress" with CUDA "system not yet initialized." Both defaults are overridable via `addons.gpuOperator.valuesOverride` ([**@kvaps**](https://github.com/kvaps) in #2780). + +For **KubeVirt VMs**, enabling `cozystack.gpu-operator` now auto-populates the KubeVirt CR — injecting the `HostDevices` feature gate and filling `permittedHostDevices` (plus `mediatedDevicesConfiguration` for vGPU) from shipped NVIDIA default tables — so GPU VMs schedule without a manual `kubectl patch`. **Action required:** the bundle now owns `spec.configuration.permittedHostDevices`, so the first reconcile after upgrade overwrites any hand-edited entries; move custom device entries into `.gpu.permittedHostDevices` before upgrading and verify each `resourceName` matches what nodes advertise ([**@lexfrei**](https://github.com/lexfrei) in #2768). A third gpu-operator `container` variant is also added for hosts where the NVIDIA driver and container-toolkit are already installed by the OS, exposing GPUs to regular containerized pods via the device plugin only ([**@lexfrei**](https://github.com/lexfrei) in #2766). + +### Deletion-Protection Guardrail + +A new deletion-protection guardrail blocks `DELETE` on critical platform objects labeled `platform.cozystack.io/no-delete=true`, evaluated in-process by the kube-apiserver via a ValidatingAdmissionPolicy — no webhook, DaemonSet, TLS, or extra image. Protected objects in this release include the `cozy-system` and `tenant-root` namespaces, the `tenant-root` HelmRelease, the `cozystack-version` ConfigMap, the `cozystack-packages` OCIRepository, the cert-manager ClusterIssuers, the LinstorCluster, and the packages CRDs; a migration backfills the label onto existing resources. + +To delete a protected object, remove the label first (`kubectl label platform.cozystack.io/no-delete-`). Requires Kubernetes 1.30+ ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2650). + +### Runtime-Populated Dashboard Dropdowns + +A new generic mechanism powers runtime-populated dropdowns in dashboard create/edit forms, so fields that reference live cluster resources — GPU devices, KubeVirt instancetypes and preferences, Multus networks, VM images, storage pools, storage classes, backup classes and plans — become accurate dropdowns instead of free text or stale static enums. It introduces a namespaced, read-only `Option` resource (`core.cozystack.io/v1alpha1`) computed on read by a privileged in-process provider registry, and an `x-cozystack-options` schema keyword that app charts declare via the cozyvalues-gen `@x-cozystack-options` directive. Tenants get read-only access to options in their own namespace (from `cozy:tenant:view:base` upward), so curated lists populate without granting broad cluster reads ([**@kvaps**](https://github.com/kvaps) in #2778). + +## Upgrade Notes and Required Actions + +Most operators can take v1.5.0 with no manual action — the in-platform migrations handle config rewrites automatically — but five changes warrant attention. + +* **Kubernetes 1.33+ required for the management cluster.** With #2602, Flux jumps v2.7.3 → v2.8.0 and helm-controller v1.5 requires Kubernetes 1.33 or newer on the management cluster (and on any tenant cluster that enables the Flux addon). Upgrade Kubernetes before taking this release if you are below 1.33. + +* **`upgrade.force` is gone — immutable-field changes no longer self-heal.** Also with #2602, the helm-controller `upgrade.force: true` knob is removed. If a chart upgrade changes an immutable field (StatefulSet `volumeClaimTemplates`, `serviceName`, etc.) the apply now fails instead of silently force-replacing; recreate the object manually with `kubectl delete sts --cascade=orphan` and let Flux re-reconcile. + +* **GPU VM operators must move custom host devices before upgrading.** With #2768, when `cozystack.gpu-operator` is enabled the platform takes ownership of `KubeVirt.spec.configuration.permittedHostDevices` and overwrites it on the first reconcile. Move any hand-edited `permittedHostDevices` / `mediatedDevicesConfiguration` entries into the `.gpu.permittedHostDevices` value before upgrading, and confirm each `resourceName` matches what your nodes advertise. + +* **MetalLB switches to the FRR-K8s BGP backend; metrics are now HTTPS-only.** With #2699, MetalLB jumps v0.15.2 → v0.16.1 and adopts the upstream-default FRR-K8s backend (the classic FRR mode is deprecated). Metrics endpoints moved from plain HTTP to HTTPS only (kube-rbac-proxy replaced by native TLS + RBAC), so any scrape config pointing at the old HTTP metrics endpoints must be updated. The host-network port denylist is rotated to match the new listener set, including the dedicated `/healthz` / `/readyz` probe port 17472. + +* **Externally published databases and messaging gain TLS automatically.** With #2681 / #2684 / #2685 / #2686, instances with `external: true` flip to TLS-on after upgrade. Because the trust anchor is a self-signed CA, external clients must retrieve and pin the CA (and PostgreSQL clients using `sslmode=verify-full` benefit from the new external-hostname SAN). Cluster-internal instances are unaffected. + +## Platform Components + +* **Flux**: v2.7.3 → v2.8.0 (flux-operator / flux-instance charts v0.33.0 → v0.50.0). helm-controller v1.5 with embedded Helm v4 brings strict Server-Side Apply and kstatus health checking; see Feature Highlights for the upgrade impact. ([release notes](https://github.com/controlplaneio-fluxcd/flux-operator/releases/tag/v0.50.0)) ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2602) + - flux-operator gained an optional **Flux Status Page Web UI** with anonymous or OIDC auth and RBAC-gated actions (suspend/resume, rollout restart, run job, pod delete) — shipped opt-in in the cozystack package (v0.37.0+). + - **Security (CVE-2026-23990 / GHSA-4xh5-jcj2-ch8q, v0.40.0)**: fixed a Web UI impersonation bypass via empty OIDC claims — relevant only if the new Web UI is exposed with OIDC. + - `ResourceSetInputProvider` gained GitLab Environments, Gitea/Forgejo, AWS CodeCommit, and ExternalService provider types; `ResourceSet` gained `checksumFrom` (rollout restart on external secret/configmap change) and `includeEmptyProviders`. + - **Breaking:** the `--disable-wait-interruption` flag and `DISABLE_WAIT_INTERRUPTION` env var were removed (v0.39.0); flux-operator's CRD migration on Flux minor upgrades was fixed (v0.41.0) and a `flux-operator migrate` command added (v0.48.0). + +* **MetalLB**: v0.15.2 → v0.16.1. FRR-K8s is now the default BGP backend (Cozystack vendors the matching `frr-k8s` subchart at v0.0.25); classic FRR mode is deprecated. See Upgrade Notes for the HTTPS-only metrics change. ([release notes](https://github.com/metallb/metallb/releases/tag/v0.16.0)) ([**@lexfrei**](https://github.com/lexfrei) in #2699) + - New BGP features: per-peer `localASN` on `BGPPeer`, configurable FRR config-reload debounce, and a `ServiceSelector` on advertisements to target a subset of services. + - v0.15.3 added a `ConfigurationState` CRD to surface configuration errors, NetworkPolicy support in the chart, and fixed **CVE-2025-22874** in the images with a hardened pod security context. + +* **SeaweedFS**: 4.05 → 4.31 (chart 4.0.405 → 4.31.0). The bump clears the upstream 4.23 hazard flagged "not safe for erasure coding and multi-disk volume servers" and lands a large batch of S3-API correctness fixes (versioned-object semantics, Hadoop S3A multipart-ETag compatibility, bucket-quota read-only enforcement) plus EC bitrot detection, volume-server write-stall fixes, and new `/healthz` / `/readyz` probes. ([release notes](https://github.com/seaweedfs/seaweedfs/releases)) ([**@lexfrei**](https://github.com/lexfrei) in #2834) + - **Follow-up fixes required after the bump** (within this release): S3 TLS and the COSI provisioner ServiceAccount were restored (#2916), and the `-lock` BucketClass / s3 service name were restored with `volumeSizeLimitMB` dropped (#2943). + +* **etcd-operator**: v0.4.3 → v0.4.5. v0.4.4 added an `EtcdBackupStatus.snapshot` field describing the created backup artifact and moved `127.0.0.1` into `ipAddresses`; v0.4.5 fixed a broken restore-datadir path (restore was non-functional before this). ([release notes](https://github.com/aenix-io/etcd-operator/releases/tag/v0.4.5)) (bumped alongside the etcd backup-strategy controller by [**@androndo**](https://github.com/androndo) in #2641) + +* **ouroboros**: v0.7.2 → v0.8.0. Now logs an explicit reason when its TCP backend readiness check fails — making stuck-proxy situations immediately diagnosable instead of silently `NotReady` — and migrates the kubectl sidecar image from Docker Hub to `mirror.gcr.io` to avoid anonymous pull rate-limits *(backported to v1.4.3 via #2835)* ([**@lexfrei**](https://github.com/lexfrei) in #2807). + +* **seaweedfs-cosi-driver**: bumped to v0.3.1 with a stale-socket self-heal — the COSI driver removes any leftover UNIX socket before binding, so the objectstorage provisioner recovers automatically from `CrashLoopBackOff` after a non-graceful exit instead of wedging on "bind: address already in use" *(backported to v1.4.3 via #2827)* ([**@lexfrei**](https://github.com/lexfrei) in #2791). + +* **kuberture**: new optional system package (v0.1.1). A controller that bridges an external-dns gap (external-dns cannot read EndpointSlices): it watches the `default/kubernetes` API-server EndpointSlice and emits annotated headless Services that external-dns consumes to publish the Kubernetes API endpoint to DNS. Off by default; enable via `bundles.enabledPackages` and declare at least one `config.outputs` entry. ([sources](https://github.com/lexfrei/kuberture)) ([**@lexfrei**](https://github.com/lexfrei) in #2647) + +## Major Features and Improvements + +* **[networking] Gateway API support via Cilium**: See Feature Highlights — opt-in `TenantGateway` CRD, platform `publishing.gateway.enabled` toggle, per-tenant opt-in with HTTP-01/DNS-01 cert modes; supersedes #2213 ([**@lexfrei**](https://github.com/lexfrei) in #2470). + +* **[fluxcd] Add flux-shard-operator for tenant helm-controller sharding**: See Feature Highlights — spreads tenant HelmReleases across helm-controller shards with `shardCount: auto`; retires the hand-rolled `flux-tenants` deployment via migration 44 ([**@kvaps**](https://github.com/kvaps) in #2821). + +* **[flux] Upgrade to Flux v2.8.0**: See Feature Highlights and Platform Components — strict SSA + kstatus, chart fixes for foundationdb/kafka/kubevirt-instancetypes/vm-instance/platform, Kubernetes 1.33+ requirement, `upgrade.force` removed (folds #2612) ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2602). + +* **[platform] Operator-provided wildcard certificate for platform and tenant ingress**: See Feature Highlights — `publishing.certificates.wildcardSecretName` serves a pre-existing wildcard cert on ingress-nginx (`--default-ssl-certificate`) and Gateway API (`existingSecret` mode); root tenant only ([**@lexfrei**](https://github.com/lexfrei) in #2819). + +* **[platform] Add default backupclass**: See Feature Highlights — platform-managed `cozy-default` BackupClass backed by the `cozy-backups` system bucket, `useSystemBucket` opt-in with projected shared credentials, default strategies for every backup-capable app ([**@androndo**](https://github.com/androndo) in #2716). + +* **[platform] Add backup-strategy controller for etcd**: See Feature Highlights — cluster-scoped `strategy.backups.cozystack.io Etcd` CRD with snapshot BackupJob and in-place RestoreJob; bumps etcd-operator to v0.4.5 ([**@androndo**](https://github.com/androndo) in #2641). + +* **[backups] Implement Job backup strategy**: A generic, application-agnostic `strategy.backups.cozystack.io/Job` strategy — the operator supplies a Kubernetes Job template that Cozystack renders and runs as a one-shot backup, emitting a `Backup` artifact on completion; restore re-renders the same template with `.Mode == "restore"` (cross-namespace restore is not supported) ([**@lllamnyp**](https://github.com/lllamnyp) in #1721). + +* **[platform] Install velero by default**: See Feature Highlights — Velero moves from optional to default so the default `backupstrategy-controller` no longer blocks the platform HelmRelease; opt out via `bundles.disabledPackages` ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2833). + +* **[api] Add Option resource and x-cozystack-options for dynamic form dropdowns**: See Feature Highlights — read-only `core.cozystack.io/v1alpha1 Option` resource computed on read, `x-cozystack-options` schema keyword, tenant-scoped read access ([**@kvaps**](https://github.com/kvaps) in #2778). + +* **[api] Expose HelmRelease generation knobs as cozystack-api flags**: A follow-up to v1.4.0's #2509 that brings the same HelmRelease generation knobs to the second HelmRelease-generating path. The `cozystack-api` `convertApplicationToHelmRelease` path previously hardcoded `Interval: 5m`, `Remediation{Retries: -1}`, and no `Strategy`/`MaxHistory`; it now honours the `--helmrelease-interval`, `--helmrelease-retry-interval`, `--helmrelease-install-timeout`, `--helmrelease-upgrade-timeout`, and `--helmrelease-max-history` flags so the operator-generated and api-generated HelmReleases behave identically ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2571). + +* **[platform] Add deletion-protection guardrail via ValidatingAdmissionPolicy**: See Feature Highlights — `platform.cozystack.io/no-delete=true` blocks DELETE on critical platform objects in-apiserver; remove the label to delete; requires Kubernetes 1.30+ ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2650). + +* **[kafka] Add TLS support via Strimzi listener configuration**: See Feature Highlights — TLS on the external listener (9094), Strimzi-managed certs, external listener decoupled from TLS ([**@Arsolitt**](https://github.com/Arsolitt) in #2681). + +* **[nats] Add TLS support via cert-manager**: See Feature Highlights — self-contained cert-manager chain for client connections and cluster routes ([**@Arsolitt**](https://github.com/Arsolitt) in #2684). + +* **[qdrant] Add TLS support via cert-manager**: See Feature Highlights — TLS for REST and gRPC endpoints via a single switch ([**@Arsolitt**](https://github.com/Arsolitt) in #2685). + +* **[postgres] Add TLS support via CNPG operator-managed certificates**: See Feature Highlights — injects the external hostname into the CNPG server certificate SANs so `sslmode=verify-full` works against the external endpoint ([**@Arsolitt**](https://github.com/Arsolitt) in #2686). + +* **[kubernetes] Enable GPU passthrough out-of-the-box**: See Feature Highlights — `gpu=on` kubelet label on GPU node-groups and `NVreg_NvLinkDisable=1` driver flag fix single-SXM-GPU passthrough ([**@kvaps**](https://github.com/kvaps) in #2780). + +* **[platform] Auto-wire KubeVirt permittedHostDevices and HostDevices feature gate**: See Feature Highlights — GPU VMs schedule without manual `kubectl patch`; the bundle now owns `permittedHostDevices` (see Upgrade Notes) ([**@lexfrei**](https://github.com/lexfrei) in #2768). + +* **[gpu-operator] Add container variant for preinstalled host driver**: New `container` gpu-operator variant for hosts where the NVIDIA driver and container-toolkit are already OS-installed — exposes GPUs to containerized pods via the device plugin only ([**@lexfrei**](https://github.com/lexfrei) in #2766). + +* **[rbac] Allow tenants to start/stop/restart their VMs**: New RBAC rule grants `update` on the `virtualmachines/start`, `/stop`, and `/restart` KubeVirt subresources at the `cozy:tenant:use:base` level, so the dashboard's VM power buttons (which previously returned 403 for every tenant role) now work ([**@kvaps**](https://github.com/kvaps) in #2777). + +* **[monitoring] Add tenant overview dashboard**: A new "Tenant Overview" Grafana dashboard for platform admins, deployed only to the root/infra Grafana in `cozy-monitoring` (never to per-tenant Grafanas, so no cross-tenant data leak). Gives a cross-tenant fleet summary, per-tenant leaderboard, top-N consumers, usage trends, and health signals ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2809). + +* **[dashboard] Add cluster-usage RBAC for the new admin page**: New `cozystack-dashboard-cluster-usage` ClusterRole (cluster-wide read on nodes/pods + `metrics.k8s.io` nodes) bound to the `cozystack-cluster-admin` group, backing the new Console → Administration → Cluster Usage page (cluster-wide and per-node utilization, including GPUs). The sidebar entry is fail-closed without the binding ([**@lexfrei**](https://github.com/lexfrei) in #2743). + +* **[apps] Mark stateful-app storageClass fields as immutable**: 16 stateful apps (clickhouse, foundationdb, harbor, http-cache, kafka, kubernetes, mariadb, mongodb, nats, openbao, opensearch, postgres, qdrant, rabbitmq, redis, vm-disk) declare `storageClass` immutable via an `x-kubernetes-validations` rule in their chart schema and the dashboard renders the field read-only on edit forms, because changing storageClass never migrates data (PVCs pin `storageClassName` at creation). **Enforcement is UI-only in this release** — the aggregated apiserver does not yet evaluate the CEL rule on `Update`, so a direct `kubectl patch` is still accepted; apiserver-level enforcement is tracked in #2657. `kubernetes.nodeGroups[].storageClass` is intentionally excluded ([**@lexfrei**](https://github.com/lexfrei) in #2639). + +* **[seaweedfs] Bump SeaweedFS to 4.31**: See Platform Components — chart 4.0.405 → 4.31.0, clears the 4.23 EC hazard ([**@lexfrei**](https://github.com/lexfrei) in #2834). + +* **[metallb] Bump to v0.16.1 and rotate host-network port denylist**: See Platform Components and Upgrade Notes — FRR-K8s default backend, HTTPS-only metrics, rebuilt controller/speaker images, rotated port denylist ([**@lexfrei**](https://github.com/lexfrei) in #2699). + +* **[ouroboros] Bump to v0.8.0**: See Platform Components ([**@lexfrei**](https://github.com/lexfrei) in #2807, *backported to v1.4.3*). + +* **[platform] Add kuberture as optional system package**: See Platform Components — publishes the Kubernetes API endpoint to DNS via external-dns ([**@lexfrei**](https://github.com/lexfrei) in #2647). + +## Bug Fixes + +* **[api] Publish open spec as x-kubernetes-preserve-unknown-fields, not additionalProperties:true**: Fixes a cluster-wide kube-controller-manager CrashLoopBackOff (nil-pointer panic) that stalled reconciliation and timed out installs/upgrades. `cozystack-api` previously published the free-form `.spec` of `apps.cozystack.io` resources as `additionalProperties: true`, whose nil inner schema crashed the VAP type-checker run by KCM. The fix publishes `x-kubernetes-preserve-unknown-fields: true` instead and adds a recursive sanitizer that rewrites any boolean `additionalProperties` anywhere — including in untrusted `ApplicationDefinition.openAPISchema` input — closing the whole crash class ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2867). + +* **[prometheus-operator-crds] Ship full upstream CRD bundle**: The package previously shipped only the four service-discovery CRDs VictoriaMetrics needs. This adds the six previously-stripped CRDs (Alertmanager, AlertmanagerConfig, Prometheus, PrometheusAgent, ScrapeConfig, ThanosRuler), so third-party apps in tenant clusters that ship their own prometheus-operator (e.g. kube-prometheus-stack) can create those CRs, or set `crds.enabled: false` and consume the platform-managed CRDs (which Flux previously kept reverting) ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2660). + +* **[info] Use root-host for Keycloak OIDC issuer URL in tenant kubeconfig**: The dashboard-issued kubeconfig for non-root tenants built `--oidc-issuer-url` from the tenant subdomain, but Keycloak's ingress and TLS cert live only at the root host, so `kubectl oidc-login` failed TLS verification and the kubeconfig was unusable. The issuer now always uses the root host (`https://keycloak./realms/cozy`), so non-root tenants can authenticate with the downloaded kubeconfig ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2704). + +* **[kubernetes] Add config_path patch for containerd 2.x**: Per-registry mirror/credential config (`/etc/containerd/certs.d`) did not take effect on containerd 2.x tenant nodes because the CRI plugin config section was renamed and the old single sed was a no-op. The fix runs version-tolerant seds and loosens quote matching (also fixing Ubuntu 24.04, which emits single-quoted section headers), so `config_path` registry config works on both containerd 1.x and 2.x without manual patching ([**@elaugaste**](https://github.com/elaugaste) in #2723). + +* **[csi] Detach orphan hot-plug volumes when VMI outlives its VM**: When a VMI outlives its owning VM, hot-plug volumes were left attached, blocking the volume from being re-attached elsewhere; the CSI wrapper now detaches the orphaned hot-plug volumes so the PVC can be reused ([**@kvaps**](https://github.com/kvaps) in #2866). + +* **[csi] Route RWX Block volumes to upstream hotplug detach**: KubeVirt live-migration disks (RWX Block PVCs) were incorrectly matched by the NFS-cleanup branch in `ControllerUnpublishVolume` and never detached, so a later attach to a different VM was rejected by linstor-csi's anti-split-brain check. A new `isNFSVolume` predicate requires both `ReadWriteMany` and `VolumeMode=Filesystem`, accurately matching only NFS-backed PVCs, and the same predicate fixes in-VM disk expansion being skipped for RWX Block volumes *(backported to v1.4.2 via #2749)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2658). + +* **[csi] Verify VMI Ready after kubevirt-csi Publish to surface stuck-PVC failures**: Upstream `ControllerPublishVolume` has a fast path that reports success as soon as the volume entry appears in `VM.spec.template.spec.volumes`, even if the backing PVC is stuck `ClaimPending` and the hotplug never completed — surfacing later as a confusing kubelet `couldn't find device by serial id`. The wrapper now re-reads `VMI.Status.VolumeStatus` and returns `codes.Unavailable` with the upstream reason if the volume is not `VolumeReady`, keeping external-attacher retrying and surfacing the real provisioning failure at the CSI layer *(backported to v1.4.2 via #2748)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2659). + +* **[capi] Add startupProbe to capi-controller-manager to survive cert provisioning delay**: The CAPI controller-manager could CrashLoop during initial certificate provisioning because the readiness probe failed while the controller was still booting; a startup probe now gives it room to come up *(backported to v1.4.x stabilization)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2946). + +* **[cluster-api] Fix kamaji OOM and set limits on unset providers**: The Kamaji control-plane-provider resource override targeted a container named `manager`, but the upstream image names it `controller`, so the intended limits were dropped and the pod OOMKilled (exit 137) on the upstream 128 Mi default. The container name is corrected and modest requests/limits are set for the core, kubeadm-bootstrap, and kubevirt-infrastructure providers that previously ran as BestEffort *(backported to v1.4.1 via #2709)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2708). + +* **[platform] Migrate ephemeralStorage to diskSize via pre-upgrade hook**: The #2454 rename of `nodeGroups[*].ephemeralStorage` to `diskSize` added a hard `{{ fail }}` guard that blocked reconciliation of any HelmRelease still carrying the legacy field. A new platform pre-upgrade migration (migration 41) walks all `kuberneteses.apps.cozystack.io` Application CRs cluster-wide and renames the field automatically before chart resources are applied; the migration is idempotent and best-effort *(backported to v1.4.1 via #2712)* ([**@IvanHunters**](https://github.com/IvanHunters) in #2688). + +* **[postgres] Accept integer values for postgresql.parameters in schema**: PostgreSQL parameters such as `max_connections` are natural integers, but the chart schema declared `{type: string}` only, so bare integers were rejected at schema-validation time. An `intOrString` alias in cozyvalues-gen now emits `anyOf: [integer, string]` plus `x-kubernetes-int-or-string: true`, making both forms valid; the `$dangerousParams` blocklist is also extended with `archive_cleanup_command` and `recovery_end_command` *(backported to v1.4.1 via #2715)* ([**@IvanHunters**](https://github.com/IvanHunters) in #2687). + +* **[kafka] Remove ZooKeeper PVCs on uninstall**: The Strimzi `Kafka` CR set `deleteClaim: true` on broker JBOD volumes but left ZooKeeper persistent-claim storage at `deleteClaim: false`, orphaning the ZooKeeper PVCs on uninstall and requiring manual cleanup before the release name could be reused. `deleteClaim: true` is now set on the ZooKeeper storage as well *(backported to v1.4.1 via #2705)* ([**@Arsolitt**](https://github.com/Arsolitt) in #2679). + +* **[opensearch-operator] Replace deprecated kube-rbac-proxy image**: The `gcr.io/kubebuilder/kube-rbac-proxy` image is no longer available since the kubebuilder GCR registry was sunset. It is replaced with `quay.io/brancz/kube-rbac-proxy` (the source already used by other Cozystack components), applied via a `values.yaml` entry instead of a vendored-chart patch *(backported to v1.4.1 via #2695)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2689). + +* **[platform] Add OpenSearch to PaaS bundle**: The OpenSearch packages existed in the repo for several releases but the PaaS bundle template never referenced the two `cozystack.opensearch-*` PackageSources, so the operator was never deployed and the dashboard catalog showed no OpenSearch entry. The two missing includes are now added *(backported to v1.4.2 via #2757)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2648). + +* **[dashboard] Grant tenant dashboard read on cozy-public PVCs**: The VM disk source-image dropdown returned 403 and stayed empty even when golden images existed, because the `cozy:tenant:dashboard` Role only granted read on Flux HelmRepositories and HelmCharts. `get/list/watch` on PersistentVolumeClaims is added so tenant identities can list the `vm-default-images-*` PVCs *(backported to v1.4.3 via #2858)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2843). + +* **[api] Emit initial-events-end bookmark for core.cozystack.io watches**: The `TenantSecret`, `TenantModule`, and `TenantNamespace` aggregated API resources never sent the `k8s.io/initial-events-end` bookmark required by the WatchList protocol, so client-go informers using `WatchListClient` (on by default since v1.35) never reached `HasSynced`. The bookmark is now emitted after initial `ADDED` events *(backported to v1.4.3 via #2844)* ([**@sunib**](https://github.com/sunib) in #2786). + +* **[networking] Point host ouroboros proxy at the root-tenant ingress**: When `publishing.proxyProtocol` was enabled, the host-level ouroboros proxy inherited a default backend FQDN that describes a managed tenant cluster and never resolved on the host. The host ouroboros `Package` is now emitted with a `proxy.target` override derived from `publishing.ingressName` *(backported to v1.4.3 via #2846)* ([**@lexfrei**](https://github.com/lexfrei) in #2800). + +* **[ingress,platform] Deliver publishing.proxyProtocol to host ingress-nginx**: Threads the `publishing.proxyProtocol` setting through to the host ingress-nginx controller so PROXY-protocol mode is actually applied on the host path ([**@lexfrei**](https://github.com/lexfrei) in #2799). + +* **[objectstorage-controller] Propagate Bucket readiness to BucketClaim**: The vendored COSI controller hardcoded `bucketReady=false` after dynamic provisioning and never re-read the `Bucket`, so `BucketAccess` was never granted and provisioned buckets ended up without credentials. The controller now re-reads the live `Bucket` and propagates its readiness *(backported to v1.4.3 via #2828)* ([**@lexfrei**](https://github.com/lexfrei) in #2792). + +* **[kubernetes] Stamp application lineage labels on worker node VMs**: Worker-node VMs created by Cluster API and the KubeVirt provider were never stamped with `apps.cozystack.io/application.{group,kind,name}` lineage labels, so the dashboard could not attribute those pods to their owning Kubernetes application. The labels are now applied to the `KubevirtMachineTemplate`, and `application.name` is quoted so a purely-numeric cluster name renders as a YAML string *(backported to v1.4.3 via #2790)* ([**@kvaps**](https://github.com/kvaps) in #2779). + +* **[kubernetes] Add spec.timeout to tenant CSI HelmRelease**: Adds an explicit `spec.timeout` to the tenant CSI HelmRelease so slow first installs are not classified as failures under the new Flux kstatus waits ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2727). + +* **[platform] Add cert-manager dependency to webhook-cert consumers**: Adds an explicit `dependsOn` cert-manager to the HelmReleases that consume webhook certificates, so they no longer race cert-manager during cold install ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2726). + +* **[platform] Order cozystack-basics after the APIs its admission policies type-check**: Reorders `cozystack-basics` so it installs after the APIs its ValidatingAdmissionPolicies reference, avoiding type-check failures when the referenced APIs are not yet registered ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2842). + +* **[dashboard] Unblock token-proxy startup when JWKS is briefly unreachable**: The dashboard token-proxy refused to start if the JWKS endpoint was momentarily unreachable; startup is now resilient to a brief JWKS outage so the dashboard comes up and recovers once JWKS is reachable ([**@lexfrei**](https://github.com/lexfrei) in #2745). + +* **[kubevirt-cdi-operator,grafana-operator] Add startupProbes**: Adds startup probes to the kubevirt-cdi-operator and grafana-operator so slow boots are not killed by the readiness probe before initialization finishes ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2725). + +* **[seaweedfs] Restore S3 TLS and COSI provisioner SA after 4.31 bump**: Restores the S3 TLS configuration and the COSI provisioner ServiceAccount that regressed during the SeaweedFS 4.31 bump ([**@kvaps**](https://github.com/kvaps) in #2916). + +* **[seaweedfs] Restore -lock BucketClass, s3 service name, and drop volumeSizeLimitMB**: Restores the `-lock` BucketClass and the s3 service name and drops the now-invalid `volumeSizeLimitMB` field after the 4.31 bump ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2943). + +* **[seaweedfs] Bump seaweedfs-cosi-driver to v0.3.1**: See Platform Components — stale-socket self-heal *(backported to v1.4.3 via #2827)* ([**@lexfrei**](https://github.com/lexfrei) in #2791). + +* **[backups] Carry dropdown option sources in CRD annotations, not schema**: Moves the backup dropdown option sources from the chart schema into CRD annotations so they are consumed by the new `x-cozystack-options` mechanism without polluting the schema ([**@lexfrei**](https://github.com/lexfrei) in #2823). + +* **[backups] Fix CI broken after merge**: Repairs the backups test suite that broke after a merge conflict between concurrent backup-strategy PRs ([**@androndo**](https://github.com/androndo) in #2762). + +## Security + +* **[deps] Bump Go to 1.26.4 and x/net to v0.55.0 to close OSV advisories**: Bumps the Go toolchain to 1.26.4 and `golang.org/x/net` to v0.55.0 to clear reported OSV/Dependabot advisories across the first-party modules ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2852). + +* **[ci] Add CodeQL workflow for Go (SAST)**: Adds a CodeQL static-analysis workflow for the Go codebase so security issues are flagged on PRs ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2851). + +* **[ci] Add OpenSSF Scorecard workflow**: Adds the OpenSSF Scorecard workflow to track and publish the project's supply-chain security posture; a follow-up pins the Scorecard workflow actions by SHA ([**@tym83**](https://github.com/tym83) in #2720, #2721). + +* **[platform] Deletion-protection guardrail**: See Feature Highlights — blocks accidental DELETE of critical platform objects in-apiserver ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2650). + +## System Configuration + +* **[platform] Velero installed by default**: See Feature Highlights — Velero moves to a default system package; opt out via `bundles.disabledPackages` ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2833). + +* **[platform] Register kuberture as optional system package**: See Platform Components — off by default, enabled via `bundles.enabledPackages` with a required `config.outputs` entry ([**@lexfrei**](https://github.com/lexfrei) in #2647). + +* **[seaweedfs] Split seaweedfs-system into seaweedfs-db + seaweedfs-system**: Extracts the SeaweedFS CNPG postgres Cluster into its own `seaweedfs-db` HelmRelease that reports Ready only when postgres actually serves connections, fixing a fresh-install race where the filer CrashLooped against an endpoint-less ClusterIP under Cilium. An automatic, data-safe migration adopts the existing postgres cluster in place (re-annotates Helm ownership and stamps `helm.sh/resource-policy: keep`); no data is moved and no PV/PVC is re-provisioned ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2601). + +## Dependencies & Version Updates + +See the **Platform Components** section near the top of this changelog for the full list of upstream bumps with user-facing impact summaries (Flux v2.8.0, MetalLB v0.16.1, SeaweedFS 4.31, etcd-operator v0.4.5, ouroboros v0.8.0, seaweedfs-cosi-driver v0.3.1, kuberture v0.1.1). + +Additional dependency-related changes: + +* **[deps] Bump Go to 1.26.4 and x/net to v0.55.0** — see Security ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2852). +* **[ci] Bump cozyvalues-gen pin to v1.5.0, then v1.6.0** — schema-generation tooling refresh ([**@lexfrei**](https://github.com/lexfrei) in #2730, #2784). + +## Development, Testing, and CI/CD + +* **[ci/build] Isolate each PR build on its own ephemeral runner VM**: Moves PR builds onto per-job ephemeral runner VMs to fix the cross-job buildkit single-writer-lock contention that caused intermittent build hangs on the shared runner host ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2939). + +* **[refactor/build] mode=max registry cache with a main-only warmer**: Switches the build cache to `mode=max` with a main-branch-only cache warmer so PR builds warm-start without each PR re-pushing cache layers ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2938). + +* **[build] Read --cache-from from ghcr :latest so PR builds warm-start**: Points `--cache-from` at the ghcr `:latest` tag so PR builds reuse the last published cache ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2855). + +* **[refactor/build] Standardize image tagging to fix concurrent PR push conflicts**: Standardizes the build image-tagging scheme so concurrent PR builds no longer collide when pushing to the registry ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2711). + +* **[ci] Create buildx builder in the build step's DOCKER_CONFIG**: Fixes buildx builder creation by scoping it to the build step's `DOCKER_CONFIG` ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2962). + +* **[feat/ci] Test-impact analysis E2E (default-on) + release E2E workflow**: Adds test-impact analysis that selects which E2E suites to run based on the PR diff (default-on), plus a dedicated release E2E workflow ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2559). + +* **[ci/tests] Fix OCIR registry login in Release E2E workflow**: Corrects the OCIR registry login step in the Release E2E workflow so the release E2E run authenticates correctly ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2973). + +* **[ci/release] Auto-patch-release only the 2 newest release lines**: The automated patch-release cron now targets only the two newest release lines, matching the supported-version window, and documents the support window ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2856). + +* **[ci/release] Repair orphaned draft tag_name on retag**: When a tag was deleted and re-created between draft creation and merge, GitHub orphaned the draft release by setting `tag_name` to `untagged-`. The finalize step now detects the orphaned form, matches by `name`, repairs `tag_name`, and publishes *(backported to v1.4.3 via #2829)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2761). + +* **[ci] Add Go-scoped Renovate configuration**: Adds a Renovate configuration scoped to the Go modules so Go dependency updates are managed automatically ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2850). + +* **[ci/tags] Include data/versions in website docs commit**: The tags workflow now includes `data/versions` in the website docs commit so version metadata stays in sync ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2707). + +* **[ci] Consolidated platform & e2e stabilization batch**: A consolidated batch of platform and E2E stabilization changes folded into a single PR ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2948). + +* **[ci/e2e] Drop 3x retry on Run E2E + Install Cozystack, wait out gateway.bats tenant teardown**: Removes the blanket 3x retry on deterministic E2E steps and instead waits out the gateway.bats tenant teardown explicitly ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2558). + +* **[ci/workflows] Add SSH breakpoint on E2E failure for debug PRs**: Adds an opt-in SSH breakpoint on E2E failure for debug PRs so failures can be inspected live ([**@kvaps**](https://github.com/kvaps) in #2535). + +* **[test/e2e] Self-heal Cilium orphaned-endpoint leak across install and apps**: Adds an in-cluster self-heal Job that recovers from the Cilium agent's orphaned-endpoint leak (the "IP already in use" flake) during install and app churn ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2874). + +* **[test/e2e] Pin Kubernetes to v1.33.12 to avoid KCM VAP type-checker panic**: Pins the E2E Kubernetes version to v1.33.12 to avoid the kube-controller-manager VAP type-checker panic on affected Kubernetes builds (the durable apiserver-side fix ships in #2867) ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2868). + +* **[test/e2e] Drive LINSTOR post-install waits off a single 15m deadline**: Replaces per-step LINSTOR waits with a single 15-minute deadline so the slowest path determines runtime ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2928). + +* **[test/e2e] Pre-pull platform images via staged-busybox DaemonSet**: Pre-pulls timing-sensitive platform images via a staged-busybox DaemonSet before install so first-install reconciliation does not race image pulls ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2724). + +* **[test/e2e] Fix bucket.bats port-forward and S3 client reliability**: Hardens the bucket.bats test's port-forward and S3 client interaction for reliability ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2944). + +* **[test/e2e] ouroboros fold + CSI/NFS ordering + OIDC keycloakInternalUrl**: Folds the ouroboros E2E case, fixes CSI/NFS ordering, and corrects the OIDC `keycloakInternalUrl` in the suite ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2728). + +* **[test/e2e] Silence helm-template render in install-cozystack trace**: Silences the noisy helm-template render output in the install trace so failures are easier to read ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2615). + +* **[tests] Exclude loop devices from host LVM scanning**: Excludes loop devices from host LVM scanning in the test harness so loop-mounted images do not get their volume groups activated ([**@kvaps**](https://github.com/kvaps) in #2798). + +* **[e2e-sandbox] Use curl -fsSL when downloading mc**: The MinIO client was downloaded without `-L`, so a 302 redirect from `dl.min.io` wrote the HTML redirect body to `/usr/local/bin/mc` and the bucket E2E test then failed trying to execute it. Adding `-fsSL` follows the redirect and fails fast on HTTP errors *(shipped in v1.4.1)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2690). + +* **[test/metallb] Assert digest-pinned image form, not exact version literal**: Updates the metallb test to assert the digest-pinned image form rather than an exact version literal, so the test survives version bumps ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2873). + +* **[test/api] Add read-path authorization tests for TenantNamespace registry**: Adds read-path authorization tests for the TenantNamespace registry to lock down the IDOR fixes ([**@lexfrei**](https://github.com/lexfrei) in #2864). + +* **[feat/hack] Rewrite check-readiness as a Go command with test suite**: Rewrites the `check-readiness` helper as a Go command with a test suite, replacing the previous shell implementation ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2755). + +* **[chore/controller] Remove dead dashboard controller**: Removes the now-dead dashboard controller code left behind after the schema-driven UI rewrite ([**@kvaps**](https://github.com/kvaps) in #2694). + +* **[chore/img] Refresh Cozystack logo and add icon/stacked variants**: Refreshes the Cozystack logo and adds icon and stacked logo variants ([**@kvapsova**](https://github.com/kvapsova) in #2746). + +* **[chore/maintainers] Update roster, add Emeritus section, mark @lllamnyp as Core Maintainer**: Updates the maintainer roster, adds an Emeritus section, documents Emeritus technical offboarding and reactivation, and marks [**@lllamnyp**](https://github.com/lllamnyp) as a Core Maintainer ([**@tym83**](https://github.com/tym83) in #2717, #2718, #2719). + +* **[docs/agents] Add E2E/CI testing conventions and consolidate build/test/verify guidance**: Adds the E2E and CI testing conventions doc and consolidates the build, test, and verification guidance for agents and contributors ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2932, #2933). + +* **[docs/release] Expand release.md into a release-engineer playbook**: Expands `release.md` into a full release-engineer playbook covering the release process end to end ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2732). + +## Documentation + +* **[harbor] Document S3 object-storage (SeaweedFS) prerequisite**: Documents that Harbor requires an S3 object-storage backend (SeaweedFS) and how to satisfy the prerequisite ([**@lexfrei**](https://github.com/lexfrei) in #2818). + +* **[gpu-operator] Document passthrough variant host-driver prerequisite**: Documents the host-driver prerequisite for the gpu-operator passthrough variant ([**@lexfrei**](https://github.com/lexfrei) in #2767). + +* **[website] KubeVirt VM Disk and VM Instance how-to**: New how-to explaining the VMDisk and VMInstance primitives — independent disk/compute lifecycles, golden images, cloning, fast provisioning, with Dashboard and kubectl examples and VM access methods (serial console, SSH, VNC) ([**@tym83**](https://github.com/tym83) in cozystack/website#586). + +* **[website] etcd-operator v1alpha2 announcement and migration guide**: Announces the donation of etcd-operator to Cozystack and the new `etcd-operator.cozystack.io/v1alpha2` API — a Membership-API lifecycle replacing the StatefulSet model — with the in-place v1alpha1 → v1alpha2 migration path ([**@kvaps**](https://github.com/kvaps) in cozystack/website#573). + +The following documentation shipped in the v1.4.x patch releases and is included for completeness: + +* **[website] Add backup and recovery guides for managed applications**: Operator and tenant guides for managed-application backups covering PostgreSQL, MariaDB, ClickHouse, and FoundationDB *(shipped in v1.4.1)* ([**@androndo**](https://github.com/androndo) in cozystack/website#536). +* **[website] Cilium Gateway API — architecture, security, and migration guide**: Comprehensive `networking/gateway-api.md` page for the Cilium-backed Gateway API feature *(shipped in v1.4.2)* ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#509). +* **[website] Publish Kubernetes API endpoint via external-dns with kuberture**: Documents exposing the managed Kubernetes API endpoint through external-dns using the kuberture system package *(shipped in v1.4.3)* ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#539). +* **[ingress] Explain how ingress works in the platform**: Adds a "How ingress works" overview to the ingress package README *(shipped in v1.4.3)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2770). +* **[website] Managed Kubernetes how-to**: Practical guide for deploying and using managed Kubernetes clusters within Cozystack *(shipped in v1.4.3)* ([**@tym83**](https://github.com/tym83) in cozystack/website#565). +* **[website] Platform-managed backups introduction**: Introduces Cozystack's platform-managed backup capabilities for stateful workloads *(shipped in v1.4.3)* ([**@tym83**](https://github.com/tym83) in cozystack/website#566). +* **[website] talm: document DRBD sysctl tuning, keepalive toggle, and etcd quota**: Documents the talm DRBD sysctl performance tuning, the DRBD keepalive toggle, and the etcd backend quota configuration that ship in talm v0.31.0 *(shipped in v1.4.3)* ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#567). +* **[website] Cozystack vs OpenStack comparison**: Vendor-neutral comparison post covering seven dimensions — architecture, compute, networking, storage, managed services, operations, and multi-tenancy — for teams evaluating private cloud options *(shipped in v1.4.4)* ([**@tym83**](https://github.com/tym83) in cozystack/website#580). + +## Other Repositories + +### talm v0.31.0 *(shipped in v1.4.4)* + +* **[talm] Add DRBD-oriented sysctl and etcd backend defaults**: Adds production-tested TCP sysctls to the `cozystack` preset that prevent TCP port exhaustion during DRBD reconnect storms, an opt-in `tcpKeepaliveTuning` group, and a configurable `etcd.quotaBackendBytes` (default 8 GiB) ([**@IvanHunters**](https://github.com/IvanHunters) in cozystack/talm#131). +* **[talm] Emit progress on stderr, enrich lookup error chain, retry transient failures**: Fixes `talm template -f X > Y` emitting a stray progress line on stdout, enriches lookup-failure errors with a six-class hint taxonomy, and retries transient connectivity failures with exponential backoff ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#212). +* **[talm] Require a template before rendering the chart**: Running `talm template` without `--file`/`--template` now fails fast with `templates are not set` instead of triggering live node discovery and a misleading `connection refused` ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#217). +* **[talm] Exclude loop devices from LVM global_filter**: Adds `/dev/loop.*` to the LVM `global_filter` in the generated Talos machine config to prevent unexpected LVM activation inside loop-mounted images ([**@kvaps**](https://github.com/kvaps) in cozystack/talm#215). + +### ansible-cozystack *(shipped in v1.4.x)* + +* **[ansible-cozystack] Exclude loop and virtual devices from host LVM scanning**: Sets an LVM `global_filter` on all prepare playbooks (Ubuntu, RHEL, SUSE) so the host does not scan or activate DRBD, device-mapper, zd, or loop-backed volume groups; exposed as `cozystack_lvm_global_filter` ([**@kvaps**](https://github.com/kvaps) in cozystack/ansible-cozystack#49). +* **[ansible-cozystack] Enable containerd device_ownership_from_security_context for CDI block imports**: Adds a k3s containerd drop-in enabling `device_ownership_from_security_context` so the KubeVirt CDI importer can write VM disk images into raw block volumes instead of failing with "Permission denied" ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#48). + +## Contributors + +We'd like to thank all contributors who made this release possible: + +* [**@androndo**](https://github.com/androndo) +* [**@Arsolitt**](https://github.com/Arsolitt) +* [**@elaugaste**](https://github.com/elaugaste) +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kvaps**](https://github.com/kvaps) +* [**@kvapsova**](https://github.com/kvapsova) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@lllamnyp**](https://github.com/lllamnyp) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@sunib**](https://github.com/sunib) +* [**@tym83**](https://github.com/tym83) + +### New Contributors + +We're excited to welcome our first-time contributor: + +* [**@elaugaste**](https://github.com/elaugaste) - First contribution! + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.4.0...v1.5.0 diff --git a/docs/changelogs/v1.5.1.md b/docs/changelogs/v1.5.1.md new file mode 100644 index 0000000000..963df43837 --- /dev/null +++ b/docs/changelogs/v1.5.1.md @@ -0,0 +1,19 @@ + + +# v1.5.1 (2026-06-23) + +A patch release that fixes a v1.5.0 regression where Windows VM preferences with persistent EFI/TPM state blocked live-migration and node drains. + +## Fixes + +* **fix(kubevirt-instancetypes): drop persistent EFI/TPM to unblock live-migration**: v1.5.0 restored persistent EFI/TPM state for the `windows.11`, `windows.2k22`, and `windows.2k25` KubeVirt instance-type preferences. With persistence enabled, KubeVirt provisions a `persistent-state-for-` backend-storage PVC on the default `replicated` StorageClass, which is **ReadWriteOnce** — this pins the VM to its current node and blocks live-migration and node drains. On clusters using `evictionStrategy: LiveMigrate`, the pin can stall cluster upgrades entirely. This fix strips persistent EFI/TPM from the affected preferences using `yq del` (the previous `sed`-based strip produced `null`, which the KubeVirt v1.8 CRD rejects). **Secure Boot and the vTPM remain present and functional — Windows VMs still boot normally; EFI/TPM state is simply not persisted across reboots.** This is a stopgap until vm-state can request RWX storage ([**@kvaps**](https://github.com/kvaps) in #3006, backport #3016). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@kvaps**](https://github.com/kvaps) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.5.0...v1.5.1 diff --git a/docs/changelogs/v1.5.2.md b/docs/changelogs/v1.5.2.md new file mode 100644 index 0000000000..f576376cf6 --- /dev/null +++ b/docs/changelogs/v1.5.2.md @@ -0,0 +1,60 @@ + + +# v1.5.2 (2026-07-03) + +A patch release with ten bug fixes covering managed Kubernetes, MariaDB, Kafka, Harbor, SeaweedFS, VictoriaMetrics-operator dependency gating, Kamaji, and platform install ordering, plus new documentation and a `talm` update adding encrypted user values. + +## Fixes + +* **fix(kamaji): backport datastore unused-deletion deadlock fix**: Deleting a Tenant whose etcd DataStore had no more referencing TenantControlPlanes left the DataStore stuck `Terminating` (blocked by the `kamaji.clastix.io/TenantControlPlane` finalizer), hanging the etcd Helm uninstall and wedging the tenant namespace. Backports the upstream fix (clastix/kamaji#1122, fixing clastix/kamaji#1115) as a source patch on the pinned kamaji build, so deleting the last control plane on a DataStore now cleans up correctly ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3146, backport #3187). + +* **fix(kubernetes): gate vmop cert-manager-crds dependency on certManager.enabled**: The victoria-metrics-operator HelmRelease listed `cert-manager-crds` in `dependsOn` unconditionally, even though that HelmRelease is only created when `certManager.enabled` is `true`. Clusters running `monitoringAgents.enabled=true` with `certManager.enabled=false` had victoria-metrics-operator permanently blocked on a dependency that would never exist, cascading to `monitoring-agents` and `vertical-pod-autoscaler` never becoming ready. The dependency is now gated on the same condition that creates it ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3130, backport #3185). + +* **fix(tenant): gate VM-resource HelmReleases on victoria-metrics-operator readiness**: The tenant `etcd`, `ingress`, and `monitoring` HelmReleases create VictoriaMetrics-operator custom resources guarded by a validating webhook with `failurePolicy: Fail`. On a cold install the operator pod could be briefly unavailable, so these releases would reconcile before the webhook was serving and fail with "connection refused" — the failed-install/rollback/retry churn could then outlast the install-readiness deadline and surface as a spurious failure. These releases now `dependsOn` the victoria-metrics-operator HelmRelease so they wait for it to become `Ready` first, eliminating the race on fresh installs ([**@lexfrei**](https://github.com/lexfrei) in #3141, backport #3166). + +* **fix(mariadb): re-guard replication so single-replica passes the operator webhook**: A prior change made `replication.enabled: true` unconditional so the operator would always create `-primary`/`-secondary` services, but the mariadb-operator validating webhook rejects replication when `spec.replicas == 1`, so single-replica MariaDB instances could no longer be created — the HelmRelease never became `Ready`. Replication is now re-guarded on `replicas > 1`; single-replica installs keep external access via a `LoadBalancer` service on the bare release name, and backups, RBAC, and dashboards are updated to recognize both topologies ([**@scooby87**](https://github.com/scooby87) in #3180, backport #3183). + +* **fix(kubernetes): make node group resources optional when instanceType is set**: A managed-Kubernetes node group sized purely by `instanceType` (with `resources` omitted) failed `values.schema.json` validation because `resources` was required on every node group. `resources` is now optional: a node group sized purely by `instanceType` passes validation and installs, while a node group setting both `instanceType` and explicit `resources` has the explicit resources take precedence, so KubeVirt does not reject a VM referencing an instancetype while overriding `domain.cpu`/`memory` ([**@lexfrei**](https://github.com/lexfrei) in #3121, backport #3156). + +* **fix(seaweedfs): enforce read-only bucket access by bumping cosi-driver to v0.3.1**: The bundled SeaweedFS COSI driver (v0.1.2) hardcoded read-write S3 permissions for every `BucketAccess` and ignored the chart's `-readonly` `BucketAccessClass`, so credentials meant to be read-only could still upload and delete objects. Bumping the driver to v0.3.1 makes it honor `accessPolicy: readonly` and issue Read/List-only credentials, closing the privilege gap for tenants relying on read-only bucket access ([**@lexfrei**](https://github.com/lexfrei) in #3136, backport #3153). + +* **fix(platform): order backupstrategy-controller after cozystack-basics**: `backupstrategy-controller` creates the `cozy-backups` Bucket in the `tenant-root` namespace, which `cozystack-basics` provisions. On a fresh install the Bucket could race namespace creation and fail with `namespaces "tenant-root" not found`, which then raced the install-wait deadline and failed unrelated fresh installs. A `dependsOn` edge on `cozystack-basics` now guarantees the target namespace exists first ([**@lexfrei**](https://github.com/lexfrei) in #3037, backport #3079). + +* **fix(kafka): set entity-operator resources to avoid OOM under tenant LimitRange**: Kafka's `entityOperator` topic-operator and user-operator containers carried no resource requests/limits, so in tenant namespaces — where the default `LimitRange` caps containers at 128Mi — the JVM processes were OOMKilled on startup. The entity-operator stayed in `CrashLoopBackOff`, `KafkaTopic`/`KafkaUser` objects never reconciled, and topics could get stuck on the `strimzi.io/topic-operator` finalizer during teardown. Explicit `256Mi`/`512Mi` requests/limits are now set so the operators no longer inherit the namespace default ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2934, backport #3054). + +* **fix(harbor): preserve jobservice and trivy storageClass on upgrade**: The `cozy-harbor` wrapper never forwarded `storageClass` to the jobservice `jobLog` PVC, so it was created without a `storageClassName` and picked up the cluster default via default-StorageClass admission — and because PVC `storageClassName` is immutable, this produced rendered-vs-live drift on every upgrade. The Trivy StatefulSet's volume claim template had the same latent issue. Both now preserve an existing PVC's class via lookup and fall back to the configured `storageClass` for fresh installs ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2930, backport #3147). + +* **fix(seaweedfs): point S3 consumers at the reachable S3 service**: Renaming the SeaweedFS S3 Service to `-s3` left several consumers — the S3 ingress backend, the iceberg ingress backend, and the in-cluster COSI provisioner endpoint — still resolving the old `-s3` name or the wrong port, so the external S3 endpoint returned `503` via ingress-nginx and in-cluster COSI paths defaulted to port 443 (or the filer service, which serves no S3) instead of the real S3 service on port 8333. All consumers now resolve `seaweedfs-s3` on the correct port ([**@lexfrei**](https://github.com/lexfrei) in #3119, backport #3127). + +## Other repositories + +### talm v0.32.0 + +* **[talm] feat: first-class encrypted user values, honored at template and apply**: Adds support for encrypted user secret values (a registry password, a KMS plugin's secret-id, etc.) that chart templates consume — encrypted at rest in git with the existing `talm.key`, decrypted in memory only at render time, and kept out of committed node files and terminal output. A value file named `*.encrypted.yaml` referenced from `templateOptions.valueFiles` is age-decrypted in memory at both `talm template` and `talm apply`; `talm init --encrypt`/`--decrypt` covers `values-secret.yaml` ↔ `values-secret.encrypted.yaml`. `talm apply` now also honors the full value-source flag set and Chart.yaml's `templateOptions`, and secret-bearing fields are redacted by default from rendered node files, stdout, and the apply drift preview, with `--show-secrets`/`--show-secrets-in-drift` to reveal them on demand ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#222). + +## Documentation + +* **[website] docs(operations): add containerized GPU workloads guide**: Publishes a new operations guide for the `container` variant of `cozystack.gpu-operator`, covering containerized GPU workloads (CUDA pods, ML training, inference) on GPU nodes that already ship the NVIDIA driver and `nvidia-container-toolkit` via the distro package manager ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#555). + +* **[website] docs(talos): warn against changing op: on the machine files block**: Adds a warning next to the Talos `machine.files` block in the bare-metal bootstrap docs, explaining that Talos rejects `op: create` for files outside `/var`, so the `/etc/lvm/lvm.conf` entry must keep `op: overwrite` — changing it silently fails the `WriteUserFiles` boot step and reboot-loops the node with only the opaque `bootstrap is not available yet` error ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#575). + +* **[website] docs(talm): document chart drift detection and strict mode**: Documents the chart drift detection shipping with `talm` (cozystack/talm#216) — the library-content and preset-baseline drift warnings, the `.talm-preset.lock` file and the advice to commit it, `talm init --update` re-sync semantics, and the `strictCharts: true`/`--strict-charts` escalation with its unverifiable-baseline behavior ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#577). + +* **[website] docs(platform): document publishing.certificates wildcard options**: Documents two previously-undocumented `publishing.certificates.*` platform-package options, including the opt-in `wildcard` mode that issues one shared `*.` certificate via a DNS-01 solver instead of a per-host ACME certificate, helping operators avoid Let's Encrypt rate limits at scale ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#588). + +* **[website] fix(docs): correct stale version pins and prevent next.yaml drift**: The version-pin data file for the v1.5 docs was a stale copy of `next.yaml`, so every `{{< version-pin >}}` on the v1.5 docs — including the `go get` example and roughly 40 install-doc references — rendered Cozystack `v1.3.0`/Talos `v1.12.6` instead of the correct `v1.5.0`/`v1.13.0`. The version pins are corrected and the release script is hardened so `next.yaml` cannot drift stale again ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#589). + +* **[website] feat(blog): add Cozystack v1.5.0 release announcement**: Publishes the release-announcement blog post for Cozystack v1.5.0, covering Gateway API via Cilium, TLS for managed databases and messaging, out-of-the-box backups with a platform-managed BackupClass, Flux v2.8, flux-shard-operator, operator-provided wildcard certificates, GPU passthrough, deletion protection, and runtime-populated dashboard docs ([**@tym83**](https://github.com/tym83) in cozystack/website#592). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@lexfrei**](https://github.com/lexfrei) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@scooby87**](https://github.com/scooby87) +* [**@tym83**](https://github.com/tym83) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.5.1...v1.5.2 diff --git a/docs/changelogs/v1.5.4.md b/docs/changelogs/v1.5.4.md new file mode 100644 index 0000000000..ee50b301a3 --- /dev/null +++ b/docs/changelogs/v1.5.4.md @@ -0,0 +1,220 @@ + + +# v1.5.4 (2026-08-19) + +v1.5.4 is the **final release of the 1.5 line**. It is a stability release: it backports fixes for a webhook certificate-renewal outage that could block all pod creation, a KubeVirt VMI validation failure, several release-blocking crashloops (velero, cert-manager, SeaweedFS, flux-shard-operator), a silent PostgreSQL restore data-integrity bug, and a raft of smaller reliability and CI fixes accumulated on the branch. It also closes the SeaweedFS 4.31 rename fallout on the 1.5.x line and pins the CAPI kubeadm bootstrap objects that a later upgrade to 1.6 would otherwise prune — both of which need an operator to act, and both of which are covered in the section below. + +**These notes are measured against v1.5.2, not v1.5.3.** v1.5.3 was tagged but its GitHub Release was left a draft and never published, so no user ever received it and every operator upgrading arrives from v1.5.2. Comparing against v1.5.3 would silently drop four commits — two user-facing fixes — that nobody has seen in a release. The range is `v1.5.2..v1.5.4`, 82 commits across 27 pull requests. + +## ⚠️ Breaking Changes and Required Actions + +There are no breaking API or values changes in v1.5.4. There are two things that need an operator, and both of them can cost data or wedge an upgrade if they are skipped. Read this section in full before applying the v1.5.4 Platform Package. + +### Pre-upgrade checks + +Run these against the management cluster **before** upgrading. + +**1. SeaweedFS 4.31 rename — classify every instance, and re-run the audit even if you have run it before.** + +Cozystack v1.5.0 bumped the vendored SeaweedFS chart from `4.0.405` to `4.31.0`. Before 4.31 the chart named its workloads after the chart (`seaweedfs-master`, `seaweedfs-filer`, `seaweedfs-volume`), ignoring the Helm release name. 4.31 names them after the release, and the data-plane release is `-system`, so every StatefulSet wanted to become `seaweedfs-system-*`. StatefulSet names are immutable, so Helm could not rename in place — it stood up a **second, duplicate set beside the running one**. + +What that duplicate does depends on the cluster. With as many nodes as master replicas the new masters cannot schedule (hard pod anti-affinity against the old ones), so the duplicate sits `Pending`/`CrashLoopBackOff` and the original keeps serving. With more nodes than masters the new, empty set comes up — and because both sets carry identical pod labels, the `seaweedfs-s3` Service load-balances across them while both filers write to the **same `seaweedfs-db` Postgres metadata store** pointing at different volume servers. That is a data-integrity incident, not a cosmetic duplicate: reads of existing objects through the new endpoint miss, new writes land on empty volumes, and two master sets hand out volume IDs from independent sequences into one shared metadata table. + +Separately, and landing on the same upgrade, the v1.5.0 database split moved the CNPG `Cluster/seaweedfs-db` — the filer metadata store, i.e. the index for every object in the tenant's S3 — out of the `-system` release into its own `-db` release. Migration 43 shipped comparing the owning release against the literal string `seaweedfs-system`, so it only ever fired for an instance named `seaweedfs`; an instance named anything else was skipped and had its Cluster pruned as a removed resource, with CNPG taking the PVC along with it. That prune is not a one-shot: Helm computes deletions by diffing the **last deployed** revision against the new manifest, so a tenant whose `-system` last succeeded on a pre-split revision recomputes the same deletion on *every* upgrade attempt, including attempts that fail for unrelated reasons. + +v1.5.4 pins `fullnameOverride: seaweedfs` in `system/seaweedfs`, so workloads are named after the chart exactly as they were before 4.31 and upgrading **adopts the running set and its volumes in place**. Migration 43 is fixed to match the `-system` suffix, and migration 45 re-runs the hand-over for clusters that already ran the hardcoded version. Two states cannot be adopted that way, and the chart **fails the render rather than guess** — the enforcing guard is `packages/system/seaweedfs/templates/naming-guard.yaml`, with a sibling copy in `extra/seaweedfs` so the refusal is visible on the SeaweedFS application itself. + +*Step 0 — `seaweedfs-db` ownership (read-only, do this first).* This one destroys data rather than duplicating it, so clear it before anything else. + +```sh +kubectl get cluster.postgresql.cnpg.io -A \ + -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name,OWNER:.metadata.annotations.meta\.helm\.sh/release-name,KEEP:.metadata.annotations.helm\.sh/resource-policy' +``` + +Read the rows where `NAME` is `seaweedfs-db`: + +| OWNER | KEEP | Meaning | +|---|---|---| +| `-db` | `keep` | Handed over. Nothing to do. | +| `-db` | *(none)* | Installed fresh on ≥ v1.5.0. Safe — `-system` never rendered the Cluster, so it is not in that release's prune baseline. | +| `-system` | *(none)* | **At risk.** Migration 45 hands it over on the next platform upgrade. Do not reconcile `-system` before the migration runs. | +| *(no row at all)* | | **Already lost.** The metadata index is gone: that tenant's S3 returns 500 and its objects are unreachable even though the volume PVCs still hold the bytes. No migration can rebuild it — restore the `seaweedfs-db` Postgres from a backup, or treat that tenant's object storage as lost. Note that `-db` may still report **Ready** while this is true; trust the `kubectl get cluster` output, not the HelmRelease status. | + +*Step 1 — classify every SeaweedFS instance (read-only, mutates nothing).* + +```sh +hack/seaweedfs-naming-audit.sh # whole cluster +hack/seaweedfs-naming-audit.sh tenant-foo # or named namespaces +``` + +| CLASS | State | Action | +|---|---|---| +| `L` | Only the chart-named generation is present. | None. The upgrade adopts it in place. | +| `S` | Only the release-named generation is present — the instance was installed fresh on 1.5.x, and its data lives on `data1-seaweedfs-system-volume-*` PVCs. | Re-bind those volumes onto the chart-named PVC names **before** upgrading. Pinning the chart name without that renames the workloads *away* from the data, and Helm cannot move data between PVCs. Runbook Step 2. | +| `MIXED` | Both generations are present. | One is an empty duplicate and one holds the data, and nothing durable in the object graph says which — so the chart refuses. Classify the tenant and delete the **empty** generation so exactly one remains; the render then adopts the survivor with no further action. Runbook Step 1, then 2a or 3. | + +**Read the exit code, not just the table.** The audit fails closed: a `kubectl` call that fails, or a Helm release payload it cannot decode, prints `FATAL` and exits non-zero, and the partial table must not be trusted. Only exit 0 means the table is the whole answer — and an empty table with exit 0 is a genuinely clean fleet. + +**If you have already run this audit, run it again on v1.5.4.** The version of the script that shipped in **v1.6.0** silenced every `kubectl` failure with `2>/dev/null`, so a timeout or an RBAC denial produced an empty, "all clean" table byte-identical to an honestly clean fleet — a false clean, on the script whose output gates a runbook step that deletes PVCs. v1.5.4 is the first release on the 1.5 line to carry the audit at all, and it carries the fail-closed version (v1.6.1 and later carry it too). A clean result from a v1.6.0 checkout, or from `main` between 2026-07-20 and 2026-07-28, is not evidence of anything. + +One unrelated filer change lands on the same upgrade and is worth knowing about while you are looking at this: the filer's `postgres2` connection pool to that same `seaweedfs-db` metadata store was unconfigured, so every metadata lookup opened a fresh PostgreSQL connection and added seconds of latency to every S3 request. That is fixed here too (see the `postgres2 connection pool` entry below), and it needs no operator action — but if you have been treating slow S3 as a symptom of the rename, it may well have been this instead. + +Recovery for `S` and `MIXED` is [`docs/operations/seaweedfs-431-rename-recovery.md`](../operations/seaweedfs-431-rename-recovery.md). Do **not** guess which generation holds the data — the runbook exists because a duplicate that briefly served writes and later crashed is indistinguishable, on every durable signal, from one that never scheduled. A tenant that went 1.4.x straight to 1.6 never renamed and is unaffected by any of this; duplicates exist only on tenants that passed through 1.5.x. + +**2. The platform migration `targetVersion` moves from 45 to 46 — which changes what a later upgrade to 1.6 runs.** + +v1.5.4 is the **first 1.5.x release stamped `targetVersion: 46`**; v1.5.0 through v1.5.3 were all stamped 45. `run-migrations.sh` loops `seq CURRENT (TARGET - 1)`, so a cluster that reaches 46 and later upgrades to v1.6 (`targetVersion: 54`) runs slots 46 through 53 — and **never executes 1.6's own slot 45**. This is a skip, not an ordering problem. + +Slot 45 does not hold the same thing on both branches. On `release-1.5` it is the SeaweedFS `seaweedfs-db` hand-over repair described above. On `main` and `release-1.6` it is the pin that stamps `helm.sh/resource-policy: keep` onto the CAPI `KubeadmConfigTemplate` objects. 1.6 drops `KubeadmConfigTemplate` from the tenant `kubernetes` chart entirely — workers move to `TalosConfigTemplate` — so on that upgrade Helm sees the object in the previous release manifest, absent from the new one, and deletes it while the kubeadm-backed MachineSet is still mid-rollover with its `bootstrap.configRef` pointing at it. controller-manager then floods with reconcile errors, and where the Talos image fetch is slow or a MachineHealthCheck remediates, workers can hang pending with nothing to bootstrap from. Tenant Kubernetes only, and a noisy broken rollover rather than data loss — but it is not self-healing. + +v1.5.4 closes this in **two places, covering two disjoint populations**, and dropping either would leave a real hole: + +* **Existing clusters** (stamped 45 or lower) pick the pin up from the migration: `release-1.5`'s slot 45 now runs the keep-pin after the SeaweedFS repair, so the annotation is already in place by the time a later 1.6 upgrade skips 1.6's slot 45. The SeaweedFS half runs first deliberately — a missed hand-over loses a tenant's filer metadata, a missed pin is recoverable by hand — and both halves fail closed ahead of the version stamp, so a half-completed attempt is safe and the Job retries the whole slot. The pin selects on the `app.kubernetes.io/managed-by=Helm` label that Helm injects, so the `KubeadmConfig` children CAPI spawns from the template (owned by their Machine, never pruned by Helm) are correctly left alone. It is idempotent: an object already carrying `keep` is skipped without a write. + +* **Fresh v1.5.4 installs** cannot be reached by any migration, so the chart covers them instead. The platform chart renders the `cozystack-version` ConfigMap directly at `targetVersion` when it does not already exist, and the migration hook emits its Job **only** when that ConfigMap is already present — so a fresh install is stamped 46 having never run slot 45, or any other slot. The `kubernetes` chart therefore stamps `helm.sh/resource-policy: keep` on the `KubeadmConfigTemplate` at render time, so the object is born pinned no matter which path a cluster took to get there. + +Verify after upgrading: + +```sh +# Should print 46. +kubectl get configmap cozystack-version -n cozy-system -o jsonpath='{.data.version}{"\n"}' + +# Every row should show keep. +kubectl get kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io -A \ + -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name,KEEP:.metadata.annotations.helm\.sh/resource-policy' +``` + +A `KubeadmConfigTemplate` row showing `` means the pin did not land on that object, and a later 1.6 upgrade will prune it out from under a live MachineSet. Recovering from that means recreating the template by hand with the right Helm ownership metadata, so fix it before upgrading to 1.6. + +### Manual actions required + +* **BEFORE upgrading — recover SeaweedFS tenants classified `S` or `MIXED`.** The chart refuses to render for them, which blocks the tenant's upgrade until an operator resolves it. Run `hack/seaweedfs-naming-audit.sh` and follow [`docs/operations/seaweedfs-431-rename-recovery.md`](../operations/seaweedfs-431-rename-recovery.md). + +* **BEFORE trusting an earlier audit result — re-run the audit from v1.5.4.** The v1.6.0 copy of the script could report a false clean (see pre-upgrade check 1). + +* **AFTER upgrading — confirm the migration stamp is 46 and every `KubeadmConfigTemplate` carries `helm.sh/resource-policy: keep`** (see pre-upgrade check 2). This is what makes the later 1.6 upgrade safe. + +* **No action, but expect one rollout each:** the cert-manager webhook (it moves off port 10250) and cainjector (raised memory limit), the kube-ovn webhook, velero (new `startupProbe`), and the CNPG instances of every Postgres release with `backup.enabled` or `bootstrap.enabled`, whose rendered `Cluster` gains one `spec.env` entry for the S3 checksum fix. + +## Fixes + +* **fix(kube-ovn): reload kubeovn-webhook serving certificate on cert-manager renewal**: `kube-ovn-webhook` only loaded its TLS certificate once at startup. Once cert-manager rotated the backing Secret and the webhook pod outlived the old certificate's expiry (roughly a year after install), the pod kept serving the expired certificate, `kube-apiserver` rejected the TLS handshake, and — because the `MutatingWebhookConfiguration` uses `failurePolicy: Fail` — every pod creation in tenant namespaces was blocked, including `virt-launcher`, so no VM could start. The webhook now reloads its certificate from disk on renewal through a `GetCertificate` closure, with `renewBefore` widened to 720h. The reload path is hardened against the cases that would have made it unreliable in practice: the certificate file's mtime is observed *before* the key pair is read, so a Secret swap racing the read is retried on the next handshake rather than cached under a newer mtime and missed; change is detected by mtime inequality rather than strictly-newer, catching equal-mtime replacements and backward clock steps; a load attempt is recorded whatever the outcome, so a persistently unreadable file is retried once its mtime advances instead of on every handshake; and `stat` failures are logged rather than swallowed, so a broken mount is visible before expiry. The same change also bounds the webhook's `http.Server` with `ReadHeaderTimeout`, `ReadTimeout`, `WriteTimeout` and `IdleTimeout` — previously it set none, so a client that opened a connection and sent headers slowly could hold a goroutine and a file descriptor indefinitely ([**@IvanHunters**](https://github.com/IvanHunters) in #3557, backport #3879). + +* **fix(flux-shard-operator): repair sharded helm-controller crashloop behind an HTTP proxy**: The cloned per-shard `helm-controller` inherited `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` from the flux-aio container and lacked a `startupProbe`, so on clusters behind a corporate HTTP proxy the shard could not reach the (proxy-unreachable) in-cluster API server and crashlooped before ever becoming ready. Proxy environment variables are now stripped from the cloned container and a `startupProbe` is added so slow starts are tolerated instead of killed. The cloned probe **inherits** its `timeoutSeconds` from the liveness handler rather than being forced to 1s, so a future flux-aio shipping a larger liveness timeout cannot end up with a stricter startup probe and recreate the very crashloop this fixes ([**@IvanHunters**](https://github.com/IvanHunters) in #3546, backport #3883). + +* **fix(velero): add startupProbe so slow startup does not crashloop the install gate**: Velero binds its health endpoint only after loading plugins and connecting to the API server, which can take minutes under a heavy parallel platform install — long enough to trip the liveness probe and crashloop the pod before the install-readiness gate ever passes. A `startupProbe` now defers liveness checks until the server has actually finished starting ([**@lexfrei**](https://github.com/lexfrei) in #3138, backport #3533). + +* **fix(cert-manager): move the webhook off the kubelet's port**: The cert-manager webhook listened on port 10250, the kubelet's own port. When the webhook Service resolved to a node IP instead of the pod IP, the connection landed on the kubelet, which answered with its own certificate, and the API server rejected every cert-manager admission call cluster-wide. The webhook now listens on a port the kubelet does not own ([**@lexfrei**](https://github.com/lexfrei) in #3359, backport #3366). + +* **fix(cert-manager): raise cainjector memory limit to unblock caBundle injection**: `cainjector` loads every webhook configuration, `APIService`, and CRD into its informer caches at startup; on a full Cozystack install that working set exceeded the 128Mi memory limit, so the single leader-elected `cainjector` pod was OOMKilled during cache population and crashlooped before it could inject any CA bundle, breaking every webhook that depends on cert-manager-issued CAs. The memory limit is raised so cache population can complete ([**@IvanHunters**](https://github.com/IvanHunters) in #3199, backport #3202). + +* **fix(mariadb): widen startup probe budget so bootstrap cannot be killed**: The MariaDB operator built its startup probe without a `failureThreshold`, so it fell back to the Kubernetes default of 3 — giving a fresh bootstrap only ~40 seconds before being killed by the liveness probe the startup probe exists to defer. The budget is widened so a legitimately slow first bootstrap is no longer mistaken for a hung container ([**@lexfrei**](https://github.com/lexfrei) in #3344, backport #3364). + +* **fix(seaweedfs): make naming audit fail closed on kubectl and payload errors**: `hack/seaweedfs-naming-audit.sh`, used as the pre-upgrade gate before PVC deletion, silenced every `kubectl` failure with `2>/dev/null`, so a transient API error produced an empty, "all clean" table indistinguishable from an honestly clean fleet. Every enumeration now routes through one fail-closed helper that names the failed query on stderr and exits non-zero, and a Helm release Secret whose payload cannot be decoded is fatal rather than a silent skip. The same change fixes a false clean reachable with no corruption at all: the chart-name extraction was a greedy (last-match) `sed`, and since Helm marshals the release's `config` after `chart`, a values subtree spelling `chart.metadata.name` shadowed the real chart name — the release then read as non-SeaweedFS and the tenant vanished from the report. It now takes the first match, folds newlines so a pretty-printed payload parses at all, and says which path and shape it expected when it cannot read one ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3436, backport #3878). + +* **fix(seaweedfs): close the 4.31 rename fallout on the 1.5.x→1.6 upgrade path**: Cozystack v1.5.0 bumped the vendored SeaweedFS chart to 4.31.0, which renames workloads after the Helm release name; because StatefulSet names are immutable, upgrading through the 1.5.x line stood up a second, empty StatefulSet beside the running one instead of adopting it. This closes the remaining gaps in the adoption path: `system/seaweedfs` pins `fullnameOverride: seaweedfs` so the upgrade adopts the running workloads and volumes in place, the naming guard moves into the chart a platform upgrade actually re-renders and refuses when both naming generations exist, cluster-scoped COSI RBAC is named per namespace again (the 4.31 release-based names collided across tenants), and the `seaweedfs-db` hand-over now runs for **every** instance name — previously an instance not named `seaweedfs` had its filer metadata database pruned on upgrade. Migration 45 repairs clusters that already ran the old hand-over, and `hack/seaweedfs-naming-audit.sh` plus [`docs/operations/seaweedfs-431-rename-recovery.md`](../operations/seaweedfs-431-rename-recovery.md) guide classification and recovery. **The refusal is expected for tenants that passed through 1.5.x — see the required-actions section above before upgrading** ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3339, backport #3370, building on the adopt-in-place pin from #3282). + +* **fix(seaweedfs): configure postgres2 connection pool for the filer**: The vendored SeaweedFS chart ships connection-pool settings only for the `mysql` store, which Cozystack does not use. The `postgres2` store, which is enabled, had none — so the Go SQL pool kept zero idle connections and every filer metadata lookup opened a fresh PostgreSQL connection (TCP + TLS + SCRAM, roughly 300ms each). At around five path lookups per S3 operation that added about two seconds of latency to every S3 request, regardless of cluster load. Measured on a 12-node cluster with two filers: `PUT` of a 4KiB object went from 10s to 0.28s (p50) and `HEAD` from 2.6s to 0.16s. `max_open` is capped at 40 so two filer replicas stay under CNPG's default `max_connections` of 100. This fix was tagged in v1.5.3, which was never published, so v1.5.4 is the first release to carry it ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2906, backport #3194). + +* **fix(cozystack-basics): gate the hostname VAP policies on the VAP API**: The route/gateway/ingress hostname `ValidatingAdmissionPolicy` templates rendered unconditionally. On a cluster missing the VAP API, an operator-generated `HelmRelease` with drift detection off would render the policies out on first install and never add them back. The templates are now gated on `.Capabilities.APIVersions.Has`, so installs on VAP-less clusters no longer silently drop hostname validation ([**@lexfrei**](https://github.com/lexfrei) in #3409, backport #3876). + +* **fix(keycloak-configure): patch HelmRelease in release namespace on teardown**: The `keycloak-configure` pre-delete teardown Job cleared the Flux `HelmRelease` finalizers with a `kubectl patch` aimed at a hardcoded namespace that didn't match the release's actual `cozy-keycloak` namespace, so the teardown Job's own RBAC could never reach the object it needed to patch and uninstalling the release could get stuck. The patch now targets the correct release namespace ([**@lexfrei**](https://github.com/lexfrei) in #3372, backport #3877). + +* **fix(platform): forward backupStorage overrides to the backupstrategy-controller Package**: The documented admin override for `cozy-default` backup S3 coordinates (`spec.components.backupstrategy-controller.values` on the platform Package) had no effect, because the `cozystack-platform` PackageSource exposes only a single `platform` component and silently ignored the override block, so admin-configured backup storage settings never reached the running backup controller. The platform chart now forwards the override through correctly ([**@androndo**](https://github.com/androndo) in #3333, backport #3402). + +* **fix(backups): request S3 checksum only when required for barman-cloud (non-AWS S3 / Ceph RGW)**: Since botocore ~1.36 (early 2025) the default `RequestChecksumCalculation` is `when_supported`, so barman-cloud's boto3 attaches a flexible checksum to every `PutObject`. AWS S3 accepts it, but several S3-compatible backends — Ceph RADOS Gateway, **the platform's own SeaweedFS system bucket**, some MinIO and Cloudflare R2 builds — reject it with `InvalidArgument: x-amz-content-sha256 must be UNSIGNED-PAYLOAD`, failing every backup and WAL-archive upload. `AWS_REQUEST_CHECKSUM_CALCULATION=when_required` is now set through the CNPG `Cluster`'s `spec.env`, which reaches the instance pods and therefore the barman-cloud subprocess the instance manager execs. On this branch that one setting covers all three S3 paths at once: the chart-rendered legacy `spec.backup.barmanObjectStore`, the same field SSA-patched by the CNPG backup driver in the `useSystemBucket` flow, and `externalClusters` recovery. Note this is a **release-1.5-native equivalent** of the fix on `main`, not a cherry-pick — `main` sets `spec.instanceSidecarConfiguration.env` on `barmancloud.cnpg.io` `ObjectStore` objects, and release-1.5 has no barman-cloud plugin and no `ObjectStore` CRD, so those objects would template cleanly and then apply to nothing. It is gated on `backup.enabled` or `bootstrap.enabled`, so a Postgres release with no S3 configured is untouched ([**@androndo**](https://github.com/androndo) in #3417, backport #3882). + +* **fix(objectstorage-controller): converge BucketClaim readiness and speed up COSI provisioner failover**: The COSI control plane (built from upstream container-object-storage-interface v0.2.2) never watched `Bucket` objects and dropped no-op resync deltas, so a `BucketClaim` whose backend `Bucket` became ready *after* the first reconcile stayed `bucketReady=false` forever, and the consuming application never got its `BucketAccess`. The controller now requeues until the backend Bucket is ready, emits a `Warning` event while it waits, and shortens the COSI leader-election lease so failover is faster ([**@lexfrei**](https://github.com/lexfrei) in #3034, backport #3532). + +* **fix(tenant): inherit full ancestor label chain from parent namespace**: Tenant namespace labelling only derived one ancestor level from `.Release.Namespace`, so a direct child of `tenant-root` never received the labels needed for its grandparent's `-egress` `CiliumClusterwideNetworkPolicy` to allow traffic, silently breaking cross-tenant egress for nested tenants more than one level deep. Namespaces now inherit the full ancestor label chain from their parent ([**@IvanHunters**](https://github.com/IvanHunters) in #2912, backport #3191). + +* **fix(backups/postgres): purge stale recovery cluster on repeat in-place restore**: A repeat in-place PostgreSQL `RestoreJob` into a target that had already been restored once silently reported `Succeeded` while doing nothing — the PVC was untouched, no recovery pods appeared, and the database kept its pre-restore contents, a silent data-integrity failure. The restore path now purges the stale recovery cluster before restoring again, so a repeat restore actually recovers data instead of quietly no-opping. The freshness discriminator that decides whether a leftover `Cluster` belongs to this restore or a previous one is deliberately **strict** (`creationTimestamp > StartedAt`), so an exact timestamp tie resolves to "not fresh" and the caller purges — the conservative direction, since a genuinely fresh Cluster is always created well after `StartedAt` ([**@IvanHunters**](https://github.com/IvanHunters) in #3318, backport #3321). + +* **fix(fluxcd): omit empty distribution.artifact in FluxInstance for guest clusters**: Guest Kubernetes clusters with the `fluxcd` addon enabled never became `Ready`, because the `flux-instance` template rendered `spec.distribution.artifact` unconditionally (unlike its sibling optional fields), and the empty string broke the intended air-gapped default of using the operator's embedded manifests. The field is now guarded like its siblings, so guest clusters with `fluxcd` enabled converge to `Ready` again ([**@IvanHunters**](https://github.com/IvanHunters) in #3284, backport #3292). + +* **fix(dashboard): unbreak CORS on expired session for k8s API calls**: The dashboard SPA broke with CORS errors once the `kc-access` cookie expired: the Keycloak client's `webOrigins` was never set (so it stuck on a stale hostname after any `rootUrl` change), and `oauth2-proxy` mishandled the expired-session case for Kubernetes API calls. Both are fixed so an expired session no longer surfaces a broken, CORS-blocked SPA ([**@IvanHunters**](https://github.com/IvanHunters) in #2788, backport #3291). + +* **fix(apps/vpn): remove invalid foo field from urls Secret**: A leftover debug `foo` field in the VPN app's `-urls` Secret template is not part of the `core/v1` Secret schema. Client-side apply tolerated it, but server-side apply's stricter validation rejected the object outright, so any VPN application upgrade failed and the release was left in a broken state. The stray field is removed ([**@IvanHunters**](https://github.com/IvanHunters) in #3281, backport #3290). + +* **fix(kubevirt): update KubeVirt to v1.8.4**: Updates the vendored `kubevirt-operator` from v1.8.2 to v1.8.4, backporting the VMI checksum status-field validation fix (uint32 range) that Kubernetes 1.36 requires — without it, strict CRD numeric-format validation could leave VMIs stuck in `Scheduled` and unable to start. The update also carries assorted upstream bug fixes and CVE remediations; the KubeVirt custom resource itself is unchanged ([**@lexfrei**](https://github.com/lexfrei) in #2940, backport #3285). + +* **fix(kubevirt-instancetypes): restore persistent EFI/TPM state**: v1.5.1 stripped `persistent: true` from both `preferredEfi` and `preferredTPM` on the six shipped Windows preferences — `windows.11`, `windows.2k22`, `windows.2k25` and their `.virtio` variants — to unblock live migration. Secure Boot and the vTPM stayed present, so the VMs still booted, but their state was no longer persisted: **EFI NVRAM and vTPM contents were discarded on every VM restart**, so anything the guest sealed to the vTPM (BitLocker being the common case) or wrote to NVRAM — Secure Boot key enrollment, boot entries — did not survive a power cycle. This affects anyone running Windows 11, Server 2022 or Server 2025 guests created from the shipped preferences on **v1.5.1 or v1.5.2**; v1.5.0 had persistence, and it is restored here. The strip was motivated by a real-looking concern that turned out to rest on an outdated premise — that the `persistent-state-for-` backend-storage PVC, ReadWriteOnce on the default `replicated` StorageClass, pins the VM to its node and blocks live migration and node drains, stalling cluster upgrades under `evictionStrategy: LiveMigrate`. KubeVirt has in fact migrated RWO-Filesystem backend storage since v1.4 (kubevirt/kubevirt#12629): it creates a fresh target state PVC and copies the small state blob during migration. Verified on the default replicated storage that a persistent-firmware VM reports `LiveMigratable=True` and gets its copy-on-target PVC, so persistence and live migration now hold at the same time. Existing VMs pick the change up on their next restart; already-running VMs are unaffected until then. This fix was tagged in v1.5.3, which was never published, so v1.5.4 is the first release to carry it ([**@kvaps**](https://github.com/kvaps) in #3154, backport #3212, reverting #3006). + +* **fix(migrations): pin KubeadmConfigTemplate on the 1.5 slot 45 migration and in the chart**: Migration slot 45 holds different things on different branches — `release-1.5` uses it for the SeaweedFS `seaweedfs-db` hand-over repair, while `main` and `release-1.6` use it for the `helm.sh/resource-policy: keep` pin on the CAPI `KubeadmConfigTemplate` objects. Cutting v1.5.4 is what first ships `targetVersion: 46`, and `run-migrations.sh` loops `seq CURRENT (TARGET - 1)`, so a cluster that reaches 46 later runs `seq 46 53` on the way to 1.6 and **never executes 1.6's slot 45 at all** — the pin is skipped, and Helm then prunes the template while the kubeadm-backed MachineSet is still mid-rollover. 1.5's own slot 45 now applies the pin (after the SeaweedFS repair, and fail-closed) so an existing cluster picks it up on the way to 46, and the `kubernetes` chart stamps the annotation at render time so the object is born pinned — which is the only thing that covers a **fresh** v1.5.4 install, since such a cluster has its `cozystack-version` ConfigMap rendered straight at 46 and the migration hook never emits a Job at all. See the required-actions section above for the verification commands ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3892). + +* **fix(postgres-operator): align CNPG CRDs with 1.27.3**: The vendored CloudNativePG CRDs were still pinned to 1.27.1 metadata while the operator image was already on 1.27.3, which prunes the new `Backup.status.instanceID.sessionID` field the newer instance-manager writes — causing false instance-manager restart errors and failed backups. The CRDs are aligned with the pinned 1.27.3 image so backups no longer fail on this mismatch ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3526). + +## Development, Testing, and CI/CD + +* **ci(release): move the v1.5.4 release path off the decommissioned runner pool**: Every job on the `release-1.5` release path was pinned to a `[self-hosted]` runner pool that no longer exists, so the release would silently sit `Queued` forever with nothing red to point at instead of failing loudly. The release, tag, and cache-warmer jobs are moved to the same runner shape `release-1.6` already uses, unblocking future patch releases on this branch ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3906). + +* **ci: warm the build cache on release-1.5 pushes**: `release-1.5`'s build-cache warmer was dead code — it triggered on pushes to `main` (which never happens on this branch) and targeted the same decommissioned `[self-hosted]` runner — so every PR build on the branch ran cold and could exceed its 30-minute timeout. The warmer now triggers on `release-1.5` pushes and targets a working runner, and the Build job is pointed at the branch-scoped cache namespace the warmer actually writes; without that the warmer would run and nothing would read it, because the `cache-from` refs still resolved to the shared namespace `main` writes, whose layers carry `main`'s sources and miss on most of this branch's 28 packages. Builds only read the cache (`WRITE_CACHE` defaults to 0), so concurrent PR builds cannot race on a cache manifest and a miss degrades to a cold build rather than an error. The timeout is raised to 75 minutes at the same time: a warm build is around 15 minutes, but a cold one is around 45, and 30 minutes only ever fit the warm case — when it did not fit, the job reported a timeout instead of whatever actually went wrong ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3469). + +* **fix(ci, kubernetes): build container disks concurrently to unblock Build**: Every `release-1.5` PR Build job was dying at its timeout, and effectively all of the budget went to one target: `image-ubuntu-container-disk` builds one disk per supported Kubernetes minor — six of them — and each spends four to six minutes inside a libguestfs appliance installing that minor's kubelet and kubeadm into its own copy of the cloud image, roughly 28 minutes serially. None of it was cached and no warmer could fix that, because only `build-main.yaml` sets `WRITE_CACHE=1` and `main` dropped this image from `make image` shortly before `release-1.5` was cut, so the buildcache refs it reads had never been written — leaving `release-1.5` as the only branch still building the image at all. The six builds share no per-version work, so they now run concurrently in a sub-make carrying its own `-j` (keeping the root `make build` serial), with buildkit deduplicating the shared guestfish and cloud-image stages across the concurrent solves and `--output-sync=target` keeping each version's log readable. The job also moves off the 4cpu/16gb runner shape, since six concurrent libguestfs appliances want roughly 9GB of RAM and a core each ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3469). + +* **ci: install flux in the release-1.5 cache warmer**: The warmer's `make build` step shells out to `flux push artifact`, which the previous self-hosted runner had baked in but the new ephemeral runner shape does not; the cache warmer is updated to install the `flux` CLI so it can complete ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3480). + +* **test(seaweedfs): make the guard-parity and fake-kubectl assertions able to fail**: The four assertions holding the invariant that neither SeaweedFS chart classifies on mutable claim timestamps or pod liveness were written as `! grep -qF X "$f"`, which cannot fail — POSIX and bash both exempt a `!`-negated pipeline from `errexit`, so the command ran, returned 1, and the test carried on reporting success. Those four were the entire body of the test, so it asserted nothing at all; and converting them naively fails, because all four names legitimately appear in the comment prose that records why each was rejected as a discriminator. They now assert against the template logic with comment blocks stripped, forbidding a *live* reference, and are proven in both directions. The fake `kubectl` used by the migration tests likewise now fails on unmodelled calls instead of returning success for anything it does not recognise ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3436, backport #3878; #3892). + +* **Regression coverage shipped alongside the fixes above**: Each of the backported fixes carried its own test, and they are listed here rather than left silent because they are what stops these bugs coming back. A reconcile-level regression drives `reconcileCNPGRestore` end to end against a fake client, covering the actual call site of the repeat-restore bug rather than only its helpers ([**@IvanHunters**](https://github.com/IvanHunters) in #3318, backport #3321); the e2e suite now asserts `BucketAccess` reaches `accessGranted=true`, asserts a `BucketClaim` converges to `bucketReady` promptly for both the bucket and Harbor paths, and dumps controller pods on a bucket failure ([**@lexfrei**](https://github.com/lexfrei) in #3034, backport #3532); `keycloak-configure` gains a run of the chart's helm-unittest suite ([**@lexfrei**](https://github.com/lexfrei) in #3372, backport #3877); the flux-shard-operator asserts the full `startupProbe` contract and rejects an alias regression, with the proxy-env drop and probe recorded in the sanitisation list ([**@IvanHunters**](https://github.com/IvanHunters) in #3546, backport #3883); the kubeadm keep-pin folded into slot 45 has migration coverage ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3892); and the PR labeler now maps the `objectstorage-controller` scope to `area/storage` ([**@lexfrei**](https://github.com/lexfrei) in #3034, backport #3532). + +## Documentation + +* **[website] fix(backup): simplify UX with default backupclass and correct the admin override path**: Rewrites the backup guides to match the corrected `backupStorage` override path shipped in #3402/#3333 above, so the documented admin override actually works and the default backup-class UX is simpler to follow ([**@androndo**](https://github.com/androndo) in cozystack/website#622). + +* **[website] docs(backups): document PostgreSQL point-in-time recovery (PITR)**: Adds a guide for restoring a PostgreSQL cluster to a specific point in time, covering the required backup configuration and the restore procedure ([**@androndo**](https://github.com/androndo) in cozystack/website#629). + +* **[website] docs(monitoring): add OIDC authentication guide for Grafana**: New guide walking operators through enabling OIDC-based single sign-on for Grafana, including the admin-group and mode-toggle caveats ([**@IvanHunters**](https://github.com/IvanHunters) in cozystack/website#597). + +* **[website] docs(operations): document the gateway.http2 platform value**: Documents the `gateway.http2` platform value so operators know how to enable/disable HTTP/2 on the ingress gateway ([**@lllamnyp**](https://github.com/lllamnyp) in cozystack/website#625). + +* **[website] docs(talm): describe the preset value knobs for network and registries**: Documents the new talm preset value knobs for tuning network sysctls and registry mirror configuration ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#633). + +* **[website] docs: fix mermaid edge label and document wildcardSecretName for v1.5**: Fixes an unrenderable mermaid diagram in the certificate documentation and documents the `wildcardSecretName` option for v1.5 ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#615). + +* **[website] docs: correct the publishing reference and document the existingSecret cert mode**: Corrects the certificate publishing reference and documents the third, `existingSecret`, certificate mode ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#619). + +* **[website] docs(gpu): drop the manual KubeVirt patch step for GPU passthrough**: The platform wires `permittedHostDevices` into the KubeVirt CR itself from `packages/core/platform/files/gpu-passthrough-defaults.yaml`, so the manual patch documented since v1.4 is obsolete; the guide now covers the pre-upgrade migration steps for hand-edited entries instead. Merged just after the v1.5.2 tag, so v1.5.4 is the first release these notes cover it in ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#556). + +* **[website] docs: import the operator guides that lived in the cozystack repo**: Moves the operator-facing guides that previously lived in the `cozystack` repository into the website's documentation, consolidating operator docs in one place ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#648). + +* **[website] docs: regenerate the managed apps reference for v1.4.5 and v1.5.2**: Refreshes the auto-generated managed-application reference pages so the published values tables match what those releases actually ship ([**@app/cozystack-ci**](https://github.com/apps/cozystack-ci) in cozystack/website#600, cozystack/website#601). + +## Other repositories + +### talm (v0.32.0 → v0.34.0) + +* **[talm] feat(charts): add preset value knobs**: Adds configurable preset value knobs for network and registry settings, letting operators tune Talos machine config presets without hand-editing templates ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#232). + +* **[talm] chore(deps): migrate to Helm 4 and drop the cozystack/talos fork**: Migrates talm's Helm library from v3 to v4 and drops the previously vendored `cozystack/talos` fork in favor of upstream Talos v1.13.7 ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#231). + +* **[talm] security: disable, then restore, KVM nested virtualization in Talos presets (CVE-2026-53359)**: talm first disabled KVM nested virtualization in its Talos presets to mitigate CVE-2026-53359, then reverted the blanket disable once the upstream fix landed, so presets keep nested virtualization available instead of being permanently disabled ([**@kvaps**](https://github.com/kvaps) in cozystack/talm#224, [**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#225). + +* **[talm] feat(version): warn when project charts drift from the talm binary**: talm now warns when a project's vendored charts have drifted from the version embedded in the talm binary, plus chart-drift-detection support in `init --update`, catching stale vendored charts before they cause a surprising render diff ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#216). + +### ansible-cozystack (v1.5.2 → v1.6.1) + +* **[ansible-cozystack] fix(prepare): keep KVM nested virtualization enabled**: Mirrors talm's CVE-2026-53359 response in the Ansible prepare role — nested virtualization is disabled to mitigate the CVE and then restored once the fix landed, matching the talm preset behavior ([**@kvaps**](https://github.com/kvaps) in cozystack/ansible-cozystack#60, [**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#61). + +* **[ansible-cozystack] fix(examples): make prepare and k3s.orchestration runnable as separate invocations**: The example playbooks can now be run as independent steps rather than requiring a single combined invocation ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#52). + +* **[ansible-cozystack] chore(deps): track the cozy-installer version**: The installer image is bumped in step with the Cozystack releases cut during this period — v1.5.3, v1.6.0 and v1.6.1 ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#59, cozystack/ansible-cozystack#63, cozystack/ansible-cozystack#65). + +No tags were released in **boot-to-talos** (latest v0.7.1, 2026-03-19), **cozyhr** (latest v1.6.1, 2026-01-27), **cozy-proxy** (latest v0.3.0, 2026-04-28) or **external-apps-example** (no tags) during this release period. + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@androndo**](https://github.com/androndo) +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@lllamnyp**](https://github.com/lllamnyp) +* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) + + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.5.2...v1.5.4 diff --git a/docs/changelogs/v1.6.0.md b/docs/changelogs/v1.6.0.md new file mode 100644 index 0000000000..b9141a1d3c --- /dev/null +++ b/docs/changelogs/v1.6.0.md @@ -0,0 +1,669 @@ + + +# Cozystack v1.6.0 + +Cozystack v1.6.0 replaces the Ubuntu + kubeadm bootstrap of **tenant Kubernetes workers with Talos Linux** driven by Cluster API, completes the **etcd-operator `v1alpha2` migration** with in-place adoption of live clusters, adds **OIDC single sign-on for tenant kube-apiservers and per-instance Grafana**, introduces the tenant-facing **`SecurityGroup` firewall API**, makes **tenant resource quotas hierarchical** so a sub-tenant can no longer escalate past its parent's budget, gives Keycloak an optional **KMS-encrypting database proxy**, a separate admin hostname and S3 backups, propagates **operator-provided wildcard certificates** down to every tenant termination point, and reworks the release pipeline around **immutable tags and rc→stable promotion** so a stable release is the byte-identical promotion of the release candidate that was tested. A ten-PR sweep also makes **application deletion actually reclaim its storage** instead of leaking PVCs. The release rolls up every fix from v1.5.1, v1.5.2 and v1.5.3. + +Platform components bumped in this release: **Talos v1.13.0 → v1.13.6** (KVM guest-escape CVE fixes), **etcd-operator v0.4.5 → v0.5.2** (new `etcd-operator.cozystack.io/v1alpha2` API), **Cilium 1.19.3 → 1.19.5**, **KubeVirt v1.8.4**, **Velero 1.17.0 → 1.18.1**, **Vertical Pod Autoscaler 1.3.0 → 1.5.0**, **Harbor 2.14.2 → 2.15.1**, **Keycloak 26.5.2 → 26.6.3**, **LINSTOR 1.33.2 → 1.33.3** with linstor-csi v1.11.2, **FoundationDB operator v2.13.0 → v2.30.0**, **HAMi 2.8.1 → 2.9.0**, **Percona MongoDB operator 1.21.1 → 1.22.0**, **OpenBao v2.5.0 → v2.5.1**, **CoreDNS chart 1.43.2 → 1.46.0**, **csi-driver-nfs 4.11.0 → 4.13.3**, **OpenCost 1.111.0 → 1.120.3**, **NATS 2.11.8 → 2.11.10**, **linstor-scheduler v0.3.2 → v0.3.6**, and **ouroboros 0.8.0 → 0.8.1**. + +> **Note:** Items marked *(shipped in v1.5.x)* were also released in the v1.5.1, v1.5.2 or v1.5.3 patch releases. + +## ⚠️ Breaking Changes and Required Actions + +This is the largest upgrade surface since v1.0. Read this section in full before upgrading — three of the items below can **block or wedge the upgrade** if their preconditions are not met. + +The platform migration `targetVersion` moves from **45 (v1.5.x) to 54**, which means migrations **45 through 53** run as pre-upgrade hooks on the way to v1.6.0. + +### Pre-upgrade checks + +Run these against the management cluster **before** applying the v1.6.0 Platform Package. + +**1. etcd adoption needs a reachable backup target (migration 50 — hard-fails the upgrade).** + +Migration 50 adopts every legacy `etcd.aenix.io/v1alpha1` cluster onto the new v1alpha2 operator and takes a mandatory pre-adoption etcd snapshot first. If it cannot resolve the platform-managed snapshot target it **exits 1 and halts the upgrade** (#2859, #3270, #3335). + +```sh +# Do you have legacy etcd clusters at all? If this prints nothing, migration 50 is a no-op. +kubectl get etcdclusters.etcd.aenix.io -A + +# The snapshot target: the cozy-default-etcd strategy and the projected credentials Secret. +kubectl get etcds.strategy.backups.cozystack.io cozy-default-etcd +kubectl get secret cozy-backups-creds -n cozy-velero \ + -o jsonpath='{.data.AWS_ACCESS_KEY_ID}{"\n"}' + +# The bucket behind it must be healthy. +kubectl get buckets.apps.cozystack.io cozy-backups -n tenant-root +``` + +If the Secret or the strategy is missing and you have legacy etcd clusters, either repair the `cozy-backups` Bucket / `backupstrategy-controller` first, or accept adopting live etcd **without** a snapshot by taking the documented escape hatch (see *Manual actions required*). + +**2. SeaweedFS naming audit (chart refuses to render for two classes of tenant).** + +The v1.5.0 SeaweedFS chart bump to 4.31 renamed workloads from chart-based (`seaweedfs-*`) to release-based (`seaweedfs-system-*`) names. v1.6.0 pins `fullnameOverride: seaweedfs` and adopts the running set in place, but two states cannot be adopted automatically and the chart **fails the render** rather than guess (#3282, #3339). + +```sh +# Step 0 — seaweedfs-db ownership. A missing row means the filer metadata is ALREADY lost. +kubectl get cluster.postgresql.cnpg.io -A \ + -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name,OWNER:.metadata.annotations.meta\.helm\.sh/release-name,KEEP:.metadata.annotations.helm\.sh/resource-policy' + +# Step 1 — classify every SeaweedFS instance (read-only). +hack/seaweedfs-naming-audit.sh +``` + +**Correction, added after release: the copy of that script shipped in v1.6.0 fails open, so a clean result from it is not evidence.** Every `kubectl` call in the v1.6.0 version is silenced with `2>/dev/null`, so a timeout, an RBAC denial or any transient API error produces an empty "all clean" table byte-identical to an honestly clean fleet — on the one script whose output gates a runbook step that deletes PVCs. Fixed in #3436 and backported as #3474, both merged 2026-07-28, so v1.6.1 and later carry a version that prints `FATAL` and exits non-zero instead. Run the audit from a v1.6.1-or-later checkout, read the **exit code** rather than the table, and re-run it if the only run you have was against the v1.6.0 copy. + +Class `L` needs no action. Class `S` (installed fresh on 1.5.x) and class `MIXED` (both naming generations present) must be recovered **before** upgrading — follow [`docs/operations/seaweedfs-431-rename-recovery.md`](../operations/seaweedfs-431-rename-recovery.md). A cluster that went 1.4.x → 1.6.0 directly never renamed and is unaffected. + +**3. Tenant Kubernetes clusters still on v1.30.** + +v1.30 leaves the Talos↔Kubernetes support matrix and the chart now refuses to render it. Migration 46 patches live CRs from `v1.30` to `v1.31`, but a **GitOps-managed** CR is re-overwritten by the next source reconcile (#2931). + +```sh +kubectl get kuberneteses.apps.cozystack.io -A \ + -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name,VERSION:.spec.version' +``` + +Any row showing `v1.30` that is reconciled from Git must have `spec.version` bumped to `v1.31` (or newer) **in Git**, before or together with the platform upgrade. + +**4. Hand-made tenant StorageClasses that collide with propagated LINSTOR classes.** + +Remote-accessible LINSTOR StorageClasses are now created inside each tenant cluster under the same name. A manually created tenant StorageClass with a colliding name (typically `replicated`) blocks the propagated class and stalls the tenant CSI release (#2872). + +```sh +# Inside each tenant cluster: +kubectl get storageclass replicated -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}{"\n"}' +``` + +Delete any such class that is not Helm-managed before upgrading. Infra classes that must stay node-local need an explicit `linstor.csi.linbit.com/allowRemoteVolumeAccess: "false"` — an absent annotation is treated as remote-accessible. + +**5. Deprecated etcd tenant-module backup values.** + +The `backup.*` block on the `etcd` tenant module is removed in this release. Confirm nothing still relies on it (#2859). + +```sh +kubectl get etcds.apps.cozystack.io -A -o jsonpath='{range .items[*]}{.metadata.namespace}/{.metadata.name}{"\t"}{.spec.backup.enabled}{"\n"}{end}' +``` + +**6. If you are coming from v1.4.x**, the v1.5.0 requirement still applies: the management cluster (and any tenant cluster enabling the Flux addon) must be on **Kubernetes 1.33+**. + +### Breaking changes + +* **Tenant Kubernetes workers move from Ubuntu + kubeadm to Talos Linux.** *What changed:* worker bootstrap is now driven by `cluster-api-bootstrap-provider-talos` (CABPT) with a `talos-csr-signer` sidecar in the Kamaji control-plane pod; workers boot the Talos openstack image streamed in by CDI. *Who is affected:* every existing tenant Kubernetes cluster. *What to do:* nothing mandatory — existing machines roll over to Talos workers automatically on the first reconcile after upgrade — but plan for a **full worker-pool replacement per tenant cluster**, and expect worker disks to be reprovisioned and container images re-pulled. The separate `disk-kubelet` PVC is gone; `nodeGroups[*].diskSize` now sizes the single Talos system disk ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2931). + +* **Kubernetes v1.30 removed from the tenant support matrix.** *What changed:* the `version` enum drops `v1.30`; the chart fails the render with an explicit guard. *Who is affected:* tenants still pinned to v1.30. *What to do:* migration 46 bumps live CRs to `v1.31`; GitOps-managed CRs must be bumped in Git (see pre-upgrade check 3) (#2931). + +* **Worker MachineHealthCheck remediation is now ON by default.** *What changed:* the worker MHC `maxUnhealthy` moved from a hard-coded `0` (remediation effectively disabled) to `nodeHealthCheck.maxUnhealthy`, defaulting to `"50%"` — CAPI now deletes and replaces unhealthy worker Machines. *Who is affected:* every tenant Kubernetes cluster. *What to do:* set `nodeHealthCheck.maxUnhealthy: "0%"` to keep the previous behaviour until your fleet is stable on Talos workers (#2931). + +* **The default `md0` node group is no longer merged into every cluster.** *What changed:* `nodeGroups` defaults to `{}` and the built-in `md0` is applied only when no node groups are configured, so it can finally be removed or replaced. *Who is affected:* clusters that defined their own node groups and were silently also getting `md0`. *What to do:* nothing — migration 47 pins `md0` explicitly on existing CRs to preserve the live topology; remove it afterwards if you do not want it. Migration 47 is **fail-closed**: a read or patch failure aborts the upgrade rather than let Helm prune a live `md0` MachineDeployment ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2936). + +* **Fresh tenant clusters with `nodeGroups: {}` come up with zero workers.** *What changed:* the chart no longer manages `MachineDeployment.spec.replicas` — the cluster-autoscaler owns it alone, seeded from `minReplicas: 0`. *Who is affected:* new clusters that enable `addons.ingressNginx.enabled: true` on a default CR. *What to do:* either supply a node group with `roles: [ingress-nginx]` and `minReplicas >= 1`, or let the autoscaler bring up `md0` once the ingress-nginx pods go `Pending`. Upside for existing clusters: `helm upgrade` no longer drains workers back to a hardcoded `replicas: 2` on every platform bump ([**@IvanHunters**](https://github.com/IvanHunters) in #3233). + +* **etcd moves to `etcd-operator.cozystack.io/v1alpha2`.** *What changed:* the legacy `etcd.aenix.io/v1alpha1` StatefulSet model is replaced by a Membership-API lifecycle; the vendored upstream chart is replaced by a Cozystack-authored one at appVersion `v0.5.2`, and CRDs move into a dedicated `etcd-operator-crds` package. *Who is affected:* every cluster running the etcd tenant module or tenant Kubernetes (Kamaji datastores). *What to do:* existing clusters are adopted **in place** by migration 50 — no data move, no pod restart — provided the pre-upgrade snapshot target resolves (see pre-upgrade check 1). A pre-upgrade hook also deletes the pre-1.6 operator Deployment, whose `spec.selector` is immutable and would otherwise fail the Helm upgrade ([**@androndo**](https://github.com/androndo) in #2859, [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3270). + +* **The deprecated `backup.*` values on the `etcd` tenant module are removed.** *What changed:* `backup.enabled`, `schedule`, `destinationPath`, `endpointURL`, `region`, `forcePathStyle`, `s3AccessKey`, `s3SecretKey` and the job-history limits no longer exist; a new `version` value (default `3.6.11`) pins the etcd release. *Who is affected:* tenants that configured legacy per-release S3 backups on the etcd module. *What to do:* migrate to the `BackupClass` flow driven by the `Etcd` strategy (`strategy.backups.cozystack.io/v1alpha1`) before upgrading (#2859). + +* **Deleting a managed application now permanently deletes its storage.** *What changed:* ten packages gained post-delete cleanup that reclaims previously-orphaned PVCs and Secrets. Deletion is **irreversible** — this is the point of the change, but it is a real behaviour change from "delete leaks the volume, and you could recover from it". *Who is affected:* ClickHouse (keeper + data + log PVCs), Qdrant (data PVC), OpenBAO (data PVC), the tenant `monitoring` module (VictoriaMetrics/VictoriaLogs storage PVCs), the tenant `seaweedfs` module (volume PVCs), the tenant `etcd` module (`data-etcd-*` PVCs), Harbor (jobservice/trivy PVCs — this force-deletes them, overriding `persistence.resourcePolicy=keep`), MariaDB (operator-generated Secrets), Bucket and Gateway (ACME solver resources). *What to do:* back up before deleting; snapshot anything you care about. Migrations 48 and 51 backfill the release label onto pre-existing ClickHouse keeper PVCs and monitoring storage PVCs so already-deployed clusters are covered too. Harbor's CNPG database PVCs are **not** touched ([**@scooby87**](https://github.com/scooby87) in #3072, #3075, #3073, #3094, #3095, #3170, #3077, #3071, #3076, #3093). + +* **Tenant resource quotas are now hierarchical.** *What changed:* a tenant's declared quota is the budget for its whole sub-tree — a child that declares a quota carves a fixed slice out of the parent's remaining budget, a child without one shares the parent's pool instead of being unbounded, and a child quota exceeding the parent's remaining budget is rejected at admission. *Who is affected:* multi-level tenant trees where a sub-tenant previously declared more than its parent, or nothing at all. *What to do:* review sub-tenant quotas before upgrading; an overcommitted pool emits a `QuotaOvercommitted` event. Set `--tenant-quota-buffer-percent` on `cozystack-controller` for a temporary grace buffer so workloads already over a freshly-binding quota keep running during rollout ([**@kvaps**](https://github.com/kvaps) in #3086). + +* **Worker node disks now default to the application-level (DRBD `replicated`) StorageClass.** *What changed:* a node group leaving `storageClass` empty previously picked the management-cluster default StorageClass; it now falls back to the application `storageClass`, because linstor-csi v1.11.2 rejects ReadWriteMany volumes on a non-DRBD class and worker VMs need RWX to live-migrate. *Who is affected:* tenant clusters relying on the empty-storageClass default. *What to do:* nothing, but the rendered worker template changes, so each tenant's worker MachineDeployment rolls once on the first reconcile after upgrade ([**@lexfrei**](https://github.com/lexfrei) in #3129, #2987). + +* **`authentication.oidc.keycloakInternalUrl` now defaults to the in-cluster Keycloak Service.** *What changed:* the default is `http://keycloak-http.cozy-keycloak.svc:8080/realms/cozy` instead of empty, so OIDC backend calls no longer leave the cluster. *Who is affected:* clusters that never set the value explicitly. *What to do:* nothing, but the dashboard and linstor-gui gatekeepers roll once. Set the value explicitly for a non-standard Keycloak placement, or to an empty string to restore external OIDC discovery ([**@lexfrei**](https://github.com/lexfrei) in #3289). + +* **Air-gapped tenant workers: `registries.mirrors` passthrough is gone.** *What changed:* the Helm-rendered `*-patch-containerd` Secret that plumbed the platform-wide `registries.mirrors` config to tenant workers has no consumer in the Talos machineconfig and was removed. *Who is affected:* tenants relying on `registries.mirrors` for in-guest container-image pulls. *What to do:* the Talos OS image and installer **are** overridable — set `talos.imageFactoryURL` and `talos.installerRepository` to your mirror — but in-guest registry mirroring is a Phase 2 follow-up (#2931, #3244). + +* **`SecurityGroup` is served with a membership-group shape.** The new `sdn.cozystack.io/v1alpha1 SecurityGroup` API landed inside this release with `spec.targetRef` replaced by a `spec.attachments[]` membership model before release, so there is no upgrade path to worry about — but anyone tracking `main` between 2026-06-30 and 2026-07-16 must rewrite their objects ([**@lexfrei**](https://github.com/lexfrei) in #2922). + +* **The `network.cozystack.io` API group (`ExposureClass` / `ServiceExposure`) was introduced and removed within this cycle.** It never shipped in a release and is **not** part of v1.6.0; native `Service` `type: LoadBalancer` + `loadBalancerClass` covers the same ground, exposed as `publishing.loadBalancerClass`. Only clusters tracking `main` that explicitly set `publishing.exposureClass` need to provision their own MetalLB/Cilium address pool and switch to `publishing.loadBalancerClass`; any orphaned `cozystack-` pool can be deleted by hand ([**@lexfrei**](https://github.com/lexfrei) in #3081, removed by [**@kvaps**](https://github.com/kvaps) in #3218). + +### Manual actions required + +* **BEFORE the upgrade — recover SeaweedFS tenants classified `S` or `MIXED`.** The chart refuses to render for them, which blocks the tenant's upgrade until an operator resolves it. Run `hack/seaweedfs-naming-audit.sh` and follow [`docs/operations/seaweedfs-431-rename-recovery.md`](../operations/seaweedfs-431-rename-recovery.md). Do **not** guess which generation holds the data — the runbook exists because a duplicate that briefly served writes is indistinguishable from one that never scheduled (#3339). + +* **BEFORE the upgrade — bump GitOps-managed tenant `Kubernetes` CRs off `v1.30`.** Migration 46 only patches live objects; Git wins the next reconcile and the HelmRelease then fails the version guard (#2931). + +* **BEFORE the upgrade — delete colliding hand-made tenant StorageClasses** (see pre-upgrade check 4) (#2872). + +* **DURING the upgrade, only if migration 50 halts** — if the cluster intentionally has no backup storage (backups disabled, external S3 without staged credentials, or SeaweedFS not yet ready) and you accept adopting live etcd **without** a safety snapshot, take the documented escape hatch: + + ```sh + kubectl edit package.cozystack.io cozystack.cozystack-platform + ``` + ```yaml + spec: + components: + platform: + values: + migrations: + etcdAdoptSkipBackup: true + ``` + + Flux re-renders the platform chart and re-runs the hook, adopting every legacy etcd without a snapshot. **A botched adoption on a tenant control-plane etcd is unrecoverable without a snapshot.** Set the flag back to `false` once the upgrade completes, or the next etcd migration will skip its snapshot too (#2859, #3335). + +* **AFTER the upgrade — review worker MachineHealthCheck tolerance.** Remediation is now on by default at `50%`; set `nodeHealthCheck.maxUnhealthy: "0%"` (cluster-wide) or per node group if you want the old behaviour while the fleet settles on Talos (#2931, #3053). + +* **AFTER the upgrade — re-check metrics scrape targets on ports 10250.** cert-manager's webhook, external-secrets' webhook and metrics-server's listener all moved off port 10250, which the kubelet also serves (#3359, #3360, #3361). + +* **No action, but expect one rollout each:** tenant worker pools (Talos rollover and the storageClass fallback), the dashboard and linstor-gui gatekeepers (OIDC internal URL default), the VPA admission controller (cert-manager-issued TLS), and the linstor-scheduler admission Deployment (migration 52 deletes it so the new immutable selector can be created cleanly). + +### Deprecations + +* **Legacy per-app S3 backup fields.** The `etcd` module's `backup.*` block is now *removed* (above); the equivalent legacy fields on Postgres and ClickHouse remain deprecated in favour of the platform-managed `BackupClass` flow introduced in v1.5.0. +* **Classic per-host ACME certificates for platform services.** Still the default, but `publishing.certificates.wildcard` (DNS-01) and `publishing.certificates.wildcardSecretName` (operator-provided) are now the recommended path at scale, and the operator-provided certificate propagates to per-tenant termination points automatically (#2988, #2990). +* **`auto-release.yaml` cron patch releases** are gone — stable releases are cut only by explicit rc→stable promotion (#3017). + +## Feature Highlights + +### Talos Linux Tenant Kubernetes Workers + +Tenant Kubernetes worker nodes no longer boot Ubuntu and bootstrap through kubeadm. Phase 1 of the Kubernetes-app split design replaces that path with **Talos Linux**, driven by `cluster-api-bootstrap-provider-talos` (CABPT v0.6.12) registered as a second bootstrap provider alongside kubeadm, and a `clastix/talos-csr-signer` sidecar embedded in the Kamaji control-plane pod. + +Talos PKI (an Ed25519 CA plus trustd TLS) is generated through cert-manager with stable, lookup-and-reuse random Talos secrets; the Kamaji control-plane provider is bumped with an upstream-bound patch that exposes `KamajiControlPlane.spec.network.additionalServicePorts` so trustd (50001/TCP) can be published on the apiserver Service. Workers boot the Talos openstack raw image via a CDI `DataVolume` streamed from the image factory, with the system disk exposed as virtio-blk using `blockSize.custom: logical=512, physical=4096` so 4Ki-native backends such as LINSTOR/DRBD behave under QEMU's `O_DIRECT` writes while SeaBIOS still boots. The separate kubelet disk is gone — Talos lays out `EPHEMERAL` itself on the single system disk that `diskSize` now sizes. + +Existing tenants roll over automatically: old machines are replaced by Talos workers without manual intervention, and platform migration 45 pins the outgoing `KubeadmConfigTemplate` objects with `helm.sh/resource-policy: keep` so Helm cannot prune them out from under the still-live kubeadm MachineSet mid-rollover. + +Two behaviour changes are worth planning for: the tenant `kubernetes` HelmRelease now reports `Ready` as soon as Helm completes (`DisableWait`) and no longer blocks on worker or addon readiness — worker-rollout health is tracked by `WorkloadMonitor` — and worker MachineHealthCheck remediation is enabled by default at `maxUnhealthy: "50%"`. See the Breaking Changes section above for the full action list ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2931). + +Follow-ups shipped in the same release make the rollover practical: per-node-group `maxUnhealthy` / `nodeStartupTimeout` overrides so a stateful group can sit at `0%` while a stateless one tolerates `50%` (#3053), overridable `talos.imageFactoryURL` and `talos.installerRepository` for air-gapped, mirrored or rate-limited environments (#3244), the `TalosConfigTemplate` moved out of a post-install hook into the main install phase so worker MachineDeployments scale reliably (#3145), and `MachineDeployment.spec.replicas` handed entirely to the cluster-autoscaler so platform bumps stop draining workers (#3233). + +### etcd-operator v1alpha2 with In-Place Adoption + +Cozystack's etcd stack moves to the donated **`etcd-operator.cozystack.io/v1alpha2`** API — a Membership-API lifecycle replacing the StatefulSet model — served by a Cozystack-authored chart at appVersion `v0.5.2`, with CRDs split into their own `etcd-operator-crds` package so they can be installed ahead of the controller. + +The interesting part is the upgrade. Platform migration 50 runs as a **pre-upgrade hook**, while the legacy operator is still the running one, and uses the `etcd-migrate` tool baked into the migrations image to rewrite ownership, labels and CRs so the new operator takes over the live data plane on its first reconcile — no data move, no pod restart. Before touching anything it takes a mandatory snapshot of every adopted cluster to the platform-managed `cozy-backups` bucket, under a system key prefix with tenant-invisible credentials, and it **fails loudly rather than adopt without one**. It also re-issues each cluster's server and peer certificates with the new operator's native member wildcard SAN first, because in `secretRef` TLS mode the operator never mints certs — an adopted cluster whose certs lack the native domain would silently fail TLS the first time a member is replaced, while still reporting `Available=True`. + +The in-cluster 1.5 → 1.6 path needed three more fixes to actually work, all folded into #3270: a chart-managed transitional `etcd-headless` Service so adopted members stay resolvable at their legacy per-pod DNS names until they roll onto the native subdomain; a pre-upgrade hook that deletes the pre-1.6 operator Deployment, whose `spec.selector` is immutable and would otherwise fail the whole HelmRelease with `field is immutable`; and a raised 256Mi memory floor so the operator does not OOM before the VPA admission webhook can scale it. Backup strategies also derive their S3 endpoint, scheme and TLS mode from the provisioned bucket Secret instead of requiring it hand-set. + +The tenant etcd module's deprecated `backup.*` values are removed in the same change — use a `BackupClass` bound to the `Etcd` strategy instead ([**@androndo**](https://github.com/androndo) in #2859, [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3270, #3335). + +### OIDC Single Sign-On for Tenant Kubernetes and Grafana + +Two symmetric Phase 1 features let a tenant opt an individual workload into the platform's Keycloak `cozy` realm through one flat selector, with no platform-level configuration. + +**Tenant kube-apiserver.** Each `Kubernetes` CR gains `spec.oidc.mode: System | CustomConfig | None` (default `None`). `System` trusts the `cozy` realm via a per-cluster public client and audience binding; `CustomConfig` accepts a tenant-supplied structured `AuthenticationConfiguration`. `spec.oidc.users[]` drives one `ClusterRoleBinding` per user inside the tenant cluster (`admin` → `cluster-admin`, `view` → `view`). In `System` mode a `-oidc-kubeconfig` Secret carrying a ready-to-use `kubectl oidc-login` exec block is surfaced through the dashboard, so a tenant user goes from "cluster exists" to "kubectl works with my SSO identity" without an operator in the loop ([**@IvanHunters**](https://github.com/IvanHunters) in #3044). The underlying passthrough — `controlPlane.apiServer.extraArgs`, `extraVolumes` and `extraVolumeMounts` on `KamajiControlPlane` — is also exposed directly for anyone wanting to hand-roll authentication ([**@lexfrei**](https://github.com/lexfrei) in #3123). + +**Grafana.** Each `Monitoring` CR gains the same `spec.oidc.mode` selector. `System` wires a per-instance confidential Keycloak client with audience binding; authorization is app-side, with `spec.oidc.users: [{email, role: Admin|Editor|Viewer}]` reconciled into Grafana's Main Org. by a chart-owned post-install/post-upgrade Job that pre-provisions users, adds them to the org, PATCHes their role and prunes stale members. `CustomConfig` accepts a tenant-supplied `[auth.generic_oauth]` payload, either inline as a map (the chart merges the two contract-critical settings on top) or as a Secret with an `auth.ini` key — in which case `spec.oidc.users` is unsupported and the chart fails the render on that combination rather than silently ignoring it. The `admin_user`/`admin_password` Secret remains a documented break-glass path in every mode ([**@IvanHunters**](https://github.com/IvanHunters) in #3176). + +Both are documented in-repo at [`docs/oidc-tenant.md`](../oidc-tenant.md) and [`docs/oidc-grafana.md`](../oidc-grafana.md). + +### SecurityGroup: Tenant-Managed Network Policy + +A new tenant-facing, namespace-scoped firewall resource — `sdn.cozystack.io/v1alpha1 SecurityGroup` — lets tenants manage their applications' network policy without any access to the `cilium.io` API group. + +A `SecurityGroup` is a **membership group**: it owns a membership label (`securitygroup.sdn.cozystack.io/`) that a new `securitygroup-controller` stamps onto the pods of every managed application listed in `spec.attachments`, and removes when an attachment is dropped. The group projects 1:1 onto a `CiliumNetworkPolicy` in the same namespace whose `endpointSelector` is exactly that membership label — so one `SecurityGroup` can cover several applications at once, and `fromSG`/`toSG` peers resolve group-to-group references live in the Cilium dataplane. A finalizer guards the backing policy so member labels are always stripped before it is removed, re-asserted by the aggregated-API REST storage on every write so a full-replace `PUT` cannot orphan them. + +Ingress and egress rules only ever **add** allowed traffic; an empty list does not isolate the member pods, because effective connectivity remains the union of every policy selecting a pod including the platform's blanket-allow baseline. Default-deny enforcement is tracked separately as future work ([**@lexfrei**](https://github.com/lexfrei) in #2922). + +### Hierarchical Tenant Resource Quotas + +`tenant.spec.resourceQuotas` previously became a plain per-namespace `ResourceQuota` that ignored the tenant tree entirely — a super-admin inside a quota'd tenant could create a sub-tenant with a larger quota, or no quota at all, and consume more than was allocated. A tenant's declared quota is now the budget for its **whole sub-tree**. + +Enforcement is two cooperating layers, mirroring OpenShift's ClusterResourceQuota split. A declaration-time gate in the aggregated apiserver deterministically rejects a sub-tenant whose declared quota exceeds the parent's remaining budget. A runtime enforcer in `cozystack-controller` maintains a per-namespace `tenant-quota-allocated` ResourceQuota that clamps each pool member to its share of the parent budget, aggregating sub-tree usage so members collectively stay within it — Kubernetes enforces the most restrictive ResourceQuota in a namespace, so this binds without fighting Flux over the chart-rendered `tenant-quota`. + +A parent quota lowered below the sum of its children emits a `QuotaOvercommitted` event, and `--tenant-quota-buffer-percent` on `cozystack-controller` temporarily inflates pool budgets during rollout so workloads already over a freshly-introduced quota keep running. No API types and no tenant chart changes — the feature works with the existing field ([**@kvaps**](https://github.com/kvaps) in #3086). + +### Keycloak: Encryption Proxy, Admin Hostname, and Backups + +Keycloak gets four independent additions this release, all opt-in. + +An optional **KMS-encrypting database proxy** provides column-level encryption for PII at rest, backed by either a static KEK or Vault Transit, and off by default ([**@kvaps**](https://github.com/kvaps) in #3003). A follow-up adds Vault **Kubernetes and AppRole auth** for the proxy, a dedicated ServiceAccount, and a shared DEK set covering both migration and HA ([**@sircthulhu**](https://github.com/sircthulhu) in #3239). + +An opt-in `ingress.adminHost` serves the **admin console and Administration REST API on a separate hostname and route**, with `publishing.ingressNameAdmin` (`_cluster.expose-ingress-admin`) attaching that route to a separate — for example private — Gateway or ingressClass. Disabled by default; existing deployments are unchanged ([**@sircthulhu**](https://github.com/sircthulhu) in #2954). + +**S3 backups** for the Keycloak CNPG database land via Barman, configurable under `backup.*` and disabled by default ([**@kvaps**](https://github.com/kvaps) in #3174). Finally, the realm **login theme** is now selectable and its theme image shipped through platform `branding` values (`branding.loginTheme`, `branding.loginThemeImage`) ([**@lexfrei**](https://github.com/lexfrei) in #3124), and the database host, port and URL properties are overridable so the connection can be routed through an external proxy ([**@sircthulhu**](https://github.com/sircthulhu) in #2907). + +### Wildcard Certificates, End to End + +v1.5.0 let operators point platform services at a pre-existing wildcard TLS Secret through `publishing.certificates.wildcardSecretName`, but only for the root tenant. That certificate is now **replicated by the platform controller into every tenant namespace that terminates TLS**, so per-tenant ingress controllers and Gateways serve it automatically — no extra operator input, and no cross-namespace Secret read ([**@lexfrei**](https://github.com/lexfrei) in #2990). + +For operators who would rather have the platform mint the wildcard, a new opt-in `publishing.certificates.wildcard` (default `false`) issues a single `*.` certificate via a DNS-01 solver on the default ingress-nginx path, instead of a per-host ACME certificate — the practical fix for hitting Let's Encrypt rate limits at scale ([**@lexfrei**](https://github.com/lexfrei) in #2988). + +### Immutable Tags and rc→stable Promotion + +A stable release is now a **renamed release candidate**: the bytes shipped as `vX.Y.Z` are bit-for-bit the bytes built and end-to-end tested as `vX.Y.Z-rc.N`. No tag is ever force-moved and stable is never rebuilt — it is promoted by retagging the rc's existing images by digest. + +The enabler was decoupling the operator's self-reported version from its image: it now reads `COZYSTACK_VERSION` from the environment (threaded through `cozystackOperator.platformVersion`), falling back to the build-time value, so the same bits can report any release name. All five force-retag sites in the release workflows are removed and replaced with write-once or fast-forward-only semantics, and the cron-driven `auto-release.yaml` patch-tag workflow is deleted outright — stable tags now exist only through explicit promotion. A `promote-rc.yaml` workflow validates the rc, `skopeo copy`s every digest-pinned image reference to the stable tag, restamps the version-stamped assets, and opens a `release-X.Y.Z` PR whose merge cuts the write-once stable tag ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3017). A companion **Cut Pre-release Tag** workflow covers the rc side of the same flow (#3253). + +### Application Deletion Reclaims Its Storage + +Deleting a managed application used to leave its volumes and operator-generated Secrets behind. A ten-PR sweep across the catalog closes that, adding post-delete cleanup hooks and PVC retention policies so a deleted application actually releases what it held: ClickHouse keeper and data/log PVCs, Qdrant's data PVC, OpenBAO's data PVC, the tenant monitoring module's VictoriaMetrics/VictoriaLogs storage and TLS Secrets, the tenant seaweedfs module's volume PVCs and cert/db Secrets, the tenant etcd module's `data-etcd-*` PVCs and five TLS Secrets, Harbor's jobservice and trivy PVCs plus ACME solver resources, MariaDB's operator-generated password Secrets, Bucket's ACME solver resources, Gateway's ACME account Secret, and the Postgres init Job. + +Two migrations backfill the release label onto storage provisioned before this change — 48 for ClickHouse keeper PVCs, 51 for monitoring storage PVCs — because the respective operators only stamp `volumeClaimTemplates` labels onto PVCs they create *after* the change and never re-label existing ones. Both are best-effort by design: the worst case if a relabel is skipped is the pre-existing leak, which is no worse than the status quo. + +**This makes deletion destructive.** See the Breaking Changes section ([**@scooby87**](https://github.com/scooby87) in #3069, #3071, #3072, #3073, #3075, #3076, #3077, #3078, #3093, #3094, #3095, #3170). + +## Platform Components + +* **Talos Linux**: v1.13.0 → v1.13.6 (management-cluster image and matchbox assets), with tenant workers on the same v1.13.6. Closes the **CVE-2026-53359** and **CVE-2026-46113** KVM guest-to-host escapes. ([release notes](https://github.com/siderolabs/talos/releases/tag/v1.13.6)) ([**@lexfrei**](https://github.com/lexfrei) in #3008, #3240, #3269) + - Kernel 6.18.29 → **6.18.38**, carrying the KVM shadow-paging use-after-free and nested-guest escape fixes. Nested virtualization remains available. + - containerd 2.2.5, runc 1.4.3, etcd 3.6.11, Kubernetes 1.36.1 in the bundled components. + - Reliability fixes across the range: DNS server and listeners recreated on host-DNS runner restart, correct `int` marshalling of kube-scheduler config, relaxed LUKS header and hostname validation, more resources marked sensitive, higher etcd open-file limit, `FailurePauseTimeout` honoured before reboot. + - System extensions refreshed to the 20260622 set, including **DRBD 9.3.2** and **ZFS 2.4.3**. Applying this rolls each tenant worker pool once, as the worker template is renamed to carry the new image. + +* **etcd-operator**: v0.4.5 (vendored upstream chart) → **v0.5.2** (Cozystack-authored chart, `etcd-operator.cozystack.io/v1alpha2`). CRDs split into a new `etcd-operator-crds` package. See Feature Highlights and Breaking Changes for the adoption path. ([sources](https://github.com/cozystack/etcd-operator)) ([**@androndo**](https://github.com/androndo) in #2859, [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3270) + +* **Cilium**: 1.19.3 → 1.19.5. ([release notes](https://github.com/cilium/cilium/releases/tag/v1.19.5)) ([**@lexfrei**](https://github.com/lexfrei) in #2972) + - Fixes `CiliumNetworkPolicy` `fromNodes`/`toNodes` when `policy-default-local-cluster` is on (the 1.19+ default), and a wildcard-namespace bypass for selectorless `ipBlock` rules. + - Gateway API: TLS-passthrough routes no longer fail silently on gateways with mixed HTTP/HTTPS/TLS listeners, weighted backend splitting is fixed for `TLSRoute` passthrough, stale route parent statuses are pruned, and a controller panic on a malformed `parametersRef` is prevented. + - Datapath: fixes node connectivity disruption when ClusterIP/LoadBalancer VIPs overlap node-local addresses, host-proxy packet routing to pods, a corrupted `NamespaceSelector` on `CiliumEgressGatewayPolicy`, and an endpoint-ID reuse race that removed a live endpoint's BPF state. + - **Removed:** the `loadBalancer.standalone` Helm option. + +* **KubeVirt**: updated to **v1.8.4**. Fixes VMIs stuck in `Scheduled` on Kubernetes 1.36 (strict CRD numeric-format validation of VMI checksum status fields). ([release notes](https://github.com/kubevirt/kubevirt/releases/tag/v1.8.4)) ([**@lexfrei**](https://github.com/lexfrei) in #2940) + - Also fixes a gRPC connection leak in `virt-handler`'s `GetLauncherClient` that caused unbounded memory growth, socket accumulation and goroutine leaks, and closes **CVE-2026-35469** via a `moby/spdystream` bump. + - Adds missing metrics, recording rules and alerts for the virt components. + +* **Velero**: 1.17.0 → 1.18.1 (chart 11.0.0 → 12.0.3), with velero plugins moved to velero-1.18-compatible releases and `kubevirt-velero-plugin` v0.8.0 → v0.9.0. ([release notes](https://github.com/vmware-tanzu/velero/releases/tag/v1.18.0)) ([**@lexfrei**](https://github.com/lexfrei) in #2974) + - **Concurrent backup processing** — multiple backups now run simultaneously without interfering, a direct win on multi-tenant clusters. + - **Cache volumes for data movers** on restore (CSI snapshot data movement and fs-backup), so data-mover pods no longer fail on limited node ephemeral disk, and restore throughput improves. + - Incremental backup size is now observable for data-mover backups. + - v1.18.1 fixes list-scoping bugs that made restores, PodVolumeRestores and CSI PVC backups list across all namespaces instead of the install namespace, and migrates `VolumeGroupSnapshot` to `v1beta2` for Kubernetes 1.34+. + +* **Vertical Pod Autoscaler**: 1.3.0 → 1.5.0 (chart 10.0.0 → 11.1.1). ([release notes](https://github.com/kubernetes/autoscaler/releases/tag/vertical-pod-autoscaler-1.5.0)) ([**@lexfrei**](https://github.com/lexfrei) in #2975) + - **In-place pod resizing is promoted to Beta**, with fallback to eviction when an in-place update fails. + - **Deprecation:** `UpdateMode: Auto` is deprecated upstream, as is the humanized-memory flag. + - New metrics: failed eviction counters, `update_mode` labels on updater metrics, VPA name/namespace on resource-update counters; Prometheus bearer-auth support. + - Dependencies bumped for Kubernetes 1.34; several race conditions in the recommender and updater fixed. + +* **Harbor**: 2.14.2 → 2.15.1 (chart 1.18.2 → 1.19.1), plus PodDisruptionBudgets for core, portal, jobservice, nginx, registry, trivy and exporter. ([release notes](https://github.com/goharbor/harbor/releases/tag/v2.15.0)) ([**@scooby87**](https://github.com/scooby87) in #2966) + - New: tag-deletion option for garbage collection, per-project upstream-connection limits (`max_upstream_conn`) with UI, and per-endpoint CA certificate support for registry endpoints. + - Audit logging to the database can now be disabled at initialization. + - 2.15.1 fixes proxy-cache serve-local on remote-not-found, prevents background polling from renewing session TTL, redacts `redis_url_reg` from GC attributes, and refreshes photon base packages for CVEs. + +* **Keycloak**: 26.5.2 → **26.6.3**. ([release notes](https://github.com/keycloak/keycloak/releases/tag/26.6.0)) ([**@lexfrei**](https://github.com/lexfrei) in #2964) + - **JWT Authorization Grant** for external-to-internal token exchange using externally signed JWT assertions. + - **Federated client authentication**, removing the need to manage individual client secrets in Keycloak. + - **Workflows** for automating realm administrative tasks such as user and client lifecycle management. + - **Zero-downtime patch releases** — rolling updates within a minor stream without service downtime. + +* **LINSTOR**: linstor-server 1.33.2 → 1.33.3, linstor-csi v1.10.6 → **v1.11.2**. ([sources](https://github.com/LINBIT/linstor-server)) ([**@lexfrei**](https://github.com/lexfrei) in #2987) + - **Upgrade impact:** linstor-csi v1.11.2 rejects ReadWriteMany volumes on a non-DRBD storage class, which is why an empty tenant worker `nodeGroups[*].storageClass` now falls back to the application-level `replicated` class. See Breaking Changes. + +* **linstor-scheduler**: v0.3.2 → v0.3.6 (chart 0.2.3 → 0.3.1), re-vendored to upstream with the native admission webhook. ([sources](https://github.com/piraeusdatastore/helm-charts)) ([**@lexfrei**](https://github.com/lexfrei) in #3201) + - The admission webhook no longer strips unknown Pod fields — the `image` volume source and native-sidecar `restartPolicy` — which had broken KubeVirt containerDisk VMs and native sidecars. + - The webhook now reloads its TLS certificate after rotation instead of silently failing. + - Migration 52 deletes the old `linstor-scheduler-admission` Deployment so the new, differently-selected one can be created (a Deployment's `spec.selector` is immutable); `failurePolicy: Ignore` means the brief gap does not block Pod creation. + +* **FoundationDB operator**: v2.13.0 → **v2.30.0** (chart 0.2.0 → 0.3.0). ([release notes](https://github.com/FoundationDB/fdb-kubernetes-operator/releases/tag/v2.30.0)) ([**@lexfrei**](https://github.com/lexfrei) in #2992) + - Seventeen upstream releases of operator fixes and reconciliation improvements; the v2.30.0 release itself is a Go 1.26 toolchain and modernization pass. + +* **HAMi**: 2.8.1 → 2.9.0. ([release notes](https://github.com/Project-HAMi/HAMi/releases/tag/v2.9.0)) ([**@lexfrei**](https://github.com/lexfrei) in #2993) + - **HAMi-DRA (NVIDIA) is ready for use**; HAMi-core performance optimized and a HAMi-core mode added for Ascend devices. + - Volcano vGPU device plugin synced to 0.19 with CDI support; new vGPU-monitor `--metrics-bind-address` flag and Prometheus `ServiceMonitor` support in the chart. + - Resource-quota checking in the webhook, plus configurable webhook `namespaceSelector`/`objectSelector`. + +* **Percona Server for MongoDB operator**: 1.21.1 → **1.22.0**, with the managed CR moved to `crVersion: 1.22.0`. ([release notes](https://github.com/percona/percona-server-mongodb-operator/releases/tag/v1.22.0)) ([**@lexfrei**](https://github.com/lexfrei) in #2968) + - Sets the Kubernetes `appProtocol: mongo` on all services, so service meshes (Istio and friends) recognize MongoDB traffic and mTLS/cluster formation work without workarounds. + - Physical and logical **restore into a cluster with different replica-set names**, via `replsetRemapping` on `PerconaServerMongoDBRestore`. + +* **OpenBao**: v2.5.0 → **v2.5.1** (chart 0.25.3 → 0.25.7). ([release notes](https://github.com/openbao/openbao/releases/tag/v2.5.1)) ([**@lexfrei**](https://github.com/lexfrei) in #2991) + - **Security:** closes **CVE-2025-68121** (Go 1.25.7) and **CVE-2026-24051** (OpenTelemetry SDK 1.40.0). + - Fixes Auto Unseal failing when upgrading to v2.5.0 across AliCloud KMS, AWS KMS, Azure Key Vault, GCP Cloud KMS and OCI KMS. + - PKI no longer returns 500 until reload on a cancelled cache-invalidation context; irrevocable-lease revocation now decrements the lease count. + - **Note:** pre-v2.5.0 corrupt namespace identity groups are removed during unseal and must be recreated by an admin — look for `deleting corrupt group` in startup logs. + +* **CoreDNS**: chart 1.43.2 → 1.46.0 (upstream appVersion 1.12.3 → 1.13.1; the shipped image is deliberately held at v1.12.4 for kube-dns alignment). ([release notes](https://github.com/coredns/coredns/releases/tag/v1.13.0)) ([**@lexfrei**](https://github.com/lexfrei) in #3004) + - Upstream fixes a Corefile infinite loop on unclosed braces and an import-cycle issue, normalizes panics on invalid origins, and prevents reload deadlocks. + +* **csi-driver-nfs**: 4.11.0 → **4.13.3**. ([release notes](https://github.com/kubernetes-csi/csi-driver-nfs/releases/tag/v4.13.0)) ([**@lexfrei**](https://github.com/lexfrei) in #2967) + - **Security:** closes **CVE-2025-52881** and **CVE-2025-58181**; CSI sidecar images upgraded. + +* **OpenCost**: 1.111.0 → 1.120.3 (chart 1.41.0 → 2.5.23). ([sources](https://github.com/opencost/opencost-helm-chart)) ([**@lexfrei**](https://github.com/lexfrei) in #3000) + +* **NATS**: 2.11.8 → 2.11.10 (chart 1.3.13 → 1.3.16). ([sources](http://github.com/nats-io/k8s)) ([**@lexfrei**](https://github.com/lexfrei) in #2976) + +* **ouroboros**: 0.8.0 → 0.8.1. The controller rollout strategy is now rendered identically in every `controller.mode`, so switching a live release into external-dns mode no longer fails server-side apply with `spec.strategy.rollingUpdate: Forbidden` ([sources](https://github.com/lexfrei/ouroboros)) ([**@lexfrei**](https://github.com/lexfrei) in #3323). + +* **Managed Kubernetes patch versions** refreshed to v1.35.6, v1.34.9, v1.33.13 and v1.32.13; v1.30 removed from the support matrix ([**@lexfrei**](https://github.com/lexfrei) in #3014). + +* **Grafana** 11.6.15, **victorialogs-datasource** v0.14.1 → v0.28.0, and **alerta-web** 9.1.0 in the monitoring stack ([**@lexfrei**](https://github.com/lexfrei) in #3011). + +* **Kamaji** rebuilt on 26.3.6-edge with the datastore unused-deletion deadlock fix and Go stdlib/module CVEs closed (#3033, #3146). + +* **New system packages:** `etcd-operator-crds` (v1alpha2 CRDs, installed ahead of the controller) and `securitygroup-controller` (backs the new `SecurityGroup` API). + +## Major Features and Improvements + +* **[kubernetes] Bootstrap tenant workers with Talos via CABPT**: See Feature Highlights — Ubuntu + kubeadm worker bootstrap is replaced by Talos Linux driven by `cluster-api-bootstrap-provider-talos` and a `talos-csr-signer` sidecar in the Kamaji control-plane pod; existing clusters roll over automatically ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2931). + +* **[platform] Migrate to etcd-operator v1alpha2**: See Feature Highlights — new `etcd-operator.cozystack.io/v1alpha2` Membership API, Cozystack-authored operator chart at v0.5.2, snapshot/BackupClass-driven backup flow, and in-place adoption of legacy clusters via migration 50 ([**@androndo**](https://github.com/androndo) in #2859). + +* **[etcd] Complete the v1alpha2 transition for in-cluster 1.5→1.6 upgrades**: Keeps the legacy `etcd-headless` Service alive so adopted members stay resolvable, deletes the pre-1.6 operator Deployment via a pre-upgrade hook to get past the immutable selector, raises the operator's memory floor so it does not OOM before the VPA scales it, and makes the adoption migration robust in-cluster ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3270). + +* **[api] Add the SecurityGroup network policy resource (`sdn.cozystack.io`)**: See Feature Highlights — a tenant-facing, namespace-scoped firewall that attaches to managed applications by reference and projects 1:1 onto a `CiliumNetworkPolicy`, letting tenants manage network policy without `cilium.io` access ([**@lexfrei**](https://github.com/lexfrei) in #2922). + +* **[tenant] Recursive (hierarchical) resource quotas for sub-tenants**: See Feature Highlights — a sub-tenant's quota is carved out of its parent's remaining budget and can no longer exceed it; sub-tenants without a quota share the parent's pool ([**@kvaps**](https://github.com/kvaps) in #3086). + +* **[apps/kubernetes] Per-cluster OIDC selector for the tenant kube-apiserver (Phase 1)**: See Feature Highlights — `spec.oidc.mode: System | CustomConfig | None` plus `spec.oidc.users[]`, with a ready-to-use `oidc-kubeconfig` Secret surfaced through the dashboard ([**@IvanHunters**](https://github.com/IvanHunters) in #3044). + +* **[monitoring] Per-instance OIDC selector for the Grafana instance (Phase 1)**: See Feature Highlights — the same flat `spec.oidc.mode` selector for Grafana, with app-side role reconciliation into Grafana's Main Org ([**@IvanHunters**](https://github.com/IvanHunters) in #3176). + +* **[kubernetes] Expose tenant apiserver authentication in app values**: Adds `controlPlane.apiServer.extraArgs`, `extraVolumes` and `extraVolumeMounts` passthrough to `KamajiControlPlane`, so operators can enable OIDC or a structured `AuthenticationConfiguration` on tenant clusters by hand. Volume sources are restricted to ConfigMap and Secret, since the control-plane pod runs on the management cluster ([**@lexfrei**](https://github.com/lexfrei) in #3123). + +* **[kubernetes] Propagate remote-accessible LINSTOR StorageClasses to tenant clusters**: Infra-cluster LINSTOR StorageClasses whose `allowRemoteVolumeAccess` is not `"false"` are auto-created inside each tenant under the same name, with the class named by `storageClass` (default `replicated`) becoming the tenant default and the legacy `kubevirt` class kept as an alias so existing PVCs are unaffected. Node-local classes are not propagated. Delete any hand-made colliding tenant class before upgrading ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2872). + +* **[platform, ingress] Propagate the operator wildcard certificate to per-tenant termination points**: See Feature Highlights — `publishing.certificates.wildcardSecretName` is now replicated into every tenant namespace that terminates TLS, with no extra operator input and no cross-namespace Secret read ([**@lexfrei**](https://github.com/lexfrei) in #2990). + +* **[platform] ACME wildcard certificate on the default ingress-nginx path**: Opt-in `publishing.certificates.wildcard` (default `false`) issues one `*.` certificate via a DNS-01 solver for system services instead of a per-host ACME certificate, avoiding Let's Encrypt rate limits at scale ([**@lexfrei**](https://github.com/lexfrei) in #2988). + +* **[keycloak] Optional KMS-encrypting database proxy**: Column-level PII encryption at rest with a static KEK or Vault Transit; off by default ([**@kvaps**](https://github.com/kvaps) in #3003). Extended with Vault Kubernetes and AppRole auth, a dedicated ServiceAccount, and a shared DEK set covering migration and HA ([**@sircthulhu**](https://github.com/sircthulhu) in #3239). + +* **[keycloak] Opt-in separate hostname and route for the admin console**: `ingress.adminHost` serves the admin console and Administration REST API on their own hostname, and `publishing.ingressNameAdmin` attaches that route to a separate (e.g. private) Gateway or ingressClass. Disabled by default ([**@sircthulhu**](https://github.com/sircthulhu) in #2954). + +* **[keycloak] S3 backup for the CNPG database**: Optional Barman-based S3 backups of the Keycloak database under `backup.*`, disabled by default ([**@kvaps**](https://github.com/kvaps) in #3174). + +* **[keycloak] Configurable login theme via platform branding**: Select the realm login theme and ship its theme image through `branding.loginTheme` / `branding.loginThemeImage` ([**@lexfrei**](https://github.com/lexfrei) in #3124). + +* **[keycloak] Allow overriding the database host/port/properties**: `db.host`, `db.port` and `db.urlProperties` plus `extraVolumes`/`extraVolumeMounts`, so the database connection can be routed through an external proxy ([**@sircthulhu**](https://github.com/sircthulhu) in #2907). + +* **[keycloak] Enable the UPDATE_EMAIL required action in the cozy realm**: Users can now change their own email address from the account portal ([**@kvaps**](https://github.com/kvaps) in #3175). + +* **[release] Immutable tags and rc→stable promotion**: See Feature Highlights — a stable release is the byte-identical promotion of its release candidate, with every force-retag site removed and the cron patch-release workflow deleted ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3017). A companion **Cut Pre-release Tag** workflow covers the rc side (#3253). + +* **[operator] Support private OCI registry credentials for the platform source**: New `--platform-source-secret` flag (and `cozystackOperator.platformSourceSecret` value) authenticates the platform package source against private OCI registries (`kubernetes.io/dockerconfigjson`) and private Git repositories (basic auth, bearer token, or SSH key). An empty value preserves the existing public-registry behaviour ([**@IvanHunters**](https://github.com/IvanHunters) in #2472). + +* **[apps/kubernetes] Overridable Talos image factory URL and installer repository**: `talos.imageFactoryURL` and `talos.installerRepository` (both defaulting to the public factory) let air-gapped, mirrored or rate-limited environments serve the worker OS image and installer from their own infrastructure ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3244). + +* **[kubernetes] Per-node-group nodeHealthCheck overrides**: Optional `nodeGroups[name].maxUnhealthy` and `nodeGroups[name].nodeStartupTimeout`, falling back to the cluster-wide `nodeHealthCheck.*` when unset — so a stateful group can sit at `0%` while a stateless one tolerates `50%`. An invalid override fails the render naming the offending group ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3053). + +* **[kamaji] Default to 2 replicas and drop the telemetry webhook**: The Kamaji controller now runs two replicas with soft pod anti-affinity, and the telemetry handler and webhook entry are removed — cutting apiserver admission p99 for `TenantControlPlane` mutating/validating webhooks by roughly 70% on multi-tenant clusters ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2671). + +* **[kubevirt-cdi] Make cloneStrategyOverride configurable**: CDI's clone strategy is now settable from values (default `csi-clone`), so operators on storage backends without efficient CSI cloning can select a different strategy ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2926). + +* **[cozy-lib] CA-only TLS trust-anchor helper for tenants**: A new `cozy-lib.tls.caCertSecret` helper renders a `ca.crt`-only Opaque Secret surfaced through the tenant-secret API, so a tenant can verify a per-app TLS endpoint without being granted read access to any object that also carries a private key — the foundation the per-app TLS series builds on ([**@lexfrei**](https://github.com/lexfrei) in #2989). + +* **[dashboard] Vendor the cozystack-ui console into the monorepo**: The console SPA is now built from in-tree source under `packages/system/dashboard/images/console` rather than a separate repository. The shipped image and digest are unchanged for cluster operators ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2963). + +## Improvements (minor) + +* **[api] Return typed apierrors from TenantNamespace registry helpers**: The TenantNamespace API now returns `401 Unauthorized` for requests without a user identity and a structured `500 InternalError` for RBAC lookup failures, instead of a generic 500 ([**@lexfrei**](https://github.com/lexfrei) in #2870). + +* **[platform] Factor the cozystack-version stamp into a shared migration helper**: Every migration now stamps `cozystack-version` through one labeled helper, so the `platform.cozystack.io/no-delete` label can no longer be silently dropped by a label-less apply — which would have removed the ConfigMap from the deletion-protection policy ([**@lexfrei**](https://github.com/lexfrei) in #2980, [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3118). + +* **[dashboard] Show the LoadBalancer external IP on the app Services tab**: Externally published services now display their assigned IP in the console instead of leaving the operator to look it up with kubectl ([**@scooby87**](https://github.com/scooby87) in #3172). + +* **[dashboard] Render error and unknown-type states instead of an infinite spinner**: The console now shows a not-found message on a failed application GET and an unknown-type message for an unrecognized resource ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3098). + +* **[objectstorage-controller] Converge BucketClaim readiness and speed up COSI failover**: `BucketClaim` readiness now converges to the backend Bucket, and single-replica COSI provisioner failover is faster ([**@lexfrei**](https://github.com/lexfrei) in #3034). + +* **[etcd] Pin tenant etcd to v3.5.31**: Restores consistent watches by letting kube-apiserver enable `RequestWatchProgress` on Kubernetes 1.35 tenants ([**@lexfrei**](https://github.com/lexfrei) in #3122). + +## Bug fixes + +* **[seaweedfs] Close the 4.31 rename fallout on the 1.5.x→1.6 upgrade path**: The 1.6 upgrade no longer renames a SeaweedFS instance away from its data. The naming guard now runs in the chart a platform upgrade actually re-renders and refuses when both pre- and post-4.31 naming generations exist; `hack/seaweedfs-naming-audit.sh` and [`docs/operations/seaweedfs-431-rename-recovery.md`](../operations/seaweedfs-431-rename-recovery.md) guide recovery, and the refusal is expected for tenants that passed through 1.5.x. Cluster-scoped COSI RBAC is named per namespace again (the 4.31 release-based names collided across tenants), and the `seaweedfs-db` hand-over now runs for **every** instance name — previously an instance not named `seaweedfs` had its filer metadata database pruned on upgrade. New migration 53 repairs clusters that already ran the old hand-over ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3339). + +* **[seaweedfs] Adopt legacy workloads in place across the 4.31 rename**: Pins `fullnameOverride: seaweedfs` so upgrading past the vendored 4.31 chart bump adopts the existing workloads and volumes instead of standing up a second, empty cluster beside them ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3282). + +* **[seaweedfs] Point S3 consumers at the reachable S3 service**: The S3 ingress backend, the iceberg ingress backend and the in-cluster COSI provisioner endpoint all resolved a stale name or the wrong port, so the external S3 endpoint returned 503 through ingress-nginx and in-cluster COSI paths hit port 443 or the filer service. All consumers now resolve `seaweedfs-s3` on port 8333 *(shipped in v1.5.2)* ([**@lexfrei**](https://github.com/lexfrei) in #3119). + +* **[seaweedfs] Configure the postgres2 connection pool for the filer**: Removes roughly 2s of per-request S3 latency caused by PostgreSQL connection churn *(shipped in v1.5.3)* ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #2906). + +* **[seaweedfs] Raise master probe timeouts above the cluster-status backoff**: A master with a transiently empty raft view is no longer killed into a permanent CrashLoop, which could time out the whole SeaweedFS install ([**@lexfrei**](https://github.com/lexfrei) in #3068). + +* **[seaweedfs] Soften master anti-affinity for small clusters**: The hard master anti-affinity becomes a preference, so the S3 stack installs on clusters with fewer than three nodes ([**@lexfrei**](https://github.com/lexfrei) in #3197). + +* **[kubernetes] Make the default md0 node group removable**: The built-in `md0` node group is applied only when no node groups are configured, so it can finally be removed or replaced; migration 47 pins `md0` explicitly on existing clusters to preserve their topology ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2936). + +* **[apps/kubernetes] Honour nodeGroup minReplicas in MachineDeployment.spec.replicas**: The chart stops managing `spec.replicas` — the cluster-autoscaler owns it, seeded by CAPI's defaulting webhook from the min-size annotation and preserved through SSA field-manager ownership. `helm upgrade` no longer drains workers back to a hardcoded `replicas: 2` on every platform bump ([**@IvanHunters**](https://github.com/IvanHunters) in #3233). + +* **[kubernetes] Run the Talos reconcile Job in the main install phase**: Creating the worker `TalosConfigTemplate` from a main-phase Job instead of a post-install hook makes worker MachineDeployments scale reliably during install ([**@lexfrei**](https://github.com/lexfrei) in #3145). + +* **[kubernetes] Default worker disk storageClass to the application storageClass**: A node group leaving `storageClass` empty now falls back to the application-level (DRBD `replicated`) class rather than the management-cluster default, so live migration of worker VMs gets the ReadWriteMany volume it requires ([**@lexfrei**](https://github.com/lexfrei) in #3129). + +* **[kubernetes] Make node group resources optional when instanceType is set**: A node group sized purely by `instanceType` no longer fails schema validation with "missing property 'resources'"; the unsupported combination of `instanceType` plus explicit `resources` is rejected at render time instead *(shipped in v1.5.2)* ([**@lexfrei**](https://github.com/lexfrei) in #3121). + +* **[kubernetes] Clear the Kamaji datastore-secret finalizer to unblock namespace deletion**: The leftover `finalizer.kamaji.clastix.io/datastore-secret` on the datastore-config Secret is now cleared during tenant teardown, so deleting a Kubernetes app or tenant no longer leaves the namespace stuck `Terminating` ([**@scooby87**](https://github.com/scooby87) in #3078). + +* **[kubernetes] Gate the vmop cert-manager-crds dependency on certManager.enabled**: Clusters running `monitoringAgents.enabled=true` with `certManager.enabled=false` had victoria-metrics-operator permanently blocked on a HelmRelease that would never exist, cascading to `monitoring-agents` and `vertical-pod-autoscaler` never becoming ready *(shipped in v1.5.2)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3130). + +* **[apps/kubernetes] Bound the pre-delete hook's tenant HelmRelease wait**: Tenant Kubernetes teardown no longer hangs when the cluster has no working nodes — the pre-delete hook bounds its wait for the in-tenant HelmReleases and force-clears their Flux finalizers on timeout ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2826). + +* **[kamaji] Backport the datastore unused-deletion deadlock fix**: Deleting a Tenant whose etcd DataStore had no more referencing TenantControlPlanes left the DataStore stuck `Terminating`, hanging the etcd Helm uninstall and wedging the tenant namespace. Backports clastix/kamaji#1122 as a source patch on the pinned build *(shipped in v1.5.2)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3146). + +* **[tenant] Inherit the full ancestor label chain from the parent namespace**: Tenant namespaces now carry the complete `tenant.cozystack.io/` chain (including `tenant-root`) at any nesting depth, restoring root-ingress reachability to nested tenants; migration 49 backfills the labels on existing clusters ([**@IvanHunters**](https://github.com/IvanHunters) in #2912). + +* **[tenant] Grant the watch verb to the cleanup Role**: Tenant deletion no longer hangs on the pre-delete hook, which needed `watch` to wait; the hook also moves to a maintained kubectl image ([**@lexfrei**](https://github.com/lexfrei) in #2923). + +* **[tenant] Gate VM-resource HelmReleases on victoria-metrics-operator readiness**: The tenant `etcd`, `ingress` and `monitoring` releases create VictoriaMetrics-operator CRs behind a `failurePolicy: Fail` webhook; on cold installs they could reconcile before the webhook was serving and fail with "connection refused". They now `dependsOn` the operator *(shipped in v1.5.2)* ([**@lexfrei**](https://github.com/lexfrei) in #3141). + +* **[monitoring] Surface missing VictoriaLogs and gate the dead-end vlinsert ExternalName**: A missing platform VictoriaLogs is now reported as not-ready instead of being masked, and the dangling `vlinsert-generic` ExternalName no longer silently drops logs. Adds a `monitoring.rootEnabled` platform value that operators must set to match their actual `Tenant/root` `spec.monitoring` ([**@scooby87**](https://github.com/scooby87) in #3215). + +* **[mariadb] Re-guard replication so single-replica passes the operator webhook**: Replication is enabled only for `replicas > 1`, matching the mariadb-operator validating webhook, so single-replica instances can be created again; the dashboard, RBAC and backups now target the bare service for single-replica setups *(shipped in v1.5.2)* ([**@scooby87**](https://github.com/scooby87) in #3180). + +* **[mariadb] Derive my.cnf sizing from the instance's own resources**: `my.cnf` is now sized from the instance's own CPU, memory and volume instead of hardcoded values assuming a large dedicated host — small presets previously advertised `max_connections=4096` they could not serve; the limit now scales with memory from the server default of 151 ([**@lexfrei**](https://github.com/lexfrei) in #3345). + +* **[mariadb] Widen the startup probe budget so bootstrap cannot be killed**: A slow first boot can no longer leave the datadir permanently unusable ([**@lexfrei**](https://github.com/lexfrei) in #3344). + +* **[mariadb] Bind the metrics exporter to IPv4**: The exporter's readiness probe is reachable again and the metrics endpoint populates reliably ([**@lexfrei**](https://github.com/lexfrei) in #3064). + +* **[kafka] Set entity-operator resources to avoid OOM under the tenant LimitRange**: The topic- and user-operator containers carried no requests/limits, so the 128Mi tenant `LimitRange` default OOMKilled them on startup — leaving `KafkaTopic`/`KafkaUser` objects unreconciled and topics stuck on the topic-operator finalizer during teardown *(shipped in v1.5.2)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2934). + +* **[harbor] Preserve the jobservice and trivy storageClass on upgrade**: The wrapper never forwarded `storageClass` to the jobservice `jobLog` PVC, so it picked up the cluster default via admission — and because PVC `storageClassName` is immutable, that produced rendered-vs-live drift on every upgrade. Both now preserve an existing PVC's class via lookup *(shipped in v1.5.2)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2930). + +* **[postgres] Purge a stale recovery cluster on repeat in-place restore**: A repeat in-place restore no longer silently succeeds without restoring — a recovery cluster left over from an earlier completed restore is purged so the restore re-bootstraps from the backup ([**@IvanHunters**](https://github.com/IvanHunters) in #3318). + +* **[keycloak] Point HTTPRoute backends at the service port**: Fixes Gateway API routing to Keycloak, which targeted the wrong port ([**@lllamnyp**](https://github.com/lllamnyp) in #3235). + +* **[kubevirt-instancetypes] Restore persistent EFI/TPM state**: v1.5.1 stripped persistent EFI/TPM from the `windows.11`, `windows.2k22` and `windows.2k25` preferences because the RWO backend-storage PVC pinned VMs to a node and blocked live migration. RWO-Filesystem backend storage already live-migrates on KubeVirt v1.4+, so persistence is restored *(dropped in v1.5.1 via #3006, restored in v1.5.3)* ([**@kvaps**](https://github.com/kvaps) in #3154). + +* **[gateway] Pin identical allowedRoutes.kinds on port 443 listeners**: All port-443 listeners now carry identical `allowedRoutes.kinds=[HTTPRoute,TLSRoute]`, preventing Cilium (cilium#45559) from collapsing the HTTPS-terminate and TLS-passthrough listeners and silently dropping HTTPRoutes ([**@shreyaabaranwal**](https://github.com/shreyaabaranwal) in #3084). + +* **[platform] Route OIDC backend calls through the in-cluster Keycloak by default**: `authentication.oidc.keycloakInternalUrl` now defaults to the in-cluster Keycloak Service, so enabling OIDC on a fresh install no longer leaves the dashboard down while the root ingress is not yet serving ([**@lexfrei**](https://github.com/lexfrei) in #3289). + +* **[platform] Order backupstrategy-controller after cozystack-basics**: On a fresh install the `cozy-backups` Bucket could race `tenant-root` namespace creation and fail, which then raced the install-wait deadline and failed unrelated fresh installs *(shipped in v1.5.2)* ([**@lexfrei**](https://github.com/lexfrei) in #3037). + +* **[platform] Make `*-rd` sources depend on cozystack-engine**: Fixes an install-ordering race where resource-definition sources reconciled before the engine that serves them ([**@lexfrei**](https://github.com/lexfrei) in #3041). + +* **[cozystack-basics] Validate legacy Ingress hostnames against the tenant apex**: Legacy Ingress hostnames in tenant namespaces are now constrained to the tenant's apex domain, closing a cross-tenant hostname-claim gap ([**@lexfrei**](https://github.com/lexfrei) in #3200). + +* **[operator] Force source-watcher drift on a stuck ArtifactGenerator status**: `cozystack-platform` could time out at 15m on fresh install when a PackageSource's derived ArtifactGenerator stalled in `Ready: Unknown` despite its artifacts being produced — an upstream `patch.Helper` race (fluxcd/pkg#934) that loses the Ready-condition write under etcd load. The reconciler now detects the stuck signature and forces source-watcher onto its drifted-branch reconcile ([**@IvanHunters**](https://github.com/IvanHunters) in #3182). + +* **[fluxcd] Omit empty distribution.artifact in FluxInstance for guest clusters**: Guest Kubernetes clusters with the fluxcd addon enabled no longer fail to install; an empty distribution artifact is omitted so the operator uses its embedded manifests instead of rendering an invalid null ([**@IvanHunters**](https://github.com/IvanHunters) in #3284). + +* **[flux] Tune source-watcher concurrency and CPU**: Prevents the platform install from timing out under load ([**@lexfrei**](https://github.com/lexfrei) in #3163). + +* **[vertical-pod-autoscaler] Migrate admission-controller TLS to cert-manager**: The in-chart `genCA`/`genSignedCert` templates are not idempotent, so every Flux reconcile minted a fresh certificate, flipped a checksum annotation and materialised a new ReplicaSet — producing 10+ ReplicaSets in minutes under load and livelocking the admission controller on its own election lease. A namespace-scoped cert-manager Issuer and Certificate replace them, with pod rotation keyed off the actual `tls.crt` content ([**@IvanHunters**](https://github.com/IvanHunters) in #3178). + +* **[cert-manager] Raise the cainjector memory limit**: CA injection into webhook `caBundle`s no longer fails under memory pressure, which previously surfaced as intermittent `x509: certificate signed by unknown authority` admission errors ([**@IvanHunters**](https://github.com/IvanHunters) in #3199). + +* **[cert-manager] Move the webhook off the kubelet's port**: The admission webhook no longer listens on port 10250, which the kubelet also serves — a connection misrouted to a node answered with the kubelet's certificate, failing every cert-manager admission call cluster-wide with a misleading x509 error ([**@lexfrei**](https://github.com/lexfrei) in #3359). + +* **[external-secrets] Move the webhook off the kubelet's port**: Same class of collision; the Service also targets the port by name so the move rolls out without an admission gap ([**@lexfrei**](https://github.com/lexfrei) in #3360). + +* **[metrics-server] Move the listener off the kubelet's port**: The default made hostNetwork mode unable to start at all; the new port also keeps node-port collisions out of the metrics path ([**@lexfrei**](https://github.com/lexfrei) in #3361). + +* **[cilium] Pass IPv6 to the kernel stack when the IPv6 datapath is disabled**: Node IPv6 (ICMPv6 Neighbor Discovery, BGP unnumbered over link-local) now works with the host firewall enabled — Cilium host policies apply to IPv4 only and node IPv6 is not filtered ([**@lexfrei**](https://github.com/lexfrei) in #2871). + +* **[kube-ovn] Raise the ovs-ovn CPU limit and request**: The co-located `ovn-controller`, `ovsdb-server` and `ovs-vswitchd` daemons get CPU headroom (limit 4 cores, request 100m) under burst installs ([**@lexfrei**](https://github.com/lexfrei) in #3050). + +* **[linstor] Set controller resource requests and relax the liveness probe**: The linstor-controller no longer crash-loops under node CPU contention, which previously left its Service with no endpoints and blocked RWX volume provisioning ([**@lexfrei**](https://github.com/lexfrei) in #3043). + +* **[opensearch-operator] Gate leader election on replica count**: Leader election is disabled on the single-replica operator Deployment, so a transient apiserver or etcd latency spike no longer crashloops the manager at install time; it re-enables automatically past one replica ([**@lexfrei**](https://github.com/lexfrei) in #3040). + +* **[monitoring-agents] Relax fluent-bit probes for loaded installs**: The DaemonSet no longer crash-loops on a loaded install when its `:2020` HTTP server binds late behind a busy kube-API ([**@lexfrei**](https://github.com/lexfrei) in #3045). + +* **[velero] Add a startupProbe so slow startup does not crashloop the install gate**: Velero's slow first start no longer trips the liveness probe and fail the platform install ([**@lexfrei**](https://github.com/lexfrei) in #3138). + +* **[kubevirt-cdi] Order behind cozystack-basics**: The `cdi-clone-dv` RoleBinding now finds `cozy-public`, fixing a cold-install ordering failure ([**@lexfrei**](https://github.com/lexfrei) in #3137). + +* **[vm-default-images] Refresh stale Alpine and openSUSE 16.0 image URLs**: Alpine moves 3.21.6 → 3.21.7 and the openSUSE Leap 16.0 URL is corrected, so default-image imports stop 404ing ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3063). + +* **[apps/vpn] Remove the invalid `foo` field from the urls Secret**: A stray field broke Helm upgrades under server-side apply ([**@IvanHunters**](https://github.com/IvanHunters) in #3281). + +* **[dashboard] Unbreak CORS on an expired session for Kubernetes API calls**: The dashboard SPA no longer fails with CORS errors after the `kc-access` cookie expires — oauth2-proxy returns 401 (not 302) for `/api` and `/apis`, and the KeycloakClient declares `webOrigins: ["+"]` ([**@IvanHunters**](https://github.com/IvanHunters) in #2788). + +* **[dashboard] Join pod phase on (namespace, pod) instead of uid**: Fixes the Capacity Planning and Namespaces dashboards failing with "duplicate time series" errors when static pods are present ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2782). + +* **[cozy-lib, harbor, mariadb] Digest-pin the cleanup-hook kubectl image and route it through cozy-lib.images-registry**: The post-delete cleanup hook image is now digest-pinned and resolved through the cluster images registry via a new `cozy-lib.image` helper, so mirrored and air-gapped installs can resolve it and uninstall no longer depends on a moving Docker Hub tag ([**@scooby87**](https://github.com/scooby87) in #3171). + +* **[migrations] Derive the etcd-adoption snapshot target from the projected bucket credentials**: Migration 50 resolves its S3 endpoint and scheme from the claimed bucket's projected Secret rather than requiring them hand-set, and makes the `etcdAdoptSkipBackup` escape hatch actually reachable from the Package CR ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3335). + +## Security + +* **[talos] Bump Talos to v1.13.6 (CVE-2026-53359)**: Ships Linux 6.18.38 with the kernel fixes for the **CVE-2026-53359** and **CVE-2026-46113** KVM guest-to-host escapes ([**@lexfrei**](https://github.com/lexfrei) in #3240). + +* **[kubernetes] Bump tenant worker Talos to v1.13.6**: The same kernel fixes reach tenant worker VMs — the KVM shadow-paging use-after-free and nested-guest escape into the worker VM. Nested virtualization remains available; applying this rolls each tenant worker pool once ([**@lexfrei**](https://github.com/lexfrei) in #3269). + +* **[seaweedfs] Enforce read-only bucket access by bumping cosi-driver to v0.3.1**: The bundled COSI driver hardcoded read-write S3 permissions for every `BucketAccess` and ignored the `-readonly` `BucketAccessClass`, so credentials meant to be read-only could still upload and delete objects. The driver now honours `accessPolicy: readonly` and issues Read/List-only credentials *(shipped in v1.5.2)* ([**@lexfrei**](https://github.com/lexfrei) in #3136). + +* **[platform] Raise the Go toolchain to 1.26 across first-party images**: Clears standard-library security advisories in every Cozystack-built binary ([**@lexfrei**](https://github.com/lexfrei) in #2941). + +* **[deps] Bump golang.org/x/crypto to v0.52.0**: Closes 13 known OSV advisories (GO-2026-5005 through GO-2026-5033) ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3019). + +* **[kamaji] Close Go stdlib and module CVEs in the shipped binary**: Rebuilds the pinned Kamaji binary on a patched toolchain and module set ([**@lexfrei**](https://github.com/lexfrei) in #3033). + +* **[build] Refresh first-party Alpine runtime bases to 3.24**: Clears the musl and zlib advisories across Cozystack-built images ([**@lexfrei**](https://github.com/lexfrei) in #3009). + +* **[kilo] Rebuild the kilo image on current upstream and Alpine 3.24**: Clears the libpng advisories ([**@lexfrei**](https://github.com/lexfrei) in #3007). + +* **[ci] Harden GitHub Actions workflows for OpenSSF Scorecard**: Pins every action by commit SHA, scopes per-job `GITHUB_TOKEN` permissions to least privilege, and adds a `zizmor` gate (pre-commit plus PR workflow) that blocks future regressions ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3223, #2849, [**@kvaps**](https://github.com/kvaps) in #3227). + +* **[ci] Enable Renovate digest pinning and OSV vulnerability alerts**: Base images and GitHub Actions are pinned to digests, indirect Go dependencies are bumped, and OSV alerts are enabled — improving the Scorecard Pinned-Dependencies and Vulnerabilities scores ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3025, #3026). + +## Dependencies & version updates + +See **Platform Components** above for the components with user-visible upstream changes. The remaining version work in this release: + +* **[linstor] Bump linstor-server to 1.33.3 and linstor-csi to v1.11.2**: Carries the RWX-on-non-DRBD rejection that drives the worker storageClass fallback (see Breaking Changes) ([**@lexfrei**](https://github.com/lexfrei) in #2987). +* **[images] Bump opencost, multus and clickhouse-backup**: OpenCost to 2.5.23, multus-cni to v4.3.0 and clickhouse-backup to 2.7.2 ([**@lexfrei**](https://github.com/lexfrei) in #3000), with clickhouse-backup later refreshed to v2.7.4 ([**@app/renovate**](https://github.com/apps/renovate) in #3226). +* **[images] Refresh stale utility image references**: kube-rbac-proxy v0.22.0, matchbox v0.11.0 and alpine/k8s 1.36.2 ([**@lexfrei**](https://github.com/lexfrei) in #2969). +* **[kubernetes] Re-pin the deleted centos:stream9 digest for kubevirt-csi-driver**: Upstream deleted the pinned base-image digest, breaking the kubevirt-csi-driver build; re-pinned twice as the upstream tag moved again ([**@IvanHunters**](https://github.com/IvanHunters) in #3303, [**@androndo**](https://github.com/androndo) in #3386). +* **[deps] Renovate updates**: kubevirt/csi-driver digest to 27b52aa (#3066), actions/checkout to v6.0.3 (#3133), and a batch of container image and action digests (#3232) ([**@app/renovate**](https://github.com/apps/renovate)). + +## System Configuration + +A sweep of platform components that previously ran without resource requests or limits — and therefore inherited namespace `LimitRange` defaults or ran unbounded — now declare them explicitly: + +* **[reloader] Set resources requests/limits on the reloader container** ([**@matthieu-robin**](https://github.com/matthieu-robin) in #2631). +* **[metrics-server] Set resources requests/limits on the metrics-server container** ([**@matthieu-robin**](https://github.com/matthieu-robin) in #2630). +* **[external-dns] Set resources requests/limits on the external-dns container** ([**@matthieu-robin**](https://github.com/matthieu-robin) in #2629). +* **[bootbox] Add resources requests/limits to the matchbox container** ([**@matthieu-robin**](https://github.com/matthieu-robin) in #2625). +* **[bucket] Add resources requests/limits to the s3manager container** ([**@matthieu-robin**](https://github.com/matthieu-robin) in #2624). +* **[goldpinger] Set resources requests/limits on the goldpinger DaemonSet** ([**@matthieu-robin**](https://github.com/matthieu-robin) in #2623). +* **[cert-manager] Set resources requests/limits on cert-manager pods** ([**@matthieu-robin**](https://github.com/matthieu-robin) in #2616). +* **[kubernetes] Add CPU/memory limits to the csi-resizer sidecar** ([**@matthieu-robin**](https://github.com/matthieu-robin) in #2614). + +## Development, Testing, and CI/CD + +* **[e2e] Migrate the app suite from BATS to Chainsaw and wire it into CI**: The managed-application end-to-end suite moves to Chainsaw, giving declarative, per-resource assertions in place of shell scripting ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2826). + +* **[e2e] Diff-driven install selector `select-install.sh`**: Computes the minimal package install set for a diff-scoped E2E run by walking the forward PackageSource dependency closure, with a `--validate` graph-consistency mode ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3279). + +* **[ci] Require an API-owner review for sizeable API changes**: A PR that introduces a new API group, a new resource, or a breaking change to an existing API schema now requires a review from an API owner. The detector discovers CRDs and APIService registrations by content rather than directory name, and builds from the base-branch tip when the merge base predates the gate ([**@lllamnyp**](https://github.com/lllamnyp) in #3167, #3184, [**@lexfrei**](https://github.com/lexfrei) in #3193). + +* **[ci] Guard migrations.targetVersion against a silent skip**: A CI check now catches a migration added without bumping `targetVersion`, which would otherwise ship a migration that never runs ([**@IvanHunters**](https://github.com/IvanHunters) in #2771). + +* **[build] Parallel matrix image builds on ephemeral runners**: Image builds fan out across an ephemeral-runner matrix instead of serializing on one host ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2983). + +* **[ci] Move remaining workflows off the persistent self-hosted runner**: All CI jobs now run on ephemeral or GitHub-hosted runners; release tag builds warm-start from the shared registry build cache ([**@kvaps**](https://github.com/kvaps) in #3268). + +* **[ci] Overlay current-main images for packages a PR did not rebuild**: PR e2e and the installer artifact now run current-main images for untouched packages instead of the last release's, so a PR is tested as current-main plus its own changes ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3148). + +* **[ci] Move the install gate to a 32 vCPU runner**: Ends the CPU oversubscription that produced most of the install-phase flakes ([**@lexfrei**](https://github.com/lexfrei) in #3251). + +* **[e2e] Stop the install pipeline flaking**: A combined pass over VPA OOM/churn and wait budgets across the install pipeline ([**@kvaps**](https://github.com/kvaps) in #2957). + +* **[e2e] Make the in-sandbox Talos image cache reachable by tenant workers**: Fixes deterministic `kubernetes-*` E2E failures where worker CDI importers could not reach the sandbox image cache ([**@lexfrei**](https://github.com/lexfrei) in #3254). + +* **[e2e] Escalate the cilium leak healer to an agent restart on leaked-IP reuse**: The only reliable recovery from a Cilium endpoint-manager IP leak under pod churn ([**@lexfrei**](https://github.com/lexfrei) in #3051), with the reserved:ingress IP variant skipped (#3142). + +* **[testing] Refresh the e2e-sandbox tools and base**: Ubuntu 24.04, kubectl 1.33.13, talosctl 1.13.5, Helm 3.21.2 ([**@lexfrei**](https://github.com/lexfrei) in #3010). + +* **[tests] Make bats assertions and hooks actually run under cozytest**: Negative assertions now fail as intended and dead `setup`/`teardown` hooks execute, closing a class of silently-passing tests ([**@lexfrei**](https://github.com/lexfrei) in #3132, #3134, #3131). + +* Additional E2E and CI work: capture host→pod CNI data-plane on failure (#3047), capture LINSTOR satellite ErrorReports in cozyreport (#3209), wait for LINSTOR pool free capacity between tenant Kubernetes tests (#3210), gate the LINSTOR node-list probe on a ready controller endpoint (#3250), drain tenant VMs/PVCs between kubernetes tests (#3159), widen tenant-root monitoring bring-up waits (#3160), widen etcd convergence and FoundationDB status budgets (#3036, #3039), raise the kafka/zookeeper preset off the 128Mi OOM threshold (#3052), bump the sandbox tenant storage quota to 200Gi (#3206), make the hairpin DNS check tolerate a tenant node recycle (#3241), size the e2e tenant apiserver to the chart default (#3351), fix the etcd metrics-probe attach race (#3350), drop the serviceexposure Chainsaw suite left after the feature removal (#3336), drop duplicate package test targets (#3352), route PR base-image pulls through mirror.gcr.io (#3042), remove the redundant Release E2E workflow (#2978), prune stale and redundant workflows (#2982), fix the backport workflow's `conflict_resolution` nesting (#3155), drop the pr-labeler `edited` trigger (#3211), map dashboard/console and cert-manager/external-secrets/metrics-server scopes in the labeler (#3117, #3362), add `needs-triage` on new issues and new area labels (#3165), force `chore(deps)` commits and an `area/dependencies` label from Renovate (#3015), resolve the nightly packages digest without image-specific skopeo operations (#3332), mirror and retag images whose digest is embedded in the values `tag` or whose registry host lives outside `repository` — ten images across eight packages were being silently skipped, so nightly published references that 404 at install and promoted releases did not carry the `:` tags they claimed (#3375), pin the console image builder to pnpm 11.1.2 (#3099), and block hard-wrapped prose in markdown and GitHub bodies (#3297). + +## Documentation + +* **[docs/security] Add a design-level threat model**: New `docs/security/threat-model.md` describing trust boundaries and attacker models for the platform ([**@lexfrei**](https://github.com/lexfrei) in #3217). + +* **[docs/security] Add a security self-assessment**: New `docs/security/self-assessment.md` prepared for the CNCF incubation due diligence ([**@tym83**](https://github.com/tym83) in #3224), alongside an alignment pass on the existing security docs and a new incident-response document (#3225). + +* **[docs] Route bugs, features, and proposals to the right repository**: Issue templates and contribution docs now direct reporters to the correct Cozystack repository ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2854). + +* **[vm-instance] Document attaching a VM to an external VLAN**: New `docs/vm-external-vlan.md` covering the Linux-bridge and bridge-type `NetworkAttachmentDefinition` path ([**@lexfrei**](https://github.com/lexfrei) in #3196). + +* **[harbor] Fix the SeaweedFS discovery key described as an annotation**: It is the `namespace.cozystack.io/seaweedfs` namespace **label** ([**@lexfrei**](https://github.com/lexfrei) in #3293). + +* **[readme] Add the OpenSSF Scorecard badge** ([**@tym83**](https://github.com/tym83) in #3237). + +* **[docs/agents] Add a downstream repository checklist to the PR template** ([**@lexfrei**](https://github.com/lexfrei) in #3298). + +* **[docs/release] Publish the patch-release changelogs**: The generated changelogs for v1.4.5, v1.5.1 and v1.5.2 land in `docs/changelogs/` so the release history on `main` is complete ([**@app/cozystack-ci**](https://github.com/apps/cozystack-ci) in #3083, #3024, #3162). + +* **[website] Multi-language proof-of-concept (ru, de, zh-cn, hi)**: Wires four additional languages into the site — localized landing page, getting-started guide, support page and navigation — with key-parity and translation-freshness lint, non-ratified localizations excluded from search indexing, and SEO blocks per language ([**@tym83**](https://github.com/tym83) in cozystack/website#593). + +* **[website] Tenant network isolation post (Cilium eBPF + Kube-OVN VPC)**: Explains how Cozystack isolates tenant traffic across the two data planes ([**@tym83**](https://github.com/tym83) in cozystack/website#614, cover image in cozystack/website#620). + +* **[website] CVE-2026-53359 (Januscape) advisory and Talos fix guide**: A security advisory for the KVM guest-to-host escape plus a step-by-step Talos remediation guide ([**@tym83**](https://github.com/tym83) in cozystack/website#603, cozystack/website#604, cozystack/website#612). + +* **[website] CVE-2026-43499 (GhostLock) exposure assessment**: Documents whether and how Cozystack clusters are exposed ([**@tym83**](https://github.com/tym83) in cozystack/website#613). + +* **[website] Call for independent Cozystack adopters for the CNCF Incubation review** ([**@tym83**](https://github.com/tym83) in cozystack/website#624). + +* **[website] Simplify the backup guides around the default BackupClass**: Corrects the admin override path and the platform-package `backupStorage` documentation for both `next` and v1.5 ([**@androndo**](https://github.com/androndo) in cozystack/website#622). + +* **[website] Document host firewall and node IPv6 behavior**: Explains that Cilium host policies apply to IPv4 only and node IPv6 is passed to the kernel stack ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#574). + +* **[website] Document the `existingSecret` certificate mode and `wildcardSecretName`**: Corrects the publishing reference and documents the third certificate mode ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#615, cozystack/website#619). + +* **[website] Drop the manual KubeVirt patch step for GPU passthrough**: The platform now auto-wires `permittedHostDevices`, so the manual patch documented since v1.4 is obsolete; includes pre-upgrade migration steps for hand-edited entries ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#556). + +* **[website] Explain why packages live in `apps` vs `extra`** ([**@kvaps**](https://github.com/kvaps) in cozystack/website#594). + +* **[website] cozyvalues-gen in the developer guide; sync the external apps guide with the example repo** ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#610). + +* **[website] talm: encrypted user values and dry-run secret redaction** ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#590). + +* **[website] Refresh the telemetry snapshot to June 2026 and fix per-period tenants** ([**@tym83**](https://github.com/tym83) in cozystack/website#598). + +The following documentation shipped in the v1.5.x patch releases and is included for completeness: + +* **[website] Add a containerized GPU workloads guide**: Operations guide for the `container` variant of `cozystack.gpu-operator` *(shipped in v1.5.2)* ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#555). +* **[website] Warn against changing `op:` on the Talos machine files block**: Talos rejects `op: create` for files outside `/var`, so the `/etc/lvm/lvm.conf` entry must keep `op: overwrite` *(shipped in v1.5.2)* ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#575). +* **[website] Document talm chart drift detection and strict mode** *(shipped in v1.5.2)* ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#577). +* **[website] Document `publishing.certificates` wildcard options** *(shipped in v1.5.2)* ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#588). +* **[website] Correct stale version pins and prevent next.yaml drift** *(shipped in v1.5.2)* ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#589). +* **[website] Cozystack v1.5.0 release announcement** *(shipped in v1.5.2)* ([**@tym83**](https://github.com/tym83) in cozystack/website#592). + +## Other Repositories + +### talm v0.33.0 / v0.33.1 + +* **[talm] Detect vendored chart and preset-template drift**: `talm` now warns when the vendored charts in a project drift from the talm binary's embedded library, and detects preset-template drift via a new `.talm-preset.lock` file (commit it). `talm init --update` becomes an exact re-sync that names the drifted paths, and `strictCharts: true` / `--strict-charts` escalates warnings to hard failures — blocking on unverifiable or missing drift baselines rather than passing silently. CRLF line endings are treated as checkout artifacts, not drift ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#216). +* **[talm] Disable and then restore KVM nested virtualization in the Talos presets**: v0.33.0 disabled nested virtualization in the `cozystack` preset as an immediate mitigation for **CVE-2026-53359**, pinning `grubUseUKICmdline=false` so the kernel args actually apply on Talos v1.12+. v0.33.1 reverts that mitigation now that the Talos v1.13.6 kernel carries the upstream fix — so nested virtualization is available again on a patched fleet ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#224, cozystack/talm#225). + +### talm v0.32.0 *(shipped in v1.5.2)* + +* **[talm] First-class encrypted user values, honored at template and apply**: A value file named `*.encrypted.yaml` referenced from `templateOptions.valueFiles` is age-decrypted in memory at both `talm template` and `talm apply`, with `talm init --encrypt`/`--decrypt` covering `values-secret.yaml` ↔ `values-secret.encrypted.yaml`. Secret-bearing fields are redacted by default from rendered node files, stdout and the apply drift preview, with `--show-secrets`/`--show-secrets-in-drift` to reveal them ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#222). + +### ansible-cozystack v1.5.1 / v1.5.2 / v1.5.3 + +* **[ansible-cozystack] Make prepare and k3s.orchestration runnable as separate invocations**: The example playbooks can now be run as independent steps rather than requiring a single combined invocation ([**@kvaps**](https://github.com/kvaps) in cozystack/ansible-cozystack#52). +* **[ansible-cozystack] Track the cozy-installer and k3s versions**: The installer image is bumped to v1.5.1, v1.5.2 and v1.5.3 in step with the Cozystack patch releases, and k3s to v1.36.2+k3s1 ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#56, cozystack/ansible-cozystack#57, cozystack/ansible-cozystack#58, cozystack/ansible-cozystack#59). + +## Contributors + +We'd like to thank all contributors who made this release possible: + +* [**@androndo**](https://github.com/androndo) +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@lllamnyp**](https://github.com/lllamnyp) +* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) +* [**@matthieu-robin**](https://github.com/matthieu-robin) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@scooby87**](https://github.com/scooby87) +* [**@shreyaabaranwal**](https://github.com/shreyaabaranwal) +* [**@sircthulhu**](https://github.com/sircthulhu) +* [**@tym83**](https://github.com/tym83) + +### New Contributors + +We're excited to welcome our first-time contributor: + +* [**@shreyaabaranwal**](https://github.com/shreyaabaranwal) - First contribution! + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.5.0...v1.6.0 diff --git a/docs/changelogs/v1.6.1.md b/docs/changelogs/v1.6.1.md new file mode 100644 index 0000000000..94f690f505 --- /dev/null +++ b/docs/changelogs/v1.6.1.md @@ -0,0 +1,72 @@ + + +# v1.6.1 (2026-08-05) + +A patch release with seven fixes covering PostgreSQL, etcd, managed Kubernetes, Keycloak, `cozystack-basics`, and SeaweedFS, plus release-pipeline reliability fixes and a `talm` update adding declarative Talos preset knobs. + +## Fixes + +* **fix(postgres-operator): align CNPG operator and CRDs to 1.28.2 for PVC resize-deadlock fix**: A simultaneous `resources` + `size` change on a single-instance PostgreSQL cluster could make the CloudNativePG operator delete the sole primary Pod, classify the PVC as `resizing`, and never recreate the Pod — wedging the cluster with zero instances and leaving the filesystem resize incomplete. Bumping the operator image and CRDs together to 1.28.2 (which carries upstream's fix, cloudnative-pg#9980 / cloudnative-pg#9981) resolves the deadlock ([**@scooby87**](https://github.com/scooby87) in #3510, backport #3542). + +* **chore(etcd-operator): bump etcd-operator to v0.5.4**: Rolls up four upstream controller bug fixes: the operator no longer exempts the bootstrap seed from crash-loop self-heal, self-heal now also covers memory-backed etcd members, `--initial-cluster-state` is derived from cluster phase instead of the seed, and each EtcdCluster's PodDisruptionBudget switches from `maxUnavailable` to `minAvailable` (existing clusters are reconciled onto the new field automatically on upgrade). Tenant etcd clusters recover more reliably from member crashes and are less likely to have their PDB block a node drain ([**@androndo**](https://github.com/androndo) in #3529, backport #3538). + +* **fix(kubernetes): render the talos-reconcile Job for the default md0 group**: On a managed Kubernetes cluster left with the default (undeclared) `nodeGroups`, the Job that creates each worker's `TalosConfigTemplate` and patches the control-plane's cert SANs only iterated the user-supplied node-group map, so it silently skipped the implicit `md0` group. Nothing failed at install time, since `md0` defaults to zero replicas, but the first scale-up (for example, autoscaler-driven growth after enabling ingress-nginx) left new Machines permanently blocked with no matching `TalosConfigTemplate`. The Job now iterates the same helper that produces the `MachineDeployment`, so `md0` gets its reconcile Job like any explicitly declared group ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3535, backport #3536). + +* **fix(keycloak-configure): patch HelmRelease in release namespace on teardown**: The `keycloak-configure` pre-delete Job cleared the Flux `HelmRelease` finalizer in a hardcoded namespace that did not match where the release actually installs (`cozy-keycloak`), so its `ServiceAccount` was forbidden to patch it, the Job retried forever, and the `HelmRelease` stuck in `Terminating` — blocking any uninstall or reinstall of Keycloak. The teardown Job now templates both the release name and namespace from the Helm release itself, so teardown completes correctly ([**@lexfrei**](https://github.com/lexfrei) in #3372, backport #3478). + +* **fix(cozystack-basics): gate the hostname VAP policies on the VAP API**: The hostname `ValidatingAdmissionPolicy` templates rendered unconditionally, so a first install on a cluster where the `ValidatingAdmissionPolicy` API is unavailable dropped the policies permanently — a later cluster upgrade that gains the API would not bring them back. The templates are now gated on `.Capabilities.APIVersions.Has`, so they render only where the API exists and are picked up automatically once it becomes available ([**@lexfrei**](https://github.com/lexfrei) in #3409, backport #3442). + +* **fix(seaweedfs): make naming audit fail closed on kubectl and payload errors**: `hack/seaweedfs-naming-audit.sh`, used by operators to classify SeaweedFS instances before the naming-migration cleanup, was fail-open — any `kubectl` failure or unreadable Helm release payload produced an empty result table indistinguishable from a genuinely clean fleet. Since the runbook uses this script as the gate before deleting old PVCs, a transient API error could have green-lit destroying live data. Every query now fails loudly on error instead of silently reporting "nothing found," and incomplete evidence now falls back to a conservative "direction cannot be established" verdict rather than a wrong deletion candidate ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3436, backport #3474). + +* **chore(release): don't activate `kubernetes-nodes` on the release-1.6 line**: The `kubernetes-nodes` app package was not ready to ship on the 1.6 line, so its single include is removed from the `iaas` platform bundle — the platform stops activating the package on this line while its code, API types, and `kubernetes-nodes-rd` system package are all kept intact for when it is ready ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3437). + +## Development, Testing, and CI/CD + +* **ci(release): carry the finalize fixes onto the 1.6 line**: `release-1.6` was cut before three release-pipeline fixes landed on `main`: dropping `persist-credentials` on checkout (which had let a stale `GITHUB_TOKEN` silently win over the app token, so the stable tag push created no workflow run and the automated changelog/docs backstops never fired, as happened for v1.6.0), publishing the GitHub release with the merged changelog as its body instead of a placeholder, and dropping a `paths-ignore` filter that could drop a changelog-only promotion PR. All three are backported so v1.6.1 releases correctly with real release notes ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3530). + +* **fix(ci): overlay images from the PR base branch, and publish per-line artifacts**: PR validation always overlaid unbuilt packages from `cozystack-packages:main`, so a `release-1.6` PR was tested against `main`'s controller binaries against its own line's charts — which is exactly what made #3437 above fail install deterministically with a schema-validation error. Each maintained `release-X.Y` branch now builds and publishes its own packages artifact, and PR validation overlays from the artifact matching the PR's own base branch ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3471, backport #3514). + +* **fix(release): make promote-retag digest verification media-type-agnostic**: The v1.6.0 finalize run aborted partway through promoting rc images to stable tags because its post-copy digest check used `skopeo inspect --format '{{.Digest}}'`, which prints nothing for OCI artifacts like `cozystack-packages`, leaving most repositories without a stable tag and skipping the installer publish. The digest is now computed as the sha256 of the raw manifest, which works identically for container images and OCI artifacts, so promotion can no longer abort mid-way on this class of artifact ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3435, backport #3473). + +## Other repositories + +### talm v0.34.0 + +* **[talm] feat(charts): add preset value knobs**: Exposes `timeServers`, control-plane component `extraArgs`, `registryMirrors`, per-host `registryTLS`, multiple Layer2 `vips`, `network.preserveExisting`, and `network.extraLinks` (bonds, VLANs, extra addresses and routes) as values on the cozystack, generic, and talm presets, so a node's Talos machine config can be described declaratively instead of via a template fork. Every knob defaults empty and a stock render stays byte-identical; each input Talos would reject fails fast at render time with a hinted error ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#232). + +* **[talm] chore(deps): migrate to Helm 4 and drop the cozystack/talos fork**: Moves talm's vendored Helm template engine from v3 to v4 and drops the `cozystack/talos` fork (carried solely for a `--skip-verify` flag, now reimplemented locally), tracking stock upstream Talos v1.13.7. Golden render snapshots confirm the generated machine config is unchanged for users ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#231). + +## Documentation + +* **[website] feat(blog): add Cozystack 1.6 release and Blockstor announcement**: Publishes the v1.6.0 release-announcement blog post — covering Talos Linux tenant workers, tenant-controlled OIDC, the `SecurityGroup` API, hierarchical quotas, and in-place etcd-operator adoption — alongside a companion post announcing the open-sourcing of Blockstor, the LINSTOR-compatible storage control plane ([**@tym83**](https://github.com/tym83) in cozystack/website#641). + +* **[website] chore(blog): repair front matter, links and bundle names**: Follow-up cleanup on the two new blog posts, fixing front matter, internal links, and page bundle names ([**@tym83**](https://github.com/tym83) in cozystack/website@4b5d1ad). + +* **[website] chore: upgrade Hugo to 0.164.0 and convert HTML content to markdown**: Upgrades the site generator to Hugo 0.164.0, requiring Node 22 under its new node permission model, and converts remaining raw-HTML content to markdown along the way, keeping the site buildable on current tooling ([**@tym83**](https://github.com/tym83) in cozystack/website#636). + +* **[website] chore(blog): make the two taxonomy axes disjoint**: Cleans up the blog's tagging so its two taxonomy axes no longer overlap, making blog post categorization and filtering more consistent ([**@tym83**](https://github.com/tym83) in cozystack/website#635). + +* **[website] docs(talm): describe the preset value knobs for network and registries**: Documents the new `talm` preset value knobs (cozystack/talm#232) for time servers, control-plane extra args, registry mirrors and TLS, VIPs, and network links, so operators can find the declarative equivalents to a template fork ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#633). + +* **[website] feat(hack): generate docs from a pre-tag ref and fail loudly on fetch errors**: Hardens the docs-generation tooling to build from a pinned pre-tag ref and to fail loudly instead of silently on a fetch error, reducing the chance of publishing docs generated from the wrong ref ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#631). + +* **[website] docs(backups): document PostgreSQL point-in-time recovery (PITR)**: Adds documentation for PostgreSQL point-in-time recovery, covering how to configure and perform a PITR restore for managed PostgreSQL clusters ([**@androndo**](https://github.com/androndo) in cozystack/website#629). + +* **[website] docs: update managed apps reference for v1.6.0**: Refreshes the managed applications reference pages to match what shipped in v1.6.0 ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#628). + +* **[website] docs(monitoring): add OIDC authentication guide for Grafana**: Adds a guide for configuring OIDC single sign-on authentication for per-instance Grafana, complementing v1.6.0's tenant OIDC support ([**@IvanHunters**](https://github.com/IvanHunters) in cozystack/website#597). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@androndo**](https://github.com/androndo) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@scooby87**](https://github.com/scooby87) +* [**@tym83**](https://github.com/tym83) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.6.0...v1.6.1 diff --git a/docs/changelogs/v1.6.2.md b/docs/changelogs/v1.6.2.md new file mode 100644 index 0000000000..a6f0a8c845 --- /dev/null +++ b/docs/changelogs/v1.6.2.md @@ -0,0 +1,50 @@ + + +# v1.6.2 (2026-08-19) + +A patch release with six fixes covering the backup-strategy controller, kube-ovn's webhook certificate, Velero CRD upgrades, CNPG barman-cloud backups, `flux-shard-operator`, and the published OpenAPI definitions, plus a release-pipeline reliability fix. + +## Fixes + +* **fix(backupstrategy-controller): repair lookup-gated backup objects**: The default backup `Strategy` CRs and the Velero `BackupStorageLocation` are gated on a Helm `lookup` performed while the referenced object is still being created; when that lookup came back empty the objects were skipped permanently, since helm-controller does not re-render a release whose chart and values are unchanged. The gate now resolves the default bucket credentials Secret through the RESTMapper, bounds each check, and tolerates an absent Secret instead of looping, so the default backup objects are created reliably instead of silently vanishing for months ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #3524, backport #3731). + +* **fix(kube-ovn): reload kubeovn-webhook serving certificate on cert-manager renewal**: `kube-ovn-webhook` loaded its TLS serving certificate once at startup and never re-read it; once cert-manager renewed the backing Secret and the old certificate expired, the apiserver's calls to the webhook failed verification and, because the `MutatingWebhookConfiguration` uses `failurePolicy: Fail`, every pod creation in tenant namespaces was rejected — including `virt-launcher` pods, blocking VMI startup. The webhook now serves its certificate through a reloading callback that re-reads the key pair when the mounted files change and widens `renewBefore` to 720h, so cert-manager renewals are honored without a pod restart ([**@IvanHunters**](https://github.com/IvanHunters) in #3557, backport #3730). + +* **fix(velero): apply CRD updates on upgrade via CreateReplace**: Velero's CRDs stayed frozen at whatever version was first installed, since Helm never touches a chart's `crds/` directory on upgrade; when the Velero image moved to a version that added new backup phases, the apiserver rejected phase transitions against the stale CRDs and backups silently stopped while the HelmRelease stayed green. The Velero package now opts into `upgradeCRDs: CreateReplace`, so CRDs are kept current on upgrade and backups keep working ([**@lexfrei**](https://github.com/lexfrei) in #3727, backport #3728). + +* **fix(backups): request S3 checksum only when required for barman-cloud (non-AWS S3 / Ceph RGW)**: CNPG's barman-cloud plugin sidecar defaulted to computing a flexible checksum on every upload, which several S3-compatible backends (Ceph RGW, some MinIO / Cloudflare R2 builds) reject outright, so every backup and WAL-archive upload to those backends failed and `ScheduledBackup`s never stored anything. Every barman-cloud `ObjectStore` Cozystack creates — Keycloak's system DB, the postgres app's backup and recovery stores, and the platform-managed system-bucket store — now sets `AWS_REQUEST_CHECKSUM_CALCULATION=when_required`, a safe default accepted by both AWS S3 and the affected backends ([**@androndo**](https://github.com/androndo) in #3417, backport #3767). + +* **fix(flux-shard-operator): repair sharded helm-controller crashloop behind an HTTP proxy**: The cloned `helm-controller-shard` Deployment inherited `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` from the flux-aio all-in-one wiring even though a standalone shard needs no external egress; behind an unreachable proxy the controller's blocking startup HTTPS call never completed, the manager never served `/healthz`, and every HelmRelease sharded to that controller was frozen. The sanitisation now also drops the inherited proxy env and adds a `startupProbe` derived from the liveness handler, so sharded HelmReleases keep reconciling in proxied environments instead of crashlooping forever ([**@IvanHunters**](https://github.com/IvanHunters) in #3546, backport #3818). + +* **fix(api): declare OpenAPIModelName for core and sdn types**: The `core` and `sdn` API groups did not declare `OpenAPIModelName` the way the `apps` group already did, so their published OpenAPI definition names were Go import paths while every `$ref` pointing at them escaped each slash — the two spellings never matched, the reference dangled, and `kubectl apply --validate` failed on any resource against a `cozystack-api` built after the underlying Kubernetes 0.35 change. Declaring `OpenAPIModelName` for `core` and `sdn` too makes every published definition name the dotted Kubernetes model name, so client-side validation against the published OpenAPI works again ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3808, backport #3812). + +## Development, Testing, and CI/CD + +* **ci(release): complete the candidate-aware promotion pipeline on release-1.6**: `release-1.6` was missing the e2e and packages-verification jobs that `Promote RC` requires on its target base, so `v1.6.1` was promoted with the rc e2e gate bypassed and the next patch release could not even be dispatched. Adds the `rc-e2e` job, the `verify-release-candidate` checks, `hack/verify-promoted-packages.sh`, `hack/validate-changelog.sh` and regression tests pinning the pipeline's contract, so future patch releases off `release-1.6` run the same e2e and package-verification gates as `main` before promoting, and the tag-time changelog is validated and ported from the tag rather than regenerated ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3893). + +## Documentation + +* **[website] docs: import the operator guides that lived in the cozystack repo**: Moves the operator-facing guides that used to live in the `cozystack` repo over to the documentation site, so operators find them alongside the rest of the docs instead of scattered across two repositories ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#648). + +* **[website] docs(oidc): document private CA and staging trust**: Documents how to configure tenant OIDC to trust a private certificate authority and staging certificates, closing a gap for operators running their own CA or testing with a staging issuer ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#650). + +* **[website] feat(community): add a Community page and link it from the main menu**: Adds a Community page linked from the site's main menu, giving visitors a single place to find how to get in touch with and contribute to the Cozystack community ([**@tym83**](https://github.com/tym83) in cozystack/website#637). + +* **[website] chore(telemetry): publish July 2026 and explain how the figures are derived**: Publishes the July 2026 telemetry figures and documents how those figures are derived, giving the community visibility into adoption trends and how the numbers are calculated ([**@tym83**](https://github.com/tym83) in cozystack/website#644). + +* **[website] feat(blog): new Blockstor banner**: Adds a new banner promoting Blockstor to the blog, improving the visibility of the storage control plane's announcement ([**@tym83**](https://github.com/tym83) in cozystack/website#646). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@androndo**](https://github.com/androndo) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@tym83**](https://github.com/tym83) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.6.1...v1.6.2 diff --git a/docs/changelogs/v1.6.3.md b/docs/changelogs/v1.6.3.md new file mode 100644 index 0000000000..333a736c12 --- /dev/null +++ b/docs/changelogs/v1.6.3.md @@ -0,0 +1,82 @@ + + +# v1.6.3 (2026-09-03) + +A patch release with a security fix restricting access to the kube-ovn admission webhook, four bug fixes covering managed Kubernetes, ClickHouse, PostgreSQL and telemetry, a reworked Marketplace/Admin dashboard, configurable ClickHouse versions, a WorkloadMonitor bucket-metrics endpoint override, a Keycloak KMS proxy update and Vault CA support, plus `ansible-cozystack` dependency bumps and a batch of new documentation. + +## Security + +* **fix(kubeovn-webhook): restrict the webhook port to the API server**: The `kube-ovn-webhook` `/mutate-pods` handler returned namespace-derived annotations to any caller that could reach its serving port, and nothing previously stopped an arbitrary in-cluster pod from being that caller. A `CiliumNetworkPolicy` now denies pod and world traffic to the webhook's port while allowing only the API server (which runs on the host network), closing the exposure tracked as [GHSA-g883-q79m-8225](https://github.com/cozystack/cozystack/security/advisories/GHSA-g883-q79m-8225) without needing the client-certificate path, which the platform's current admission configuration cannot use ([**@kvaps**](https://github.com/kvaps) in #3955, backport #3975). + +## Features and Improvements + +* **feat(dashboard): rework the Marketplace entry point and the Admin portal**: The Console is now the dashboard's entry point instead of the Marketplace, so opening the dashboard shows what a tenant already runs rather than a "choose a product" landing page; the Marketplace opens the grouped application list directly, one click away. Administration (Tenants, Modules, External IPs, per-tenant Info) moves from Console into an always-visible Admin tab, while the two cluster-wide operator areas (Capacity, Backup Classes) keep their own gates. Tenants and Modules are now rendered as a tenant hierarchy tree instead of a flat table, so sub-tenants and inherited modules are visible instead of missing or erroring, and External IPs lists LoadBalancer services across every accessible tenant in one table. Secret values can be copied without being revealed on screen first ([**@kvaps**](https://github.com/kvaps) in #3928, backport #3952). + +* **feat(clickhouse): add configurable version parameter**: The ClickHouse server and Keeper images were hardcoded to `24.9.2.42`, so — unlike postgres, mariadb, mongodb, opensearch, rabbitmq and redis — ClickHouse instances carried no engine `version` in the Cozystack API. A `version` values key (`v25.8`, `v25.3`, `v24.9`, default `v24.9` for byte-identical existing renders) now selects the server and Keeper image tags, with `hack/update-versions.sh` refreshing the supported-version map from Docker Hub ([**@scooby87**](https://github.com/scooby87) in #3476, backport #4054). + +* **feat(workloadmonitor): bucket metrics endpoint override**: `cozystack-controller` meters S3 bucket sizes for billing by discovering a VictoriaMetrics endpoint through a namespace label, which assumed the monitoring stack scraping SeaweedFS always runs in the same cluster. A new `cozystackController.seaweedfsMetricsEndpoint` chart value lets a deployment point bucket-size queries at a remote Prometheus-compatible endpoint instead, with no behavior change when unset. This also fixes a latent billing bug where any metrics-query failure silently zeroed out recorded bucket sizes; the controller now retains the last known values and surfaces the failure via a `BucketMetricsUnavailable` event and error metrics instead of wiping billing data ([**@lllamnyp**](https://github.com/lllamnyp) in #3443, backport #3558). + +* **feat(keycloak): let the KMS proxy trust a private Vault CA**: The KMS-encrypting DB proxy could previously only reach a Vault whose certificate chains to a publicly trusted root, so a Vault behind an internal load balancer with a self-signed certificate was reachable only over plain HTTP. `encryption.kms.vault` now accepts a CA either inline as `caBundle` or via an existing `caSecretName`/`caSecretKey`, letting the proxy verify a private Vault over HTTPS; existing installations are unaffected when neither key is set ([**@sircthulhu**](https://github.com/sircthulhu) in #3874, backport #3981). + +## Fixes + +* **fix(postgres): let in-place restore re-bootstrap instead of wedging**: An in-place PostgreSQL `RestoreJob` (restoring an app back into itself) never came back up — the cluster stayed in `Setting up primary` with crash-looping recovery pods, or the HelmRelease upgrade failed outright. Two independent bugs in the restore patch caused this: the restored cluster's WAL-archive `serverName` collided with the recovery source's, and `useSystemBucket` stayed enabled alongside explicit restore S3 coordinates, tripping the chart's own compatibility guard. Both are fixed, so in-place restores now re-bootstrap to a healthy cluster instead of wedging ([**@androndo**](https://github.com/androndo) in #3959, backport #3988). + +* **fix(clickhouse): scheme the backup S3_ENDPOINT on the system-bucket flow**: On the default `useSystemBucket` backup flow, the ClickHouse chart passed a scheme-less host straight into the `clickhouse-backup` sidecar's `S3_ENDPOINT`, which `altinity/clickhouse-backup`'s AWS SDK endpoint resolver rejects outright — so every ClickHouse backup and restore on the default flow failed with `was not a valid URI`. The chart now prepends `https://` on that path, matching how the platform already schemes the equivalent endpoint for other backup strategies, so default-flow ClickHouse backups and restores work again ([**@androndo**](https://github.com/androndo) in #3963, backport #3987). + +* **fix(kubernetes): retry the tenant CNI install instead of uninstalling it**: The tenant Cilium HelmRelease had no install strategy, so Flux used its default `RemediateOnFailure`, which uninstalls Cilium — the tenant cluster's only CNI — between retry attempts whenever the initial install ran long (for example while waiting for the first worker node to register), then reinstalls it on the next attempt, repeating without end. The strategy is now `RetryOnFailure`, which keeps the applied manifests in place and retries a failed install as an upgrade, so the CNI is no longer torn down and the agent DaemonSet rolls out as soon as a worker registers ([**@lexfrei**](https://github.com/lexfrei) in #3552, backport #4033). + +* **fix(telemetry): make the documented opt-out cover both collectors**: Telemetry is reported by two components — `cozystack-operator` and `cozystack-controller` — but only the operator had a reachable opt-out switch; the documented `cozystackOperator.disableTelemetry` setting could never reach the controller, which kept posting application counts every 15 minutes even after an operator followed the published opt-out procedure. The platform chart now threads a `telemetry.disabled` key into the controller as well, so the documented opt-out actually disables both collectors ([**@kvaps**](https://github.com/kvaps) in #3791, backport #3916). + +## Dependencies + +* **chore(keycloak): bump KMS proxy to 0.2.2**: Updates `keycloak-kms-proxy` to v0.2.2 and pins the image to its published digest, fixing decryption for reads whose result set spans a join ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3995, backport #3996). + +* **[ansible-cozystack] chore(deps): update k3s to v1.36.3+k3s1 and v1.36.4+k3s1**: Renovate dependency bumps of the k3s Kubernetes distribution used by the management-cluster bootstrap role ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#66, #69). + +* **[ansible-cozystack] chore(deps): track cozystack installer v1.6.2**: Renovate dependency bump of the `cozy-installer` Docker image tag to follow the latest Cozystack patch release ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#68). + +## Documentation + +* **[website] fix(telemetry): document the opt-out for both collectors**: Documents both opt-out steps now that the fix above threads `telemetry.disabled` through the platform, replacing the previously incomplete single-step procedure ([**@kvaps**](https://github.com/kvaps) in cozystack/website#651). + +* **[website] docs(home): refresh the screenshot gallery for the reworked console**: Retakes the home-page screenshot gallery against the reworked console (Marketplace, Modules, Tenants, Admin), replacing slides that showed a header and sidebar layout that no longer exist ([**@kvaps**](https://github.com/kvaps) in cozystack/website#662). + +* **[website] docs(images): refresh the product screenshot and its dark variant**: Retakes the standalone product screenshot referenced by the announcement post and outside links, which still showed a console two redesigns old ([**@kvaps**](https://github.com/kvaps) in cozystack/website#663). + +* **[website] feat(site): Live Demo — the real console, mocked, in the browser**: Adds a `/demo` page running the real `cozystack-ui` console with a mocked Kubernetes API, so a visitor can explore the actual marketplace and console without a cluster, signup, or install ([**@tym83**](https://github.com/tym83) in cozystack/website#669). + +* **[website] feat(demo): self-refreshing console demo + Resources nav grouping**: Adds the demo's source as an overlay with a weekly-refresh workflow so the mocked bundle tracks upstream `cozystack-ui` automatically, and groups secondary navigation links under a Resources menu ([**@tym83**](https://github.com/tym83) in cozystack/website#670). + +* **[website] fix(demo): embed console under site nav, fix refresh + deep create**: Fixes the live demo breaking on a hard refresh of any deep route by serving the SPA at a separate path and embedding it in an iframe under a real Hugo page, and keeps the site's own navigation visible above the demo ([**@tym83**](https://github.com/tym83) in cozystack/website#676). + +* **[website] fix(demo): create works for every catalog type**: Fixes demo creation hanging on an endless spinner for most marketplace types (Kubernetes cluster, FoundationDB, Harbor, MariaDB, and others) by resolving the mocked kind from the application catalog instead of a hardcoded list of seven types ([**@tym83**](https://github.com/tym83) in cozystack/website#677). + +* **[website] fix(page): render page title and fix top spacing on type:page pages**: Fixes page titles not rendering and content sitting flush against the navbar on `type: page` pages, most visible on the Compliance section ([**@tym83**](https://github.com/tym83) in cozystack/website#668). + +* **[website] docs(virtualization): guide for building a Windows golden image with Packer**: Adds a guide for building a Windows golden image with Packer on KubeVirt, covering the CDI clone flow and the published HashiCorp KubeVirt Packer plugin ([**@scooby87**](https://github.com/scooby87) in cozystack/website#666). + +* **[website] docs(compliance): add Kubernetes conformance results page**: Adds a Kubernetes conformance page documenting the platform's CNCF conformance runs, later extended with the v1.31 result and corrected submission scope ([**@tym83**](https://github.com/tym83) in cozystack/website#658, #660). + +* **[website] fix(blog): restore Community Meeting Calendar link in Wizard post**: Fixes an empty link in the Wizard announcement blog post ([**@tym83**](https://github.com/tym83) in cozystack/website#659). + +* **[website] docs(faq): drop the retired GitOps example**: Replaces a FAQ pointer to an archived, pre-1.0 GitOps example repository with a direct answer explaining that Cozystack applications are ordinary Kubernetes objects, so no platform-specific GitOps tooling is needed ([**@kvaps**](https://github.com/kvaps) in cozystack/website#661). + +* **[website] docs(licenses): list kuberture and ouroboros**: Adds `kuberture` and `ouroboros` — both shipped with the platform and maintained by a Cozystack maintainer — to the Licenses page, which otherwise only enumerates upstream components ([**@kvaps**](https://github.com/kvaps) in cozystack/website#649). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@androndo**](https://github.com/androndo) +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@lllamnyp**](https://github.com/lllamnyp) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@scooby87**](https://github.com/scooby87) +* [**@sircthulhu**](https://github.com/sircthulhu) +* [**@tym83**](https://github.com/tym83) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.6.2...v1.6.3 diff --git a/docs/changelogs/v1.6.4.md b/docs/changelogs/v1.6.4.md new file mode 100644 index 0000000000..2858cd80a1 --- /dev/null +++ b/docs/changelogs/v1.6.4.md @@ -0,0 +1,154 @@ + + +# v1.6.4 (2026-09-28) + +A patch release with backported fixes for VM live migration during eviction and scheduling, Kafka release deletion, KubeVirt-related billing labels, ingress client-IP trust, PostgreSQL-adjacent LINSTOR/CDI/ZFS reliability, MongoDB and RabbitMQ defaults, and a batch of e2e hardening, plus a new `talm` documentation site, an NVIDIA vGPU restore role for `ansible-cozystack`, and a large Marketplace documentation section on the website. + +## Fixes + +* **fix(cluster-api): backport live migration before host eviction**: `cluster-api-provider-kubevirt` now attempts live migration before falling back to drain and delete when a guest node is evicted, uses the guest node's own name for the drain fallback instead of the wrong identifier, and uncordons a recreated guest — so an eviction no longer forces an unnecessary VM restart when a live migration would have sufficed ([**@yankawai**](https://github.com/yankawai) in #4291, backport #4474). + +* **fix(kubeovn): retry VM migration setup after scheduling**: Retries the VM migration network setup when the target Pod or source/target scheduling information is not yet available, instead of failing outright, and folds the Kube-OVN image into the regular in-tree build matrix — so a migration that starts before Kube-OVN has finished scheduling both endpoints no longer aborts ([**@yankawai**](https://github.com/yankawai) in #4253, backport #4328). + +* **feat(kubevirt): expose migration configuration through platform values**: Cluster-wide KubeVirt migration settings (bandwidth per migration, parallel migrations per cluster, parallel outbound migrations per node) can now be set through the platform value `kubevirt.migrations` instead of hand-patching the generated `Package`, which the platform undoes on every render; leaving the value unset preserves existing defaults ([**@yankawai**](https://github.com/yankawai) in #4254, backport #4402). + +* **fix(linstor): add opt-in graceful satellite shutdown on Talos**: Adds an opt-in Talos satellite `preStop` hook that releases unused Secondary DRBD resources, ZFS zvols included, before the satellite stops, without forcing, demoting or calling the Kubernetes API. Enable it with `talos.gracefulShutdown.enabled` for sequential node maintenance; existing installations keep today's behavior by default, and the hook also runs on ordinary pod restarts so it should be enabled only where node restarts are serialized ([**@yankawai**](https://github.com/yankawai) in #4292, backport #4401). + +* **fix(workloadmonitor): populate VM instance-profile label from the KubeVirt preference annotation**: `WorkloadMonitor` read a `kubevirt.io/cluster-instanceprofile-name` annotation that KubeVirt never writes; the real annotation is `kubevirt.io/cluster-preference-name`. Reading the correct key means a VM's instance profile — a Windows guest, for example — now reaches the billing metric pipeline instead of being silently dropped ([**@IvanHunters**](https://github.com/IvanHunters) in #4448, backport #4454). + +* **fix(opensearch-operator): name the component reconciler a stall sits in**: The OpenSearch cluster controller now logs a watchdog report naming which of its nine component reconcilers has been running for over a minute, and keeps reporting on an interval while it stays stuck, instead of leaving a stalled reconcile with no trace in the operator log at all ([**@IvanHunters**](https://github.com/IvanHunters) in #4333, backport #4377). + +* **fix(manifestutil): report one message when the CRD wait is cancelled**: A CRD wait cut short by a cancelled context now always reports one error naming the CRD it was waiting on, instead of sometimes including the name and sometimes not, depending on which of two exit paths the scheduler happened to pick ([**@lexfrei**](https://github.com/lexfrei) in #3742, backport #4475). + +* **fix(kafka): delete release topics before the topic operator is removed**: Deletes release-owned `KafkaTopic`s with a pre-delete hook before the Strimzi topic operator is removed, and fixes the hook's own `--request-timeout` flag, which made it skip in-cluster credentials and fail every deletion outright. Together these mean a Kafka release with the `strimzi.io/topic-operator` finalizer no longer leaves stray topics behind on reinstall, and release deletion itself no longer wedges ([**@yankawai**](https://github.com/yankawai), [**@IvanHunters**](https://github.com/IvanHunters) in #3938, #4280, backport #4456). + +* **fix(ingress): trust CF-Connecting-IP only from Cloudflare ranges**: With `cloudflareProxy` enabled, `CF-Connecting-IP` is now trusted only from Cloudflare's published address ranges. Previously the ranges were passed under a key `ingress-nginx` ignores, so any client that reached the load balancer directly could set the header itself and bypass the ingress IP `whitelist` ([**@lexfrei**](https://github.com/lexfrei) in #4444, backport #4445). + +* **fix(apps): let the cleanup hooks watch the objects they delete**: The Harbor, ClickHouse, MariaDB and Qdrant cleanup hooks now wait for the objects they delete to actually disappear, with the wait bounded so a stuck claim cannot hang the whole uninstall ([**@yankawai**](https://github.com/yankawai) in #4135, backport #4369). + +* **fix(registry): preserve write options in aggregated storage**: The application, tenant-secret and security-group APIs now preserve Kubernetes write options (dry-run, field manager, field validation, delete propagation, grace periods, preconditions) when delegating to controller-runtime, instead of silently dropping them; a rejected write also keeps its machine-readable status, so a quota or admission failure now answers 403/409 with a reason instead of a generic 500 ([**@yankawai**](https://github.com/yankawai) in #3937, backport #4370). + +* **fix(dashboard): let dashboard users read backup CRD schemas**: Grants dashboard users read access to the backup CRD schemas, fixing the backup create/edit forms that previously returned 403 ([**@androndo**](https://github.com/androndo) in #4236, backport #4307). + +* **fix(backups): give the barman-cloud sidecar its own resources**: The barman-cloud sidecar previously had no resource requests or limits, so in tenant namespaces it inherited a 128Mi `LimitRange` default and was OOMKilled during backups even while the surrounding `ObjectStore` and `Cluster` stayed healthy. It now requests 100m/256Mi and is capped at 1Gi on both the chart and the platform's Go rendering path ([**@yankawai**](https://github.com/yankawai) in #4220, backport #4336). + +* **fix(linstor): reconnect only the stalled peer in the satellite plunger**: The satellite plunger disconnected a stalled DRBD peer but then tried to reconnect every peer of the resource at once, which failed with "Device has a net-config" for peers that were never disconnected and left the stalled one permanently unreconnected. It now reconnects only the peer pair it disconnected, so a stalled resync actually recovers ([**@yankawai**](https://github.com/yankawai) in #4184, backport #4319). + +* **fix(foundationdb): let tenant roles read the connection ConfigMap**: Tenants previously had no way to obtain their FoundationDB connection string: the cluster file lives in a ConfigMap tenant subjects could not read and the dashboard resource map did not expose. The ConfigMap is now in the dashboard resource map with a Role/RoleBinding granting the tenant `use` access level get/list/watch on it ([**@yankawai**](https://github.com/yankawai) in #4148, backport #4315). + +* **fix(opensearch): remove the data PVCs when the application is deleted**: Adds a post-delete hook that reclaims an OpenSearch application's data PVCs when it is removed, instead of leaving them orphaned ([**@yankawai**](https://github.com/yankawai) in #4136, backport #4368). + +* **fix(nats): merge config.merge.accounts with the generated accounts map**: `config.merge.accounts` is now merged into the generated NATS accounts map instead of being rendered as a second, conflicting copy of the same key ([**@yankawai**](https://github.com/yankawai) in #4134, backport #4321). + +* **fix(clickhouse): keep the Keeper name inside the StatefulSet volume-name limit**: Shortens the generated Keeper name for application names longer than 15 characters, fixing the Keeper `StatefulSet` failing to create for those names ([**@yankawai**](https://github.com/yankawai) in #4133, backport #4314). + +* **fix(linstor): wait for temporary probe devices**: Backports a LINSTOR upstream fix that waits for a ZFS probe device to appear before reading its I/O properties, instead of racing udev; without the wait, a pool could be left with incorrect block sizes and failed replica placement on 4K pools ([**@yankawai**](https://github.com/yankawai) in #4100, backport #4335). + +* **fix(mongodb): fill the dashboard credentials on a first install**: The dashboard credentials Secret is now filled with the database password and connection URI on first install; previously it shipped empty until someone ran `helm upgrade` by hand, because the lookup it relied on could only see users the Percona operator had not yet created ([**@yankawai**](https://github.com/yankawai) in #4014, backport #4330). + +* **fix(rabbitmq): right-size the default resources preset to s1.nano**: Raises the default `resourcesPreset` from `t1.nano` (128Mi) to `s1.nano` (250m CPU, 512Mi), because a default-sized RabbitMQ 4.2 broker was OOMKilled on `t1.nano` and never became Ready. Brokers with no stored preset are restarted on upgrade and gain a larger guaranteed memory reservation — check node headroom, and tenants near a `resourceQuotas` memory ceiling, before upgrading; brokers with an explicit `resourcesPreset` already set keep their existing size ([**@yankawai**](https://github.com/yankawai) in #3936, backport #4393). + +* **fix(kafka): make topics[].config optional**: Kafka topics can now omit the optional Strimzi config map entirely, instead of the chart requiring one even when there is nothing to set ([**@yankawai**](https://github.com/yankawai) in #3935, backport #4334). + +* **fix(kubernetes): pin KubeVirt CSI sidecars to SIG Storage releases**: Pins the KubeVirt CSI sidecars to versioned SIG Storage images, fixing sidecars that previously would not run on pre-x86-64-v3 CPUs ([**@yankawai**](https://github.com/yankawai) in #3934, backport #4313). + +* **fix(cdi): update CDI to v1.66.1**: Updates CDI to v1.66.1, fixing HTTP imports that block volumes on 4Kn devices (DRBD/LINSTOR) via a repaired `qemu-img`, and a `resourceVersion` update storm on prime PVCs during `WaitForFirstConsumer` imports ([**@lexfrei**](https://github.com/lexfrei) in #3920, backport #4156). + +* **feat(etcd-operator): bump to v0.5.5 and vendor EtcdDefragPolicy CRDs**: Bumps `etcd-operator` to v0.5.5, adding the `EtcdDefrag` and `EtcdDefragPolicy` APIs for on-demand and scheduled etcd defragmentation ([**@androndo**](https://github.com/androndo) in #3968, backport #4151). + +## Development, Testing, and CI/CD + +* **test(e2e): run the merge-gating lanes on Talos containers**: Moves the merge-gating E2E lanes onto Talos containers, fixing tenant node-join failures that were happening in 8 of the last 11 runs because tenant workers sat one virtualization level too deep on the shared runners and had their CSR signed after the readiness window closed ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #4020, backport #4437). + +* **chore(e2e): ignore the artifacts the sandbox bringup writes to the repo root**: Adds `.gitignore` entries for the Talos secrets bundle, rendered machine configs, kubeconfigs, boot image and per-VM directories a local e2e sandbox bringup writes to the repo root, closing a path where a broad `git add` could have staged cluster private keys ([**@lexfrei**](https://github.com/lexfrei) in #3891, backport #4438). + +* **fix(e2e): budget tenant scheduling separately from workload readiness**: Waits for a tenant node to actually accept a Pod, on its own budget, before creating the workload used by the kubernetes suites, instead of sharing one 300s deadline between node scheduling and workload readiness — fixing intermittent timeouts where scheduling alone consumed most of the shared window ([**@lexfrei**](https://github.com/lexfrei) in #3579, backport #4435). + +* **fix(e2e): budget each linstor wait separately in post-install-prep**: Gives each of the two LINSTOR waits in the post-install-prep script its own deadline, started when the previous one finishes, instead of sharing a single 15-minute window anchored at the script's start — fixing failures where a slow HelmRelease left the following Deployment wait too little time ([**@lexfrei**](https://github.com/lexfrei) in #3715, backport #4434). + +* **fix(reloader): add startupProbe to survive slow first start under load**: Adds a startup probe to `cozy-reloader` so a slow first start is no longer killed by the liveness probe and left crash-looping, which also unblocks the LINSTOR stack that depends on it starting cleanly ([**@lexfrei**](https://github.com/lexfrei) in #3470, backport #4433). + +* **fix(e2e): stop the sandbox cluster CIDR overlapping the pod range**: Moves the e2e sandbox's Kubernetes cluster CIDR off the range Kube-OVN uses for pod IPAM, fixing intermittent Cilium/Kube-OVN IP collisions that either refused a pod outright with "IP already in use" or silently broke host-to-pod traffic ([**@lexfrei**](https://github.com/lexfrei) in #3750, backport #4432). + +* **fix(e2e): disable guest fsync on ephemeral CI VM disks**: Boots e2e QEMU guests with `cache=unsafe` so ephemeral CI VM fsyncs no longer stall the management-cluster etcd, removing cross-operator leader-election flakiness that was failing a different chainsaw suite on every run ([**@kvaps**](https://github.com/kvaps) in #3455, backport #4431). + +## Documentation + +* **[website] feat(compliance): surface AI Conformance page, mark submission accepted**: Publishes the AI Conformance results page and marks the platform's submission as accepted ([**@tym83**](https://github.com/tym83) in cozystack/website#706). + +* **[website] fix(docs): refresh trunk version pins to Cozystack v1.6.3**: Updates the documentation's trunk version references to Cozystack v1.6.3 ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#618). + +* **[website] docs(platform-package): sync the publishing.\* table with the platform values**: Keeps the platform values reference's `publishing.*` table in sync with the actual platform values ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#679). + +* **[website] docs(networking): GatewayClass selection, edge TLS mode, and the Cloudflare Tunnel package**: Documents GatewayClass selection, the edge TLS certificate mode, and the Cloudflare Tunnel Gateway API package ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#655). + +* **[website] docs(platform-package): document KubeVirt migration settings**: Documents the new `kubevirt.migrations` platform value from the corresponding fix above ([**@yankawai**](https://github.com/yankawai) in cozystack/website#697). + +* **[website] fix(demo): keep the screen when the demo is reloaded**: Fixes the live console demo losing its current screen on a browser reload ([**@kvaps**](https://github.com/kvaps) in cozystack/website#701). + +* **[website] docs(platform-package): document kubevirt.disabledFeatureGates**: Documents the `kubevirt.disabledFeatureGates` platform value ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#678). + +* **[website] chore(demo): refresh the console demo from the cozystack monorepo**: Refreshes the live demo's bundled `cozystack-ui` console from the monorepo ([**@kvaps**](https://github.com/kvaps) in cozystack/website#698). + +* **[website] fix(site): point the Certified Kubernetes badge at the live landscape entry**: Fixes the Certified Kubernetes badge linking to a stale landscape entry ([**@tym83**](https://github.com/tym83) in cozystack/website#699). + +* **[website] docs(compliance): add AI Conformance page, update Kubernetes Conformance**: Adds the initial AI Conformance page and updates the Kubernetes Conformance page ([**@tym83**](https://github.com/tym83) in cozystack/website#693). + +* **[website] docs(blog): route independent-adopter replies to the project**: Redirects independent-adopter reply channels on the blog to the project instead of an individual ([**@tym83**](https://github.com/tym83) in cozystack/website#696). + +* **[website] fix(site): resolve /roadmap and move the vendor logo off a project path**: Fixes the `/roadmap` link and moves a vendor logo off a project-owned path ([**@tym83**](https://github.com/tym83) in cozystack/website#695). + +* **[website] feat(home): add a "where Cozystack fits" section**: Adds a "where Cozystack fits" section to the home page ([**@tym83**](https://github.com/tym83) in cozystack/website#694). + +* **[website] fix(site): give the live-demo band its own layout**: Gives the live-demo band on the home page its own layout instead of sharing one with unrelated content ([**@tym83**](https://github.com/tym83) in cozystack/website#692). + +* **[website] feat(site): verify the @cozystack.io Bluesky handle over HTTPS**: Serves the Bluesky handle DID at `/.well-known/atproto-did` to verify the `@cozystack.io` Bluesky handle over HTTPS ([**@kvaps**](https://github.com/kvaps) in cozystack/website#691). + +* **[website] docs(marketplace): add Application Marketplace section**: Adds a large new Application Marketplace documentation section covering the CLI reference, package taps, subscriptions, trust model, and catalog behavior ([**@lexfrei**](https://github.com/lexfrei), [**@IvanHunters**](https://github.com/IvanHunters) in cozystack/website#672). + +* **[website] fix(site): replace vendor social accounts in the footer with project channels**: Replaces vendor-owned social accounts in the site footer with the project's own channels ([**@tym83**](https://github.com/tym83) in cozystack/website#690). + +* **[website] fix(networking): correct what disabledPackages does to ouroboros**: Corrects the documented effect of `disabledPackages` on the `ouroboros` package ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#683). + +* **[website] docs(virtualization): add a VMware vSphere migration guide**: Adds a guide for migrating VMware vSphere VMs into Cozystack, including the VDDK image build ([**@kvaps**](https://github.com/kvaps) in cozystack/website#673). + +* **[website] docs(kubernetes): add OIDC authentication guide**: Adds a guide for configuring OIDC authentication on managed Kubernetes clusters ([**@IvanHunters**](https://github.com/IvanHunters) in cozystack/website#596). + +* **[website] ci(telemetry): put the monthly refresh back on a schedule**: Restores the scheduled monthly refresh of the telemetry dashboard data ([**@tym83**](https://github.com/tym83) in cozystack/website#645). + +* **[website] docs: update managed apps reference for v1.6.3**: Refreshes the managed applications reference for the v1.6.3 release ([**@app/cozystack-ci**](https://github.com/apps/cozystack-ci) in cozystack/website#681). + +* **[talm] docs: publish the manual as a documentation site**: Publishes `talm`'s manual as a documentation site at talm.cozystack.io, with a CLI reference generated from the Cobra command tree ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#235). + +* **[talm] ci: unbreak the lint job under Go 1.27**: Fixes the `talm` lint job under Go 1.27 and requires Go 1.27 for the build ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#237). + +* **[talm] chore(deps): bump Talos to v1.14.0**: Bumps the Talos dependency to v1.14.0, along with fixes so `applycheck` and `meta --insecure` keep working against it and a rendered config keeps every document instead of dropping some ([**@sircthulhu**](https://github.com/sircthulhu) in cozystack/talm#223). + +* **[ansible-cozystack] feat(roles): restore NVIDIA vGPU host state across reboots**: Adds a role that restores NVIDIA vGPU host state after a reboot, instead of requiring it to be reconfigured by hand ([**@lexfrei**](https://github.com/lexfrei) in cozystack/ansible-cozystack#70). + +* **[ansible-cozystack] chore(deps): track cozystack installer v1.6.3**: Renovate dependency bump of the `cozy-installer` Docker image tag to follow the v1.6.3 Cozystack release ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#71). + +* **[ansible-cozystack] chore(deps): update k3s to v1.37.0+k3s1**: Renovate dependency bump of the k3s Kubernetes distribution used by the management-cluster bootstrap role ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#72). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@androndo**](https://github.com/androndo) +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@sircthulhu**](https://github.com/sircthulhu) +* [**@tym83**](https://github.com/tym83) +* [**@yankawai**](https://github.com/yankawai) + +### New Contributors + +We're excited to welcome our first-time contributor: + +* [**@yankawai**](https://github.com/yankawai) - First contribution! + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.6.3...v1.6.4 diff --git a/docs/changelogs/v1.6.5.md b/docs/changelogs/v1.6.5.md new file mode 100644 index 0000000000..c9158e5a3a --- /dev/null +++ b/docs/changelogs/v1.6.5.md @@ -0,0 +1,157 @@ + + +# v1.6.5 (2026-10-16) + +A patch release with backported fixes for tenant Kubernetes cluster bootstrap, Application updates through the API, suspended `HelmRelease` handling, Cilium and Kube-OVN address allocation, `tcp-balancer` on HAProxy 3.3+, Alertmanager delivery, and a batch of PostgreSQL, MongoDB, NATS, etcd, Velero and dashboard fixes, plus optional email alert delivery, OpenBao static-key auto-unseal and zone placement for Keycloak. **This release contains a breaking change on the Kube-OVN platform variants — read the upgrade notes before upgrading.** + +## Features and Improvements + +* **feat(monitoring): add optional email receiver to alertmanager**: Alerta alerts can now be delivered by email through an optional receiver configured in the monitoring application values, with the alert severity guard tightened so rules using non-Alerta severities are caught ([**@yankawai**](https://github.com/yankawai) in #3800, backport #4614). + +* **feat(openbao): optional static-key auto-unseal and apiserver egress label**: OpenBao can now be auto-unsealed with a static key instead of manual Shamir unseal, with the seal values constrained so only valid settings reach the server config, and the pods can carry an apiserver egress label for network-policy setups. The static seal trade-off and the Shamir migration path are documented in the chart ([**@yankawai**](https://github.com/yankawai) in #4766, backport #4906). + +* **feat(keycloak): expose zone placement for Keycloak and the database proxy**: Keycloak, its database proxy and the `keycloak-db` instances can now be spread across zones, so a zone outage no longer takes the identity provider down with it ([**@sircthulhu**](https://github.com/sircthulhu) in #4606, backport #4856). + +## Fixes + +* **fix(kubernetes): break bootstrap-token Job out of Helm hook to unblock fresh installs**: The bootstrap-token Job is no longer a `post-install,post-upgrade` Helm hook, which deadlocked the chart: the release waited for workers, workers could not register without the bootstrap-token Secret, and the Secret was only created after the wait succeeded. Tenant Kubernetes clusters on fresh installs and upgrades no longer get stuck with `Provisioned` machines that never join ([**@IvanHunters**](https://github.com/IvanHunters) in #3887, backport #4541). + +* **fix(api): keep a suspended HelmRelease suspended across an app update**: Updating an Application through the API no longer resumes a `HelmRelease` that an operator or controller suspended. This fixes backup restores of PostgreSQL stalling at `TargetPurged` because the release was resumed and re-rendered before the purge finished ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #4411, backport #4928). + +* **fix(api): preserve foreign HelmRelease metadata and repair empty required fields**: `Update` on an Application now carries over finalizers, owner references and foreign labels/annotations from the live `HelmRelease`, so a delete racing an update no longer orphans every resource of the release; and a required field written with nothing under it (`replicas:`) is dropped so the chart default applies, with a warning to the client, instead of failing the release with a "missing property" error ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri), [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #2914, #3956, backport #4919). + +* **fix(api): let the garbage collector finalize Applications**: Deleting an Application no longer leaves the garbage-collection finalizer behind, so Applications finalize instead of hanging in `Terminating` ([**@yankawai**](https://github.com/yankawai) in #4610, backport #4683). + +* **fix(cilium)!: take cilium's own addresses from a block kube-ovn does not hand out**: On the Kube-OVN variants (`isp-full`, `isp-full-generic`) Cilium now runs `ipam.mode: cluster-pool` with a /29 per node out of the last /21 of `networking.podCIDR`, and the same block is excluded from Kube-OVN, so Kube-OVN can no longer hand a pod an address Cilium already holds for its router or Gateway API ingress. A post-upgrade hook migrates existing clusters. **Breaking, see "Migration and Upgrades" below** ([**@yankawai**](https://github.com/yankawai) in #4778, backport #4918). + +* **fix(tcp-balancer): rename backends and pin HAProxy so the chart starts on 3.3+**: Backend names no longer collide with frontend names, HAProxy is pinned to 3.4.4, and an empty whitelist leaves Kubernetes and Talos frontends unfiltered. Enabling `whitelistHTTP` without a whitelist now fails at render time instead of producing a configuration that rejects all traffic ([**@yankawai**](https://github.com/yankawai) in #4366, backport #4516). + +* **fix(monitoring): send alerts to every Alertmanager replica**: Alerts are now sent to every Alertmanager replica instead of a single one, so alerts are not lost when the replica receiving them is not the active one ([**@yankawai**](https://github.com/yankawai) in #4889, backport #4890). + +* **fix(velero): bump velero-plugin-for-aws to v1.14.4 so backups work on R2**: Updates the AWS plugin so Velero backups to Cloudflare R2 (and compatible S3 stores) work again ([**@yrahman82**](https://github.com/yrahman82) in #4834, backport #4854). + +* **fix(postgres): read the live Cluster when deciding the WAL-archive prefix**: The WAL-archive prefix is now decided from the live CNPG `Cluster`, so upgrades and restores keep archiving WAL to the right location instead of switching prefix ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #4299, backport #4592). + +* **fix(postgres): cap the default shared_buffers at a quarter of the memory limit**: The default `shared_buffers` is capped at a quarter of the memory limit (and at the memory request), so small PostgreSQL instances no longer start with a buffer size that causes out-of-memory kills ([**@yankawai**](https://github.com/yankawai) in #4724, backport #4725). + +* **fix(mongodb): set the s3 bucket and prefix on the bootstrap restore**: The bootstrap restore now gets the S3 bucket and prefix, so restoring a new MongoDB from a backup actually finds the data ([**@yankawai**](https://github.com/yankawai) in #4779, backport #4780). + +* **fix(nats): enable JetStream for the generated account**: JetStream is now enabled for the generated NATS account, so streams and consumers work out of the box ([**@yankawai**](https://github.com/yankawai) in #4675, backport #4677). + +* **fix(etcd): leave compaction to etcd and pass the real quota to defrag**: etcd compaction is left to etcd itself and defragmentation is given the real backend quota, so etcd no longer compacts redundantly or defragments against the wrong size ([**@yankawai**](https://github.com/yankawai) in #4873, backport #4874). + +* **fix(platform): keep the legacy etcd-operator stopped after etcd adoption**: After etcd is adopted by the new operator, the legacy `etcd-operator` stays stopped instead of being scaled back up and fighting over the same clusters ([**@yankawai**](https://github.com/yankawai) in #4738, backport #4739). + +* **fix(linstor): use severity warning in piraeus-datastore alerts**: The `piraeus-datastore` alerts now use the `warning` severity instead of `warn`, which Alerta did not recognise, so these alerts arrive with the correct severity ([**@yankawai**](https://github.com/yankawai) in #3799, backport #4740). + +* **fix(dashboard): allow editing free-form objects in application forms**: Free-form object fields in application forms can now be edited, and a field is marked invalid when the schema rejects it, instead of being locked or silently accepted ([**@yankawai**](https://github.com/yankawai) in #4682, backport #4905). + +* **fix(dashboard, foundationdb): show the connection ConfigMap to tenants**: The application page gets a ConfigMaps tab, with a copy button per value, for ConfigMaps in the application's resource map, and tenant roles may read the FoundationDB connection ConfigMap — so tenants can obtain their FoundationDB connection string. The tab goes away if the resource map becomes unreadable instead of showing stale names ([**@yankawai**](https://github.com/yankawai) in #4612, backport #4684). + +* **fix(cert-manager): enable the Gateway API solver in the host cert-manager**: The host cert-manager controller now has the Gateway API solver enabled, so ACME HTTP-01 challenges can be solved through Gateway API routes ([**@lexfrei**](https://github.com/lexfrei) in #4039, backport #4808). + +* **fix(kubernetes): apply worker TalosConfigTemplate server-side**: The worker `TalosConfigTemplate` is applied server-side, avoiding conflicts and field ownership problems when it is updated on existing tenant clusters ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #4163, backport #4575). + +* **fix(kubernetes): probe Keycloak CRDs by output in oidc-bootstrap cleanup**: The OIDC bootstrap cleanup now probes for Keycloak CRDs by their output, so it no longer fails on clusters where those CRDs are absent ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #3520, backport #4571). + +* **fix(cozy-lib): round the sanitized CPU request up to a whole milli-CPU**: Sanitized CPU requests are rounded up to a whole milli-CPU, so fractional requests no longer produce invalid values that Kubernetes rejects ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #3522, backport #4590). + +## Dependencies + +* **chore(hetzner-robotlb): update robotlb to 0.1.0**: Updates `hetzner-robotlb` to robotlb 0.1.0 ([**@sircthulhu**](https://github.com/sircthulhu) in #4557, backport #4560). + +## Development, Testing, and CI/CD + +* **ci(e2e): run nightly and release-tag e2e on the container lane**: Nightly and release-tag E2E runs now use the Talos container lane, with the nightly runner given headroom over its nodes, matching the merge-gating lanes ([**@lexfrei**](https://github.com/lexfrei) in #4828, backport #4913). + +* **fix(e2e-sandbox): install mc from its GitHub release**: The sandbox image installs the final `mc` release from its pinned GitHub asset and verifies its per-architecture checksum, because `dl.min.io` now returns HTTP 410 and the release-line image build was failing ([**@lexfrei**](https://github.com/lexfrei) in #4447, backport #4535). + +## Documentation + +* **[website] docs(platform-package): note the /21 the kube-ovn variants keep for cilium**: Documents the /21 of `networking.podCIDR` that the Kube-OVN variants now reserve for Cilium ([**@yankawai**](https://github.com/yankawai) in cozystack/website#739). + +* **[website] docs(platform-package): document kubevirt.migrations for v1.6**: Documents the `kubevirt.migrations` platform value in the v1.6 docs ([**@yankawai**](https://github.com/yankawai) in cozystack/website#705). + +* **[website] docs(platform): document the monitoring platform values**: Documents the monitoring platform values ([**@scooby87**](https://github.com/scooby87) in cozystack/website#738). + +* **[website] docs(platform-package): add publishing.serviceDomain**: Adds `publishing.serviceDomain` to the platform values reference ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#724). + +* **[website] docs(variants): document the slim platform variants**: Documents the `isp-slim`, `isp-slim-generic` and `isp-hosted-slim` variants ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#718). + +* **[website] docs(talm): document the ZFS export-on-shutdown knob**: Documents the `talm` knob controlling ZFS pool export on shutdown ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#721). + +* **[website] docs(install): note the talm version pins on the update path**: Notes the `talm` version pins that apply when updating ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#703). + +* **[website] docs(gpu): select the container variant from bundle values**: Documents selecting the GPU container variant from bundle values ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#714). + +* **[website] fix(operations): replace the component removal recipe that undoes itself**: Replaces a component removal recipe that reverted itself, and warns what removing a v0 component destroys ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#684). + +* **[website] fix(docs): correct stale hairpin-proxy uninstall claim and package names**: Corrects a stale hairpin-proxy uninstall claim and package names in `enabledPackages` examples ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#709). + +* **[website] docs(development): describe the multi-arch default of make image**: Describes the multi-arch default of `make image` ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#711). + +* **[website] docs: move the next docs to Talos v1.14.2**: Moves the next docs version to Talos v1.14.2 ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#720). + +* **[website] fix(versions): pin the trunk to published releases and fix released pin headers**: Pins the docs trunk to published releases and fixes released pin file headers ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#710). + +* **[website] docs(marketplace): point the Repositories view to Admin**: Points the Marketplace docs at the Admin page for the Repositories view ([**@kvaps**](https://github.com/kvaps) in cozystack/website#700). + +* **[website] fix(llms): point llms.txt docs links at the current release**: Makes `llms.txt` link to the current release docs ([**@themoriarti**](https://github.com/themoriarti) in cozystack/website#737). + +* **[website] fix(footer): use the LF Projects copyright notice on every page**: Uses the LF Projects copyright notice in the footer on every page ([**@themoriarti**](https://github.com/themoriarti) in cozystack/website#736). + +* **[website] docs(community): drop the Russian-language Telegram channel**: Removes the Russian-language Telegram channel from the community pages ([**@tym83**](https://github.com/tym83) in cozystack/website#744). + +* **[website] feat(i18n): automated translation pipeline with machine review gate**: Adds an automated documentation translation pipeline with a machine review gate ([**@tym83**](https://github.com/tym83) in cozystack/website#623). + +* **[website] feat(blog): announce the CNCF General Technical Review and compliance results**: Publishes news and a page on the CNCF General Technical Review and conformance results ([**@tym83**](https://github.com/tym83) in cozystack/website#722). + +* **[website] blog: Oberon RISC5 machine as a Cozystack catalog app**: Publishes a news post on the Oberon RISC5 machine as a catalog app ([**@tym83**](https://github.com/tym83) in cozystack/website#723). + +* **[website] docs: update managed apps reference for v1.6.4**: Refreshes the managed applications reference for v1.6.4 ([**@app/cozystack-ci**](https://github.com/apps/cozystack-ci) in cozystack/website#712). + +* **[cozy-proxy] feat(image): publish a linux/amd64 and linux/arm64 index**: `cozy-proxy` v0.4.0 publishes a multi-architecture image index for linux/amd64 and linux/arm64 ([**@lexfrei**](https://github.com/lexfrei) in cozystack/cozy-proxy#22). + +## Migration and Upgrades + +**Breaking change on the Kube-OVN variants (`isp-full`, `isp-full-generic`)** — introduced by #4778 / #4918 (`fix(cilium)!`): + +* `networking.podCIDR` must be /20 or wider; otherwise the platform refuses to render and the upgrade stops before anything changes. Widening the value later does not help, because the migration hook refuses a block outside the subnet the cluster was installed with. +* A /21 of /29 blocks caps these variants at 256 nodes. +* Every Cilium agent restarts twice, once for the configuration change and once in the migration hook. While the `cilium` release is failed, packages that depend on networking wait for it. +* If Kube-OVN has already handed out an address from the last /21, the hook names that address and stops before moving any node. Recreate the pods holding it and the next retry succeeds. +* Rolling back to v1.6.4 restores `ipam.mode: kubernetes`, but the exclusions stay in the `ovn-default` subnet and have to be removed by hand. +* `isp-hosted` is not affected. + +Other upgrade notes: + +* `tcp-balancer`: enabling `whitelistHTTP` now requires a non-empty whitelist; rendering fails otherwise. +* Tenant Kubernetes clusters stuck on v1.6.3/v1.6.4 because of the missing bootstrap-token Secret recover on upgrade; no manual Secret creation is needed. +* PostgreSQL restarts on upgrade for small clusters: a cluster with a memory limit below 512Mi and no explicit `shared_buffers` (the `t1.nano` preset and the default `t1.micro`) now gets `shared_buffers` set to a quarter of the limit, 64MB for `t1.micro`, where PostgreSQL used its built-in 128MB before. The setting needs a restart, so CloudNativePG restarts the replicas first and then the primary in place: writes pause for one restart, and a single-instance cluster is unavailable during it. Upgrade in a quiet window, or set `postgresql.parameters.shared_buffers` yourself beforehand to take the restart on your own schedule. Clusters with 512Mi or more are unchanged. + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@scooby87**](https://github.com/scooby87) +* [**@sircthulhu**](https://github.com/sircthulhu) +* [**@themoriarti**](https://github.com/themoriarti) +* [**@tym83**](https://github.com/tym83) +* [**@yankawai**](https://github.com/yankawai) +* [**@yrahman82**](https://github.com/yrahman82) + +### New Contributors + +We're excited to welcome our first-time contributors: + +* [**@themoriarti**](https://github.com/themoriarti) - First contribution! +* [**@yrahman82**](https://github.com/yrahman82) - First contribution! + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.6.4...v1.6.5 diff --git a/docs/deprecating-applications.md b/docs/deprecating-applications.md new file mode 100644 index 0000000000..4935a8685c --- /dev/null +++ b/docs/deprecating-applications.md @@ -0,0 +1,89 @@ +# Retiring a catalog application + +Removing an application from `packages/apps/` is not a deletion. The platform stops publishing its chart, but the instances people already created keep running, and nothing in the tree tells you what happens to them. This document is the procedure: how to decide what shape the retirement takes, what breaks if you skip the migration, and what the migration has to do. + +## Decide the shape first + +| Situation | Shape | Precedent | +| --- | --- | --- | +| A successor application exists | Migrate each instance onto it | migrations `28` (mysql to mariadb), `29` (virtual-machine to vm-disk plus vm-instance) | +| No successor, and instances exist in the field | Freeze them on the artifact they already render from | migration `58` (http-cache) | +| No successor, and no instance anywhere | Remove the platform side outright | the second branch of migration `58` | + +You cannot know which of the last two applies fleet-wide, so do not choose between them in the tree. The migration decides per cluster, at upgrade time, by looking for instances. A cluster that never used the application keeps nothing; a cluster that still runs one keeps a working application. + +Telemetry tells you how many instances exist across reporting clusters, which is the input to the decision to retire at all. It is not a substitute for the per-cluster branch: an application with four instances still has four owners who did not agree to lose them. + +## What removal breaks + +Deleting the chart and its `PackageSource` breaks every surviving instance, in a way that produces no error at merge time and no error at upgrade time either. The chain: + +1. `packages/core/platform/templates/sources.yaml` globs `sources/*.yaml` and emits each file verbatim, with no `helm.sh/resource-policy: keep`, so Helm prunes the `PackageSource` on upgrade. +2. `internal/operator/packagesource_reconciler.go` stamps a controller `ownerReference` on the `ArtifactGenerator` it creates, so garbage collection takes the generator with the `PackageSource`. +3. The generator owns both `ExternalArtifact` objects, so they go too. +4. Those are the `chartRef` targets of `-rd` in `cozy-system` and of every tenant `-` release. Both sit `Ready=False` permanently. +5. `pkg/registry/apps/application/rest.go` mirrors HelmRelease readiness into the instance status, so the instance reads not ready in `kubectl` and unhealthy in the dashboard while its pods keep serving. `packages/system/monitoring-agents/alerts/flux.yaml` fires `HelmReleaseNotReady` at severity major, and `check-readiness` never reports green again. + +The `Package` object survives, because `packages/core/platform/templates/_helpers.tpl` stamps `helm.sh/resource-policy: keep` on every one. Do not tell operators to delete it by hand: `internal/operator/package_reconciler.go` sets a controller `ownerReference` from each generated HelmRelease to its `Package`, so deleting it cascades into uninstalling `-rd`, which takes the `ApplicationDefinition` with it. From that moment the kind stops being served, and a surviving instance disappears from the API with its workloads still running and its owner unable to remove them. + +The changelog for `v1.0.0-rc.2` records this exact failure after the ferretdb, mysql and virtual-machine renames, and migrations `28`, `29` and `33` were written to clean it up. Treat it as established, not hypothetical. + +## Freezing on the cluster's own artifact + +An application with no successor can be kept alive without a second repository and without the platform publishing its chart, because the chart a running instance renders from is already resolved on that cluster as an object. Copy that object and pin the copy. + +Read the digest off the live `cozystack-packages` `OCIRepository` rather than writing one into the migration. The migration runs as a `pre-upgrade` hook, before the upgrade re-points that object, so the digest it reads is by definition the release the instances are already rendering from. That makes the freeze a no-op by construction, and it asks no question about which release was the last to ship the chart. + +Take the digest from `spec.ref.digest` when the installer pinned one, and otherwise from the digest half of `status.artifact.revision`, which source-controller writes as `@sha256:`. Never use `status.artifact.digest`: that is the checksum of the downloaded tarball, not of the manifest, and a source pinned to it resolves to nothing. A source with neither must abort the migration rather than publish a source that points nowhere. + +Copy the whole `spec` rather than rebuilding it. The url, the pull secret, the interval and any `verify` block come across with it, so an air-gapped or mirrored registry keeps serving exactly what it already served, under the verification the operator configured. A digest written into the script has none of those properties and resolves only where that exact manifest was mirrored. + +Write the copy with no `ownerReferences` and no Helm metadata, and give it the `platform.cozystack.io/no-delete` label: it becomes the only chart source those instances have, so a stray delete strands them the same way the prune would. + +Then repoint the `PackageSource` at the copy, add `helm.sh/resource-policy: keep`, and null both `meta.helm.sh/release-name` and `app.kubernetes.io/managed-by`. Helm re-reads the keep annotation off the live object during the prune, which is what lets a `pre-upgrade` hook exempt an object from the upgrade that follows it, and nulling the release metadata stops a later release from adopting it. + +Do not rename anything. The `ApplicationDefinition` inside the frozen artifact hardcodes its `chartRef`, so a renamed `PackageSource` leaves every release pointing at an artifact nobody produces. Because the names hold, the rd release and every tenant release keep the `chartRef` they already have, and nothing goes not ready. + +## The migration + +Select instances by both the `chartRef` name and the `apps.cozystack.io/application.kind` label that migration `22` stamped, so a release whose `chartRef` was never rewritten is still found. + +On the branch where instances exist, freeze as above and delete nothing. On the branch where none exist, remove the platform side: the `Package` first, and block on it rather than passing `--wait=false`, because the operator reconciles off an informer cache and a reconcile in flight against the pre-delete entry re-creates the release it owns. Then the rd HelmRelease, its `ApplicationDefinition`, and its Helm storage secrets. + +Fail closed on every path. Migrations never re-run, so a swallowed error stamps the version past a fleet that is half migrated. No `|| true`. + +Number the migration one past the highest that exists **on main**, not one past the highest in your branch, and set `migrations.targetVersion` one past that. Check for other pull requests in flight that add migrations: whichever lands second has to renumber, and `run-migrations.sh` refuses to advance past a number with no file, so a hole fails the upgrade on every cluster below it. + +## Tenant RBAC + +`cozy:tenant:admin:base` in `packages/system/cozystack-basics/templates/clusterroles.yaml` enumerates the `apps.cozystack.io` resources a tenant owner may write. Remove the retired application's plural from it: the list tracks the catalog, and an entry for an application nobody ships reads as dead config to the next person pruning it. + +The clusters that froze still need the grant, so have the migration apply its own `ClusterRole` carrying `rbac.cozystack.io/aggregate-to-tenant-admin`, with the same verbs the base role granted. Apply it in the hook, before the upgrade drops the rule from the chart, so the access never lapses. + +Without it the owner of a surviving instance can see it and not remove it. The wildcard over `apps.cozystack.io` lives in `cozy:tenant:base`, which aggregates into `cozy:tenant` and not into `cozy:tenant:admin`; `cozy:tenant:view:base` is read only; and no role in the tenant chain carries any verb on `helm.toolkit.fluxcd.io`, so there is no way around it from inside a tenant. + +## What comes out of the tree + +The chart under `packages/apps//`, its ResourceDefinition under `packages/system/-rd/`, the values type under `api/apps/v1alpha1//` together with its generated deepcopy, the `PackageSource` under `packages/core/platform/sources/`, the bundle line that installs it, the image build line in the root `Makefile`, the dashboard icon mapping in the console's `sidebar-icons.tsx`, and the plural in `cozy:tenant:admin:base`. + +Regenerate rather than hand-delete the generated files: the values type is `cozyvalues-gen` output driven by the chart's own `generate` target, and deepcopy is `controller-gen` over the whole `api/apps/v1alpha1` module. + +Leave history alone. Published changelogs are a record, and earlier migrations that name the retired kind replay what those clusters actually did. + +## Downstream + +Two repositories carry per-application code that nothing here checks. `cozystack/website` lists each app in a hardcoded `Makefile` list and keeps one reference page per documentation version, so the `next` page and the list entry go while released versions stay. `cozystack/terraform-provider-cozystack` hand-writes a resource and a data source per kind, plus a per-kind schema guard that imports this repository's `api/apps/v1alpha1` module and breaks when that pin moves past the removal. + +## Checklist + +- [ ] The decision to retire is recorded, with usage numbers and the maintenance case +- [ ] The shape is chosen: successor migration, or freeze plus conditional removal +- [ ] The migration branches per cluster and deletes nothing on the branch where instances exist +- [ ] The frozen source copies the live `OCIRepository` and pins the digest read at hook time +- [ ] The `PackageSource` is repointed, annotated `keep` and disowned; names are unchanged +- [ ] The tenant grant is dropped from the chart and re-applied by the migration +- [ ] Every failure path stops before the version stamp +- [ ] The migration number is one past main's highest, `targetVersion` one past that, with in-flight PRs checked +- [ ] Generated files are regenerated, not edited +- [ ] The release note says what an upgraded cluster experiences, on both branches +- [ ] Downstream follow-ups are open and linked in the PR diff --git a/docs/gateway-classes.md b/docs/gateway-classes.md new file mode 100644 index 0000000000..5aafb31b2f --- /dev/null +++ b/docs/gateway-classes.md @@ -0,0 +1,125 @@ +# GatewayClass selection and TLS termination for tenant Gateways + +A tenant that owns a Gateway (`tenant.spec.gateway: true`) gets one `Gateway` object bound to one `GatewayClass`. Which class that is decides who programs the dataplane and, with it, whether TLS is terminated by the Gateway itself or by something in front of it. This page covers the operator-facing surface: the platform values that bound the choice, the tenant field that makes it, what each resulting certificate mode implies, what a move between classes does to certificates and Secrets, and how the mechanism fails when the configured names and the installed classes disagree. + +The chart-level reference for the per-tenant Gateway — inheritance, listener shapes, the security model, ACME rate limits — is [`packages/extra/gateway/README.md`](../packages/extra/gateway/README.md). + +## Platform values + +Three keys under `gateway` in the platform values bound which class a tenant Gateway can land on: + +```yaml +gateway: + enabled: true + className: cilium + tenantSelectableClasses: [] + edgeTerminatedClasses: [] +``` + +- `className` — the class every tenant Gateway uses unless the tenant names another one. Defaults to the Cilium class the platform ships. +- `tenantSelectableClasses` — the additional class names a tenant may name for itself. Empty (the default) means a tenant cannot name anything but the platform default. +- `edgeTerminatedClasses` — the class names whose provider terminates TLS upstream of the Gateway. Membership in this list is the only thing that puts a tenant into edge mode. + +All three reach the per-tenant gateway chart through the platform values channel, as `_cluster.gateway-class-name`, `_cluster.gateway-tenant-classes` and `_cluster.gateway-edge-classes`. The two lists are joined with commas on the way, so a name containing a comma is rejected at platform render time rather than silently split into two names. + +`gateway.enabled` is a separate switch and not a prerequisite for any of this: it moves the platform's own published services from Ingress onto Gateway API and points the ACME `ClusterIssuer`s' HTTP-01 solver at the publishing tenant's Gateway. A tenant Gateway is created from `tenant.spec.gateway` alone. + +## How a tenant picks a class + +```yaml +apiVersion: apps.cozystack.io/v1alpha1 +kind: Tenant +metadata: + name: acme + namespace: tenant-root +spec: + gateway: true + gatewayClass: edge-tunnel +``` + +`edge-tunnel` throughout this page stands for the name of a GatewayClass whose controller terminates TLS outside the cluster and forwards cleartext to the Gateway. Substitute the name your own edge-terminating controller registers. + +The field is read only for a tenant that owns a Gateway. A tenant that inherits — `gateway` left unset or set to `false` — publishes through the nearest ancestor that owns one and runs on that ancestor's class, so its own `gatewayClass` has no effect. Leaving `gatewayClass` empty selects `gateway.className`. + +The set a tenant may name is `gateway.tenantSelectableClasses` plus the current `gateway.className`, so a tenant may always name the default explicitly. Anything else fails that tenant's own gateway release at render time, with a message naming the class and the allowed set, and reaches no other tenant. The allowlist exists because `Tenant` is a tenant-writable object while a `GatewayClass` is cluster-scoped: without it a tenant could attach its Gateway to any class the cluster registers, including one an admin keeps for internal-only traffic. + +## What the class decides: the certificate mode + +`TenantGateway.spec.certMode` is derived, never written by the tenant. The gateway chart resolves it in this order, first match winning: + +| Condition | `certMode` | What the controller issues | +| --- | --- | --- | +| the resolved class is in `gateway.edgeTerminatedClasses` | `edge` | nothing | +| `publishing.certificates.wildcardSecretName` is set | `existingSecret` | nothing; the operator's Secret is referenced | +| `publishing.certificates.solver: dns01` | `dns01` | one wildcard `Certificate` per tenant | +| otherwise | `http01` | one `Certificate` per published hostname, except the passthrough names below | + +Under `http01` a hostname held by a TLS-passthrough listener is the exception to the per-hostname rule. The hostname earns neither an HTTPS-terminate listener nor a `Certificate`, whether or not a `TLSRoute` is attached to that listener yet, because the backend presents its own certificate and the Gateway never holds a key for the name. The shipped `tlsPassthroughServices` defaults put `api`, `vm-exportproxy` and `cdi-uploadproxy` in that position on the publishing tenant's Gateway, for the names `publishing.exposedServices` lists while `gateway.enabled` is on, and the `Certificate` objects that covered them are collected on upgrade. No other tenant renders those listeners, so a child tenant's `HTTPRoute` on one of those names keeps its listener. `packages/extra/gateway/README.md` carries what that changes and the ways forward. + +Edge therefore wins over an operator-supplied wildcard Secret, which wins over the solver. In edge mode the solver, DNS-01 provider and issuer-name inputs are not read at all, so an unsupported value in any of them does not fail the tenant's release. `existingSecret` skips the same validations; only `http01` and `dns01` reject a bad solver, provider or issuer name. + +## What edge mode changes on the Gateway + +- The Gateway serves port 80 only. Its listeners are `edge` for `*.`, `edge-apex` for ``, and one `edge-child-*` per inheriting child tenant carrying that child's `*.`. There is no port-443 listener, and no hostname-less port-80 listener either. +- No per-tenant ACME `Issuer` and no `Certificate` are created. +- No http-to-https redirect `HTTPRoute` is created. The edge is expected to perform that redirect. +- `tlsPassthroughServices` is not written onto the `TenantGateway` and no passthrough listener is rendered, so the tenant Kubernetes API, VM export and CDI upload endpoints are not published through such a Gateway. The packages that own those endpoints still create their `TLSRoute` objects, which then match no listener on that Gateway. +- Application `HTTPRoute`s attach by hostname exactly as in the other modes — the edge listeners carry the same namespace selector the HTTPS listeners carry elsewhere. + +That last point has a blast radius worth stating on its own, and it applies only where those endpoints are published through the Gateway at all. Each of them renders its `TLSRoute` only when `gateway.enabled` is on and its name is still in `publishing.exposedServices`; at the default `gateway.enabled: false` there is no such route and nothing to lose. Where both hold, setting `gateway.className` to an edge-terminated class puts the publishing tenant on it too, and then the endpoints that go unpublished are the platform's own: the `TLSRoute` for `api.` is pinned to a `tls-api` listener, an edge Gateway has no such listener, so the tenant Kubernetes API stops being served — VM export and CDI upload go the same way. Nothing fails at render time, and the loss surfaces only on the routes themselves, which the controller flips to `Accepted=False` with reason `NoMatchingParent`. Putting only child tenants on the edge class avoids it, because the platform's routes keep pointing at the publishing tenant's Gateway. + +Listing a class in `gateway.edgeTerminatedClasses` is an assertion about that class, and nothing verifies it. Put the bundled `cilium` class in the list and its Gateways serve every application hostname over plain HTTP on a public address, with no redirect and no certificate. Only list a class whose provider really does terminate TLS in front of the Gateway. + +## Moving a tenant between classes + +Onto an edge-terminated class, on the next reconcile: + +| Object | What happens | +| --- | --- | +| the per-tenant ACME `Issuer` | deleted | +| per-listener and wildcard `Certificate` objects | deleted | +| the TLS Secrets those `Certificate`s issued | left in the namespace, unreferenced | +| the http-to-https redirect `HTTPRoute` | deleted | +| the replica of `publishing.certificates.wildcardSecretName` | deleted from that tenant's namespace | +| route status conditions the controller wrote earlier | an attached `TLSRoute` that declares hostnames is flipped to `Accepted=False`, reason `NoMatchingParent`; on a hostname-less `TLSRoute`, and on every attached `HTTPRoute`, the controller's own entry is removed instead | + +Every deletion is guarded by ownership: an object of the same name that the controller did not create is left in place. + +The TLS Secrets survive because cert-manager ships here with `enableCertificateOwnerRef: false`, so a `Certificate` does not own the Secret it issued into. That is how every mode transition behaves and is not something edge introduces. Removing the leftovers is an operator action. + +The wildcard replica is withdrawn because the namespace stops counting as a TLS termination point. Three things keep it anyway: a namespace that also owns an ingress controller still terminates locally; a namespace holding more than one `TenantGateway` keeps the replica while any of them is not on an edge-terminated class; and a namespace whose `TenantGateway` objects cannot be listed, or which carries the Gateway owner label but has no `TenantGateway` yet, keeps it as well, so that a transient read failure never withdraws a key a Gateway may still be serving. + +Moving back off an edge-terminated class restores whatever the new mode calls for: the ACME `Issuer` and its `Certificate`s under `http01` or `dns01`, the redirect route under any non-edge mode, and the wildcard replica whenever `publishing.certificates.wildcardSecretName` is set. The `TLSRoute` conditions go back too, since the passthrough listeners do: `http01` judges those routes again from the hostname claims it collects, while `dns01` and `existingSecret` collect none and so drop the entry rather than leave a withdrawal standing on a listener that is serving. `existingSecret` mints no `Issuer` and no `Certificate` of its own, so a move from `edge` into it brings back the replica and the redirect route and nothing else. + +The two halves of the move are not ordered against each other. The wildcard-secret controller watches the `TenantGateway` and prunes the replica as soon as `certMode` changes, while the `Gateway` itself is re-rendered when the tenant's HelmRelease reconciles, and the class controller reprograms after that. A tenant leaving `existingSecret` can therefore be briefly serving HTTPS listeners whose Secret has already gone. + +## When the configured names and the installed classes disagree + +Nothing in this mechanism checks that a class name corresponds to a `GatewayClass` the cluster actually has. Run `kubectl get gatewayclass` before putting a name into any of the three values. + +- **A name in `className` or `tenantSelectableClasses` that matches no installed class.** The tenant's release renders and the `Gateway` object is created, but no controller claims it, so it gets no address and no listener status. Nothing fails at render time; the symptom is the `TenantGateway` holding `Ready=False` with reason `GatewayNotAccepted`. +- **A misspelled name in `edgeTerminatedClasses`.** It matches no class, so edge mode never engages for it and nothing says so. A tenant whose TLS really is terminated upstream stays in whichever mode the solver and wildcard-secret settings select, and its Gateway keeps terminating TLS itself. +- **A class removed from `tenantSelectableClasses` while a tenant still names it.** That tenant's gateway release starts failing to render. Other tenants are unaffected. +- **`className` changed while a tenant pins the outgoing name.** The allowed set is the *current* default plus `tenantSelectableClasses`, so the pin stops being valid and that tenant's gateway release fails. Add the outgoing class to `tenantSelectableClasses` before changing the default, or update the tenant first. +- **A class whose `status.supportedFeatures` lists features but not `TLSRoute`.** A tenant Gateway that renders TLS-passthrough listeners on it reads `Ready=False` with reason `GatewayClassUnsupported`, naming the class, because nothing would serve those listeners. A class that lists no features at all is not judged, since implementations predating the field leave it empty. +- **The publishing tenant selecting a class that terminates TLS differently from `className`.** Refused at render time, in both directions. The cluster-wide ACME `ClusterIssuer`s point their HTTP-01 solver at the publishing tenant's Gateway, and whether that solver pins `sectionName: http` is decided from `gateway.className` alone — a tenant's own choice does not travel on that channel. A publishing tenant that moved itself onto an edge class would keep a solver pinned to the `http` listener its Gateway no longer renders, and every HTTP-01 certificate in the cluster would stop issuing with nothing in any status naming the cause. Change `gateway.className` instead. The publishing tenant is the namespace named by `publishing.ingressName`, `tenant-root` by default. +- **A comma or a non-string entry in either list.** The platform chart fails to render, naming the entry. + +## Checking what a tenant landed on + +```bash +kubectl get gatewayclass + +kubectl --namespace tenant-acme get tenantgateway cozystack \ + --output jsonpath='{.spec.gatewayClassName}{"\t"}{.spec.certMode}{"\n"}' + +kubectl --namespace tenant-acme get tenantgateway cozystack \ + --output jsonpath='{range .status.conditions[?(@.type=="Ready")]}{.status}{"\t"}{.reason}{"\n"}{end}' + +kubectl --namespace tenant-acme get gateway cozystack \ + --output jsonpath='{range .spec.listeners[*]}{.name}{"\t"}{.port}{"\t"}{.hostname}{"\n"}{end}' + +kubectl --namespace tenant-acme get certificate,issuer +``` + +A tenant on an edge-terminated class reports `certMode: edge`, shows port-80 listeners only, and has no `Certificate` or `Issuer` that this controller created. Leftover TLS Secrets from an earlier mode are not shown by the last command — list Secrets of type `kubernetes.io/tls` in the namespace to find those. diff --git a/docs/gpu-vgpu.md b/docs/gpu-vgpu.md index 68bbb7ec3c..af6e574f24 100644 --- a/docs/gpu-vgpu.md +++ b/docs/gpu-vgpu.md @@ -1,8 +1,8 @@ -# GPU Operator: vGPU Support +# GPU Operator: passthrough and vGPU This document describes how to configure the GPU Operator package with NVIDIA vGPU support so that a single physical GPU can be sliced and shared across multiple virtual machines. -**Last verified:** 2026-04-29 against KubeVirt `main` (`virt-handler` nightly `20260429_74d7c52588`) + this PR's `vgpu` variant + NVIDIA vGPU 20.0 host driver `595.58.02` + GRID guest driver `595.58.03`. +**Last verified:** 2026-04-29 against KubeVirt `main` (`virt-handler` nightly `20260429_74d7c52588`) + the `vgpu` variant + NVIDIA vGPU 20.0 host driver `595.58.02` + GRID guest driver `595.58.03`. ## Two driver models @@ -17,16 +17,140 @@ This guide focuses on the **SR-IOV path**, which is the only model NVIDIA suppor - An Ada Lovelace (or newer) NVIDIA GPU that supports SR-IOV vGPU (L4, L40, L40S, etc.). - Ubuntu 24.04 host OS. Older Ubuntu releases also work if the upstream `gpu-driver-container` repo has a matching `vgpu-manager/` Dockerfile. **Talos Linux is not recommended** for vGPU. NVIDIA does not publicly distribute the vGPU guest driver — it requires NVIDIA Enterprise Portal access — and Sidero [closed siderolabs/extensions#461](https://github.com/siderolabs/extensions/issues/461) noting that they cannot support vGPU "unless NVIDIA changes their licensing terms or provides us a way to obtain, test, and distribute the software". Building a Talos system extension that includes the driver in-tree is therefore not feasible without a private fork that violates the EULA. -- KubeVirt with [kubevirt/kubevirt#16890](https://github.com/kubevirt/kubevirt/pull/16890) ("vGPU: SRIOV support", merged to `main` 2026-04-10). Targeted at the next minor release (v1.9.0); track the PR for the actual release tag. Released tags up to and including v1.8.x do not include the patch and backports are not planned. If you need vGPU before v1.9.0 lands you have to run a `main`-based nightly build of `virt-handler`; the rest of the operator can stay on the latest released tag. +- KubeVirt v1.9.0 or newer, which is the release the platform bundles. [kubevirt/kubevirt#16890](https://github.com/kubevirt/kubevirt/pull/16890) ("vGPU: SRIOV support") shipped in v1.9.0; v1.8.x and older do not include it and backports are not planned. - An NVIDIA vGPU Software / NVIDIA AI Enterprise subscription (the `.run` is not redistributable). - A reachable NVIDIA Delegated License Service (DLS) instance and a matching `client_configuration_token.tok` file. ## Variants -The `gpu-operator` package exposes two variants: +The `gpu-operator` package exposes three variants. This document is vGPU-focused; the variant inventory is shared. - **`default`** — passthrough mode (`vfio-pci`). Whole GPU goes to a single VM. Talos is supported here; the kernel module is the open-source `vfio-pci`, no proprietary driver is needed on the host. - **`vgpu`** — SR-IOV vGPU mode. One physical GPU is sliced into multiple VFs, each VF bound to a vGPU profile that the guest sees as its own GPU. +- **`container`** — containerized GPU workloads (CUDA pods, ML training) via the standard NVIDIA device plugin on hosts that already provide both the NVIDIA driver and `nvidia-container-toolkit` (the typical apt-installed Ubuntu/Debian shape). Sandbox workloads are off, `devicePlugin` is on, `driver` / `toolkit` / `vfioManager` / `cdi` are off so that the operator does not fight the host install. Orthogonal to the two VM variants — it does not pass GPUs to KubeVirt VMs. Note that `apt install nvidia-container-toolkit` installs binaries only — it does not configure containerd. Because this variant disables the operator's toolkit component (which would normally do that wiring), the host must additionally have run `nvidia-ctk runtime configure --runtime=containerd` (followed by a containerd restart) and exposed the `nvidia` runtime as the default or via a RuntimeClass before the device plugin can serve GPUs. + +## Passthrough variant: host preparation + +**Last verified:** 2026-05-28 against `cozystack.gpu-operator` chart `gpu-operator` v26.3.1 + image `nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.10.0` (distroless image with a Go ELF binary `/usr/bin/driver-manager`, entrypoint `["driver-manager", "preflight_check"]`; verified via `crane export … | tar -x usr/bin/driver-manager` and `strings` over the resulting binary) + Ubuntu 24.04 host with `nvidia-driver-580-open` 580.82.07 pre-installed. Tracks [#2763](https://github.com/cozystack/cozystack/issues/2763). + +The `default` (passthrough) variant assumes the GPU is **owned by the host kernel's `vfio-pci` driver and nothing else**. + +**How the operator detects a pre-installed host driver.** The chart's `vfio-manager` DaemonSet runs the upstream NVIDIA `k8s-driver-manager` init container with the `uninstall_driver` subcommand; that path calls the Go method `(*DriverManager).isHostDriver`, which runs `chroot /host nvidia-smi --query-gpu=driver_version --format=csv,noheader` and treats any non-empty stdout as "host driver present" (file-existence is not pre-tested — if `nvidia-smi` is missing, the chroot exec errors and `isHostDriver` returns false, which is the intended path on a clean host: the operator then proceeds with the uninstall flow and `vfio-manager` binds `vfio-pci` as designed). On a positive detection the binary logs `Host driver detected: `, labels the node `nvidia.com/gpu.deploy.driver=pre-installed`, exits, and `nvidia-sandbox-validator` then crashloops with `device not bound to 'vfio-pci'`. + +**`FORCE_REINSTALL` does not bypass this.** `k8s-driver-manager` v0.10.0 exposes a `FORCE_REINSTALL` / `--force-reinstall` env+flag pair (visible in the binary's strings table), but it gates a later "same-config already loaded" branch inside `uninstallDriver`, not the `isHostDriver` short-circuit at the top — operators who set it and expect a bypass will report a false bug. There is currently no opt-out for the `isHostDriver` guard itself. + +**Current mitigation.** The only mitigation that keeps VM passthrough working is to remove the host NVIDIA stack before enabling the variant (the "Clean-host workaround" section below). + +**Alternatives and scope.** The `container` variant of `cozystack.gpu-operator` is the no-purge path: it keeps the host driver and exposes GPUs to pods rather than VMs, so it does not replace passthrough for anyone who actually needs a GPU inside a VM. Talos is unaffected because the Talos image ships only the `vfio-pci` extension; this section applies to Linux distributions where you installed the host driver yourself (typically Ubuntu / Debian / RHEL with `apt install nvidia-driver-*` or equivalent). See also [`packages/system/gpu-operator/examples/README.md`](../packages/system/gpu-operator/examples/README.md) for the Talos native-pod-workload reference flow. + +### Symptom + +The first thing you see is `kubectl get pods -n cozy-gpu-operator` showing `nvidia-vfio-manager-*` stuck in `Init:Error` / `Init:CrashLoopBackOff` — the init container exits non-zero after detecting the host driver, so kubelet keeps restarting it. + +Its log shows it skipped the bind step: + +```text +Host driver detected: 580.82.07 +NVIDIA GPU driver is already pre-installed on the node, + disabling the containerized driver +Labeling node with nvidia.com/gpu.deploy.driver=pre-installed +``` + +`nvidia-sandbox-validator` then crashloops: + +```text +Error: error validating vfio-pci driver installation: + device not bound to 'vfio-pci'; device: 0000:18:00.0 driver: 'nvidia' +``` + +`lspci -nnk -d 10de:` still shows `Kernel driver in use: nvidia` on every target GPU, the node carries `nvidia.com/gpu.deploy.driver=pre-installed`, and `kubectl get node -o json | jq '.status.allocatable | with_entries(select(.key | startswith("nvidia.com/")))'` reports `{}` — no GPU resource was registered. + +### Clean-host workaround + +Purge the NVIDIA host stack and blacklist the kernel modules so the host never re-claims the GPU. Two pitfalls to avoid: `apt autoremove` is dangerous here because the `nvidia-` prefix is shared with NVIDIA DOCA / Mellanox / InfiniBand userspace, so a blanket autoremove can take RDMA out on a converged GPU + RDMA host; and a hardcoded `apt purge 'nvidia-*' 'cuda-*'` pattern list is fragile — `apt` treats `*` as a cache-wide regex and **aborts the entire transaction, purging nothing**, if any pattern matches nothing in the cache (e.g. `cuda-*` on a host without NVIDIA's CUDA repo). Build the list from what is actually installed instead: + +```bash +# dpkg-query patterns are true globs over INSTALLED packages: no +# zero-match abort (unlike apt's cache-wide regex) and no accidental +# substring over-match. List first, then review before purging. +dpkg-query -W -f '${Package}\n' 'nvidia-*' 'libnvidia-*' 'cuda-*' 2>/dev/null +``` + +Review the list before purging: + +- `nvidia-dkms-*` and `nvidia-kernel-*` are the load-bearing kernel pieces — without removing them DKMS rebuilds `nvidia.ko` on the next reboot and the blacklist below is bypassed by any explicit `modprobe`. +- On a **converged GPU + RDMA host**, drop any `libnvidia-*` that belong to NVIDIA DOCA / Mellanox OFED — purging those breaks RDMA. +- If the list is **empty**, the driver was installed with NVIDIA's `.run` installer rather than apt — run `sudo nvidia-uninstall` instead of the purge below. + +Then purge the reviewed list, blacklist the modules, and rebuild the initramfs: + +```bash +# Replace with the packages you kept from the list above. +sudo apt purge nvidia-driver-580-open nvidia-dkms-580-open <...> + +sudo tee /etc/modprobe.d/blacklist-nvidia.conf > /dev/null <<'EOF' +blacklist nouveau +blacklist nvidia +blacklist nvidia_drm +blacklist nvidia_modeset +blacklist nvidia_uvm +blacklist nvidia_peermem +EOF + +# -k all rebuilds EVERY installed kernel's initramfs (plain -u touches +# only the running kernel), so a just-upgraded kernel also boots with +# the blacklist and cannot re-claim the GPU. +sudo update-initramfs -u -k all +sudo reboot +``` + +After the reboot, confirm the host is clean. The init container exits non-zero after detecting a host driver, so both DaemonSet pods sit in `Init:CrashLoopBackOff` and **will retry on their own** — deleting them just skips the up-to-5-minute backoff window: + +```bash +# Should print nothing. +lsmod | grep -E '^(nvidia|nouveau)' + +# command -v matches what isHostDriver does (PATH lookup inside the +# chroot), so it also catches /usr/local/bin/nvidia-smi left by a .run +# or CUDA-toolkit install. Prints "ok: gone" on success. +command -v nvidia-smi >/dev/null && echo "STILL PRESENT — purge incomplete" || echo "ok: gone" + +# A leftover DKMS module rebuilds nvidia.ko on the next kernel update +# and an explicit modprobe bypasses the blacklist — should print nothing. +dkms status | grep -i nvidia + +# Skip the CrashLoopBackOff backoff window by deleting the stuck pods. +# The DaemonSet labels are operator-managed and not stable across +# gpu-operator versions, so delete by name pattern — anchored so the +# match is the two DaemonSets and nothing else. +kubectl -n cozy-gpu-operator get pods -o name \ + | grep -E '^pod/(nvidia-vfio-manager|nvidia-sandbox-validator)-' \ + | xargs -r kubectl -n cozy-gpu-operator delete +``` + +Within a couple of minutes `vfio-manager` should bind every target GPU to `vfio-pci` and the node's `allocatable` will gain the registered resource: + +```bash +lspci -nnk -d 10de: | grep 'Kernel driver in use' +# Kernel driver in use: vfio-pci + +kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{": "}{.status.allocatable}{"\n"}{end}' | grep nvidia.com +``` + +One label is **not** cleaned up automatically. The init container set `nvidia.com/gpu.deploy.driver=pre-installed`, and the operator's success path restores only the operand labels — it never resets this one (`rescheduleGPUOperatorComponents` in `k8s-driver-manager` v0.10.0 touches the validator / toolkit / device-plugin / etc. labels, not `deploy.driver`). So the node keeps `pre-installed` indefinitely: it is the same label the Symptom section uses as evidence, and it would disable the containerized-driver DaemonSet if the node later switches to the `container` workload. Clear it once recovery is confirmed: + +```bash +# Nothing on the success path resets this label; clear it so it does not +# mislead future debugging or block a later container-workload switch. +kubectl label node nvidia.com/gpu.deploy.driver- +``` + +### Known limitation + +The skip-on-pre-installed behavior lives in the upstream [`NVIDIA/k8s-driver-manager`](https://github.com/NVIDIA/k8s-driver-manager) Go binary at `cmd/driver-manager/main.go`: the method `(*DriverManager).isHostDriver` is called from `(*DriverManager).uninstallDriver` and has no in-band opt-out in `:v0.10.0`. `FORCE_REINSTALL` exists as an env / CLI flag in the same binary but gates the later "same-config already loaded" branch inside `uninstallDriver`, not the `isHostDriver` short-circuit. Hosts that need to keep the NVIDIA host driver installed for non-Kubernetes workloads cannot currently share the same GPU with the passthrough variant. Two mitigation paths: + +- **`container` variant** — a third variant of `cozystack.gpu-operator` that targets the apt-installed-driver host shape and exposes GPUs to pods (not VMs) without unbinding the host driver. Available now, and selectable from platform values with `bundles.iaas.gpuOperatorVariant: container`. It solves the host-driver conflict by giving up VM passthrough, so it is a mitigation only for workloads that can run in a pod. +- **Upstream override** — an env-var override of `isHostDriver` is the only structural fix that would let the passthrough variant coexist with a host driver. Not available today; requested in [NVIDIA/k8s-driver-manager#191](https://github.com/NVIDIA/k8s-driver-manager/issues/191). ## Building the vGPU Manager image @@ -38,30 +162,30 @@ The proprietary `.run` is the **Linux KVM** variant (not the Ubuntu KVM `.deb`, ## Deploying with the vgpu variant -Create a `Package` CR pointing at your image coordinates: +The platform's `iaas` bundle deploys the gpu-operator Package CR when `cozystack.gpu-operator` is in `bundles.enabledPackages` and `bundles.iaas.gpuOperatorVariant: vgpu` is set. The vGPU Manager image is proprietary and not redistributable, so the bundle does not ship a default tag — the operator builds the container per the upstream [`gpu-driver-container`](https://github.com/NVIDIA/gpu-driver-container) recipe and supplies the private-registry coordinates through platform values: ```yaml -apiVersion: cozystack.io/v1alpha1 -kind: Package -metadata: - name: cozystack.gpu-operator -spec: - variant: vgpu - components: - gpu-operator: - values: - gpu-operator: - vgpuManager: - repository: registry.example.com/nvidia - image: vgpu-manager - version: "595.58.02-ubuntu24.04" - # imagePullSecrets lives per-component (vgpuManager, - # driver, validator, dcgmExporter, …). The value is a - # list of strings, not [{name: ...}]. - imagePullSecrets: - - nvidia-registry-secret +bundles: + iaas: + enabled: true + gpuOperatorVariant: vgpu + enabledPackages: + - cozystack.gpu-operator + +gpu: + vgpuManager: + repository: registry.example.com/nvidia + image: vgpu-manager + version: "595.58.02-ubuntu24.04" + # imagePullSecrets lives per-component (vgpuManager, driver, + # validator, dcgmExporter, …). The value is a list of strings, + # not [{name: ...}]. + imagePullSecrets: + - nvidia-registry-secret ``` +The platform forwards `gpu.vgpuManager` into the emitted gpu-operator Package CR's `components.gpu-operator.values.gpu-operator.vgpuManager`, so the bundle handles the variant + image coordinates in one place. If you need to override anything else on the gpu-operator chart (driver, validator, dcgmExporter, custom node selectors), hand-craft a `Package` CR named `cozystack.gpu-operator` with the full `components.gpu-operator.values` block and put that name in `bundles.disabledPackages`. A hand-written Package CR does not take precedence over the bundle: as long as the platform emits its own copy of that package, the next platform render replaces whatever you wrote. + The `nvidia-registry-secret` should be a docker-registry Secret created beforehand in `cozy-gpu-operator`. Verify the DaemonSet is running and `nvidia.ko` loads on every GPU node: @@ -94,24 +218,56 @@ For Pascal–Ampere GPUs (V100, T4, A100, A30) the mdev model still applies. Fli ## KubeVirt configuration -After [kubevirt#16890](https://github.com/kubevirt/kubevirt/pull/16890), `virt-handler` recognises SR-IOV VFs bound to the `nvidia` driver as candidates whenever a vGPU profile is configured (`current_vgpu_type` ≠ 0). PFs are skipped automatically. +When `cozystack.gpu-operator` is in `bundles.enabledPackages` (and not also in `bundles.disabledPackages`), the platform mirrors the chosen GPU variant into the `KubeVirt` CR automatically. There is no manual `kubectl patch` step. This applies to the two VM variants only: `container` serves GPUs to pods and never to VMs, so selecting it emits the gpu-operator Package with no host-device wiring on the KubeVirt side. + +If you opt out of bundle management and hand-craft a `cozystack.gpu-operator` Package CR directly — typically to apply overrides the bundle does not expose (driver settings, custom node selectors, validator / dcgmExporter tweaks, etc.) — the platform does NOT auto-wire `HostDevices` or `permittedHostDevices` into the KubeVirt CR. In that flow you also hand-craft a `cozystack.kubevirt` Package CR with `components.kubevirt.values.extraFeatureGates: [HostDevices]` and the appropriate `permittedHostDevices` block. The escape-hatch values shape under `.gpu` (below) is intentionally documented in the bundle-managed flow only. Both hand-crafted Package CRs need their names in `bundles.disabledPackages`: the iaas bundle emits `cozystack.kubevirt` on every render and `cozystack.gpu-operator` on every render that has it in `bundles.enabledPackages`, and that render replaces the hand-written spec rather than merging with it. + +- `developerConfiguration.featureGates` gets `HostDevices` appended (current KubeVirt splits this from the `GPU` gate; the admission webhook rejects `spec.template.spec.domain.devices.hostDevices` without it). +- `permittedHostDevices.pciHostDevices` is filled from `packages/core/platform/files/gpu-passthrough-defaults.yaml` when `bundles.iaas.gpuOperatorVariant: default` (the package default) and `gpu.replaceDefaults` is left at `false`. The table covers Hopper (H100/H200), Ada Lovelace (L4/L40/L40S), Ampere (A100 PCIe/SXM, A40, A30, A10), Turing (T4), Volta (V100/V100S). All entries carry `externalResourceProvider: true` because the resource names come from `nvidia-sandbox-device-plugin`, not from KubeVirt's in-tree device plugin. +- `permittedHostDevices.mediatedDevices` is filled from `packages/core/platform/files/gpu-vgpu-defaults.yaml` when `bundles.iaas.gpuOperatorVariant: vgpu` **and** `gpu.vgpuDeviceManager.enabled: true` — that knob defaults to `false`, so a stock vgpu cluster ships no mdev table at all (see the Ada Lovelace / Blackwell note below for why). `gpu.replaceDefaults: true` also suppresses it. This list only EXPOSES, by profile name (`mdevNameSelector`), mdevs that the GPU Operator's vGPU Device Manager CREATES on the node; the platform does not ship a numeric `mediatedDevicesConfiguration` default (those `nvidia-NNN` type ids are per-SKU/driver sysfs indices with no portable value — set `.gpu.mediatedDevicesConfiguration` yourself, with host-verified ids, only if you want KubeVirt rather than the Device Manager to create mdevs). The starter set covers Pascal–Ampere mdev profiles (A100-40C/80C, A40-24Q/48Q, A30-24C, A10-24Q, V100D-32C, T4-16Q) — the same family range as the upstream `vgpu-device-manager` walks `/sys/class/mdev_bus/` for. Ada Lovelace / Blackwell SR-IOV vGPU is out of scope for the chart's default list; advertise those VFs via the user-override hook below. + +### Extending or replacing the default table -Patch the `KubeVirt` CR to permit the resource: +The platform exposes three knobs under `.gpu`: ```yaml -spec: - configuration: - permittedHostDevices: - pciHostDevices: - - pciVendorSelector: "10DE:26B9" # L40S — same device ID for PF and VF - resourceName: nvidia.com/L40S-24Q +gpu: + # Extend the platform defaults with cluster-specific entries. Read by + # the two VM variants only; the container variant reads neither key. + # pciHostDevices is read in both of them — it feeds the passthrough + # (vfio-pci) path AND the post-kubevirt#16890 SR-IOV vGPU VF path on + # Ada Lovelace / Blackwell. mediatedDevices is read in vgpu alone, for + # the pre-#16890 mdev path on Pascal–Ampere; the passthrough variant + # ignores it. Both render into the same KubeVirt CR. + permittedHostDevices: + pciHostDevices: + - pciVendorSelector: "10DE:26B9" # L40S, advertised as a VF for SR-IOV vGPU + resourceName: nvidia.com/L40S-24Q + # externalResourceProvider is intentionally omitted here: after + # kubevirt/kubevirt#16890, virt-handler's in-tree device plugin + # advertises the resource directly, no sandbox plugin in the loop. + mediatedDevices: [] + # mediatedDevicesConfiguration makes KubeVirt itself create mdevs (vgpu mode). No platform default: mdev creation is normally delegated to the vGPU Device Manager (name-based), and these mediatedDeviceTypes are host/driver-specific nvidia-NNN sysfs indices (look yours up via /sys/bus/pci/devices//mdev_supported_types/*/name). Set this only to opt into KubeVirt-driven creation; mergeOverwrite REPLACES a supplied top-level key wholesale. + mediatedDevicesConfiguration: {} + # Wipe the platform defaults entirely and ship only the cluster's + # curated lists. Useful for non-NVIDIA-only clusters and strict + # allowlist requirements. + replaceDefaults: false ``` +`replaceDefaults: false` (the default) appends user entries to the NVIDIA defaults. `replaceDefaults: true` drops the NVIDIA table entirely — if you don't then supply your own pciHostDevices / mediatedDevices list, the rendered KubeVirt CR has no `permittedHostDevices` block and the admission webhook will reject every GPU VM. + +### Notes on `nvidia-sandbox-device-plugin` resource names + +The `resourceName` strings in `gpu-passthrough-defaults.yaml` are what `nvidia-sandbox-device-plugin` (`nvcr.io/nvidia/kubevirt-gpu-device-plugin`) advertises: it derives each slug mechanically from the device's PCI-IDs database name by uppercasing it, turning `/`, `.` and whitespace into `_`, and stripping the remaining non-alphanumerics (the `[` / `]`). So `TU104GL [Tesla T4]` becomes `nvidia.com/TU104GL_TESLA_T4` and `GA100GL [A30 PCIe]` becomes `nvidia.com/GA100GL_A30_PCIE` — the slug carries every token the PCI-IDs string holds (the `GL` die suffix, the `Tesla` brand on Turing/Volta, form factor, memory), not a tidy `_`. The names track the pci.ids snapshot bundled in the plugin image, so a different plugin build can publish a different string — check with `kubectl describe node | grep nvidia.com/` and override via `.gpu.permittedHostDevices.pciHostDevices` (or wipe the table with `replaceDefaults: true` and curate it yourself). PCI vendor:device IDs themselves are stable across driver versions. + +### SR-IOV PF vs VF (Ada Lovelace and newer) + On L40S (and other Ada-Lovelace cards) the SR-IOV VFs report the same PCI device ID as the PF — `lspci -nn -d 10de:` on the host shows both as `[10de:26b9]`. `virt-handler` distinguishes them by `is-VF + has-vGPU-profile`, so a single `pciVendorSelector` matches the right set. Verify on your specific GPU before assuming this — some other generations split PF/VF IDs. -`externalResourceProvider: true` is **not** required here (and should be omitted) — the device plugin lives inside `virt-handler` itself, no external sandbox-device-plugin advertises this resource. +`externalResourceProvider: true` is **not** required when the resource is advertised by `virt-handler`'s in-tree device plugin (the SR-IOV path post kubevirt#16890). The platform passthrough defaults include the flag because that path is driven by the external sandbox plugin. -Verify allocatable capacity: +### Verifying allocatable capacity ```bash kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{": "}{.status.allocatable.nvidia\.com/L40S-24Q}{"\n"}{end}' @@ -231,10 +387,12 @@ Other GPU families have analogous tables in the [NVIDIA Virtual GPU Software Doc ## OS support summary -| Host OS | passthrough (`default`) | vGPU (`vgpu`) | -| --- | --- | --- | -| Ubuntu 24.04 | ✅ supported upstream | ✅ supported upstream (`vgpu-manager/ubuntu24.04`) | -| Ubuntu 22.04 | ✅ | ✅ | -| Ubuntu 20.04 | ✅ | ✅ | -| Ubuntu 26.04 | ⚠️ patch needed in `nvidia-driver` for usr-merge | ⚠️ same patch + own Dockerfile fork | -| Talos Linux | ✅ (open `vfio-pci`) | ❌ NVIDIA does not grant redistribution rights for the proprietary `.run`; we tried and the path is blocked | +The `container` variant column assumes the host already ships the NVIDIA driver and `nvidia-container-toolkit` via the distro package manager, with the `nvidia` runtime registered in containerd (`nvidia-ctk runtime configure --runtime=containerd`). With `driver.enabled=false` the operator uses the pre-installed host driver at its standard location, so a stock apt install needs no `hostPaths.driverInstallDir` override. Talos installs the driver under a non-standard prefix, so the operator does not find it at the default location — see `packages/system/gpu-operator/examples/` for the Talos-specific path with a compat DaemonSet and an explicit `hostPaths.driverInstallDir` override. + +| Host OS | passthrough (`default`) | vGPU (`vgpu`) | container (`container`) | +| --- | --- | --- | --- | +| Ubuntu 24.04 | ⚠️ supported upstream, but the host must be clean of any apt-installed NVIDIA driver — see ["Passthrough variant: host preparation"](#passthrough-variant-host-preparation) | ✅ supported upstream (`vgpu-manager/ubuntu24.04`) | ✅ apt-installed driver + nvidia-container-toolkit | +| Ubuntu 22.04 | ⚠️ same clean-host requirement as 24.04 | ✅ | ✅ | +| Ubuntu 20.04 | ⚠️ same clean-host requirement as 24.04 | ✅ | ✅ | +| Ubuntu 26.04 | ⚠️ same clean-host requirement as 24.04, plus a `nvidia-driver` patch for usr-merge (details pending) | ⚠️ same patch + own Dockerfile fork | ✅ | +| Talos Linux | ✅ (open `vfio-pci`; the Talos image ships no host NVIDIA stack, so the clean-host check passes trivially) | ❌ NVIDIA does not grant redistribution rights for the proprietary `.run`; we tried and the path is blocked | ⚠️ host driver lands in a non-standard prefix — use `examples/values-native-talos.yaml` (compat DaemonSet + `hostPaths.driverInstallDir` override) as a starting point instead | diff --git a/docs/observability/distributed-tracing.md b/docs/observability/distributed-tracing.md new file mode 100644 index 0000000000..d4e6ad80a6 --- /dev/null +++ b/docs/observability/distributed-tracing.md @@ -0,0 +1,123 @@ +# Distributed tracing + +Distributed tracing is the third observability signal in Cozystack, alongside metrics (VictoriaMetrics) and logs (VictoriaLogs). This guide explains what the platform provides, how to send traces from your workloads, what each managed engine can and cannot emit on its own, and how tenants can share one traces store instead of running their own. The design is [cozystack/community `design-proposals/distributed-tracing`](https://github.com/cozystack/community/tree/main/design-proposals/distributed-tracing). + +## What the platform provides + +A tenant's Monitoring application stands up two pieces once tracing is configured: + +- A **VictoriaTraces backend**, one per `tracingStorages` entry (a `VTCluster` in `cluster` mode, a `VTSingle` in `single` mode), with a Grafana Jaeger datasource named `vtraces-` for viewing its spans. +- A **per-tenant OpenTelemetry Collector**, the app-facing OTLP ingest gateway, reachable in the tenant namespace at: + - **`otel-traces:4317`**: OTLP over gRPC + - **`otel-traces:4318`**: OTLP over HTTP + +The collector applies head sampling (10% by default) and optional attribute redaction, then forwards spans to the backend. It lives in your tenant namespace, so traffic from your workloads to it stays inside the namespace. With the default per-tenant backend the spans stay there too; with [shared-central tracing](#shared-central-tracing) the collector forwards them to a store in tenant-root. + +What the platform does **not** do is emit spans for you: producing spans is the workload's job. The platform delivers the transport, storage and tenancy; the *depth* of what shows up is a property of how each application or engine is instrumented. + +## Sending traces from your application (client-side) + +This is the primary path and it works for any workload with OpenTelemetry instrumentation (an OTel SDK in your code, or an auto-instrumentation agent). You do **not** hard-code the endpoint: point your workload at the in-namespace collector through the standard OpenTelemetry environment variables: + +```yaml +env: + # Send to the per-tenant collector over OTLP/HTTP (use :4317 for gRPC). + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://otel-traces:4318" + - name: OTEL_EXPORTER_OTLP_PROTOCOL + value: "http/protobuf" + # Name this service and the environment it runs in. + - name: OTEL_SERVICE_NAME + value: "my-app" + - name: OTEL_RESOURCE_ATTRIBUTES + value: "deployment.environment.name=production" +``` + +The variables are defined by the OpenTelemetry specification, and the official SDKs and agents read them, but support for each variable differs per language: check your SDK against the [specification compliance matrix](https://github.com/open-telemetry/opentelemetry-specification/blob/main/spec-compliance-matrix.md). For a JVM workload you can add the OpenTelemetry Java agent (`-javaagent:/otel/opentelemetry-javaagent.jar`), and the same variables drive it with no code changes. To use gRPC instead, set the endpoint to `http://otel-traces:4317` **and** `OTEL_EXPORTER_OTLP_PROTOCOL=grpc`; with [shared-central tracing](#shared-central-tracing), stay on HTTP, since spans refused over the tenant's write rate are retried only over HTTP, and keep each export request below 3 MiB. + +You do not need to tag your tenant: the backend a tenant writes to is its own, and in shared-central mode the platform attributes every span to its tenant itself. `service.namespace` is for grouping your own services (for example by team), not for the tenant. + +Once spans arrive, they are visible in Grafana under **Explore**, in the traces datasource. + +> **Correlation is not wired yet.** Cross-signal pivots, span to logs by `trace_id` and span to RED metrics, are part of the design but not configured: the traces datasource carries no links to logs or metrics, and no `spanmetrics` connector produces span metrics. Traces are viewable on their own until that follow-up work lands. + +## What each managed engine can emit + +Tracing depth differs per engine, because a sidecar cannot see *inside* a server process. Most managed servers do **not** emit OTLP spans themselves; their tracing is client-side (instrument the application that talks to them). + +| Engine | Server-side OTLP spans? | How you get traces | +| --- | --- | --- | +| **ClickHouse** | Partial: native span *table*, no push, disabled | The server can record internal query spans into `system.opentelemetry_span_log`, which the Cozystack chart removes (see below). Client-side propagation gives end-to-end traces. | +| **PostgreSQL** | No (extension-gated) | Statement-level spans need the `pg_tracing` extension, which is not in the `sharedPreloadLibraries` allowlist, and `shared_preload_libraries` cannot be set through `parameters`. Use client-side instrumentation. | +| **Kafka** | No (brokers) | Brokers do not emit request spans; instrument producers/consumers (client-side). JVM clients can use the OTel Java agent. | +| **RabbitMQ** | No (non-OTLP) | The `rabbitmq_tracing` plugin emits RabbitMQ-format events, not OTLP. Instrument publishers/consumers (client-side). | +| **NATS** | No | Server message-tracing emits NATS-format events, not OTLP. Instrument clients (context is propagated in message headers). | +| **MariaDB** | No | No server-side OTLP. Instrument the application (client-side). | +| **Redis** | No | No server-side OTLP. Instrument the application (client-side). | + +The takeaway: for every engine here, the way to get a useful request trace is to **instrument the application** that issues the queries or messages and point it at `otel-traces` as shown above. The engine then appears as a span in your application's trace when context is propagated. + +## ClickHouse: internal query spans + +ClickHouse is the one engine that produces spans natively: it can record query-execution spans into the `system.opentelemetry_span_log` system table and take part in a trace whose context a client propagates through the W3C `traceparent` header. The Cozystack ClickHouse chart removes that log in its server configuration, alongside the other system logs it disables, and the ClickHouse application exposes no setting to turn it back on, so these spans are not available today. + +Even with the log enabled, ClickHouse has **no native OTLP push**: getting those spans into a collector, and so into Grafana, needs a shipper that reads `system.opentelemetry_span_log` and forwards OTLP, which neither Vector nor Fluent Bit does. Both are follow-up work; end-to-end request traces come from client-side instrumentation, as for every other engine. + +## Sampling, redaction, and tenancy + +- **Sampling:** the collector head-samples traces at 10% by default (`tracingCollector.samplingPercentage`). The decision is made per trace ID, so a kept trace keeps all of its spans and several collector replicas agree on it. A busy application keeps about one trace in ten, so the backend is not overwhelmed. +- **Redaction (PII):** span attributes can carry sensitive data (SQL text with literals, parameters, connection strings). List such keys in `tracingCollector.redactAttributes` to drop them before export. +- **Tenancy:** with the default per-tenant backend, spans never leave the tenant namespace, and only the tenant's own Grafana reads them. With shared-central tracing they are stored in tenant-root; see below for how tenants are kept apart there. + +## Shared-central tracing + +Instead of a VictoriaTraces backend per tenant, tenants can send their spans to one store in tenant-root. Applications do not change: they keep sending to `otel-traces` in their own namespace, and the collector forwards to the shared store. + +### Hosting the store (tenant-root) + +tenant-root opts in with `tracingCentralHost: true` on its Monitoring application, next to a `tracingStorages` entry named `generic` in `cluster` mode; that store is then shared with every tenant that opts in. A vmauth in tenant-root fronts it, and every central tenant's collector and Grafana go through that vmauth. + +All central tenants share the store's disk and retention with tenant-root, and nothing reserves space per tenant. Every traces store is bounded by disk usage: without `retentionDiskUsageBytes` it drops its oldest day once its volume is 80% full. VictoriaTraces always keeps the last two days, though, so the disk cap alone does not stop a fast writer from filling the volume. + +There is no per-tenant allow-list: with hosting on, every tenant that sets `tracingCentral` is admitted, nested tenants included. A tenant-root that needs to limit who shares its store keeps `tracingCentralHost` off. + +### The per-tenant write rate + +Because the disk cap alone does not bound a fast writer, each central tenant is also held to a write rate, which the platform sets with `monitoring.tracingCentralTenantBytesPerSecond` in the platform package, as a whole number of bytes per second up to 1099511627776 (1 TiB/s); any other value fails the platform render. By default it is about 97 KiB/s, at which one tenant writes at most the disk cap of a default 10Gi store over the two days, so raise it with the store. The rate is counted uncompressed and before the collector samples, so the store receives only the sampled share of it and keeps that compressed. A change reaches every tenant's collector through the `cozystack-values` Secret, and upgrades every release in every tenant once. + +The collector Cozystack ships is built with a rate-limit processor, and each central tenant's collector enforces the rate, split across its replicas: a sender whose connection stays on one replica gets that replica's share. Data above the rate is refused. An OTLP/HTTP exporter gets a 429 and retries; an OTLP/gRPC exporter gets `RESOURCE_EXHAUSTED` without the `RetryInfo` the OTLP specification requires for a retry, so it drops those spans. A request larger than 3 MiB, decompressed, is refused with a 400, which exporters do not retry. Applications of central tenants should therefore export over HTTP, in batches below 3 MiB. + +The limiter keeps its state in each collector pod's memory, and every pod starts with a full allowance, so a pod start lets through above the rate up to 5 MiB, or one second of the pod's share of the rate when that is larger. Shared-central tracing therefore allows at most 10 collector replicas, and the tenant's Monitoring render fails above that. The rate bounds each tenant, not their sum: several tenants writing at their limits together can still exceed the cap, so size the volume for the tenants you admit. + +### Opting a tenant in + +Set `tracingCentral: true` on the Tenant, which needs `monitoring: true` on the same tenant. The tenant chart then renders the egress rules the collector and Grafana need to reach the vmauth, and switches the tenant's Monitoring to the shared store. If tenant-root does not host the store, the tenant's Monitoring render fails with a message naming the prerequisite, rather than dropping spans. + +In the tenant's Grafana the shared store appears as the `traces-central` datasource. The tenant's own local stores, if it lists any, keep their datasources, so traces stored before the switch stay readable until the tenant removes those entries. + +### Isolation and the trust boundary + +The store itself authorizes nothing; it takes the tenant from request headers. Each central tenant gets two VMUsers that pin one account, derived from its namespace name and UID: the collector's, which may only write, and its Grafana's, which may only read, each with its own Secret (`traces-central-credentials` and `traces-central-reader-credentials`). vmauth replaces any account a client sends. Tenant pods can reach only the vmauth proxy port, and only from pods carrying the collector's or Grafana's label, with the tenant's credentials. So tenants sharing the store cannot read or write each other's traces. + +Within its own account, a tenant writes only through its collector, so the write rate holds. Grafana's login cannot write, so a write sent through the Grafana datasource proxy, which vmauth would otherwise route to the store, finds no route. The collector and Grafana images are the platform's and cannot be set from the Monitoring application, and of a Secret named in `oidc.customConfig.secretRef` only its `auth.ini` key is mounted into Grafana, so the collector's password does not reach a pod the tenant can shape. + +tenant-root is inside the store's trust boundary. Its workloads reach the store directly, and one that learns a tenant's account can read and write it. Whoever tenant-root admits to its Grafana can read every account through tenant-root's own datasource for the store, which does not go through vmauth. Who that is follows tenant-root Grafana's OIDC mode: with `System`, tenant-root's own groups, which already hold the same access level in every tenant under it; with `CustomConfig` or `None`, whoever that configuration admits. + +### Lifecycle + +- **The account follows the namespace.** It is derived from the namespace name and UID, so a re-created namespace, a restored one included, starts a new account and does not see the spans stored before. +- **Turning central on** keeps the tenant's local stores and their datasources. Right after it, spans the collector sends before the vmauth has loaded the tenant's login get a 401, which the exporter does not retry, so those spans are dropped. The same happens after the tenant's `traces-central-credentials` Secret is re-created with a new password; after `traces-central-reader-credentials` is, the tenant's Grafana gets a 401 on reads until the vmauth loads the new login. +- **Turning central off** removes the tenant's login to the shared store. The spans it wrote there stay until the store's retention drops them and cannot be read from the tenant while it is off; turning it on again on the same namespace makes them readable again. Spans sent between turning it off and the collector switching back to a local store may be dropped, since the egress rule to the shared store goes first. +- **Deleting a tenant** leaves its spans in the shared store until retention drops them. + +### Stopping hosting + +Turn `tracingCentral` off on every central tenant and let its Monitoring reconcile before tenant-root stops hosting. tenant-root's render refuses to stop hosting while a central tenant's VMUsers remain and lists the tenants still holding them. Removing tenant-root's Monitoring altogether is not a render and is not refused; until the central tenants turn central off, every Monitoring render of theirs fails, their metrics, logs and alerting changes included. + +The two refusals read each other's objects when they render, so a tenant turning central on while tenant-root stops hosting can pass both. That tenant's collector then exports to a vmauth that is gone while every release reports Ready, until its Monitoring renders again. Recover by hosting again; turning the tenant's `tracingCentral` off instead stops the silent drop, and turning it on again is then refused until tenant-root hosts. + +## Prerequisites + +- The tenant's Monitoring application with a `tracingStorages` entry, or shared-central tracing turned on for the tenant, and `tracingCollector.enabled` left on (the default). +- For shared-central tracing: tenant-root hosting the store, as described above, and at most 10 collector replicas (`tracingCollector.replicas`). +- A workload with OpenTelemetry instrumentation (SDK or auto-instrumentation agent). diff --git a/docs/oidc-grafana.md b/docs/oidc-grafana.md new file mode 100644 index 0000000000..d5d89b7e33 --- /dev/null +++ b/docs/oidc-grafana.md @@ -0,0 +1,405 @@ +# OIDC for the Grafana instance (Phase 1) + +Cozystack ships one Grafana codebase deployed twice, both released as `monitoring-system` and differing only by namespace: the platform instance runs in `tenant-root` and a per-tenant instance runs in every tenant namespace. Both instances opt in to OIDC authentication through the same flat selector on the `Monitoring` CR. This document covers the operator-facing surface: what the modes mean, what the chart provisions, and how a user signs in. + +The architectural rationale lives in +[cozystack/community#24](https://github.com/cozystack/community/pull/24) +— in particular why per-tenant Keycloak realms are deliberately +deferred to Phase 2. This Grafana integration follows the same shape +as the tenant kube-apiserver's Phase 1 +([cozystack/cozystack#3044](https://github.com/cozystack/cozystack/pull/3044)), +adapted for a server-side (confidential) OAuth client instead of a +public kubectl one: authentication is wired up by the chart, but +authorization is an operator-supplied `users:` map reconciled into +the target (Grafana orgs here, ClusterRoleBindings in #3044) by a +chart-owned Job. The chart does not own any directory objects. + +## Selector + +```yaml +apiVersion: apps.cozystack.io/v1alpha1 +kind: Monitoring +metadata: + name: monitoring + namespace: tenant-acme +spec: + oidc: + mode: System # System | CustomConfig | None (default) + users: + - email: alice@acme.example + role: Admin + - email: bob@acme.example + role: Viewer +``` + +Three modes: + +- **None** — the only user-facing path is the `grafana-admin-password` + Secret, exposed to the tenant through + `packages/system/monitoring-rd`. This is the default; existing + instances render byte-identical to before. +- **System** — the Grafana instance trusts the platform `cozy` Keycloak + realm via a per-instance confidential client. Users are the ones a + platform admin already provisioned in `cozy`; the tenant does not + manage a directory of its own. +- **CustomConfig** — Grafana trusts a tenant-supplied OIDC issuer + directly. `cozy` is not in the path. Use for BYO IdPs (Okta, Auth0, + a customer's own Keycloak). + +The `grafana-admin-password` Secret stays a documented break-glass +path in every mode; `disable_login_form` is not flipped by this +selector. Hardening that further (e.g. locking the form off in +`System` mode) is a follow-up. + +## System mode + +What the chart provisions, in the release namespace of the Monitoring +CR: + +1. **KeycloakClient** named `-` in the `cozy` + realm — `public: false`, `directAccess: false`. The `secret` field + points at a chart-owned Kubernetes Secret via the EDP-Keycloak + `$:` syntax so the operator provisions the same + confidential secret to Keycloak that Grafana reads at runtime. + `redirectUris` is locked to + `https://grafana./login/generic_oauth`. +2. **KeycloakClientScope** named `--audience` + carrying an `oidc-audience-mapper` that pins `id_token.aud` to the + per-instance clientId. This is the per-instance isolation + primitive: a token minted for one Monitoring release is rejected + by another's Grafana. +3. A **Secret** `-oidc-client` carrying a random 32-char + `client-secret`. Generated on first install (`lookup` + random + fallback, same pattern as `packages/system/dashboard`), preserved + across upgrades. +4. The Grafana CR's `spec.config.auth.generic_oauth` section wired to + the platform realm issuer (`cozy` by default, `authentication.oidc.realmName`) + the per-instance audience scope. Two + independent gates ride on top of the raw OAuth wiring: + + **Tenant-membership gate — unconditional in System mode.** The chart + requests the `groups` OIDC scope (already provisioned platform-wide + by `packages/system/keycloak-configure` as a realm-default + KeycloakClientScope with an `oidc-group-membership-mapper`) and sets + + ``` + allowed_groups = -view -use -admin -super-admin + groups_attribute_path = groups + ``` + + `groups_attribute_path` is REQUIRED alongside `allowed_groups` on + Grafana v11.x+: without it Grafana leaves the extracted + `user.Groups` slice empty regardless of what the userinfo endpoint + returns, and the gate rejects every login with "user not a member + of one of the required groups" — even when the token carries the + correct claim (verified end-to-end on a test cluster with Grafana v11.6.15). + The JMESPath expression `groups` reads the top-level `groups` + array from userinfo, matching the shape Keycloak's + `oidc-group-membership-mapper` emits. + + The four groups match the KeycloakRealmGroups the tenant chart + provisions in the `cozy` realm — see + `packages/apps/tenant/templates/keycloakgroups.yaml`. Grafana + rejects any login whose token does not carry at least one of these + groups. This is the isolation primitive that stops tenant-alice + from signing into tenant-bob's Grafana even though both authenticate + against the same flat `cozy` realm. The audience mapper in point 2 + protects against *replay* of a minted token across releases; this + gate protects against *issuance* of a usable token to a caller + outside the release's tenant. + + **Users-map contract — active only when `spec.oidc.users` is + non-empty.** Three additional keys are merged on top: + - `skip_org_role_sync = true` — a login never overwrites the + users-Job's role assignments; + - `oauth_allow_insecure_email_lookup = true` — the OIDC identity + binds to the pre-provisioned local account by email; + - `allow_sign_up = false` — a cozy-realm identity whose email is + NOT in `spec.oidc.users` is rejected at login (even if they are + a tenant member and pass the group gate) instead of being + admitted as `auto_assign_org_role` Viewer and waiting for the + next helm-upgrade prune pass to clean them up. + + When `spec.oidc.users` is empty (or unset) the three users-map + keys are NOT forced. The tenant-membership gate still applies, so + only tenant members can log in — they land at Grafana's + `auto_assign_org_role` default (`Viewer`). This is the intended + hands-off contract: the chart owns nothing app-side, but the + Keycloak-layer gate still isolates the release. Cross-tenant + users are rejected outright; in-tenant users get a self-service + Viewer with no org-role management by the chart. +5. A `GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET` env on the Grafana + Deployment sourced from the Secret in step 3. +6. A post-install/post-upgrade **users-Job** — see the "Users and + RBAC" section — that reconciles `spec.oidc.users` into Grafana's + Main Org. + +No `KeycloakRealmGroup`s and no `role_attribute_path`. Directory +objects (users, groups, group memberships) stay owned by whoever +operates the `cozy` realm; the Monitoring release only requests +authentication wiring, it does not act as a user directory. + +Server-level `GrafanaAdmin` promotion (`allow_assign_grafana_admin`) +is out of scope for Phase 1. All Grafana instances — platform and +tenant — cap at org-level `Admin`. + +## CustomConfig mode + +The tenant supplies the entire `[auth.generic_oauth]` payload. Two +paths: + +```yaml +spec: + oidc: + mode: CustomConfig + customConfig: + config: + client_id: my-grafana + client_secret: xxxxxxxx + auth_url: https://idp.acme.example/protocol/openid-connect/auth + token_url: https://idp.acme.example/protocol/openid-connect/token + api_url: https://idp.acme.example/protocol/openid-connect/userinfo + scopes: openid email profile +``` + +…or via an existing Secret in the tenant namespace whose `auth.ini` +key holds a ready-made `[auth.generic_oauth]` fragment: + +```yaml +spec: + oidc: + mode: CustomConfig + customConfig: + secretRef: + name: acme-byo-grafana-auth +``` + +When `spec.oidc.users` is non-empty the chart merges three settings +on top of the operator's inline map so the users-Job contract holds +in `CustomConfig` mode too: + +- `skip_org_role_sync = true` — a login never overwrites the Job's + org-role assignments; +- `oauth_allow_insecure_email_lookup = true` — the OIDC identity + binds to the pre-provisioned local account by email; +- `allow_sign_up = false` — an OIDC identity whose email is not in + the users-map is rejected at login instead of being admitted with + the default `auto_assign_org_role`. + +All three keys are chart-forced (`merge` semantics, chart wins) — +setting them in the operator's map is a no-op, and +`role_attribute_path` would just be dead config since the Job +manages roles. + +When `spec.oidc.users` is empty (or unset) the chart forces none of +the three keys — a BYO-IdP operator who does not use the users-map +gets their inline `auth.generic_oauth` applied verbatim. This +matters because `oauth_allow_insecure_email_lookup` against a BYO +issuer that does not verify emails is a needless account-linking +surface, and `allow_sign_up = false` with no pre-provisioned +accounts would lock everyone out. If you want the isolation +posture without the users-map, set the keys explicitly in your +inline map. + +**`secretRef.name` mode disables the users-map.** The chart mounts the +operator's `auth.ini` fragment verbatim via `GF_PATHS_CUSTOM_INI` and +cannot inject either of the two chart-forced settings. Setting +`spec.oidc.users` alongside `customConfig.secretRef.name` fails the +render with an explicit message — the alternatives are: (a) use +`customConfig.config` (inline) and let the chart merge, or (b) leave +`spec.oidc.users` empty and include both `skip_org_role_sync = true` +and `oauth_allow_insecure_email_lookup = true` in your ini fragment +plus manage the OIDC → Grafana user mapping yourself. + +Setting both `config` and `secretRef.name` (or neither) fails the +render. In `CustomConfig` mode no Keycloak objects are provisioned in +`cozy` and no chart-owned client-secret Secret is created — the +operator manages their own credentials end-to-end. + +## Users and RBAC + +Grafana's built-in identity model exposes three org-level roles: +`Admin` / `Editor` / `Viewer`. The chart drives them from an +operator-supplied map on the CR: + +```yaml +spec: + oidc: + users: + - email: alice@acme.example + role: Admin + - email: bob@acme.example + role: Editor + - email: carol@acme.example + role: Viewer +``` + +A chart-owned Job runs on every `helm install` / `helm upgrade` and +reconciles that list into Main Org. membership via Grafana's admin +API: + +1. Each listed email gets a pre-provisioned local Grafana account + with a random password. The account is a shell; the operator + never uses that password. When they sign in with the OIDC flow + Grafana looks up the pre-provisioned account by email + (`oauth_allow_insecure_email_lookup = true`) and attaches the + OIDC identity to it. +2. Each listed email is added to Main Org. with the requested role. + Re-runs of the Job converge role changes (`Editor` → `Admin`, + demotions, etc.) via `PATCH /api/orgs/{orgId}/users/{userId}`. +3. Every Main-Org member whose email is neither in the list nor the + break-glass `grafana-admin-password` login is removed. Removing + an entry from `users:` and re-reconciling revokes access. + +The Job renders only when **both** `mode != None` and +`spec.oidc.users` is non-empty. Three empty-users states all resolve +to the same hands-off contract — the chart owns nothing in Grafana +orgs: + +- `mode: None` — OIDC is off, no chart-managed org membership at + all. This matters on upgrade: existing tenants who ran the + pre-PR Monitoring chart may have added users to Grafana manually + through the UI; if the Job ran in `mode: None` its prune pass + would silently delete those accounts on the next Flux reconcile. +- `customConfig.secretRef` — the users-map is forbidden by the + render-time assert; the operator's mounted `auth.ini` fragment is + authoritative. +- `mode: System | CustomConfig-inline` with `users:` unset — the + operator opted into OIDC but not into the chart-managed users-map; + they manage org membership themselves. + +Setting `users: [...]` on the CR means the chart owns Main-Org +membership — anything added by hand (Grafana UI, admin API, other +tooling) gets pruned on the next reconcile. The reverse edge is +worth calling out: taking `users:` back to `[]` (or unsetting it) +does NOT prune the last entry — the chart stops managing membership +and any accounts the Job provisioned are left in place. Operators +who want to clean up OIDC-provisioned accounts after switching +`System | CustomConfig → None` (or after emptying `users:`) do it +themselves through the Grafana UI or admin API. The +`activeDeadlineSeconds` and `ttlSecondsAfterFinished` notes below +only apply when the Job actually renders. + +While the users-map is active, users NOT listed in `spec.oidc.users` +who try to log in through OIDC are rejected at the door +(`allow_sign_up = false`) — no account is minted, no default +`Viewer` role is assigned. `skip_org_role_sync = true` in the +Grafana config additionally makes sure the Job's assignments +outlive the next login for users who ARE listed. + +For `System` mode, the operator provisions the corresponding +Keycloak user in `cozy` in whatever way they already do (Keycloak +UI, a `KeycloakRealmUser` CR, an identity broker) and adds them to +one of the tenant's `-{view,use,admin,super-admin}` +groups — Grafana's `allowed_groups` rejects the login otherwise. +The Monitoring release does not create Keycloak users and does not +provision `KeycloakRealmGroup`s — group directory ownership stays +with the tenant chart (`packages/apps/tenant`); group membership is +curated by whoever operates the `cozy` realm. + +## How a user logs in + +The user opens `https://grafana.` in a browser and picks the +"Sign in with Keycloak" button under the login form. Grafana runs the +Authorization Code + PKCE flow against the cozy realm, receives a +token whose `aud` claim matches this Monitoring instance's clientId, +and — if the user's email is in `spec.oidc.users` — binds the OIDC +identity to the pre-provisioned local account with the role the Job +already set. + +If the user's email is not in `spec.oidc.users` (and the users-map +is active — `mode != None` and the list is non-empty) the login is +refused by Grafana at the OAuth callback — `allow_sign_up = false` +does not mint an account for them. Add them to `spec.oidc.users` +and re-apply the CR; the Job will run on the next helm-upgrade, +pre-provision their local account, and the next login attempt will +succeed. When the users-map is inactive Grafana's default +`allow_sign_up = true` applies and any authenticated cozy-realm +identity lands with `auto_assign_org_role` (Viewer). + +The break-glass `admin_user` / `admin_password` field on the form +stays wired to the `grafana-admin-password` Secret and continues to +work — useful when Keycloak is down or misconfigured. + +## Failure modes + +- **`mode: System` without the platform-level OIDC feature** — chart + render hard-fails: + `spec.oidc.mode: System requires the platform-level OIDC feature`. + Flip `authentication.oidc.enabled: true` in the platform values, or + switch to `CustomConfig`. On the platform side that flag also + swings the `cozystack.monitoring-application` PackageSource from + variant `default` (baseline dependsOn only) to variant `oidc` + (additionally waits for `cozystack.keycloak-operator` so the + `v1.edp.epam.com` CRDs consumed by the Keycloak-side of System + mode are registered before the monitoring chart reconciles). +- **`mode: System` without the Keycloak operator CRDs registered** — + chart render hard-fails: + `spec.oidc.mode: System requires the Keycloak operator CRDs + (v1.edp.epam.com/v1)`. Symmetric across + `templates/grafana/oidc-keycloak.yaml` and + `templates/grafana/grafana.yaml` so Grafana never comes up with an + `auth.generic_oauth` block pointing at a client that never gets + provisioned. For the platform `monitoring-system` release the + `oidc` variant's `dependsOn: cozystack.keycloak-operator` prevents + the race; for tenant releases (`Monitoring` CR with + `mode: System`) verify the keycloak-operator package is deployed + before creating the CR. +- **`CustomConfig` with an unreachable issuer or wrong claim + mappings** — Grafana rejects the callback and the login screen + shows an error; the `admin_user`/`admin_password` Secret keeps + working as break-glass. +- **User successfully logs in but sees no dashboards.** Their email + is missing from `spec.oidc.users` — no org role was assigned. + Add the entry and re-apply the CR; the users-Job runs on the + next helm-upgrade and grants access. +- **users-Job fails.** The Job caps at `activeDeadlineSeconds: 900` + and `backoffLimit: 6`. Common causes: Grafana never becomes + ready (check `kubectl -n get pods -l app=grafana`), or the + `grafana-admin-password` Secret is missing its `user`/`password` + keys. `ttlSecondsAfterFinished: 3600` keeps the failed hook Job + (and its Pod's logs) around for **one hour** after the terminal + Failed condition — check its logs with + `kubectl -n logs job/-oidc-users` inside that window. + After 1h the Kubernetes TTL controller garbage-collects the Job + and the Pod together, so grab the logs quickly; if you missed + the window, re-trigger the hook with `helm upgrade` on the + release and reproduce. +- **`emailVerified` on Keycloak users is a prescriptive requirement, + not a chart-enforced one.** The chart does not emit any + `claimValidationRules` — the layered guarantees you rely on + instead are: + 1. Provision users with `emailVerified: true` (via + `KeycloakRealmUser` or the Keycloak UI's email-verify flow) so + no unverified identity ever holds a given email. + 2. The `cozy` realm keeps Keycloak's default + `duplicateEmails: false`, so a second account cannot claim an + already-registered address to impersonate an existing operator. + 3. Grafana's own login flow rejects tokens the apiserver rejects, + so on the wire the same guarantee still holds through the JWT + signature and audience checks. + Adding a CEL `claimValidationRules` entry is a reasonable + follow-up hardening; it is out of scope for Phase 1. +- **`CustomConfig` with both or neither payload set** — chart render + hard-fails: `set exactly one of 'config' (inline) or + 'secretRef.name' — they are mutually exclusive`, or `CustomConfig + requires either spec.oidc.customConfig.config … or …secretRef.name`. + +## What is NOT in Phase 1 + +- **Per-tenant Keycloak realms.** Phase 2 candidate; tracked in + cozystack/community#24. +- **Full-logout through Keycloak's end-session endpoint.** + `auth.generic_oauth` native to Grafana does the OAuth part; wiring + `--backend-logout-url` and the corresponding Keycloak client + attribute is a subsequent hardening. +- **`disable_login_form: true` under `mode: System`.** Kept off so + the `admin_user`/`admin_password` Secret remains a documented + break-glass path; hardening it is a follow-up. +- **Server-level `GrafanaAdmin` promotion.** All Grafana instances + cap at org-level `Admin`. Not exposed on the CR. +- **CEL `claimValidationRules` enforcing `email_verified`.** See the + failure-modes note above. +- **Multi-issuer / BYO alongside `cozy`.** `mode: System` and + `mode: CustomConfig` are mutually exclusive on a single Monitoring + release — no composition. Follow the tenant kube-apiserver PR's + structured-config path if you need this later. diff --git a/docs/oidc-tenant.md b/docs/oidc-tenant.md new file mode 100644 index 0000000000..7fc7fa952b --- /dev/null +++ b/docs/oidc-tenant.md @@ -0,0 +1,309 @@ +# OIDC for tenant Kubernetes clusters (Phase 1) + +Cozystack tenant Kubernetes clusters opt in to OIDC authentication on their +kube-apiserver through a flat selector on the `Kubernetes` CR. This document +covers the operator-facing surface: what the modes mean, what the chart +provisions on either end, and how to give a user kubectl access. + +The architectural rationale lives in +[cozystack/community#24](https://github.com/cozystack/community/pull/24) — +in particular why per-tenant Keycloak realms are deliberately deferred to +Phase 2. + +## Selector + +```yaml +apiVersion: apps.cozystack.io/v1alpha1 +kind: Kubernetes +metadata: + name: prod + namespace: tenant-acme +spec: + oidc: + mode: System # System | CustomConfig | None (default) + users: + - email: alice@acme.example + role: admin # admin → cluster-admin + - email: bob@acme.example + role: view # view → view +``` + +Three modes: + +- **None** — the only user-facing path is the static + `-admin-kubeconfig` Secret (the Kamaji-minted `super-admin.svc` + kubeconfig). This is the default; existing clusters render byte-identical + to before. +- **System** — the apiserver trusts the platform Keycloak realm (`cozy` by default, set via `authentication.oidc.realmName`) via a per-cluster public client. Authenticates the users already in that realm (the one cozystack provisions). Zero-config default. +- **CustomConfig** — the apiserver trusts a tenant-supplied OIDC issuer + directly. `cozy` is not in the path. Use for BYO IdPs (Okta, Auth0, a + customer's own Keycloak). + +The `users[]` map is independent of the mode and drives per-user +`ClusterRoleBinding`s inside the tenant cluster. + +## System mode + +What the chart provisions, in the management cluster: + +1. **KeycloakClient** named `-` in the `cozy` realm — + `public: true`, PKCE required, redirect URIs locked to `localhost:8000` + and `localhost:18000` (the kubelogin / `kubectl oidc-login` defaults). +2. **KeycloakClientScope** named `--audience` carrying + an `oidc-audience-mapper` that pins `id_token.aud` to the per-cluster + clientId. This is the per-cluster isolation primitive: a token minted + for cluster A is rejected by cluster B's apiserver. +3. A **Secret** `-oidc-authn-config` carrying a structured + `apiserver.config.k8s.io/v1beta1` `AuthenticationConfiguration` with + the cozy realm issuer and the per-cluster audience. +4. A `--authentication-config=` flag, mount, and volume on the + `KamajiControlPlane` referencing the Secret above. +5. A **bootstrap Job** (Helm `post-install,post-upgrade` hook) that + applies one `ClusterRoleBinding` per `users[]` entry inside the + tenant cluster and writes a ready-to-use OIDC kubeconfig Secret on + the management side (see below). +6. A `-oidc-kubeconfig` Secret in the tenant namespace carrying + a kubeconfig with a `kubectl oidc-login` exec block, exposed to the + dashboard via `packages/system/kubernetes-rd`. + +The structured authentication-config form (rather than the legacy +`--oidc-*` flags) is intentional: it accepts multiple issuers in a list, +inline private-CA PEM, and future Phase-2 issuers extend the same Secret +instead of fighting the chart shape. + +## CustomConfig mode + +The tenant supplies the entire `AuthenticationConfiguration`. Two paths: + +```yaml +spec: + oidc: + mode: CustomConfig + customConfig: + config: | + apiVersion: apiserver.config.k8s.io/v1beta1 + kind: AuthenticationConfiguration + jwt: + - issuer: + url: https://idp.acme.example + certificateAuthority: | + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- + audiences: + - cozystack-prod + claimMappings: + username: + claim: email + prefix: "" + groups: + claim: groups + prefix: "" + users: + - email: alice@acme.example + role: admin +``` + +…or via an out-of-band Secret the operator has already created in the +tenant namespace: + +```yaml +spec: + oidc: + mode: CustomConfig + customConfig: + secretRef: + name: acme-byo-authn-config # has key config.yaml +``` + +`config` and `secretRef.name` are mutually exclusive; the chart fails the +render if both — or neither — are set. + +No Keycloak objects land in `cozy`; the chart writes only the +AuthenticationConfiguration Secret (or mounts the operator's) and the +RBAC Job. The `-oidc-kubeconfig` helper Secret is NOT written +in CustomConfig mode: the issuer and clientId are inside the +operator-supplied config and are not knowable to the chart. Distribute +the OIDC kubeconfig out-of-band. + +## Aggregated API servers + +Authenticating a user is only half of the path. A call that lands on an aggregated API server — an `APIService` backed by an extension server rather than by the core apiserver — reaches it over the aggregation layer, which forwards the caller's identity in request headers. The UID travels in `X-Remote-Uid`, which the aggregation layer sends whenever the `RemoteRequestHeaderUID` gate is on, and the extension server trusts that header only when the tenant kube-apiserver has published it under `requestheader-uid-headers` in the `extension-apiserver-authentication` ConfigMap. Without that key the extension server ignores the header, and the call is served as if the caller had no UID, even though `kubectl` against the core apiserver works fine. Anything on the far side that reads the UID, an authorizer or an audit record, sees an empty value. + +Carrying the UID across that hop is a property of the aggregation layer, not of OIDC: a caller authenticated by any means loses their UID the same way. The chart nonetheless renders the header flag only when `spec.oidc.mode` is not `None`, which keeps the blast radius to clusters opting into a new feature. A cluster that has never left `mode: None` therefore still loses the UID at the extension server. + +Be clear about who this helps. The flag forwards a UID the caller already has; it does not create one. ServiceAccount tokens carry a UID, and so do other identities that set one, and for those the aggregation hop starts working. An OIDC user gets one only from a claim mapping: in `spec.oidc.mode: System` the AuthenticationConfiguration the chart generates maps `uid` to the `sub` claim, the Keycloak account UUID, so those users reach an aggregated API server under a stable identifier. `sub` rather than the username claim, because the platform realm lets a user change the address `email` carries while the subject stays where it is. In `CustomConfig` mode the configuration is yours: map `uid` and the header carries what you mapped; leave it out and upstream resolves an empty UID (`plugin/pkg/authenticator/token/oidc/oidc.go`, `getUID`, read at v0.35.0), with nothing for the header to carry. + +On a cluster that was running before the mapping existed, those users gain a UID where they had none. Tenant RBAC does not notice — a role binding matches a username or a group, never a UID — but an audit record on the tenant apiserver now carries `user.uid`, `kubectl auth whoami` shows it, and an admission webhook or extension server inside the tenant that read the empty value now reads the subject. + +Where `spec.oidc.mode` is not `None`, and outside the cases further down that stop it, the chart renders the flag that publishes it alongside `--authentication-config`: + +| `spec.version` | rendered | +| --- | --- | +| `v1.31` | nothing — the flag does not exist upstream, and an unknown flag stops the apiserver from starting | +| `v1.32` | nothing — `RemoteRequestHeaderUID` is alpha and off, and the apiserver refuses the flag while the gate is off | +| `v1.33` and newer | `--requestheader-uid-headers=X-Remote-Uid` | + +The chart never renders `--feature-gates` itself. From `v1.33` `RemoteRequestHeaderUID` is beta and on by default, so the header flag is all the apiserver needs. On `v1.32` the chart does not switch on an alpha gate for you, so aggregated API servers do not see the caller's UID there; if you need them to on `v1.32`, write both `--feature-gates=RemoteRequestHeaderUID=true` and `--requestheader-uid-headers=X-Remote-Uid` into `controlPlane.apiServer.extraArgs` yourself, and the chart passes them through untouched. + +The version in that table is the effective one, not the image tag. `--emulated-version` in `controlPlane.apiServer.extraArgs` moves it: the gate resolves through the versioned spec at the emulated version, so `--emulated-version=1.32` puts `RemoteRequestHeaderUID` back to alpha-and-off on a `v1.35` cluster, and the chart follows by rendering nothing, as it would on a `v1.32` image. The chart reads the dot-separated numbers at the start of the value, after an optional `v`, and ignores what follows them, so it reads `v1.32`, `1.32.0` and `1.32-rc` all as `1.32`. + +Two values are still not harmless, whatever the chart does with them. An element naming another component, `--emulated-version=wardle=1.2`, says nothing about the tenant kube-apiserver, but the apiserver rejects a component it has not registered, `component not registered: wardle`, and does not start. A value whose minor is above your `spec.version` is ignored too, because emulation only moves the effective version down, and there the apiserver checks against a range rather than a ceiling: it rejects one above its own binary or below the floor it still supports, naming the range it accepts. In both cases nothing the chart renders makes any difference. + +Adding the header flag through `controlPlane.apiServer.extraArgs` by hand is no longer necessary from `v1.33`, and an existing cluster that does keeps working as long as its entries would start an apiserver: the chart skips its own copy when your entries already carry `X-Remote-Uid`, and refuses to render the shapes listed below. + +Every entry you write on one of these three flags reaches the tenant kube-apiserver. None of them is among the flags the tenant control plane sets itself and drops from `extraArgs`; the chart moves or refuses an entry naming one of those, and the description of `controlPlane.apiServer.extraArgs` lists them. Where your entry does not already satisfy the requirement the chart therefore adds its own beside it rather than failing. `--requestheader-uid-headers` is a repeatable flag whose occurrences append into one list, so a list of your own that omits `X-Remote-Uid` gets a second entry carrying it and the union is what the apiserver validates; your header keeps working alongside the standard one. + +The chart does not check how you spell kube gate terms. The apiserver collects `--feature-gates` terms per component across all your entries and refuses a configuration that names the kube component both bare and `kube:`-prefixed; the chart passes that through like any other gate value the apiserver rejects on its own terms. + +If you turned the gate off on purpose, with `RemoteRequestHeaderUID=false` or a blanket `AllBeta=false`, the chart does not render the header flag and the render is byte-identical to the one before this feature existed. You keep losing the UID at the aggregation layer, which is what the opt-out asks for. Adding the opt-out or removing it again needs no second edit on Kamaji 26.9.4-edge: the tenant control plane compares a hash of `extraArgs` against the one it stored last time and rebuilds the argument list from scratch whenever they differ, so nothing is left behind from the previous shape. A rollback of Kamaji to an earlier build loses that. The older code expects a number in the `kube-apiserver.kamaji.clastix.io/args` annotation and gives up when it finds the hash. From then on, a flag removed from `extraArgs` stays on the running control plane until someone sets the annotation back to a number by hand. + +The render fails where entries of your own describe a kube-apiserver that already refuses to start: a uid-headers list with an element that is empty once trimmed, which a trailing comma leaves behind, and a uid-headers list alongside a gate setting that turns `RemoteRequestHeaderUID` off. The chart cannot rescue either, because the value the apiserver rejects is yours. The error message names the fix, which beats a crash-looping control plane. More refusals sit outside that group and outside the conditions below: the `--oidc-*` and `--authentication-config` collisions further down, each on its own terms; an unquoted `--emulated-version=value` entry whose kube version has a non-zero patch component, such as `1.32.4`, which the apiserver refuses with `patch version not allowed`; an entry naming a flag the tenant control plane sets itself, refused in every `spec.oidc.mode` unless the chart moves it into a field, as the description of `controlPlane.apiServer.extraArgs` explains; and the list ending in one of these three flags, where the apiserver may well start — it is the silence that earns that one. + +Those first two checks run only where the chart renders something. Three things switch them off with the header flag: an effective version of `v1.32` or below, a quoted `--emulated-version`, and two `--emulated-version` entries that both name kube. Each of those leaves the chart either with nothing to render or unable to say what the apiserver will read, so it renders nothing and judges nothing, and a shape that would otherwise be refused here passes silently through to a control plane that does not start. Take the unreadable entry out if you want the checks back. `--feature-gates` or `--emulated-version` written as a flag and a value in separate entries hides a value too, but a narrower one, and the chart keeps what holds whatever that value is. The empty element is still refused. An explicit `RemoteRequestHeaderUID=false` is still refused beside a header entry behind a hidden `--emulated-version`, because it turns the gate off at every version; a blanket `AllBeta=false` is not, because at a lower emulated version the gate can be alpha, where `AllBeta` does not reach it. Neither is refused behind a hidden `--feature-gates` value, which can turn the gate back on. A uid-headers list of yours without `X-Remote-Uid` still gets the chart's element beside it, which the apiserver needs with the gate on and which changes nothing with the gate off, since your list alone is refused then. Behind a hidden value and without a list of yours, the chart renders no `X-Remote-Uid` entry at all. + +Otherwise nothing stops rendering: every other `extraArgs` shape reaching this code either renders as it did or gains the header flag. + +The gate check reads your entries the way the apiserver does. + +- An element may name a component, because `--feature-gates` is a colon-separated multimap: `kube:RemoteRequestHeaderUID=false` is the same opinion as `RemoteRequestHeaderUID=false`, while an element naming another component is not about the tenant kube-apiserver, and the chart ignores it when it works out the state of the UID gate. That is the chart's reading only. Whether the apiserver accepts the component is up to its own component registry, as it is for `--emulated-version` above. +- Past that it is one `Name=value` element at a time, spaces trimmed on both sides, and the value resolved the way `strconv.ParseBool` resolves it, so `1`, `t` and `T` mean the same as `true`, and `0`, `f` and `F` the same as `false`. +- Blanket settings count, because the apiserver applies them to every gate you have not named explicitly. `AllBeta=false` switches `RemoteRequestHeaderUID` off, so the chart reads it as your opt-out; `AllAlpha` covers alpha gates only, and from `v1.33` this gate is beta, so the chart does not read it. +- Naming the gate wins over a blanket: `AllBeta=false,RemoteRequestHeaderUID=true` leaves the gate on, because an explicit entry beats a blanket one whichever order they appear in. +- Gate names are compared exactly, as the apiserver looks them up, so `remoterequestheaderuid=false` is not an opinion about `RemoteRequestHeaderUID`; the apiserver would reject that spelling as an unrecognized gate anyway. +- Several `--feature-gates` entries settle a gate between them, last term wins, exactly as the apiserver settles them: an earlier entry naming the gate still counts when a later entry says nothing about it. +- An entry with an empty value carries no term, so it settles nothing, and it does not cancel what an earlier entry said. Drop the entry you do not want instead of emptying it. + +`--emulated-version` accumulates rather than replaces, and the apiserver refuses two elements naming the same component with `duplicate version flag`. Two entries of yours that both name kube, explicitly or by naming nothing, stop the control plane from starting; the chart does not render the header flag there, because there is no effective version left to decide against. Entries naming different registered components are fine and all of them apply, so an emulated kube version stays in force when a later entry names something else. + +Quotes stop the chart from reading the entry they are on. `--requestheader-uid-headers` and `--emulated-version` are both CSV-parsed by the apiserver, which strips a surrounding pair of double quotes before it sees the value, and the chart does not reproduce that. On `--emulated-version` that hides the version the whole decision rests on, so the chart does not render the header flag, exactly as it did before that flag existed. On `--requestheader-uid-headers` it only means the chart cannot tell whether that entry already carries `X-Remote-Uid` or an empty element: it renders its own element, which is safe whatever your value turns out to be, and does not refuse the render over an element it may be misreading. `--feature-gates` is not CSV-parsed, so quotes there are neither stripped nor understood: the apiserver rejects the term as an unrecognized gate name and the chart, reading the same text, does not take it for an opinion about `RemoteRequestHeaderUID` either. Drop the quotes if you want the chart to read the entry. + +Writing a flag and its value as two list entries has the same effect, for the same reason. The apiserver pairs them, but the value sits in an entry the chart reads separately, so it cannot be attached to the flag. On `--feature-gates` and `--emulated-version` the chart therefore renders its own `X-Remote-Uid` element only beside a uid-headers list of yours that lacks it, as described above. On `--requestheader-uid-headers` it counts the flag as present, which is what the opt-out check needs, and renders its own `X-Remote-Uid` entry beside it. Write `--flag=value` in one entry if you want the chart to read the value. + +One arrangement of that shape the chart refuses outright: one of those three flags as the last entry in the list. The apiserver pairs a valueless flag with whatever argument follows it, and what follows yours is the `--authentication-config` the chart appends, so the flag would swallow it. The AuthenticationConfiguration is then never applied, and what that costs depends on which flag took it: `--requestheader-uid-headers` accepts the path as a header name, so the control plane starts and fails every login with nothing to show for it, while `--feature-gates` and `--emulated-version` reject it and the apiserver exits instead. Give the flag its value in the same entry. Moving it earlier works too: the entry after it is then one of yours, and it reaches the apiserver right behind the flag, since the chart refuses every entry the control plane would drop or moves it out of the list it renders. + +The uid-headers list is read without trimming, because the apiserver does not trim it either: `--requestheader-uid-headers=X-Remote-Uid , Other` does not contain `X-Remote-Uid` as far as the apiserver is concerned, so the chart does not read it as one and renders its own element, which is what makes that list start. The emptiness check is the one that trims, again because the apiserver's does: in `X-Remote-Uid, ` the second element is empty once trimmed, and the apiserver refuses the whole list with `empty value in "requestheader-uid-headers"` although `X-Remote-Uid` is in it, which is why that shape fails the render instead. + +On `v1.31` and `v1.32` the chart renders no UID flag, and the UID guards above do not run, so a uid-headers or `--feature-gates` entry of yours passes through untouched. These checks still apply there as everywhere else, because none of them depends on the effective version: the `--oidc-*` and `--authentication-config` collisions, the refusal of a patch-level `--emulated-version`, the move or refusal of an entry naming a flag the tenant control plane sets itself, which the description of `controlPlane.apiServer.extraArgs` explains, and the refusal of a list ending in one of the three flags, which is about the `--authentication-config` the chart appends on every version rather than about the UID flags. Aggregated API servers cannot see the caller's UID on `v1.31`; on `v1.32` they can only through the alpha gate you enable yourself, as described above. Move the cluster to `v1.33` or newer if you need them to without it. Writing `--requestheader-uid-headers` yourself does not work around it on `v1.31` — the flag does not exist there, and an unknown flag stops the apiserver from starting. + +## Users and RBAC + +`users[]` is a flat list. Each entry produces a single +`ClusterRoleBinding` inside the tenant cluster, labelled +`app.kubernetes.io/managed-by=cozystack-oidc` and +`app.kubernetes.io/instance=`. The CRB name is a +deterministic hash of `-`, so the same email always +maps to the same binding. + +| `role:` | `ClusterRole` bound | +| --- | --- | +| `admin` | `cluster-admin` | +| `view` | `view` | + +The CRB `User:` subject is the literal `email` value; it must match the +`email` claim emitted by the issuer. The chart-generated OIDC +kubeconfig requests `--oidc-extra-scope=email` in the +`kubectl oidc-login` exec block, so the token includes `email` +regardless of whether the per-cluster client lists `email` in its +default client scopes. The `email` scope itself is a built-in OIDC +scope available in every conformant issuer (Keycloak included), so no +extra realm-side configuration is required for `System`. For +`CustomConfig`, verify your BYO issuer emits the `email` claim when +the client requests the `email` scope; every conformant OIDC provider +does. + +Toggling a user out of `users[]` revokes their access on the next chart +reconcile — the bootstrap Job prunes any CRBs labelled by the release +that are no longer in the desired list. + +The static admin kubeconfig stays as the documented break-glass path +regardless of mode. + +## How a user logs in (System mode) + +The user installs `kubectl oidc-login` once: + +```bash +kubectl krew install oidc-login +``` + +The operator hands them the OIDC kubeconfig. The Secret name follows +the Helm release pattern `kubernetes--oidc-kubeconfig` (the +cozystack-api prefixes the cluster name with `kubernetes-` when it +materialises the `HelmRelease`): + +```bash +kubectl --namespace tenant-acme get secret kubernetes-prod-oidc-kubeconfig \ + --output=jsonpath='{.data.kubeconfig}' | base64 -d > prod.kubeconfig +``` + +First `kubectl --kubeconfig prod.kubeconfig get …` call triggers the +Keycloak browser flow on localhost:8000 (then 18000 as fallback) and +caches the token locally. Subsequent calls are silent until the token +expires. + +## Failure modes + +- **`mode: System` without the platform-level OIDC feature** — chart + render hard-fails: `spec.oidc.mode: System requires the platform-level + OIDC feature (authentication.oidc.enabled) — enable it in + cozystack-values, or use mode: CustomConfig for a tenant-supplied + issuer.` +- **`CustomConfig` with an unreachable issuer or wrong claim mappings** — + the apiserver rejects tokens at request time; the admin kubeconfig + keeps working as break-glass. +- **`emailVerified` on Keycloak users is a prescriptive requirement, + not a chart-enforced one.** The chart emits a + `claimValidationRules` entry keyed on `groups` (see next bullet), + but not one keyed on `email_verified`. The layered guarantees you + rely on instead: + 1. Provision users with `emailVerified: true` (via `KeycloakRealmUser` + or the Keycloak UI's email-verify flow) so no unverified identity + ever holds the email you name in `users[].email`. + 2. The `cozy` realm keeps Keycloak's default `duplicateEmails: false`, + so a second account cannot claim an already-registered address to + impersonate an existing operator. + 3. As a k8s side-effect, if the issuer explicitly emits + `email_verified: false` on a token the apiserver rejects it with + `oidc: email not verified` — but a *missing* claim is treated as + verified. + Adding a CEL `claimValidationRules` entry + (`!has(claims.email_verified) || claims.email_verified == true`) to + the rendered System-mode config would elevate item 3 to a hard gate; + that is a reasonable follow-up hardening but is out of scope for + Phase 1. + +- **Cross-tenant isolation gate.** The rendered + `AuthenticationConfiguration` carries a CEL `claimValidationRules` + entry on the `groups` claim that rejects any token whose caller is + not a member of at least one of this tenant's four Keycloak groups: + + ``` + -view -use -admin -super-admin + ``` + + The tenant chart (`packages/apps/tenant/templates/keycloakgroups.yaml`) + provisions these groups in the shared `cozy` realm. Without this + gate, any authenticated cozy-realm user could point `kubectl + oidc-login` at another tenant's cluster, land at + `system:authenticated`, and enumerate the discovery surface + (`kubectl api-resources`, OpenAPI schemata) — RBAC would default-deny + named-resource access, but discovery still leaks. Symmetric with the + Grafana-side `allowed_groups` gate on the Monitoring chart's + `auth.generic_oauth` block (see `docs/oidc-grafana.md`). CustomConfig + mode does NOT get this gate injected — the tenant's own + AuthenticationConfiguration is authoritative and the tenant is + responsible for their own claim-side guards. +- **`kubectl` without the `oidc-login` plugin** — the exec block errors + out client-side; install the plugin. + +## What is NOT in Phase 1 + +- **Per-tenant Keycloak realms.** Phase 2 candidate; tracked in + cozystack/community#24 (Option B — retained as the strongest isolation + answer; Option A is Keycloak Organizations). +- **Federating an external IdP into `cozy`.** Out of scope. +- **Cross-cluster SSO inside one tenant.** Each cluster has its own + audience — that is the per-cluster isolation primitive. +- **Custom credential plugin / RFC 8693 token exchange.** Possible future + optimisation; not required for the per-cluster client + audience model. diff --git a/docs/operations/backup-classes.md b/docs/operations/backup-classes.md index f60cba0ad4..2cfde8c84f 100644 --- a/docs/operations/backup-classes.md +++ b/docs/operations/backup-classes.md @@ -1,6 +1,6 @@ # Backup Classes -Cozystack ships a single platform-managed `BackupClass` named `cozy-default`. It is provisioned automatically when the `backupstrategy-controller` package is installed and references the system-managed S3 bucket `cozy-backups` in the `tenant-root` namespace. +Cozystack ships a single platform-managed `BackupClass` named `cozy-default`. It is provisioned automatically when the `backupstrategy-controller` package is installed and references the system-managed bucket provisioned through the `apps.cozystack.io/Bucket` CR `cozy-backups` in the `tenant-root` namespace (the real S3 bucket name is the COSI-assigned one from `BucketClaim.status.bucketName`). Tenants reference `cozy-default` from `BackupJob`, `Plan`, and `RestoreJob` resources — they do **not** supply S3 credentials, endpoints, or paths. The platform projects the system-managed credentials Secret into the tenant namespace per BackupJob (or, for long-lived references like Velero's `BackupStorageLocation`, into a fixed list of system namespaces on a periodic tick), and the default strategy templates encode `/` into every S3 path so two tenants with the same application name never collide. @@ -13,10 +13,17 @@ Tenants reference `cozy-default` from `BackupJob`, `Plan`, and `RestoreJob` reso | `apps.cozystack.io/Postgres` | CloudNativePG (barman) | `strategy.backups.cozystack.io/CNPG` `cozy-default-cnpg` | | `apps.cozystack.io/MariaDB` | mariadb-operator dump | `strategy.backups.cozystack.io/MariaDB` `cozy-default-mariadb` | | `apps.cozystack.io/ClickHouse` | Altinity `clickhouse-backup` sidecar | `strategy.backups.cozystack.io/Altinity` `cozy-default-altinity` | +| `apps.cozystack.io/MongoDB` | Percona psmdb operator (pbm) dump | `strategy.backups.cozystack.io/MongoDB` `cozy-default-mongodb` | +| `apps.cozystack.io/Kafka` | Kafka Admin API (topic metadata) | `strategy.backups.cozystack.io/Kafka` `cozy-default-kafka` | | `apps.cozystack.io/Etcd` | etcd-operator snapshot | `strategy.backups.cozystack.io/Etcd` `cozy-default-etcd` | +| `apps.cozystack.io/RabbitMQ` | RabbitMQ definitions (management API) | `strategy.backups.cozystack.io/Rabbitmq` `cozy-default-rabbitmq` | +| `apps.cozystack.io/Redis` | RDB dump Job (sentinel-discovered master) | `strategy.backups.cozystack.io/Redis` `cozy-default-redis` | +| `apps.cozystack.io/Valkey` | RDB dump Job (sentinel-discovered master) | `strategy.backups.cozystack.io/Redis` `cozy-default-redis` | | `apps.cozystack.io/VMInstance` | Velero + kubevirt-velero-plugin | `strategy.backups.cozystack.io/Velero` `cozy-default-velero-vminstance` | | `apps.cozystack.io/VMDisk` | Velero | `strategy.backups.cozystack.io/Velero` `cozy-default-velero-vmdisk` | +Valkey runs on the same RedisFailover operator as Redis, so `cozy-default-redis` serves both: the driver hands the template `.Release.Prefix` (`redis-` or `valkey-`, after the app's Kind), from which it builds the Sentinel Service and the auth and CA Secret names. A Valkey backup restores only into a Valkey app, a Redis backup only into a Redis app. The Valkey chart has no TLS option, so a Valkey dump always connects in plaintext. + ### Shipped but NOT bound (admin opt-in required) | Application Kind | Driver | Strategy CR | @@ -27,18 +34,21 @@ The FoundationDB strategy CR is rendered by the chart so admins can reference it ### Endpoint format per driver -Different operators expect different endpoint shapes; the strategy templates rendered by `backupstrategy-controller` adapt the single `backupStorage.endpoint` value (a full URL like `http://seaweedfs-s3.tenant-root.svc:8333`) to each consumer's contract: +Different operators expect different endpoint shapes; the strategy templates rendered by `backupstrategy-controller` resolve one S3 endpoint (via the `backupstrategy-controller.endpoint` helper) and adapt it to each consumer's contract. For a provisioned bucket (`provisionBucket: true`, the default) the endpoint is **derived from the COSI bucket's system-credentials Secret** (`backupStorage.systemSecretName`) and forced to `https://` — the external S3 ingress with an ACME cert, the only endpoint the backup operators can verify. SeaweedFS's in-cluster S3 serves TLS on `:8333` behind the self-signed "SeaweedFS CA", and the Etcd Strategy S3 schema has no `caCert` field, so the in-cluster endpoint cannot be targeted directly. The `backupStorage.endpoint` chart value (a full URL like `http://seaweedfs-s3.tenant-root.svc:8333`) is the **fallback**, used for external S3 (`provisionBucket: false`) and for offline `helm template`/pre-reconcile renders where the Secret lookup returns nothing. The resolved endpoint is adapted per consumer: | Driver | Strategy template field | Form | |--------|-------------------------|------| | CNPG (Postgres) | `barmanObjectStore.endpointURL` | full URL (scheme preserved) | | Etcd | `destination.s3.endpoint` | full URL (scheme preserved) | | MariaDB | `storage.s3.endpoint` | bare host:port (scheme stripped); `tls.enabled` derived from the scheme | +| MongoDB | `s3.endpointUrl` | full URL (scheme preserved); on the default flow the app's `backup.endpointURL`, on `useSystemBucket` the strategy coordinate injected by the driver | | FoundationDB | `blobStoreConfiguration.accountName` + `urlParameters.secure_connection` | bare host:port + derived secure flag | | Velero | `BackupStorageLocation.spec.config.s3Url` | full URL (scheme preserved) | | ClickHouse sidecar | `S3_ENDPOINT` env | bare host:port (from projected Secret) | +| Redis (dump Job) | `S3_ENDPOINT` env | bare host:port (from projected Secret); `https://` prepended when unscheme'd | +| Kafka | `S3_ENDPOINT` env on the strategy Job | full URL (scheme preserved); the Job's `curl --aws-sigv4` prepends `https://` if the endpoint carries no scheme, honours `backupStorage.forcePathStyle`, and (for the image's curl 7.76.1) signs a portless URL with `--connect-to` to the real port so a ported endpoint still verifies | -The projected `cozy-backups-creds.endpoint` key is **stripped of scheme** so chart-emitted sidecars (ClickHouse) consume it directly. Drivers that need the full URL pull from `backupStorage.endpoint` in chart values, not from the Secret. +The projected `cozy-backups-creds.endpoint` key is **stripped of scheme** so chart-emitted sidecars (ClickHouse) consume it directly. Drivers that need the full URL receive the resolved endpoint described above — derived from the COSI system Secret (forced `https://`) for a provisioned bucket, or the `backupStorage.endpoint` fallback for external S3. VM-driven (Velero) backups land in the same `cozy-backups` bucket under the `velero/` prefix. A `BackupStorageLocation` named `cozy-default` is shipped by the `backupstrategy-controller` chart (`packages/system/backupstrategy-controller/templates/velero-bsl.yaml`) so endpoint/bucket/region come from the same `backupStorage` values block used by Strategy CRs and the projector. @@ -46,7 +56,34 @@ VM-driven (Velero) backups land in the same `cozy-backups` bucket under the `vel The strategy CR `cozy-default-foundationdb` is shipped, but it is **not** bound by `cozy-default` yet. Restore runs `fdbrestore` from inside the `cozy-foundationdb-operator` Deployment, which does not yet mount `cozy-backups-creds`. Until the operator deployment is updated to mount the projected Secret, FDB platform-default restore silently fails — admins who need it today should keep using a per-app `Bucket` plus a custom `BackupClass`, or wire the credentials file into the operator deployment themselves. -**Cleanup gotcha (zombie backup_agent).** Unlike CNPG/MariaDB/Altinity (one-shot operator-side Backup CRs), the FoundationDB driver creates a `foundationdb.org/FoundationDBBackup` CR that drives a **long-lived** `backup_agent` Deployment streaming continuously to S3. Deleting a Cozystack `Backup` (e.g. via retention sweeping) does NOT stop that Deployment — the agent keeps writing until the next BackupJob's `stopOtherFoundationDBBackups` call swaps it out, until an admin invokes `examples/backups/foundationdb/cleanup.sh`, or until the operator-side CR is deleted by hand. If a tenant deletes their last Cozystack Backup and never submits another BackupJob, the agent pods will continue running indefinitely and accumulate S3 PUTs. This is intentional today (the driver has no RBAC verb to stop the operator-side CR on Cozystack-Backup deletion) but admins should be aware of it. +**Cleanup gotcha (zombie backup_agent).** Unlike CNPG/MariaDB/Altinity (one-shot operator-side Backup CRs), the FoundationDB driver creates an `apps.foundationdb.org/FoundationDBBackup` CR that drives a **long-lived** `backup_agent` Deployment streaming continuously to S3. Deleting a Cozystack `Backup` (e.g. via retention sweeping) does NOT stop that Deployment — the agent keeps writing until the next BackupJob's `stopOtherFoundationDBBackups` call swaps it out, until an admin invokes `examples/backups/foundationdb/cleanup.sh`, or until the operator-side CR is deleted by hand. If a tenant deletes their last Cozystack Backup and never submits another BackupJob, the agent pods will continue running indefinitely and accumulate S3 PUTs. This is intentional today (the driver has no RBAC verb to stop the operator-side CR on Cozystack-Backup deletion) but admins should be aware of it. + +## Postgres and MariaDB: a restored copy converges its application passwords shortly after recovery + +Managed Postgres no longer accepts a plaintext `users[].password`; every application-user password is chart-generated into `-credentials` and read only from there. The CNPG backup snapshot therefore carries no passwords — deliberate, since a snapshot that did was the one place a tenant password sat in cleartext. + +**Residue from before the upgrade — this is forward-looking, not retroactive.** The removal scrubs new renders and new snapshots; it does not rewrite copies made earlier, and upgrading preserves the live password (the chart reads the existing `-credentials` through `lookup`), so those older copies still expose the *current* credential. Two sinks retained it: a Postgres `Backup` object created before this release snapshotted `app.spec.users` while that spec still carried a plaintext `password`, so its `status.underlyingResources` held that password in cleartext — a field any tenant with `get backups` in the namespace can read (see the `underlyingResources` caution below); and every pre-upgrade Helm revision of `-init-script` rendered `ALTER ROLE … WITH PASSWORD ''` into its manifest, retained for `MaxHistory` revisions. There is no in-band rotation to invalidate the exposed value (chart-based rotation was intentionally not added — a chart-rendered Secret cannot revoke a leaked credential; rotation is being reworked as a controller, cozystack/community#72). The backup strategy controller removes `users.*.password` from every CNPG `Backup` snapshot each time its leader starts, so the first sink is cleaned in-cluster after the upgrade, including a `Backup` written by a pre-upgrade replica during the rollout; the start-up log line `legacy snapshot password scrub finished` reports how many it changed. That does not reach copies made outside the cluster — apiserver audit logs, etcd backups, or object-level exports of the `Backup` resources — and the old init-script Secret persists in Helm history until those revisions age out of `MaxHistory`. So after upgrading, treat any password that was ever set through `users[].password` as exposed until it is rotated. + +The consequence for a **restore into a copy** (a differently-named target): the recovered database keeps the source roles with their source password hashes, but the target chart generates fresh random passwords into the target's `-credentials`, and the role-reconciling init-job is skipped while `bootstrap.enabled` is set on the restored app. The restore driver clears `bootstrap.enabled` automatically once recovery is healthy, so the next HelmRelease reconcile runs the init-job, which applies `ALTER ROLE … WITH PASSWORD` for every `spec.users` entry and converges the recovered roles onto the target Secret. Convergence is therefore automatic but not instant: between the RestoreJob reaching `Succeeded` and that init-job completing, the passwords advertised in `-credentials` do not yet match the recovered roles, so a client reading that Secret cannot log in as an application user for that brief window. The chart flags the Secret `postgres.cozystack.io/credentials-pending: "true"` and the NOTES print the same caution while bootstrap is still on, so a reader does not treat the advertised password as usable mid-restore; both clear on their own once the driver clears bootstrap. The data is intact and reachable as the CNPG superuser throughout. After a to-copy restore, wait for the application login to succeed rather than assuming the advertised password works the instant the RestoreJob succeeds. The driver clears `bootstrap.enabled` by writing it onto the Postgres app's own spec, so a GitOps source (Argo CD, Flux, Terraform) that pins `bootstrap.enabled: true` for that app will revert the clear on its next apply and the copy's credentials never converge — exclude the restored app from such a source until convergence completes, or clear `bootstrap.enabled` in the source itself. If the driver cannot clear the flag at all (the app was deleted or renamed mid-restore), the RestoreJob fails with reason `BootstrapDisableFailed` once its deadline elapses rather than sitting `Running` forever with no signal. An in-place restore is unaffected: the release keeps its existing `-credentials`, whose passwords already match the roles the recovery brings back. One-time migration: an app restored to a copy under an earlier release — before this auto-clear existed — may still be sitting with `bootstrap.enabled` set and its init-job permanently skipped; the driver clears the flag only for a restore it drives to completion, never retroactively for one that already finished, so such an app keeps advertising unconverged credentials until you clear `bootstrap.enabled` on it once by hand. + +Databases and users after a Postgres restore, in place or into a copy: the driver sets the app's `spec.databases` and `spec.users` to the snapshot taken when the Backup was created, while recovery replays WAL past that point (to the end of the archive when no `recoveryTime` is set), so the recovered cluster can hold databases and users the snapshot does not list. On the first init-job run after the restore, a database the chart manages but the restored spec does not declare is released instead of dropped: its data stays, and it and its `_admin` and `_readonly` roles lose the `managed by helm` comment, so the chart no longer grants on it or drops it. Add it to `databases` to manage it again. Users are not released: a recovered user the restored spec does not declare, including a source user the target of a copy does not declare, is dropped on that run, so a client still logging in as that user loses the login. Every object it owned is reassigned to `cozystack:orphaned`, a role that cannot log in and that the chart grants nothing. Functions it owns are switched to `SECURITY INVOKER`, and on PostgreSQL 15 and later its views to `security_invoker`. A materialized view, a rule, and a view on PostgreSQL 13 or 14 have no such switch and keep reading with the rights of `cozystack:orphaned`. The user's grants and memberships do not carry over, so a view or function of that user can lose access it had through them. Declare the user in `users` to keep it; the chart then generates a new password for it into `-credentials`. + +**Objects reassigned by earlier releases stay as they were.** Before this change the chart reassigned a dropped user's or role's objects to the `postgres` superuser, and upgrading does not move them. After upgrading, run the query below in each database: it lists the relations and routines `postgres` owns outside the system catalog and extensions, leaving out a sequence that belongs to a column, which moves with its table. Of those, the chart itself owns only its `auto_grant_schema_privileges()` event-trigger function, so anything else came from a tenant role or was created by hand. Hand what a tenant role created to `cozystack:orphaned` with the matching `ALTER TABLE`, `ALTER VIEW`, `ALTER ROUTINE` or other `ALTER … OWNER TO "cozystack:orphaned"`; the next init-job run switches its functions and, on PostgreSQL 15 and later, its views as above. Handle first the rows with `security_definer` set, and views, materialized views and tables with rules, since those run with their owner's rights: + +```sql +SELECT c.oid::regclass::text AS object, NULL::boolean AS security_definer FROM pg_class c +WHERE c.relowner = 'postgres'::regrole AND c.relkind NOT IN ('i', 'I', 't') + AND c.relnamespace NOT IN ('pg_catalog'::regnamespace, 'information_schema'::regnamespace) + AND NOT EXISTS (SELECT FROM pg_depend d WHERE d.classid = 'pg_class'::regclass AND d.objid = c.oid + AND (d.deptype = 'e' OR (c.relkind = 'S' AND d.refclassid = 'pg_class'::regclass AND d.deptype IN ('a', 'i')))) +UNION ALL +SELECT p.oid::regprocedure::text, p.prosecdef FROM pg_proc p +WHERE p.proowner = 'postgres'::regrole + AND p.pronamespace NOT IN ('pg_catalog'::regnamespace, 'information_schema'::regnamespace) + AND NOT EXISTS (SELECT FROM pg_depend d WHERE d.classid = 'pg_proc'::regclass AND d.objid = p.oid AND d.deptype = 'e'); +``` + +MariaDB avoids the same problem at backup time: the driver sets `ignoreGlobalPriv` on every operator `Backup` it creates, so `mysql.global_priv` — the grant table holding every user's password hash, root included — is excluded from the logical dump and a restore into a copy never overwrites the target's chart-managed users. The same exclusion is also a limitation, and it applies to every restore, not only a to-copy one: because the grant table is never in the dump, only accounts Cozystack declares through `User` CRs (the chart's `users` map) are recreated on restore — an account created out of band, directly in MySQL rather than through `users`, is in no backup, so a restore brings back its data and its `mysql.db` grants but not the account that owns them, and that login fails until the account is recreated. Cozystack treats the `User` CRs as the source of truth for accounts precisely so the grant table need not be, so keep managed accounts in `users` rather than creating them by hand. That matters most for `root`, which the operator only applies at datadir bootstrap via `rootPasswordSecretKeyRef` and could never repair afterwards; keeping the source's grants out of the dump is what lets the target's advertised `root` password stay correct through a restore. This is a **backup-side** fix, so it only protects backups captured from this release onward. A restore from a backup taken **before** this change still replays the source's `mysql.global_priv` and overwrites the target's application users and `root` with the source's hashes; after restoring such an older backup, treat the target's advertised credentials as unreliable and reset them — application users converge on the next reconcile through their `User` CRs, but `root` must be reset out of band (`ALTER USER 'root' … IDENTIFIED BY …` against the restored server) since the operator applies `rootPasswordSecretKeyRef` only at datadir bootstrap. That once-only write is also why the Secret itself now matters: `root` lives solely in `-credentials` with no plaintext escape hatch left in values, so if that Secret is lost after bootstrap — a tenant deletes it, or a partial Velero restore drops it — the advertised `root` password and the live server's diverge with no in-band way to reconcile them; back the Secret up alongside the data, and reset `root` out of band if it is ever lost. ## ClickHouse: opt-in to the system bucket @@ -66,6 +103,73 @@ When `useSystemBucket: true`: `s3Region`, `s3Bucket`, `endpoint`, `s3AccessKey`, `s3SecretKey`, and `s3CredentialsSecret` are ignored in this mode. +## MongoDB: backup storage + +MongoDB supports two storage flows. By default the S3 target lives on the app CR and the tenant supplies its own credentials; alternatively the tenant opts into the platform system bucket and the driver injects the storage onto the live cluster at backup time. + +### Default: the application owns the backup storage + +The Percona psmdb operator runs the percona-backup-mongodb (pbm) agents whenever the `PerconaServerMongoDB` cluster has `spec.backup.enabled: true` (a declared storage is not needed for the agents to start), and services a `PerconaServerMongoDBBackup` only once the storage it references by name is declared on the cluster. On the default flow the MongoDB application **opts into backups** and points at the bucket in its own chart values: + +```yaml +apiVersion: apps.cozystack.io/v1alpha1 +kind: MongoDB +metadata: + name: orders-db +spec: + backup: + enabled: true + destinationPath: "s3:///orders-db/" + endpointURL: "https://seaweedfs-s3.tenant-root:8333" + insecureSkipTLSVerify: true # self-signed in-cluster seaweedfs; psmdb s3 storage has no CA-bundle field + s3AccessKey: "" + s3SecretKey: "" +``` + +The chart declares that storage as `s3-storage`, which the `cozy-default-mongodb` strategy names (`storageName: s3-storage`, `type: logical`). The driver then drives on-demand backups and restores through the unified `BackupJob` / `RestoreJob` / `Plan` interface on top of the operator's storage config — adding restore-to-differently-named instances (via `PerconaServerMongoDBRestore.spec.backupSource`) and point-in-time recovery that the raw scheduled-task / `mongodump` paths do not offer. When a target cluster has backups disabled, the driver surfaces a clear `Ready=False` precondition on the BackupJob/RestoreJob rather than hanging until the deadline. + +For a to-copy restore the operator reads the dump from the **source** backup's bucket/endpoint. The driver re-points the restore at the **target** cluster's own S3 credentials only when the target declares a storage on that same bucket (the common case where both share one bucket), since the source release's Secret dies with it in a DR scenario; a system-bucket source keeps its projected `cozy-backups-creds` reference, which outlives the source app. A target whose storages are all on a different bucket gets no credential swap: the restore keeps the source's Secret reference and is held at `Ready=False RestoreCredentialsMissing` naming that Secret until it exists in the restore namespace, so a cross-bucket restore never runs with a credential for the wrong bucket. Re-create the source release's `-s3-creds` Secret by name to let it proceed. + +A full, scripted example (write a marker document, back up, restore to a copy, assert the round-trip while the source stays untouched) is in [`examples/backups/mongodb/`](../../examples/backups/mongodb/) — driven by `run-all.sh`. + +### Opt-in to the system bucket + +Unlike ClickHouse's sidecar, PBM takes the bucket/endpoint/prefix as static fields on the `PerconaServerMongoDB` CR, and the platform bucket name is only known at BackupJob time — so the chart cannot render them. Instead the `cozy-default-mongodb` driver injects the storage onto the live cluster. To back up to the platform bucket without supplying S3 credentials, set: + +```yaml +backup: + enabled: true + useSystemBucket: true +``` + +When `useSystemBucket: true`: + +- The chart-emitted `-s3-creds` Secret is no longer rendered, and the chart leaves `spec.backup.storages`, `spec.backup.tasks` and `spec.backup.pitr` unset on the `PerconaServerMongoDB`. +- On every BackupJob the driver SSA-injects the `s3-storage` entry from the strategy coordinates (bucket/endpoint from the platform system bucket, `credentialsSecret: cozy-backups-creds`, prefix `/`) under its own field manager, so a Flux re-render never reverts it and a later change to the coordinates is picked up on the next backup rather than frozen at the first. The backup itself is not started until the operator reports that it has pushed that `spec.backup` into pbm's own configuration (`status.observedGeneration` caught up with the cluster's generation and the `PBMReady` condition `True`): pbm writes to whatever its config last said, not to the spec, so a backup started before that push would land at the previous coordinates while its `Backup` recorded the new ones. The job waits at `Ready=False PerconaServerMongoDBPBMConfigPending` meanwhile, and fails at the backup deadline if the operator never gets there. +- The injection waits for the release to actually render the flag. The app's `backup.useSystemBucket` is a desired value (it reads `true` as soon as it is written, before helm-controller renders the revision), while the tenant's own storage, scheduled tasks and PITR stream from the previous render stay on the `PerconaServerMongoDB` until it does. As long as the cluster still carries `spec.backup.tasks` or `spec.backup.pitr`, the driver does not inject and holds the BackupJob at `Ready=False` (`PerconaServerMongoDBLegacyRender`), failing it at the backup deadline; a release wedged on an old revision therefore never has its nightly dump and oplog redirected into the shared bucket. `useSystemBucket: true` together with `enabled: false` is refused by the chart at render time; the schema carries the same rule as an `x-kubernetes-validations` marker, which the aggregated apiserver does not evaluate yet ([#2657](https://github.com/cozystack/cozystack/issues/2657)), so until then the render-time refusal is what stops such a release, and a release wedged on it shows up as a failed HelmRelease rather than a redirected backup. + +`destinationPath`, `endpointURL`, `insecureSkipTLSVerify`, `s3AccessKey` and `s3SecretKey` are ignored in this mode. + +Because the chart drops `spec.backup.tasks` and `spec.backup.pitr`, the psmdb operator's own scheduled backups and oplog PITR do **not** run on this flow — `backup.schedule` and `backup.retentionPolicy` stay in the schema for the default flow but have no effect here, and the `recoveryTime` PITR restores described below are not available. Migrate scheduled backups to a `backups.cozystack.io/Plan` against `cozy-default` instead. **Flipping this flag on a running app** removes the storage, tasks and pitr the app had, so a release that was taking scheduled backups + PITR before the flip stops until a `Plan` takes over. + +Dropping `spec.backup.tasks` also drops the psmdb `keep` retention it carried, so on this flow **the driver owns the archive** the way the Rabbitmq/Redis drivers own theirs (an on-demand `PerconaServerMongoDBBackup` is not covered by psmdb task retention). The driver stamps the operator's `percona.com/delete-backup` finalizer onto each backup it mints against a storage that carries the platform coordinates (the strategy's bucket and `credentialsSecret`, as read back from the live cluster after the injection; a strategy without an `s3` block, or a storage pointing anywhere else, holds the job at `Ready=False` instead of minting an unowned dump), and deleting a Cozystack `Backup` (by hand today; by platform retention once it lands) deletes that `PerconaServerMongoDBBackup` CR — whose finalizer removes the pbm object from the shared bucket before the `Backup` is released, so deleting `Backup` objects is what reclaims `cozy-backups`. A legacy backup writes to the tenant's own bucket and keeps the pre-existing hands-off contract (its lifecycle stays the tenant's / operator's; nothing is deleted on `Backup` removal). Note this prunes only when a `Backup` is deleted: unlike CNPG, whose shipped strategy sets a `retentionPolicy` the engine trims on, MongoDB has no engine-side scheduled trim on this flow, so keeping `cozy-backups` bounded depends on a `Plan`'s retention actually deleting old `Backup` objects. **A `Plan` has no retention field yet** (`PlanSpec` carries only `applicationRef`, `backupClassName` and `schedule`), so today a nightly `useSystemBucket` Plan grows the shared bucket unbounded — `schedule` and `retentionPolicy` on the app are inert here (see [Opt-in to the system bucket](#opt-in-to-the-system-bucket)), and platform-level retention is tracked in the [`BackupRetentionPolicy` design (#4192)](https://github.com/cozystack/cozystack/pull/4192). Teardown GC of orphaned `Backup`s / S3 data is [#3966](https://github.com/cozystack/cozystack/issues/3966). Until those land, bound `cozy-backups` with an S3 lifecycle policy on the object store. Two cases release the `Backup` without waiting for the object to be pruned, so a wedged prune never pins a namespace in `Terminating`: a terminating namespace (where the operator's delete-backup step cannot run) and the `backups.cozystack.io/skip-artifact-cleanup: "true"` annotation. In both the driver strips its own finalizer and leaves the object for a bucket lifecycle policy to reclaim. A down operator or unreachable storage in a live namespace is NOT released automatically — the `Backup` stays `Terminating` as a visible signal until the operator recovers or you set that annotation to force the release. + +A dump that has started streaming on this flow is not cancelled on the 30-minute backup deadline (a real dataset routinely outruns it, and deleting the CR mid-stream would prune a partial), but it is not unbounded either: after 24 hours in `running` the `BackupJob` fails with a `BackupAbandoned` warning naming the `PerconaServerMongoDBBackup`, which is left in place. If that dump later completes, its archive is reachable only through that CR; delete the CR to prune it. + +**Opting back out** (`useSystemBucket` true→false) is not automatic in reverse: the driver stops injecting, but the `s3-storage` entry it already applied stays on the live `PerconaServerMongoDB` under the driver's own field manager (`cozystack-psmdb-backup-driver`) — the chart does not own it and so never prunes it. It is inert once `backup.enabled=false`; to fully return a release to the legacy flow, delete that storage entry from the `PerconaServerMongoDB` by hand before re-declaring the tenant's own `backup.*` S3 values. Until it is removed, a new `BackupJob` on that release is refused rather than written into the platform bucket unowned: the driver recognises the injected entry by its `credentialsSecret` and holds the job at `Ready=False` (`PerconaServerMongoDBStorageStale`) naming that manual step, failing it at the backup deadline. A backup already streaming when the flag was flipped is unaffected — its flow is fixed by the finalizer on its own operator CR. + +### Point-in-time recovery (MongoDB) + +psmdb records an oplog stream between logical backups. A MongoDB `RestoreJob` recovers to a timestamp via `spec.options.recoveryTime` — the same option name and RFC3339 format the Postgres/CNPG driver uses, so the two are uniform: + +```yaml +spec: + options: + recoveryTime: "2026-08-05T12:34:56Z" # RFC3339 (UTC), same as the CNPG driver +``` + +The driver converts `recoveryTime` to the psmdb oplog target internally. Unlike CNPG (whose empty `recoveryTime` replays WAL to the latest archived point), an empty `recoveryTime` here restores the backup snapshot as taken — a psmdb logical backup is already a consistent point, so "restore this backup" is the safe default. `spec.options.restoreTimeoutSeconds` caps how long the driver waits for the operator restore before failing (default 30m), matching the CNPG option. Any unrecognised key under `spec.options` (e.g. a `recoverytime` typo) is ignored but surfaced as a `UnknownRestoreOption` Warning event on the RestoreJob rather than silently dropped. + ## Inspecting the defaults ```bash @@ -76,26 +180,75 @@ kubectl -n tenant-root get secret bucket-cozy-backups-system-credentials kubectl -n cozy-velero get backupstoragelocation cozy-default ``` -The bucket lives in `tenant-root` and is provisioned through the `apps.cozystack.io/Bucket` CR. The system-managed credentials Secret never leaves that namespace. The backupstrategy-controller projects a copy under the name `cozy-backups-creds` into a tenant namespace right before each BackupJob runs, and refreshes the same Secret in `cozy-velero` (and any other namespace listed in `backupStorage.systemNamespaces`) on a 1-minute tick. The projected Secret carries multiple key formats so each driver finds what it needs in one place: +The bucket lives in `tenant-root` and is provisioned through the `apps.cozystack.io/Bucket` CR. The system-managed credentials Secret never leaves that namespace. The backupstrategy-controller projects a copy under the name `cozy-backups-creds` into a tenant namespace right before each BackupJob or RestoreJob runs, and refreshes the same Secret in `cozy-velero` (and any other namespace listed in `backupStorage.systemNamespaces`) on a 1-minute tick. The projected Secret carries multiple key formats so each driver finds what it needs in one place: | Key | Consumer | |-----------------------------------------------|-------------------------------------------| -| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | CNPG, MariaDB, Etcd | +| `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` | CNPG, MariaDB, Etcd, RabbitMQ, Redis | | `accessKey` / `secretKey` (plus `bucketName`, `endpoint`, `region`) | ClickHouse sidecar | | `cloud` | Velero (AWS credentials file format) | | `blob_credentials.json` | FoundationDB backup_agent | +The Redis dump Job additionally reads the `endpoint`, `bucketName`, and `region` keys (alongside the `AWS_*` pair) via `secretKeyRef`, the same set the ClickHouse sidecar consumes. + +The dump Job also speaks to the Redis app's own Sentinel and master, which is a separate TLS axis from the S3 endpoint above. When the `Redis` app has `tls.enabled`, the operator moves both to a TLS-only listener and publishes the CA at `redis-.ca-cert`; the Job mounts that Secret optionally and connects with `--tls --cacert` exactly when its `ca.crt` is present, so a plaintext (non-TLS) app is unaffected. This covers the default `tls.authClients: no`. Mutual TLS (`authClients: yes`) is not supported for backup — it requires a client certificate signed by the release CA, which the platform does not issue. + ### Bootstrap window -On a fresh-cluster install, the Velero `BackupStorageLocation` `cozy-default` is rendered before the credentials projector has had a chance to copy `cozy-backups-creds` into `cozy-velero`. The BSL reports `Unavailable` until the projector's first synchronous round completes (which happens immediately when the `backupstrategy-controller` Pod becomes Ready — typically tens of seconds after `helm install` returns, not minutes). Velero rejects new `Backup` AND `Restore` requests against `storageLocation: cozy-default` during that window. Plan VM backup automation accordingly, or wait for `kubectl -n cozy-velero get bsl cozy-default -o jsonpath='{.status.phase}' = Available` before submitting backups. +On a fresh-cluster install, the Velero `BackupStorageLocation` `cozy-default` is rendered before the credentials projector has had a chance to copy `cozy-backups-creds` into `cozy-velero`. The BSL reports `Unavailable` until the projector's first synchronous round completes (which runs as soon as the `backupstrategy-controller` acquires leadership — in practice moments after the Pod becomes Ready, typically tens of seconds after `helm install` returns, not minutes). Velero rejects new `Backup` AND `Restore` requests against `storageLocation: cozy-default` during that window. Plan VM backup automation accordingly, or wait for the BSL to become ready before submitting backups: `kubectl -n cozy-velero wait backupstoragelocation cozy-default --for=jsonpath='{.status.phase}'=Available --timeout=5m`. **Note on controller restarts.** The BSL flickers `Unavailable` on every `backupstrategy-controller` pod restart while the projector replays its first synchronous round. The window is short (single-digit seconds) but operators who alert on BSL availability should suppress alerts during the controller's `kube_pod_container_status_restarts_total{container=backupstrategy-controller}` events or use a longer evaluation window than the projector tick (60s). ### Cozy-default Bucket bootstrap -`cozy-default` ships an `apps.cozystack.io/Bucket cozy-backups` CR in `tenant-root`, which the bucket-application chart turns into a `BucketClaim`; the COSI driver then assigns the real S3 bucket name and writes it to the BucketClaim's `.status.bucketName`. The strategy templates and the Velero BSL all read that real bucket name (Helm `lookup` against the BucketClaim). On a fresh install the BucketClaim takes a short reconcile cycle to populate its status — until it does, the strategy templates render empty and only the `Bucket` CR + `BackupClass` are present in the cluster. Flux re-renders the HelmRelease on its standard interval (default 10 minutes), at which point the populated BucketClaim status causes the missing strategy templates to materialise. +`cozy-default` ships an `apps.cozystack.io/Bucket cozy-backups` CR in `tenant-root`, which the bucket-application chart turns into a `BucketClaim`; the COSI driver then assigns the real S3 bucket name (`bucket-`, so it cannot be computed in advance) and writes it to the BucketClaim's `.status.bucketName`. The strategy templates and the Velero BSL all read that real bucket name (Helm `lookup` against the BucketClaim). On a fresh install the BucketClaim takes a reconcile cycle to populate its status — until it does, the strategy templates render empty and only the `Bucket` CR + `BackupClass` are present in the cluster. + +**That skip does not repair itself on a reconcile.** helm-controller re-renders a release only when its chart or values change; the interval reconcile is a no-op for a healthy release, and drift detection is off on operator-generated HelmReleases. A cluster that lost the race at install time therefore keeps `BackupClass cozy-default` with **no** `Strategy` CRs and **no** Velero BSL indefinitely — which also fail-closes the pre-adoption snapshot in the v1.6.0 etcd migration. + +The same trap sits one release earlier, on the Secret everything else depends on. `bucket-cozy-backups-system-credentials` is rendered by the `bucket-cozy-backups-system` release behind a `lookup` of the COSI Secret, and is skipped just as permanently when that lookup is empty. Without it the credentials projector has no source at all, so no strategy, no Velero and no v1.6.0 etcd migration can resolve — and the gate cannot even determine the bucket name, which it reads from that Secret. + +Convergence is driven instead by the controller's default-objects gate (`backupStorage.reconcileDefaultObjects`, on by default), which repairs both, in order. While the credentials Secret is absent or carries no bucket name, the gate stamps `reconcile.fluxcd.io/forceAt` + `requestedAt` on the **bucket's** `bucket--system` HelmRelease. Once the bucket name resolves it checks that every object `cozy-default` routes to exists and stamps the same pair on the **`backupstrategy-controller`** HelmRelease, forcing the real Helm upgrade that re-runs the lookups. The two are throttled independently (one force per 5 minutes each), so the second is not delayed by the first. Expect the objects within a minute or two of the bucket becoming ready. -If you need the BackupClass functional immediately (e.g. an e2e), trigger a Flux reconcile (`flux reconcile helmrelease backupstrategy-controller`) once you see `kubectl get bucketclaim -n tenant-root bucket-cozy-backups -o jsonpath='{.status.bucketName}'` non-empty. +Each default `Strategy` CR also waits for its own CRD. The chart ships the `strategy.backups.cozystack.io` CRDs as templates in the same release, and Helm resolves every object's kind before it applies any, so a CR rendered next to a CRD the apiserver does not serve yet would fail the whole release, CRDs included. Each strategy template therefore renders only once its CRD is `Established`. That holds the CRs back on the first render of every install, including an external-S3 one where the bucket name is known up front, and on the upgrade that first ships a new strategy kind: that revision creates the CRDs, and the gate's forced upgrade above renders the CRs. + +The gate does not force a suspended HelmRelease — helm-controller ignores both annotations while `spec.suspend` is true, so it logs the skip and leaves the force counter alone. A release left suspended (`cozyhr suspend`) is therefore never repaired until it is resumed. + +Watch it with: + +```bash +kubectl -n cozy-backup-controller logs deploy/backupstrategy-controller | grep default-objects-gate +``` + +#### Manual recovery on an affected cluster + +Only needed on a cluster running a version without the gate (or with `reconcileDefaultObjects: false`). A plain `flux reconcile helmrelease` does **nothing** here — it does not re-render. You need a forced upgrade, and **both** annotations: `forceAt` is what makes helm-controller run a real Helm upgrade, and it is only honoured together with `requestedAt`. + +First confirm the bucket name is actually resolvable — forcing before that just re-runs the same empty lookup: + +```bash +kubectl -n tenant-root get bucketclaim bucket-cozy-backups -o jsonpath='{.status.bucketName}' +``` + +Then force the two releases, **in this order**: + +```bash +# 1. The credentials Secret. It is rendered by the -system release, not by +# bucket-cozy-backups — easy to miss, and the projector (hence every +# strategy and Velero) has no source without it. +ts=$(date -u +%Y-%m-%dT%H:%M:%SZ) +kubectl -n tenant-root annotate helmrelease bucket-cozy-backups-system \ + reconcile.fluxcd.io/forceAt="$ts" reconcile.fluxcd.io/requestedAt="$ts" --overwrite +kubectl -n tenant-root get secret bucket-cozy-backups-system-credentials + +# 2. The Strategy CRs and the Velero BSL. +ts=$(date -u +%Y-%m-%dT%H:%M:%SZ) +kubectl -n cozy-backup-controller annotate helmrelease backupstrategy-controller \ + reconcile.fluxcd.io/forceAt="$ts" reconcile.fluxcd.io/requestedAt="$ts" --overwrite +kubectl get $(kubectl get crd -o name | grep strategy.backups.cozystack.io) 2>/dev/null +kubectl -n cozy-velero get backupstoragelocation cozy-default +``` + +The race is nested: step 2's `lookup` resolves from the BucketClaim status, but the projector — and the v1.6.0 etcd migration — read the Secret from step 1, so a cluster missing both needs both. ### Observability @@ -104,11 +257,17 @@ The credentials projector emits two Prometheus counters labelled by `namespace` - `cozystack_backup_credentials_projection_successes_total` - `cozystack_backup_credentials_projection_failures_total` -Alert on `rate(failures_total) > 0` or `absent_over_time(successes_total[10m])` to catch a stale BSL credential or a malformed source Secret without log scraping. +Alert on `rate(cozystack_backup_credentials_projection_failures_total[5m]) > 0` or `absent_over_time(cozystack_backup_credentials_projection_successes_total[10m])` to catch a stale BSL credential or a malformed source Secret without log scraping. + +The default-objects gate emits three more: + +- `cozystack_backup_default_objects_missing{backupclass="cozy-default"}` — how many of the objects the platform default backups depend on are absent: the credentials Secret, the Strategy CRs `cozy-default` routes to, and the Velero BSL. **This is the alert that would have caught the missing Strategy CRs**: it is non-zero whatever the HelmRelease's `Ready` condition says. Alert on `min_over_time(cozystack_backup_default_objects_missing[15m]) > 0`. +- `cozystack_backup_default_objects_check_errors_total{backupclass="cozy-default"}` — checks that could not reach a conclusion (an API error reading the source Secret, the BackupClass, or one of the routed objects). The gauge above is deliberately **not** written on those ticks, so that it does not flap on a transient API error — which means that while this counter climbs, the gauge is stale and a `0` on it proves nothing. Pair the two: `min_over_time(cozystack_backup_default_objects_missing[15m]) > 0 or rate(cozystack_backup_default_objects_check_errors_total[15m]) > 0`. +- `cozystack_backup_default_objects_force_reconciles_total{namespace,name}` — forced Helm upgrades issued, labelled by the release forced (this chart's own, or the bucket's `-system` release). A counter that keeps climbing means the forced render is not producing the objects (a missing CRD, for instance), which is a different problem from the install-time race. A suspended release is skipped before the patch and is **not** counted here, so a paused release cannot masquerade as a render that keeps failing — look for the `skipped forcing a suspended HelmRelease` log line instead. ## Admin overrides for `cozy-default` -`cozy-default` is rendered by the `backupstrategy-controller` chart and owned by Flux's helm-controller. **Direct `kubectl edit backupclass cozy-default` is overwritten on the next helm reconcile** — the same applies to its companion `strategy.backups.cozystack.io/*` CRs (`cozy-default-cnpg`, `cozy-default-etcd`, `cozy-default-mariadb`, `cozy-default-altinity`, `cozy-default-foundationdb`, the two `cozy-default-velero-*`). The supported override path is the cozystack `Package` CR, which lets admins inject Helm values into platform components: +`cozy-default` is rendered by the `backupstrategy-controller` chart and owned by Flux's helm-controller. **Direct `kubectl edit backupclass cozy-default` is overwritten on the next helm reconcile** — the same applies to its companion `strategy.backups.cozystack.io/*` CRs (`cozy-default-cnpg`, `cozy-default-etcd`, `cozy-default-mariadb`, `cozy-default-altinity`, `cozy-default-mongodb`, `cozy-default-foundationdb`, `cozy-default-rabbitmq`, `cozy-default-redis`, the two `cozy-default-velero-*`). The supported override path is the `backupStorage` block on the **`platform` component** of the `cozystack.cozystack-platform` Package CR: ```yaml apiVersion: cozystack.io/v1alpha1 @@ -117,7 +276,7 @@ metadata: name: cozystack.cozystack-platform spec: components: - backupstrategy-controller: + platform: values: backupStorage: provisionBucket: true # default; set false for external S3 @@ -130,12 +289,31 @@ spec: - cozy-velero ``` +The platform chart forwards this block into the child `Package cozystack.backupstrategy-controller` as `components.backupstrategy-controller.values.backupStorage` (`packages/core/platform/templates/bundles/system.yaml`), from where the cozystack operator merges it into the `backupstrategy-controller` HelmRelease over the chart defaults. Two paths that look plausible do **not** work: `spec.components.backupstrategy-controller` on the `cozystack.cozystack-platform` Package is silently ignored (the only component under that PackageSource is `platform`), and patching the child `Package cozystack.backupstrategy-controller` directly is reverted whenever the platform helm-reconcile re-renders it. + +A sibling `backupStrategyController` block on the same `platform` component is forwarded the same way, for the controller's own knobs rather than the bucket. The one an operator reaches for is `redisBackupResources`, which sizes the Redis strategy Pod — restore loads the whole dataset into a throwaway loader, so a large Redis needs more memory / ephemeral-storage than the defaults (the loader-not-ready log line points here): + +```yaml +spec: + components: + platform: + values: + backupStrategyController: + redisBackupResources: + limits: + memory: 8Gi + ephemeral-storage: 32Gi +``` + | Knob | Effect | |---|---| | `provisionBucket` | Toggle creation of the in-cluster `apps.cozystack.io/Bucket` CR. Set `false` for external S3 (see [Disabling the platform-managed bucket](#disabling-the-platform-managed-bucket)). | -| `bucketName` | K8s name of the Bucket CR + lookup key for the COSI BucketClaim. The actual S3 bucket name is the COSI-assigned UUID, surfaced through `BucketClaim.status.bucketName`. | +| `bucketName` | Two modes. With `provisionBucket: true` (default): K8s name of the Bucket CR + lookup key for the COSI BucketClaim — the actual S3 bucket name is the COSI-assigned UUID, surfaced through `BucketClaim.status.bucketName`. With `provisionBucket: false`: taken **verbatim as the real S3 bucket name** and baked into every strategy CR + the Velero BSL. | +| `namespace` | Namespace the Bucket CR (and its system-credentials Secret) lives in — `tenant-root` by default. Must be a tenant namespace (`tenant-*`): the Bucket chart's RBAC helper fails the Helm render for any other prefix. | | `bucketNameOverride` | Escape hatch for offline `helm template` renders — bypasses the live-cluster BucketClaim lookup. Leave empty in production. | -| `endpoint` | S3 endpoint baked into every default strategy CR + the Velero BSL. Switching to `https://` silently enables TLS in the MariaDB strategy — ensure the CA bundle is reachable to the relevant operator/driver Pods before flipping it. | +| `endpoint` | **Fallback** S3 endpoint. For a provisioned bucket the strategy CRs + Velero BSL derive the endpoint from the COSI system Secret (external ACME ingress, forced `https://`) instead; this value is used only for external S3 (`provisionBucket: false`) and offline renders. For external S3, switching it to `https://` enables TLS in the MariaDB/FoundationDB strategies, which derive TLS from the scheme. The Redis dump Job is not scheme-driven: the projector delivers a bare host and the script prepends `https://` unconditionally, so the Job always connects over TLS — point `endpointCASecretName` (below) at the private CA rather than relying on the scheme, and ensure that CA is reachable to the relevant Pods. | +| `endpointCASecretName` | Optional Secret (key `ca.crt`) in the app namespace the Redis Job trusts for a self-signed S3 endpoint. Empty by default: the projected endpoint is always `https://` and the platform bucket's ACME cert verifies against the image's system CA store unaided. Set it only for a private CA. The Job mounts it optionally (nothing projects this Secret automatically), so a name typo does not wedge the Pod on `FailedMount`: a missing `ca.crt` falls through to the system CA store and the Job fails fast at the TLS handshake with a legible error — it does not skip verification, which still needs the explicit `insecureSkipTLSVerify` opt-in. | +| `insecureSkipTLSVerify` | Disables S3 certificate verification for the Redis Job (`curl -k`). `false` by default and an explicit opt-in, never a fallback — an untrusted-cert endpoint fails closed unless this is set. Prefer `endpointCASecretName`. | | `region` | Re-projected into `cozy-backups-creds` on the next reconcile. Pod-restart required for chart-emitted clients consuming the region via env (ClickHouse sidecar today). | | `forcePathStyle` | Path-style addressing; SeaweedFS S3 requires it, AWS S3 typically doesn't. | | `systemSecretName` | Name of the human-friendly Secret produced by the Bucket app (or pre-created manually for external S3). The projector also accepts the raw COSI Secret format. | @@ -145,29 +323,42 @@ When the override needs to go beyond storage coordinates — different retention ## Tuning via a custom BackupClass -The defaults aim at a reasonable middle (30-day retention, gzip compression where applicable). To override for a specific tenant or workload, create your own `BackupClass` pointing at the same strategy CRs but with tweaked `parameters`, or a fresh strategy CR. Common knobs: +The defaults aim at a reasonable middle (gzip compression where applicable; engine-side retention only where the strategy carries it, which today is the CNPG `retentionPolicy: "30d"` and the two Velero `ttl: 720h`; every other shipped strategy, MongoDB included, sets none, so its archives on the shared bucket are bounded only by an S3 lifecycle policy or a manual `Backup` delete: a `Plan` only creates BackupJobs and never deletes a `Backup`). To override for a specific tenant or workload, create your own `BackupClass` pointing at the same strategy CRs but with tweaked `parameters`, or a fresh strategy CR. Common knobs: - **CNPG strategy**: `barmanObjectStore.retentionPolicy`, `data.compression`, `wal.compression`. - **MariaDB strategy**: `compression`, `maxRetention`, `databases[]`. -- **Altinity strategy**: tune the `clickhouse-backup` sidecar via `backup.*` values on the ClickHouse release; the strategy Pod is a thin HTTP client. +- **MongoDB strategy**: `storageName` (which `spec.backup.storages` entry on the psmdb cluster to use), `type` (`logical`), `compressionType` / `compressionLevel`, and — for the `useSystemBucket` flow — the `s3` coordinates the driver injects. On the default flow the S3 target is tuned via `backup.*` values on the MongoDB release instead (see [MongoDB: backup storage](#mongodb-backup-storage)). +- **Altinity strategy**: tune the `clickhouse-backup` sidecar via `backup.*` values on the ClickHouse release; the strategy Pod is a thin HTTP client. When the S3 endpoint's certificate is signed by a private CA rather than a publicly-trusted one — SeaweedFS's in-cluster `:8333` being the case in point — point `backup.endpointCA` at a Secret holding that CA bundle; the chart mounts it into the sidecar and adds it to the trust store via `SSL_CERT_DIR`, which supplements the system CA set rather than replacing it. - **FoundationDB strategy**: `snapshotPeriodSeconds`, `agentCount`, `urlParameters[]`. +- **Rabbitmq strategy**: `artifactURITemplate` (the `///definitions.json` object-key layout) and the pod `template` (image, resources). The backup is a definitions export over the management API — vhosts, users, permissions, queues, exchanges, bindings, policies, parameters — so message payloads are out of scope and there is no point-in-time recovery; a full message-data backup is a Velero volume snapshot instead. **Restore is a merge, not a reset**: importing definitions (`POST /api/definitions`) creates or updates what the export contains and never deletes, so anything created after the backup survives the restore and the broker is not returned to its exact backup-time state. A to-copy restore imports the source's users (with password hashes) into the target, so those source credentials become valid logins there. Unlike the operator-backed strategies (whose engine owns archive retention), the Rabbitmq driver owns its object outright and deletes it from the bucket when its `Backup` is deleted — via a one-shot Job the Backup's removal waits on — so a retention-pruned `Plan` does not normally accumulate objects. The delete is **best-effort**: when it cannot run from here the Backup is released without deleting the object (with a `Warning` Event naming what was left behind), so the Backup — and any namespace being torn down — never wedges. The named give-up conditions are: the namespace is terminating (`CREATE` is forbidden there), the strategy CR or the projected credentials are unavailable, the rendered Job name is invalid, or the skip annotation below is set. Otherwise a genuinely failing delete (object storage unreachable) keeps the `Backup` in `Terminating` (a visible signal to act on) and retries. To release such a stuck `Backup`, annotate it `backups.cozystack.io/skip-artifact-cleanup: "true"` — cleanup then skips the delete and lets the `Backup` go, leaving the object in the bucket to be reclaimed manually or by a bucket lifecycle policy (`kubectl annotate backup -n backups.cozystack.io/skip-artifact-cleanup=true`). - **Velero strategy (VMInstance / VMDisk)**: `ttl`, `includedResources[]`, `excludedResources[]`. -- **Etcd strategy**: today the strategy is path-only; combine with `Plan.spec.retentionPolicy` for trim cadence. +- **Etcd strategy**: today the strategy is path-only, and a `Plan` carries no retention field yet (see [#4192](https://github.com/cozystack/cozystack/pull/4192)), so nothing trims etcd snapshots on the shared bucket automatically either. The system-managed credentials Secret is the **only** way for in-cluster strategies to reach `cozy-backups`. Do not embed access keys in `BackupClass.parameters` — the security model relies on Secret references, and `parameters` end up in `Backup.status.underlyingResources`, which tenants can read. ## Disabling the platform-managed bucket -If a deployment runs against an external S3 (no SeaweedFS), set `backupStorage.provisionBucket: false` in the `backupstrategy-controller` values and create the source credentials Secret in `tenant-root` manually (flat-key format: `accessKey` / `secretKey` / `endpoint` / `bucketName`; or the raw COSI `BucketInfo` JSON). Update `backupStorage.endpoint`, `backupStorage.region`, and (for VM backups) the chart's Velero BSL settings to point at the external S3. +If a deployment runs against an external S3 (no SeaweedFS), set `backupStorage.provisionBucket: false` via the `platform` component override described above and create the source credentials Secret in `tenant-root` manually (flat-key format: `accessKey` / `secretKey` / `endpoint` / `bucketName`; or the raw COSI `BucketInfo` JSON). In the same `backupStorage` block, update `endpoint`, `region`, **and `bucketName`**: with `provisionBucket: false` the strategies and the Velero BSL take `bucketName` verbatim as the real S3 bucket name (no COSI lookup), so it must name the actual bucket on the external S3 — the `bucketName` key inside the Secret alone is not enough. The Velero `BackupStorageLocation` picks the same values up automatically (the chart renders it from the same `backupStorage` block), so no separate BSL configuration is needed. Note that disabling the cluster-default BSL itself (the chart's `velero.bslEnabled` value) is **not** carried by the `backupStorage` override path — the platform Package forwards only the `backupStorage` block. ## Upgrade notes from chart-managed backups -> **Postgres `backup.enabled: true` with placeholder credentials no longer renders `barmanObjectStore` on upgrade.** +> **Postgres backups now use the CloudNativePG Barman Cloud plugin, not native `spec.backup.barmanObjectStore`.** +> +> Postgres backups were migrated off the deprecated native `barmanObjectStore` (removed in CNPG 1.29) onto the [Barman Cloud plugin](https://github.com/cloudnative-pg/plugin-barman-cloud). The `barman-cloud-*` binaries the native path needs are absent from the `standard` image variant `keycloak-db` pins; `apps/postgres` pins bare (system-flavor) tags that still ship them — its breakage was the operator/CRD skew below, not missing binaries — but the system flavor is deprecated upstream, so both move to the plugin. The chart now renders `spec.plugins` on the `cnpg.io/Cluster` plus a `barmancloud.cnpg.io/ObjectStore` CR that carries the S3 configuration, and the CNPG backup driver SSA-applies an `ObjectStore` and patches `spec.plugins` (instead of `spec.backup.barmanObjectStore`) for the platform flow; `Backup`/`ScheduledBackup` use `method: plugin`. On `helm upgrade` a legacy chart-managed Cluster's `spec.backup.barmanObjectStore` is replaced by `spec.plugins` + an `ObjectStore`. This only works once the `plugin-barman-cloud` operator is installed and co-located with the CNPG operator in `cozy-postgres-operator` (shipped by the `postgres-operator` chart) and its cert-manager dependency is present — otherwise the Cluster references a plugin that is not registered and backups / WAL archiving stall. +> +> **Mixed-version window: BackupJobs fail until the app re-renders with the new chart.** A `BackupJob` that fires while a `Cluster` still carries the pre-upgrade chart's native `spec.backup.barmanObjectStore` is rejected by the CNPG admission webhook (`Cannot enable a WAL archiver plugin when barmanObjectStore is configured`) — the driver cannot attach `spec.plugins` alongside the native config. This resolves itself once Flux re-renders the Postgres app with the new chart (which replaces `barmanObjectStore` with `spec.plugins`); re-run the BackupJob afterward. Upgrading only the operators/driver without the app charts leaves every `backup.enabled: true` Postgres in this state permanently. +> +> **Postgres `backup.enabled: true` with placeholder credentials no longer wires backups on upgrade.** +> +> The pre-v1.5 defaults for `backup.s3AccessKey` / `backup.s3SecretKey` in `packages/apps/postgres/values.yaml` were the literal `""` / `""` placeholders, so the Postgres chart still rendered a backup block on the `cnpg.io/Cluster` (with junk credentials, WAL archiving failing at runtime). Starting with v1.5 those defaults are empty strings and the chart NO LONGER renders any backup wiring when the placeholders are unmodified. Tenants on the legacy chart-managed flow who relied on those placeholders see `spec.plugins` and the `ObjectStore` disappear from the live `Cluster` on `helm upgrade`. Action — pick one: +> +> - **Move to the platform flow (recommended).** Set `backup.useSystemBucket: true`; the chart leaves `spec.plugins` unset and the CNPG backup driver SSA-applies the `ObjectStore` and patches `spec.plugins` onto the live `Cluster` at first BackupJob time. No tenant-side keys required. +> - **Stay on the legacy chart-managed flow.** Supply real `backup.s3AccessKey` / `backup.s3SecretKey` (or a pre-existing `backup.s3CredentialsSecret.name`); the chart renders `spec.plugins` + the `ObjectStore` from those coordinates. > -> The pre-PR defaults for `backup.s3AccessKey` / `backup.s3SecretKey` in `packages/apps/postgres/values.yaml` were the literal `""` / `""` placeholders, so the Postgres chart still rendered `spec.backup.barmanObjectStore` on the `cnpg.io/Cluster` (with junk credentials, `archive_command` failing at runtime). After this PR those defaults are empty strings and the chart NO LONGER renders the backup block at all when the placeholders are unmodified. Tenants on the legacy chart-managed flow who relied on those placeholders see their `barmanObjectStore` disappear from the live `Cluster` on `helm upgrade`. Action — pick one: +> **Two platform-level changes ride along with this migration (they are required for the plugin to run):** > -> - **Move to the platform flow (recommended).** Set `backup.useSystemBucket: true`; the chart leaves `barmanObjectStore` unset and the CNPG backup driver SSA-patches it onto the live `Cluster` at first BackupJob time. No tenant-side keys required. -> - **Stay on the legacy chart-managed flow.** Supply real `backup.s3AccessKey` / `backup.s3SecretKey` (or a pre-existing `backup.s3CredentialsSecret.name`); the chart renders `barmanObjectStore` exactly as before. +> - **CloudNativePG operator bumped to 1.28.1** (`postgres-operator` chart). A minor operator upgrade for the whole fleet; the CNPG CRDs move with it. This also fixes a latent bug where the operator image was newer than its bundled CRDs (`status.instanceID.sessionID` pruned), which made the operator believe the instance manager restarted and fail **every** backup cluster-wide with `instance manager was restarted during backup`. The vendored chart ships the CNPG CRDs as ordinary release manifests (gated by `crds.create`), so a normal `helm upgrade` updates them together with the operator — the `helm.sh/resource-policy: keep` annotation only prevents their deletion, not updates. The historical skew came from the `image.tag` pin outrunning the vendored chart's CRDs, which this PR removes; no out-of-band CRD apply is needed unless the HelmRelease itself is held back. +> - **LINSTOR scheduler admission webhook must not strip native-sidecar fields.** Older `linstor-scheduler` extender builds (≤ v0.3.5's predecessors) decoded every pod through a stale vendored Kubernetes API and re-encoded it, stripping `initContainers[].restartPolicy` cluster-wide — which breaks the plugin's restartable barman sidecar ("startupProbe forbidden without restartPolicy=Always"). The platform already ships the fixed webhook (`linstor-scheduler` re-vendored to upstream chart 0.3.1 / extender v0.3.6, which stopped stripping unknown pod fields), so no action is needed on an up-to-date platform; just do not hold the `linstor-scheduler` package back on an older version when rolling out the plugin. > > The same `useSystemBucket` opt-in applies to ClickHouse — see [ClickHouse: opt-in to the system bucket](#clickhouse-opt-in-to-the-system-bucket). When `useSystemBucket: true` is set on ClickHouse, the legacy `-backup` CronJob, credential Secret, and backup script are no longer rendered (they are mutually exclusive with the platform flow); migrate scheduled backups to a `backups.cozystack.io/Plan` against `cozy-default`. @@ -188,3 +379,83 @@ spec: kind: Postgres name: orders-db ``` + +An application takes one restore at a time. A RestoreJob that targets an application another unfinished RestoreJob is already restoring fails with reason `ConcurrentRestore` before it changes anything, and its message names the job that goes first. It is not queued: once that job has finished, create a new RestoreJob if the restore is still wanted. A restore that has started always goes first. Between two that have not, the older one runs; creation times have one-second precision, so of two created in the same second, the one whose name sorts first runs. + +## Point-in-time recovery (PostgreSQL) + +A `RestoreJob` restores a `Postgres` application from a `Backup`. Omit `spec.options.recoveryTime` to recover to the latest point in the WAL archive; set it (RFC3339) to recover the database to an exact instant — a point-in-time recovery (PITR). The CNPG barman-cloud plugin restores a base backup and replays archived WAL from it: up to `recoveryTime` when one is set, so the restored cluster reflects the database as of that instant and later writes are absent, and to the end of the archive otherwise. + +The base backup a restore starts from is the one the `Backup` refers to: the driver passes its barman backup ID to CNPG as `bootstrap.recovery.recoveryTarget.backupID`. There are two exceptions, and for both the choice is left to the plugin. The first is a `recoveryTime` earlier than one second after that backup ended, which the backup cannot serve (recovery would have to stop before the backup is consistent); the plugin then takes the newest base backup that ended at or before `recoveryTime`. The second is a `Backup` whose `cnpg.io/Backup` no longer exists: retention (`barmanObjectStore.retentionPolicy`) deletes it together with the base backup it describes, so the archive can no longer start from that backup, and the `RestoreJob` records a `BaseBackupGone` warning event. A base backup removed from the bucket by anything else — a lifecycle rule, a manual delete — is still pinned for as long as its `cnpg.io/Backup` stays. The plugin deletes a `cnpg.io/Backup` whose base backup has left the catalog on its next maintenance pass (every `instanceSidecarConfiguration.retentionPolicyIntervalSeconds` of the `ObjectStore`, 30 minutes by default, retention policy or not), but only from the primary of the cluster that took it, running under the same UID. Until then a restore from it fails after the target has been purged, with `no backup found with ID` in the recovery log, and a retry fails the same way; deleting that `cnpg.io/Backup` by hand, which leaves the object store untouched, lets the next `RestoreJob` go through the plugin's choice. Left to itself for every restore, the plugin would start from the newest base backup in the archive whenever `recoveryTime` is empty, and from the newest one ending by `recoveryTime` otherwise, whichever `Backup` the `RestoreJob` names and on any timeline — see [Archives with several timelines](#archives-with-several-timelines) for why that matters. + +```yaml +apiVersion: backups.cozystack.io/v1alpha1 +kind: RestoreJob +metadata: + name: orders-db-pitr + namespace: tenant-acme +spec: + backupRef: + name: orders-db-adhoc # a completed Backup of the source + targetApplicationRef: # omit for a destructive in-place restore + apiGroup: apps.cozystack.io + kind: Postgres + name: orders-db-copy # a pre-deployed, empty Postgres app + options: + recoveryTime: "2026-07-21T09:30:00Z" +``` + +A full, scripted example (write a marker, capture the timestamp, restore to it, assert what survived) is in [`examples/backups/postgres/`](../../examples/backups/postgres/) — `45-restorejob-pitr.yaml`, driven by `run-all.sh`. + +### The recoverable window + +`recoveryTime` must fall inside the window the archive can reconstruct: + +- **Earliest** — the completion of the oldest base backup still in the archive. You cannot recover to an instant before the first base backup: WAL replay always starts from a base backup, and retention (`barmanObjectStore.retentionPolicy`) eventually trims the oldest ones together with the WAL that predates them. Aim at least one second after a backup's `status.stoppedAt`: `stoppedAt` is truncated to the second, while the plugin compares `recoveryTime` with the end time barman records in `backup.info`, to the microsecond, and keeps only a backup that ended at or before it. A `recoveryTime` equal to `stoppedAt` therefore falls before that backup's actual end: an older backup is used instead, and when there is none the restore fails with `no target backup found`. +- **Latest** — the last transaction, commit or abort, contained in the WAL shipped to object storage. It is not the newest WAL segment: a time-targeted recovery ends just before the first transaction recorded after `recoveryTime`, and a segment switch, a checkpoint or the end of a base backup is not one. A `recoveryTime` at or after the last archived transaction leaves PostgreSQL nothing to stop on. On a busy database the last transaction trails "now" by the archive lag; on a database at rest it is its last write, however recent the WAL archive and however recent the latest base backup. Once the source is deleted, the window is frozen at the last transaction that made it to S3 before it went away. + +A `recoveryTime` **at or after the last archived transaction cannot converge**: PostgreSQL replays every available WAL, never finds a transaction after the target, exits with `FATAL: recovery ended before configured recovery target was reached`, and CNPG re-creates the recovery instance in a loop. Such a wedged restore is failed by the restore deadline (`spec.options.restoreTimeoutSeconds`, default 30m) — and the driver reads the recovery pod's log at that point, so instead of a generic timeout the RestoreJob ends `status.phase: Failed` with conditions `RecoveryConverged=False` and `Ready=False`, both reason `RecoveryTargetUnreachable`, and a message naming the target. The driver deliberately does **not** fail earlier off that FATAL: a *reachable* near-now target hits the identical FATAL transiently — often repeatedly, on a slow archiver — before it converges, so an earlier trip would reject a recoverable restore. To reject an unreachable target quickly, set a short `restoreTimeoutSeconds`; otherwise pick a time before the last archived transaction. The state as of the last write needs no `recoveryTime` at all: omitting it replays the whole archive, and that is the state it recovers. + +Restoring to a **very recent** instant is safe as long as the database keeps writing and WAL archiving is current: the segment covering the target, or the transaction after it, may not be in object storage yet and the same FATAL fires transiently, but because the driver only fails at the deadline (not on that FATAL), the recovery has the whole window to catch up — the next attempt promotes once the WAL ships and the cluster goes healthy, which completes the restore. Only a target that never becomes reachable within the deadline is failed: on a database that has stopped writing, that is every target after its last transaction. If archiving is badly behind (a stalled `archive_command`, an overloaded cluster) a near-now target can exceed even the default 30m window; restore to a point you can confirm is followed by an archived transaction (see the discovery section below). + +A `recoveryTime` **before the earliest base backup** fails differently: PostgreSQL cannot begin replay before the base backup it restored, so recovery never reaches a consistent state and never emits the "recovery ended before …" FATAL the driver classifies on. That case also surfaces at the deadline, but with the generic reason `RestoreFailed` rather than `RecoveryTargetUnreachable`. Choosing a `recoveryTime` at least one second after the oldest base backup's `stoppedAt` (below) avoids it. + +### Discovering the earliest / latest restorable time + +CNPG's `Cluster.status.firstRecoverabilityPoint` exists in the status schema but is unreliable under the barman-cloud plugin — it was observed empty on every plugin-backed cluster in a test environment (CNPG 1.28.1) — so read the window from the backup catalog instead. Each completed base backup records its WAL range and timestamps on the underlying `cnpg.io/Backup`: + +```bash +# Completed base backups, oldest first: STOP plus one second is the earliest +# instant that backup alone can restore to; the oldest one bounds the window. +kubectl -n get backups.postgresql.cnpg.io \ + --sort-by=.status.stoppedAt \ + -o custom-columns=NAME:.metadata.name,PHASE:.status.phase,ID:.status.backupId,START:.status.startedAt,STOP:.status.stoppedAt,BEGINWAL:.status.beginWal,ENDWAL:.status.endWal + +# The cozystack Backup points at its underlying cnpg.io/Backup and S3 prefix. +kubectl -n get backup.backups.cozystack.io -o jsonpath='{.spec.driverMetadata}' +``` + +The upper bound is the timestamp of the last `COMMIT` or `ABORT` record in the archived WAL, which nothing in the cluster status reports. `pg_waldump` prints it from the newest segments under `//wals/`, working back from the newest segment until one holds such a record; on a database at rest that can be several segments back, because every segment switch and every base backup adds one with no transaction in it. Any `recoveryTime` before that timestamp is reachable; for the state as of that last transaction, omit `recoveryTime`. + +### Archives with several timelines + +A WAL archive can hold several PostgreSQL timelines under one `serverName`. Every failover starts one, and so did every in-place restore made before restored clusters archived to a fresh prefix (`bootstrap.newServerName`): those wrote the new timeline back into the prefix they had recovered from, often without its `.history` file, so the archive ends up with branches that are siblings rather than a single line of descent. + +Recovery follows `recovery_target_timeline: latest`, which CNPG leaves as PostgreSQL's default: starting from the base backup's timeline, PostgreSQL looks for the history file of the next timeline number, then the next, stops at the first one missing, and targets the last one it found, replaying only the segments on that line of descent. Without any history file that is the base backup's own timeline alone. When the timeline it targets does not descend from the base backup — a sibling branch whose history file happens to carry the next number — recovery does not fall back to another timeline: it stops with `FATAL: requested timeline N is not a child of this server's history` (or `... is not in this server's history`). Two consequences: + +- the last archived transaction that bounds a restore is the last one on the timeline that restore follows, not the newest one in the archive; +- a restore must start from a base backup on the branch it means to bring back. The plugin's own choice ignores timelines — CNPG passes no `targetTLI` — so the newest backup, or the newest one ending by `recoveryTime`, can lie on another branch and restore that branch's data instead. The driver pins the `Backup`'s own base backup with `recoveryTarget.backupID` for that reason; a `recoveryTime` earlier than that backup's end still goes through the plugin's time-based choice, and on such an archive can land on another branch. + +### Idempotency under GitOps + +An in-progress restore is safe to reconcile. The driver purges the target `Cluster` + PVCs exactly once per RestoreJob (guarded by the `TargetPurged` condition and by the `postgres.cozystack.io/restored-server-name` annotation, which identifies the RestoreJob whose purge produced the live recovery `Cluster`), suspends the target's HelmRelease across the purge so Flux cannot race the bootstrap swap, and resumes it once the recovery cluster is rendered. A Flux reconcile (or a controller restart) mid-restore therefore re-attaches to the recovering cluster rather than deleting it and starting over. + +## Kafka: topic metadata only + +The `cozy-default-kafka` strategy backs up **topic metadata**, not message data. Its Job talks to the Kafka Admin API (`kafka-topics --describe` on backup, `kafka-topics --create` + `kafka-configs --alter` on restore) and stores one small object per run at `s3://////kafka-metadata.txt`. It captures every non-internal topic's partition count, replication factor and non-default configs; message payloads, consumer-group offsets, ACLs, quotas and `KafkaUser`s are out of scope, and there is no point-in-time recovery. The driver gates each run on the Strimzi `Kafka` cluster reporting `Ready` and never mutates it. + +This is the "reconstruct a cluster's topic topology" flow: an in-place restore recreates dropped topics into the source, and a to-copy restore applies the source's topics onto a freshly-bootstrapped empty `Kafka` (partitions and configs preserved). Restore is additive at the topic-set level — it never deletes topics that exist live but not in the backup — but it does not silently accept a divergent existing topic: it grows the partition count when the backup asks for more, and fails loudly (naming the topic and both values) when the backup asks for fewer partitions or a different replication factor, since Kafka cannot shrink partitions or change RF in place and a restore that cannot reach the recorded state must not report `Succeeded`. Restored topics are created directly through the Admin API, so on the target they are **unmanaged** (no `KafkaTopic` CR); that is the accepted tradeoff of an Admin-API-only driver. Restoring a topic whose replication factor exceeds the target cluster's broker count fails at `--create`. Topic configs are restored **additively** (`kafka-configs --add-config`): a config the backup recorded is re-applied, but a config added to a live topic after the backup is not removed, so an in-place restore does not return a topic's config set to its exact backup-time state (partition count and replication factor are the only shape the restore reconciles or fails loudly on). + +One consequence of the Admin-API approach: if a topic is declared in the Kafka app's `spec.topics`, the Strimzi Topic Operator owns it and reconciles its config from the `KafkaTopic` CR, reverting any dynamic config an in-place restore sets that the CR does not list. So this strategy is durable for **out-of-band** topics (created directly, no CR); for CR-declared topics the `KafkaTopic` CR / GitOps is the source of truth. A to-copy restore onto a fresh empty cluster is unaffected. For message-data durability, use a volume-snapshot strategy instead. See `examples/backups/kafka-metadata/` for the end-to-end flow. + +Restore is not transactional: it applies topics one at a time as it reads the object, so a failure part-way through (an RF mismatch, an unreachable broker) leaves the topics already created behind — re-running the restore is safe (existing topics are reconciled, not recreated). Like the Rabbitmq driver, this one owns its object outright and deletes it from the bucket when its `Backup` is deleted — via a one-shot Job the Backup's removal waits on — so a retention-pruned `Plan` does not accumulate objects; the delete is **best-effort** with the same give-up conditions and `backups.cozystack.io/skip-artifact-cleanup` escape hatch documented for Rabbitmq above. One caveat specific to this driver: its image ships an older `curl` (7.76.1) whose `--aws-sigv4` omits a non-default port from the SigV4 canonical host, which every S3 backend then rejects with `403 SignatureDoesNotMatch` on a ported endpoint (fixed in curl 7.86.0). The strategy script works around it with `--connect-to` — it signs and sends a portless URL and redirects the connection to the real port — so a ported endpoint (the `backupStorage.endpoint` fallback `…:8333`, `provisionBucket: false`, or SeaweedFS with `s3.ingress.enabled: false`) works too; the sibling drivers run curl 8.x and need no such workaround. Export time scales with topic count — the backup spends two Kafka CLI invocations (each a fresh JVM, ~1.5–3s) per topic — so a cluster with many hundreds of topics takes tens of minutes; the run is bounded at the Job layer by `activeDeadlineSeconds` (default 30 minutes, from the same wait the readiness precondition uses), so a wedged or unschedulable run is failed by Kubernetes as `DeadlineExceeded` — the pod is killed before the script's final upload, so no object is left orphaned — rather than requeuing forever. A cluster whose export legitimately runs longer raises the bound with the `backupTimeout` BackupClass parameter (a Go duration, e.g. `2h`), which round-trips through the `Backup` so a restore is bounded by the same value. diff --git a/docs/operations/seaweedfs-431-rename-recovery.md b/docs/operations/seaweedfs-431-rename-recovery.md new file mode 100644 index 0000000000..f26a671ebc --- /dev/null +++ b/docs/operations/seaweedfs-431-rename-recovery.md @@ -0,0 +1,385 @@ +# SeaweedFS 4.31 rename — audit & recovery + +This runbook covers clusters affected by the SeaweedFS chart-rename regression introduced when the vendored chart was bumped from `4.0.405` to `4.31.0` (Cozystack v1.5.0). Use it to classify each tenant, and to recover the ones that need an operator before they can be upgraded. + +**Scope — the default instance name is assumed throughout.** The supported way to run SeaweedFS is the tenant module: the tenant chart creates the instance under the fixed name `seaweedfs` (`packages/apps/tenant/templates/seaweedfs.yaml` hardcodes it; a tenant only enables or disables the module). Every shell selector below assumes that name. The API does not yet enforce it, so an instance created directly against `seaweedfses.apps.cozystack.io` under another name can exist — the audit script still classifies it (its release is `-system`), but **do not run the shell loops here against it: escalate instead**, because the name-based `grep seaweedfs` filters cannot see claims whose names the chart truncated (instance names of roughly 30+ characters), and the chart guard's reconstruction does not cover the zone/pool volume components of long-named instances. One accepted limit applies even to default-named instances: a zone or pool **key** of roughly 40+ characters pushes `seaweedfs-system-volume-` past the chart's truncation limit and similarly out of the guard's reconstruction — do not use keys that long. + +## Background + +Before 4.31 the chart named workloads after the chart (`seaweedfs-*`), ignoring the release name. 4.31 names them after the release, and the data-plane HelmRelease is `-system`, so every StatefulSet wanted to become `seaweedfs-system-*`. StatefulSet names are immutable, so the upgrade could not rename in place — Helm stood up a second, duplicate set beside the running one. Depending on cluster size the duplicate either deadlocks or splits: + +- **D-wedged** — with as many nodes as master replicas, the new masters cannot schedule (hard pod anti-affinity against the old masters). The new set stays `Pending`/`CrashLoopBackOff`, the old set keeps serving. Usually no data at risk — but "usually" is not something a Helm render can verify (a duplicate that served writes and later crashed or was scaled down is indistinguishable from one that never started), so the chart refuses these too rather than adopt on an assumption. +- **D-split** — with more nodes than masters, the new (empty) set comes up. Both sets carry identical pod labels, so the `seaweedfs-s3` Service load-balances across them, and both filers write to the **same `seaweedfs-db` Postgres** metadata store while pointing at different volume servers. This is a data-integrity incident, not just a duplicate: reads of existing objects through the new endpoint miss, new writes land on empty volumes, and the two master sets hand out volume IDs from independent sequences into one shared metadata table. + +The fix pins `fullnameOverride: seaweedfs` in `system/seaweedfs` values, so workloads are always named after the chart, exactly as they were before 4.31. Upgrading past the bump therefore **adopts the running set and its volumes in place**. + +Two states cannot be adopted that way, and the charts refuse to render for both rather than guess: + +- A tenant installed **fresh on 1.5.x**, whose data was written under the release-based names and lives on `data1-seaweedfs-system-volume-*` PVCs. Pinning the chart name there would rename the workloads *away* from that data, and Helm cannot move data between PVCs. Re-bind its volumes (Step 2) before upgrading. +- A tenant where **both** naming generations exist. One of them is an empty duplicate and one holds the data — but nothing durable in the object graph says which. Claim timestamps are not evidence: Step 2's own re-bind deletes and recreates claims, so a tenant interrupted mid-recovery has a brand-new claim holding real data. StatefulSets are recreated by the adoption hook. `readyReplicas: 0` does not prove a duplicate never served. Rendering would adopt the chart-named set, so a wrong guess strands or destroys data. Step 1 classifies these with signals a template does not have; once the empty generation is deleted, exactly one remains and the render proceeds on its own. + +The **enforcing** guard lives in `system/seaweedfs` (`templates/naming-guard.yaml`) — the `-system` HelmRelease pulls that chart straight from a platform-managed ExternalArtifact, so a platform upgrade re-renders it directly and nothing else stands between the upgrade and the tenant's workloads; `extra/seaweedfs` carries a sibling copy so the refusal is also visible on the SeaweedFS application itself. + +A tenant upgrading **1.4.x straight to 1.6 never renames**, so it only ever has one generation and is unaffected by any of this. Duplicates exist only on tenants that passed through 1.5.x. + +## Step 0 — `seaweedfs-db` ownership check (read-only, do this FIRST) + +Unrelated to the rename, but it lands on the same upgrade and it destroys data rather than duplicating it, so clear it before anything else. + +The v1.5.0 db split moved the CNPG `Cluster/seaweedfs-db` — the filer metadata store, i.e. the index for every object in the tenant's S3 — out of the `-system` release into its own `-db` release. Migration 43 performs the hand-over: it re-owns the Cluster to `-db` and stamps `helm.sh/resource-policy: keep` so the `-system` upgrade, whose chart no longer renders the Cluster, does not delete it as a removed resource. **Migration 43 shipped comparing the owning release name against the literal `seaweedfs-system`**, so it only ever fired for an instance named `seaweedfs`. `SeaweedFS` is a user-creatable kind: an instance named `foo` is owned by `foo-system`, was skipped, and had its Cluster pruned — CNPG takes the PVC with it. + +The prune is not a one-shot. Helm computes deletions by diffing the **last deployed** revision against the new manifest, so a tenant whose `-system` last succeeded on a pre-split revision recomputes the same deletion on *every* upgrade attempt — including attempts that fail for unrelated reasons and never become the new deployed revision. Such a tenant re-deletes the Cluster each time `-db` recreates it. + +Migration 43 is fixed to match the `-system` suffix, and migration 53 re-runs the hand-over for clusters that already ran the hardcoded version. Both are pre-upgrade hooks, so they land before `-system` re-renders. Audit anyway — a Cluster already deleted cannot be recovered by either: + +```sh +kubectl get cluster.postgresql.cnpg.io -A \ + -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name,OWNER:.metadata.annotations.meta\.helm\.sh/release-name,KEEP:.metadata.annotations.helm\.sh/resource-policy' +``` + +Read the rows for `NAME=seaweedfs-db`: + +| OWNER | KEEP | Meaning | +|---|---|---| +| `-db` | `keep` | Handed over. Nothing to do. | +| `-db` | *(none)* | Installed fresh on ≥1.5.0. Safe: `-system` never rendered the Cluster, so it is not in that release's prune baseline. | +| `-system` | *(none)* | **At risk.** Migration 53 hands it over on the next upgrade. Do not reconcile `-system` before the migration runs. | +| *(no row at all)* | | **Already lost** — see below. | + +A tenant with a SeaweedFS instance but **no `seaweedfs-db` row** has already had its metadata deleted. Its S3 returns 500 and its objects are unreachable even though the volume PVCs still hold the bytes. Nothing in this runbook or in any migration can rebuild that index: the Cluster and its PVC are gone. Restore the `seaweedfs-db` Postgres from a backup, or treat the tenant's object storage as lost. Note that `-db` may report **Ready** while this is true — it rendered its Cluster successfully and a later `-system` prune removed it; Flux has not re-checked. Trust the `kubectl get cluster` output, not the HelmRelease status. + +## Step 1 — Audit the fleet (read-only) + +```sh +hack/seaweedfs-naming-audit.sh # whole cluster +hack/seaweedfs-naming-audit.sh tenant-foo # or named namespaces +``` + +**Read the exit code, not just the table.** The audit fails closed: any error it cannot interpret — a kubectl call that fails, a Helm release payload it cannot decode — makes it print `FATAL` and exit non-zero, and the table it printed up to that point is incomplete. A non-zero exit means you do not yet know the state of the fleet, so none of the steps below may be taken on the strength of it. Only a zero exit means the table is the whole answer; an empty table with exit 0 is a genuinely clean fleet. + +It mutates nothing. Earlier revisions of this runbook inlined the classification as a shell snippet here; it is a tested script now (`hack/seaweedfs-naming-audit.bats`), because it is what the chart's refusal hands you to and acting on it deletes PVCs. Two inline versions shipped wrong — one whose selector matched both generations at once and so inverted its own primary rule, one that could not see a long instance name at all — so it is not a snippet any more. + +It reports one class per SeaweedFS instance, matching exactly what the chart's guard decides: + +| CLASS | Meaning | Action | +|---|---|---| +| `L` | Only the chart-named generation. | None. The upgrade adopts it in place. | +| `S` | Only the release-named generation — installed fresh on 1.5.x, or a long instance name. | Step 2, before upgrading. | +| `MIXED` | Both generations. The chart **refuses**. | Below. | + +For `MIXED` it also names which generation is **original**, from two independent durable signals: the naming scheme revision 1 of the `-system` release was installed with, and each generation's **PersistentVolume** creation timestamps. It reads PV timestamps, never claim timestamps — Step 2 deletes each release-named claim and recreates it under the chart name against the same PV, so claim age is not durable and inverts for a tenant interrupted mid-re-bind. The direction rule is **relative, never a clock**: a generation is the candidate duplicate only when *every* one of its bound PVs is strictly newer than every bound PV of the other generation — the same precondition Step 2a enforces. A tenant interrupted mid-re-bind has both generations on original-vintage PVs, so their ranges **overlap** and the audit names no candidate: finish Step 2, do not run Step 2a. + +**Read the audit's own warning.** "Original" is not "the other one is empty", and the gap is exactly where it matters. A duplicate that never scheduled (safe to delete) and one that served writes and later crashed or was scaled down (holds unique objects, deleting destroys them) are **identical on every durable signal** — same revision-1 scheme, same `first_deployed` deltas. The audit narrows the question to one generation; it does not answer it. Before deleting anything, establish that the candidate is empty: + +Empty means: no volume files (`.dat`/`.idx`/`.vif`) in any data directory. Do **not** `kubectl exec` into the candidate's own pods to check — a wedged duplicate's pods never start, so a check that needs the pod running is unexecutable exactly for the class where it matters most. Mount the claims instead: + +```sh +ns= +# 1. Stop the candidate's workloads so its RWO claims can be mounted elsewhere. +# Reversible, and it is Step 3's first action anyway (Step 2a's for S-damaged). +# 2. A candidate claim still Pending has no PV and therefore no data — empty by +# construction; skip it. +# 3. Mount each BOUND candidate claim read-only in a scratch pod: +pvc= # e.g. data1-seaweedfs-system-volume-0; repeat per claim +kubectl -n "$ns" apply -f - < -- weed shell -c "volume.list" +``` + +If a candidate claim cannot be inspected, or the two views disagree, **stop and escalate**. Both generations holding real data is recoverable; deleting the wrong one is not. + +**D-wedged** — a duplicate that never scheduled — is `MIXED` too, and the chart refuses it like any other duplicate. On main it rendered through, because a duplicate reading `readyReplicas: 0` was taken as proof it never served. That is not proof: a duplicate that served writes and later crashed or was scaled down reads identically. Confirm emptiness as above, then remove it via Step 3 **before** upgrading. + +On a cluster with no spare nodes (nodes ≤ replicas) a wedged duplicate also blocks the adoption rollout even once the render passes: its pods are still *scheduled*, carry the same labels as the adopted set (including `app.kubernetes.io/instance`), and their hard pod anti-affinity keeps the adopted set's rolled pods from landing anywhere (observed as `seaweedfs-filer-1`/`seaweedfs-master-2` stuck Pending on `didn't match pod anti-affinity rules`, wedging the `-system` HelmRelease in upgrade/rollback loops). Step 3 removes it, which resolves that too. + +## Step 2 — `S` tenants: re-bind the volumes before upgrading + +The data is on `data1-seaweedfs-system-volume-N`; the fixed chart expects `data1-seaweedfs-volume-N`. Rather than copying objects, re-point the same PersistentVolume at a PVC with the new name. Nothing is written or moved; only the claim is renamed. Expect downtime for this tenant's S3. + +```sh +ns= + +# 1. Stop the operator from fighting the change. +app= # the SeaweedFS resource name; `seaweedfs` unless you renamed it +kubectl -n "$ns" patch helmrelease "${app}-system" --type merge -p '{"spec":{"suspend":true}}' +# Select the renamed workloads precisely. An S tenant has only the renamed set, +# but exclude the pre-4.31 chart-named seaweedfs-master/-filer/-volume[-] +# anyway so the same selector is reused in Step 3, where both sets coexist. +# seaweedfs-system-* and -system-seaweedfs-* are kept. +renamed_sts() { + kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs -o name \ + | sed 's|statefulset.apps/||' | grep -vE '^seaweedfs-(master|filer|volume)($|-)' +} +for sts in $(renamed_sts); do + kubectl -n "$ns" scale sts "$sts" --replicas=0 +done +kubectl -n "$ns" scale deploy -l app.kubernetes.io/name=seaweedfs,app.kubernetes.io/component=s3 --replicas=0 + +# 2. For every volume PVC: protect its PV, then re-bind it under the new name. +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' \ + | grep -E '^data1-.*volume' | grep seaweedfs | grep -vE '^data1-seaweedfs-volume'); do + # data1--volume[-]-N -> data1-seaweedfs-volume[-]-N + new_pvc=$(echo "$pvc" | sed -E 's/^data1-.*-volume-/data1-seaweedfs-volume-/') + pv=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.volumeName}') + sc=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.storageClassName}') + size=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.resources.requests.storage}') + # Stash the PV's own reclaim policy ON THE PV before changing it, so a volume the + # cluster deliberately set to Retain does not silently come back as Delete -- and + # so the record survives this loop being interrupted, which a shell variable would + # not. Step 5 restores it from the annotation once the tenant is verified healthy. + reclaim=$(kubectl get pv "$pv" -o jsonpath='{.spec.persistentVolumeReclaimPolicy}') + if [ -z "$(kubectl get pv "$pv" -o jsonpath='{.metadata.annotations.cozystack\.io/original-reclaim-policy}')" ]; then + kubectl annotate pv "$pv" "cozystack.io/original-reclaim-policy=${reclaim}" || { echo "FAILED to record the original reclaim policy on $pv; aborting before any PVC is deleted" >&2; break; } + fi + + # Keep the PV (and the data) when the claim goes away. + kubectl patch pv "$pv" -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' || { echo "FAILED to set Retain on $pv; aborting before deleting its PVC" >&2; break; } + kubectl -n "$ns" delete pvc "$pvc" + # A Released PV cannot be re-bound until its old claimRef is cleared. + kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' + + # The labels matter: the post-delete cleanup hook selects volume PVCs by + # app.kubernetes.io/name + instance, and StatefulSet reconciliation does NOT + # retrofit claim-template labels onto an existing PVC. A claim recreated without + # them is invisible to that hook forever, so a recovered tenant would leak its + # volumes on a later app deletion. + kubectl -n "$ns" apply -f - <-system-seaweedfs-volume-*` for an instance running under another name, because 4.31's name helper appends the chart name when the release name does not contain it. + +## Step 2a — `S-damaged` tenants: remove the empty chart-named set first + +An `S` tenant that an unguarded 1.6.0 upgrade already reached has an extra problem: the upgrade **created** chart-named workloads (`seaweedfs-master/-filer/-volume`, an s3 Deployment) and **empty** `data1-seaweedfs-volume-*` PVCs beside the live renamed set, and deleted the renamed `-s3` Service (only `seaweedfs-s3` remains — its endpoints may still resolve to the renamed set's pods because both sets carry identical labels, which is luck, not design: if the chart-named pods ever become Ready, the Service splits reads across a live set and an empty one). + +> **STOP if you are resuming an interrupted Step 2.** This step deletes every `data1-seaweedfs-volume-*` claim. If Step 2 already re-bound some of them, those claims hold your data and are bound to the original PVs — deleting them is the data-loss path this step used to be reachable through by misclassification. Finish Step 2 instead. The check below refuses in that case, but read the Step 1 classification first and be sure. + +Verify the direction before touching anything. The chart-named generation must be the **newer** one, judged on the **PV** ages (claims are recreated by Step 2's re-bind, so their timestamps prove nothing). The precondition below refuses unless every chart-named claim is bound to a PV strictly newer than every release-named PV: + +```sh +#!/usr/bin/env bash +set -euo pipefail +ns= +app= + +# PRECONDITION. Every chart-named claim must be bound to a PV strictly newer than +# every release-named PV. A chart-named claim sitting on an OLD PV means Step 2 +# already re-bound it: it holds data, and this step would delete it. +pv_age() { kubectl get pv "$(kubectl -n "$ns" get pvc "$1" -o jsonpath='{.spec.volumeName}')" \ + -o jsonpath='{.metadata.creationTimestamp}'; } +newest_release_pv="" +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' \ + | grep -E '^data1-.*volume' | grep seaweedfs | grep -vE '^data1-seaweedfs-volume'); do + a=$(pv_age "$pvc"); [ "$a" \> "$newest_release_pv" ] && newest_release_pv="$a" +done +[ -n "$newest_release_pv" ] || { echo "REFUSING: no release-named volumes found; this is not an S-damaged tenant"; exit 1; } +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-seaweedfs-volume'); do + a=$(pv_age "$pvc") + if [ ! "$a" \> "$newest_release_pv" ]; then + echo "REFUSING: $pvc is bound to a PV created $a, NOT newer than the newest release-named PV ($newest_release_pv)." + echo "That claim is not an empty duplicate — Step 2 has most likely already re-bound it. Finish Step 2; do not run this step." + exit 1 + fi +done +echo "ok: every chart-named claim is bound to a strictly newer PV — safe to clear the duplicate" + +kubectl -n "$ns" patch helmrelease "${app}-system" --type merge -p '{"spec":{"suspend":true}}' +# The chart-named workloads were created by the aborted upgrade and never held +# data. Delete them so the re-bind can take over their names. +kubectl -n "$ns" delete sts seaweedfs-master seaweedfs-filer --ignore-not-found +kubectl -n "$ns" get sts -o name | sed 's|statefulset.apps/||' | grep -E '^seaweedfs-volume($|-)' \ + | xargs -r -I{} kubectl -n "$ns" delete sts {} +# The EMPTY chart-named claims block Step 2's re-bind (same names). +kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-seaweedfs-volume' \ + | xargs -r -I{} kubectl -n "$ns" delete pvc {} +``` + +Leave the HelmRelease suspended and continue with Step 2 (skip its suspend line; the renamed StatefulSets it scales down are the ones holding the data, exactly as in a plain `S` tenant). Do NOT scale down or delete anything named `*-system-*` before its volumes are re-bound. + +## Step 3 — `MIXED` tenants: remove the duplicate before upgrading + +The chart refuses while both generations exist, so the duplicate must be **gone before** the upgrade, not cleaned up after it. Do not skip to Step 4: it cannot run until this is done. + +Do **not** start by deleting PVCs. If the duplicate ever served, they may hold objects written through the split endpoint. + +1. **Take the duplicate out of service** so nothing else is written to it. Select it precisely — the generation Step 1 named as ORIGINAL is authoritative and must keep serving. For the common case (chart-named original, release-named duplicate): + + ```sh + ns= + kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs -o name | sed 's|statefulset.apps/||' \ + | grep -vE '^seaweedfs-(master|filer|volume)($|-)' \ + | xargs -r -I{} kubectl -n "$ns" scale sts {} --replicas=0 + # The renamed s3 Deployment is the one that is NOT the chart-named `seaweedfs-s3`. + kubectl -n "$ns" get deploy -l app.kubernetes.io/name=seaweedfs,app.kubernetes.io/component=s3 -o name \ + | sed 's|deployment.apps/||' | grep -v '^seaweedfs-s3$' \ + | xargs -r -I{} kubectl -n "$ns" scale deploy {} --replicas=0 + ``` + + If Step 1 named the **release-named** generation as original, this tenant is `S-damaged`: the duplicate is the chart-named set. Use **Step 2a** instead, then Step 2 — the selectors are inverted there. + +2. **Confirm the authoritative set is serving**, and that the duplicate is out of the `seaweedfs-s3` endpoints: + + ```sh + kubectl -n "$ns" get endpoints seaweedfs-s3 -o yaml + ``` + +3. **If the duplicate ever served, escalate.** Step 1's emptiness check is what decides this. Recovering objects that exist only on a duplicate is not a procedure this runbook can give you: both master sets allocate volume IDs from independent sequences into one shared `seaweedfs-db`, so a fid written through the split endpoint can collide with a fid on the authoritative set, and there is no supported tool that reconciles two volume-ID spaces against one metadata store. Do not improvise it. Involve someone who can plan a per-object export, and treat the tenant as an incident. + +4. **Delete the duplicate's StatefulSets, Deployments and PVCs.** Only once (3) is settled, and only for a duplicate confirmed empty (or whose contents have been exported): + + ```sh + ns= + # StatefulSets and Deployments of the duplicate generation. + kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs -o name | sed 's|statefulset.apps/||' \ + | grep -vE '^seaweedfs-(master|filer|volume)($|-)' \ + | xargs -r -I{} kubectl -n "$ns" delete sts {} + kubectl -n "$ns" get deploy -l app.kubernetes.io/name=seaweedfs -o name | sed 's|deployment.apps/||' \ + | grep -vE '^seaweedfs-(s3|objectstorage-provisioner)$' \ + | xargs -r -I{} kubectl -n "$ns" delete deploy {} + # Duplicate volume PVCs, BY NAME. Never by label: the live data PVCs carry the + # same app.kubernetes.io/instance=-system label and would match too. + kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-.*volume' \ + | grep seaweedfs | grep -vE '^data1-seaweedfs-volume' | xargs -r -I{} kubectl -n "$ns" delete pvc {} + ``` + + Never delete `data1-seaweedfs-volume-*` here — those are the live data PVCs of the authoritative set. (For an `S-damaged` tenant it is the other way round; that is Step 2a's job, and it has its own precondition check.) + +5. **Re-run the audit.** The tenant must now read `L` (or `S`, if you are on the Step 2 path), and the audit must exit zero — a `FATAL` here means the re-check never completed, not that the tenant is fine. Only then upgrade. + + ```sh + hack/seaweedfs-naming-audit.sh "$ns" + ``` + +## Step 4 — Upgrade, then clear the leftovers + +Every tenant must read `L` or `S` in the audit before you start: the chart refuses to render while both generations exist, so a `MIXED` tenant does not upgrade at all — Steps 2/2a/3 come first, not after. Once the fleet is clean, upgrade to a Cozystack version carrying the fix. On reconcile the `-system` release renders the chart-based names and adopts the running workloads and their volumes in place. + +A duplicate's StatefulSets and PVCs are removed in Step 3, before the upgrade. What can still be left behind afterwards are objects no release templated and nothing owns: cert-manager Secrets have no owner reference, and PVCs Helm never templated are never GC'd. Remove those once the tenant is verified healthy: + +```sh +ns= +# Duplicate volume PVCs, BY NAME. Never by label: the live data PVCs carry the +# same app.kubernetes.io/instance=seaweedfs-system label and would match too. +kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-.*volume' \ + | grep seaweedfs | grep -vE '^data1-seaweedfs-volume' | xargs -r -I{} kubectl -n "$ns" delete pvc {} +# Orphaned certificate/db secrets of the duplicate set. cert-manager names them +# --cert, so the renamed set's are everything matching the cert/db +# suffix EXCEPT the adopted chart-named seaweedfs--cert / seaweedfs-db-secret. +# This covers the default (seaweedfs-system-*) and any non-default (-system- +# seaweedfs-*) instance without hardcoding the name. +kubectl -n "$ns" get secret -o name | sed 's|secret/||' \ + | grep -E '(-cert|-db-secret)$' | grep seaweedfs \ + | grep -vE '^seaweedfs-(admin|ca|client|filer|master|volume|worker)-cert$|^seaweedfs-db-secret$' \ + | xargs -r -I{} kubectl -n "$ns" delete secret {} --ignore-not-found +``` + +Never delete `data1-seaweedfs-volume-*` — those are the live data PVCs of the adopted set. + +The same 4.31 bump also renamed the tenant's four **cluster-scoped** RBAC objects, and those leftovers are cluster-wide rather than namespaced. Pre-4.31 they were named after the per-namespace service account (`-seaweedfs-*`); 4.31 named them after the Helm release, which is identical in every tenant, so the fleet collided on one object. The fix puts them back on the service account name, which is where a 1.4.x tenant's objects already are — those are adopted in place and need no cleanup. A tenant that passed **through** 1.5.x, however, leaves behind whatever name that release used, and because more than one tenant claimed it, it may not be pruned by any release's manifest: + +```sh +# Stale shared/renamed cluster-scoped RBAC. The go-forward names all start with a +# tenant namespace; anything on the release-based names is a leftover. The fourth +# pre-4.31 object is matched separately: 4.31 renamed the master-rw BINDING from +# upstream's username-shaped system:serviceaccount::default to -rw-crb, so +# the old one carries neither suffix and no tenant prefix. +kubectl get clusterrole,clusterrolebinding \ + -o custom-columns='KIND:.kind,NAME:.metadata.name,OWNER:.metadata.annotations.meta\.helm\.sh/release-name,NS:.metadata.annotations.meta\.helm\.sh/release-namespace' \ + | grep -E 'objectstorage-provisioner|-rw-cr|^\S+\s+system:serviceaccount:.*:default' \ + | grep -vE '^\S+\s+tenant-' +``` + +A `system:serviceaccount:-seaweedfs:default` row is the pre-4.31 master-rw binding. It is superseded by `-seaweedfs-rw-crb` and is pruned automatically by the tenant's own upgrade (it is in that release's manifest), so it should not survive — if it does, the tenant has not upgraded yet. + +Each surviving row is inert once every tenant is upgraded and verified — no release renders those names any more. Confirm the tenant listed in `OWNER`/`NS` is healthy on the go-forward names first, then delete by name. Do not delete anything named `-seaweedfs-*`: those are live. + +**During** a rolling fleet upgrade there is a window, and it is worth expecting rather than debugging: Helm prunes by name without checking ownership, so the first tenant to reconcile onto the fixed chart deletes the shared `seaweedfs-objectstorage-provisioner` / `seaweedfs-rw-cr` objects that a not-yet-upgraded tenant is still bound through. Those tenants' COSI provisioners get 403s on bucket operations until they reconcile onto their own per-namespace RBAC. Existing buckets keep serving — S3 traffic does not go through the provisioner — so this is a provisioning outage, not a data-plane one, and it closes on its own as the remaining tenants reconcile. + +If a tenant's `seaweedfs-system` HelmRelease was suspended during triage, resume it so the fix can reconcile: + +```sh +kubectl -n patch helmrelease seaweedfs-system --type merge -p '{"spec":{"suspend":false}}' +kubectl -n annotate helmrelease seaweedfs-system reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite +``` + +## Step 5 — Verify + +```sh +ns= +# Exactly one set, named after the chart, healthy. +kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs +# All three HelmReleases Ready. +kubectl -n "$ns" get helmrelease seaweedfs seaweedfs-system seaweedfs-db +# S3 endpoints resolve only to the adopted set's ready pods. +kubectl -n "$ns" get endpoints seaweedfs-s3 +# Objects are readable through S3 (bucket list via any tenant bucket). +``` + +A recovered tenant has a single `seaweedfs-*` set, all three HelmReleases `Ready`, no `seaweedfs-system-*` StatefulSets, and no `data1-seaweedfs-system-volume-*` PVCs left behind. + +Only once that is true, restore the reclaim policy Step 2 stashed on each PV. Until this runs the volumes are `Retain`, which is deliberate: it is what makes an accidental claim deletion during recovery survivable. + +```sh +ns= +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-seaweedfs-volume'); do + pv=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.volumeName}') + orig=$(kubectl get pv "$pv" -o jsonpath='{.metadata.annotations.cozystack\.io/original-reclaim-policy}') + [ -n "$orig" ] || continue + kubectl patch pv "$pv" -p "{\"spec\":{\"persistentVolumeReclaimPolicy\":\"${orig}\"}}" && kubectl annotate pv "$pv" cozystack.io/original-reclaim-policy- +done +``` diff --git a/docs/operations/seaweedfs-capacity-for-backups.md b/docs/operations/seaweedfs-capacity-for-backups.md new file mode 100644 index 0000000000..eb6e5c1891 --- /dev/null +++ b/docs/operations/seaweedfs-capacity-for-backups.md @@ -0,0 +1,37 @@ +# SeaweedFS capacity for backups + +A SeaweedFS instance starts with two volume servers of `volume.size: 10Gi` each and `replicationFactor: 2`, so every byte it stores lands on both servers and the instance holds a little under 10 GiB of objects. That is enough for occasional backups and restores. It is not enough for continuous WAL archiving from a Postgres cluster that writes steadily: WAL is archived around the clock and kept for the whole retention window, so the bucket grows until the window is full, and an instance sized by default runs out long before then. + +A volume server stops taking writes in one of two ways. Once its free disk falls below `minFreeSpacePercent` (5%), which it checks every minute, every volume on it turns read-only and it takes no new volume, so every bucket with a copy there stops at once. Before that, it can run out of slots: the volumes that already exist keep taking writes until each reaches `master.volumeSizeLimitMB`, a bucket whose volumes are all full needs a new one, and a server with no room left for another volume of the size limit cannot host it. When no server can, the master logs `Not enough data nodes found` and every upload into that bucket fails. The slots can run out while the disk is still half empty: a volume server counts every volume it holds at the full size limit, so the nearly empty volumes of other buckets keep one bucket from growing. `barman-cloud-wal-archive` then fails on every segment, WAL segments stay on the Postgres volume, and the database goes down when that volume is full. + +## Estimating what a bucket needs + +For each Postgres cluster that archives into the instance, the bucket holds the WAL of the whole retention window plus every base backup inside it: + +```text +bucket ≈ WAL per day × retention days + database size × (base backups in the window + 1) +``` + +The WAL rate of a running cluster, in bytes per day before compression, is roughly the number of segments it archived over a day times the 16 MiB segment size. The count also includes the small history and backup-label files, and the sum covers every pod that was primary during the day: + +```promql +sum by (namespace, job) (increase(cnpg_pg_stat_archiver_archived_count{namespace=""}[1d])) * 16 * 1024 * 1024 +``` + +The platform's default backup strategy compresses both WAL and base backups with gzip, so the stored size is lower, by an amount that depends on the data; a cluster backed up to an object store of its own choosing stores them uncompressed unless its ObjectStore says otherwise. The default `retentionPolicy` is `30d`, so a cluster that writes 4 GB of WAL a day needs about 120 GB of WAL alone before its base backups are counted. + +Each volume server must hold the sum of those buckets, since at `replicationFactor: 2` with two servers each server carries a full copy. Size `volume.size` so that this sum stays below 95% of it, and leave room on top for every bucket that is still growing: at `replicationFactor: 2` a grow claims three volumes of `master.volumeSizeLimitMB` on each server. + +## Alerts that watch the instance + +The generic `KubePersistentVolumeFillingUp` alert does not cover this. It pages below 3% free, while a volume server stops at 5% or, out of slots, with its disk half empty, and its early warning needs the disk to keep filling, which it stops doing once writes fail. Each instance carries its own rules instead: + +- `SeaweedFSVolumeServerDiskLow` (warning): a volume server has less than 10% of its disk left before the `minFreeSpacePercent` reserve. +- `SeaweedFSVolumeServerFull` (warning): a volume server cannot host another volume, because it is at the reserve, where all its volumes are read-only, or has no slot left, where buckets whose volumes on it are full stop taking writes. +- `SeaweedFSWritesFailing` (critical): the master has failed to place a write in every 5-minute window for 15 minutes. Other writes may still succeed in between. The usual cause is a bucket with no writable volume left that the master cannot grow, whose uploads all fail; a steady stream of newly created buckets, each failing a few picks while its first volumes grow, looks the same. + +On the Postgres side, `LastFailedArchiveTime` fires while WAL archiving keeps failing, and `KubePersistentVolumeFillingUp` fires as the WAL backlog fills the Postgres volume. + +## Growing an instance + +Raise `volume.size` on the SeaweedFS application. The upgrade expands each volume server's PVC in place, which needs a StorageClass that allows volume expansion. Postgres keeps retrying the segments that waited on its volume, so archiving catches up once the master can grow the bucket again. diff --git a/docs/operations/seaweedfs-lowering-the-volume-size-limit.md b/docs/operations/seaweedfs-lowering-the-volume-size-limit.md new file mode 100644 index 0000000000..6d3d4d0178 --- /dev/null +++ b/docs/operations/seaweedfs-lowering-the-volume-size-limit.md @@ -0,0 +1,157 @@ +# SeaweedFS volume size limit — lowering it on an existing instance + +New SeaweedFS instances start with `master.volumeSizeLimitMB: 400`. An instance that existed before that default was lowered keeps 1000 unless its limit was already set: on a variant that runs platform migrations, migration 57 wrote 1000 into every instance that had not chosen one, because lowering it on an instance that holds data is not free. The one exception is a release whose `master` is not an object, which no v1 chart renders; it is covered below. The `default` variant runs none, so there that step is yours, before the upgrade, as described under [Before upgrading the platform](#before-upgrading-the-platform). A value the new chart refuses, anything but a whole number from 1 to 30000, counts as not chosen and was replaced with 1000, which is what the previous chart ran in its place. A value from 1 to 30000 that was set before the upgrade stays, and since the previous chart ignored it, it takes effect with the upgrade. Where the migration runs, it names each such instance in its Job's log, but that Job runs as part of the upgrade, too late to act on, so find the ones whose limit it changes beforehand, as described under [Before upgrading the platform](#before-upgrading-the-platform). The value lives in the instance's own spec, so a tool that replaces the whole resource from a manifest without the field drops it and the instance takes 400. If an instance is managed from a manifest, keep `master.volumeSizeLimitMB` in that manifest. Every volume already at or above the new limit leaves the master's writable set on the next heartbeat, so every bucket whose volumes have all crossed that line needs a grow before its next write lands. This page is the check to run before setting a lower limit on such an instance, and the way back if a lowered instance stops taking writes. What decides the outcome is free space measured against what the buckets on each server will claim, not the size of the volume. An instance that comes out short keeps serving reads and loses no data, but the affected buckets stop taking writes until an operator intervenes. + +An instance left at 1000 still has more room than it used to. At `replicationFactor: 2` a grow now claims three volumes rather than six, so a bucket's budget there is 3000 MB instead of 6000 and the instance has room for roughly twice as many new buckets as it had. The same change costs write spread: a bucket that grows gets three writable volumes instead of six, and a volume server takes the writes into each volume one at a time, so a heavily parallel writer spreads over fewer volumes until the bucket grows again. The growth count is not a field, so there is no setting that takes this back. At any other replication factor the batch is unchanged. + +**Leave the limit at 1000 if the instance runs `topology: MultiZone`, or if more of its volume servers share one disk type than its `replicationFactor`.** The master places a new volume by disk type, not by group: it picks among all servers of the requested type with room, whether they belong to the default group or to a pool, and it treats an empty `diskType` and `hdd` as the same type. Under `MultiZone` it also grows each zone on its own. In both cases it can put a new volume on a server where the bucket that asked for it has nothing yet, which a per-server listing cannot account for. So count the servers of all groups together for each disk type, taking `volume.diskType` for the default group and `diskType` for each pool. The shipped layout, two volume servers at `replicationFactor: 2` and no pool, is one the check covers, and `kubectl get statefulset` in the instance's namespace shows how many servers each group runs. + +## Before upgrading the platform + +An instance whose limit was already set to a valid value other than 1000, a whole number from 1 to 30000, changes limit with the upgrade itself. List those instances, and the releases that need a closer look, while the previous release still runs; this needs `jq`: + + +```sh +kubectl get helmreleases.helm.toolkit.fluxcd.io -A -o json \ + -l apps.cozystack.io/application.kind=SeaweedFS,apps.cozystack.io/application.group=apps.cozystack.io \ + | jq -r '"instances \(.items | length)", + (.items[] | "\(.metadata.namespace) \(.metadata.name)" as $id | .spec.values as $s + | [(.spec.valuesFrom // [])[] | select(. != {kind: "Secret", name: "cozystack-values"})] as $extra + | (if $extra != [] then "valuesfrom \($id) \($extra | tojson)" else empty end), + (if $s != null and ($s | type) != "object" then "unreadable \($id) \($s | tojson)" + elif $s.master != null and ($s.master | type) != "object" then "unrenderable \($id) \($s.master | tojson)" + else $s.master.volumeSizeLimitMB as $v + | select(($v | type) == "number" and $v == ($v | floor) and $v >= 1 and $v <= 30000 and $v != 1000) + | "changes \($id) \($v)" end)), + "complete \(.items | length)"' +``` + +**The last line must read `complete` with the same number as the first.** If it is missing or `jq` printed an error, `jq` stopped at a release it could not read, and the list says nothing about the releases after it, whatever it printed: stop. + +**The `instances` count, less the number of `unreadable` lines, must equal the number of instances `kubectl get seaweedfses.apps.cozystack.io -A` lists.** The API leaves out a release it cannot convert and only logs it, and a release whose `spec.values` is not an object is one it cannot convert. Any other difference means the labels in this command are not the ones the API selects instances by, or the API is leaving out a release for another reason: stop and find out why before reading the list. + +Every other line names one release, by namespace and name after its first word: + +- `changes`: the upgrade changes its limit to the number at the end of the line. Run [the check](#the-check) on it with `LIMIT` set to that number. +- `unreadable`: its `spec.values` is not an object, and the line shows what it holds. helm-controller ignores such values and renders the chart's defaults, so the instance runs at 1000 today. The migration, or the step below on the `default` variant, pins it there and replaces what it held, which drops nothing that applies. +- `unrenderable`: its `master` is not an object, and the line shows what it holds. No v1 chart renders that, so the release does not reconcile today, and nothing pins it. When you repair `master`, set `volumeSizeLimitMB` in the same edit, or the instance comes up at 400. +- `valuesfrom`: it also takes values from the entries on the line, besides the platform's `cozystack-values` Secret. helm-controller merges `spec.values` over them, so a `master.volumeSizeLimitMB` an entry sets without `targetPath` does not apply, and one set through `targetPath` can, depending on the entries before it. If an entry sets it that way, check the instance with that value as for a `changes` line. + +For a `changes` line: + +- Below 1000 the upgrade lowers the limit, which is what the rest of this page is about. +- Above 1000 it raises the limit. Existing buckets keep writing, but every volume below the new limit is charged up to that limit, so the room new buckets grow into shrinks, possibly to none. Measured on a local SeaweedFS 4.31 setup: a server holding seven nearly empty volumes had no free slot left once the limit was raised tenfold, and a new bucket was refused while the existing one kept writing; setting the limit back let the new bucket grow. The budget is an upper bound here too, and what the `df` figure has left after it, divided by the limit, is roughly how many volumes new buckets can still get, a batch for each. + +If one comes out short and you cannot free space before the upgrade, set its limit to 1000 in its HelmRelease with the same patch the migration applies, taking `NS` and `NAME` from its line in the list: + +```sh +kubectl -n "$NS" patch helmreleases.helm.toolkit.fluxcd.io "$NAME" --type=merge \ + -p '{"spec":{"values":{"master":{"volumeSizeLimitMB":1000}}}}' +``` + +The running chart ignores the value, and the new one applies it, so the instance stays at 1000. Patch the HelmRelease, not the SeaweedFS resource. The API applies a patch to the instance as it reads it, with every default filled in, and writes that whole spec back into `spec.values`, so every field left at its default would stop following the chart's default. The HelmRelease patch writes this one key, as the migration does. + +On the `default` variant, which installs packages by hand and runs no platform migrations, nothing writes 1000 into the instances that have not chosen a limit, and the upgrade would lower every one of them whose `master` is an object to 400. Before upgrading, apply the migration's patch yourself. It writes only `master.volumeSizeLimitMB` into each instance's HelmRelease, and only where no valid value is set: + + +```sh +kubectl get helmreleases.helm.toolkit.fluxcd.io -A -o json \ + -l apps.cozystack.io/application.kind=SeaweedFS,apps.cozystack.io/application.group=apps.cozystack.io \ + | jq -r '"instances \(.items | length)", + (.items[] | "\(.metadata.namespace) \(.metadata.name)" as $id | .spec.values as $s + | [(.spec.valuesFrom // [])[] | select(. != {kind: "Secret", name: "cozystack-values"})] as $extra + | (if $extra != [] then "valuesfrom \($id) \($extra | tojson)" else empty end), + (if $s != null and ($s | type) != "object" then "unreadable \($id) \($s | tojson)", "pin \($id)" + elif $s.master != null and ($s.master | type) != "object" then "unrenderable \($id) \($s.master | tojson)" + else $s.master.volumeSizeLimitMB as $v + | select(($v | type) != "number" or $v != ($v | floor) or $v < 1 or $v > 30000) + | "pin \($id)" end)), + "complete \(.items | length)"' \ + | while read -r what ns name detail; do + case "$what" in + pin) ;; + instances|complete) echo "$what $ns"; continue ;; + *) echo "$what $ns $name $detail"; continue ;; + esac + echo "pinning $ns/$name" + kubectl -n "$ns" patch helmreleases.helm.toolkit.fluxcd.io "$name" --type=merge \ + -p '{"spec":{"values":{"master":{"volumeSizeLimitMB":1000}}}}' _.dat`, or `.dat` for the unnamed default collection. You should get one block per server, as many as the count, each with a `df` line and a budget line. + +**Read `volume files` before anything else.** It counts every volume on that server, from the same listing the budget is computed from, so a server that holds any bucket's data cannot show zero there. If every server shows zero on an instance that stores data, the listing is not reading the volume files and a budget of 0 MB means nothing: stop. Zero `below the limit` is a legitimate answer on its own. + +**A server is clear when its available `df` figure is at least its budget, and the instance is clear when every server is.** For an instance the top of this page leaves to the check, a server's budget is an upper bound on what the new limit claims of its free space when it lands: one limit for each volume below it, since that is the most such a volume can be charged, and a whole batch for each collection with no volume at or below 90 percent of the limit, 360 MB at 400, since that collection is grown whether or not it writes. `BATCH` is at least the two volumes the master's own growth adds. The listing reads `.dat` sizes only, while the volume server charges each volume the room left after its `.dat` and `.idx` together, so leaving the index out can only overstate the charge. The master draws that 90 percent line against the `.dat` length the volume server reports, rounded up to the 8-byte needle padding, so a file within seven bytes of the line can fall on the other side of it; it also adds bytes it has handed out but not yet seen written, so run the check while the instance is not being written to heavily. The master also counts only writable volumes as roomy, and the listing cannot tell which ones are read-only: a server under its five percent reserve has made all of its volumes read-only, and an operator can mark one by hand. If either applies to a server, take its budget as `LIMIT` × (`BATCH` × collections + volumes below the limit) MB, from the same line, instead. If a server falls short, free space as described below before lowering the limit, or leave it at 1000. A server that clears one limit but not its budget is the trap this check exists for: the first bucket to write grows into what there is, and the ones behind it stay refused while `df` still shows room. + +## If the check comes out short + +Delete buckets the tenant no longer needs, then re-run the check. A deleted bucket gives back a slot for every volume it held, including one it barely wrote to. Deleting objects inside a bucket helps less and later: that room comes back only after a vacuum, and only for what was deleted. + +## Lowering the limit + +Set the new value in the instance's HelmRelease, with the patch used above and for the same reason; the master StatefulSet rolls with it: + +```sh +kubectl -n "$NS" patch helmreleases.helm.toolkit.fluxcd.io "$NAME" --type=merge \ + -p '{"spec":{"values":{"master":{"volumeSizeLimitMB":400}}}}' +``` + +`NAME` is the instance's name, which its HelmRelease carries unchanged. The tenant module creates its instance under the fixed name `seaweedfs`; an instance created directly against the API can carry another one. + +## If a lowered instance stops taking writes + +The symptom is some buckets writing and others failing on the same instance, while the disk reads as far from full and the master logs a grow that places nothing. The ones that write are those whose grow went through first; the rest are behind them in the same queue for the same free space. No data was lost on the way in, since the volumes stay readable throughout. Freeing space helps as described above, takes effect on the next heartbeat and restarts nothing. Setting the limit back to 1000, with the same HelmRelease patch and 1000 in place of 400, frees every bucket at once, and it rewrites the master's command line, so the master StatefulSet rolls; give it a window. diff --git a/docs/operations/system-memory-limits.md b/docs/operations/system-memory-limits.md new file mode 100644 index 0000000000..af3076f0e3 --- /dev/null +++ b/docs/operations/system-memory-limits.md @@ -0,0 +1,179 @@ +# System Component Memory Limits + +Cozystack defaults container memory in eligible system namespaces and sets explicit requests and limits on the node DaemonSets. This page explains why that is not merely resource hygiene, how to tune it, and what it deliberately does not cover. + +## Why a memory limit, and not a PriorityClass + +Talos Linux v1.12 introduced a userspace OOM handler, enabled by default, that reacts to memory pressure before the kernel OOM killer does. Its only input from the machine is its own config document; everything else it reads straight out of cgroupfs, and it holds no Kubernetes client at all. `PriorityClass` — `system-node-critical` included — therefore has no bearing on which cgroup it picks. + +Setting a priority on a system component to stop these kills looks like a fix and changes nothing about them. Take that claim narrowly, because priority is far from inert elsewhere: it still drives scheduler preemption, kubelet eviction ordering, and the `oom_score_adj` kubelet hands the kernel OOM killer. None of those three is what kills these pods. The one ordering input this handler has is the Kubernetes **QoS class**, inferred from the cgroup path rather than from any pod object — a different axis from priority, so a `system-node-critical` pod with no memory limit is an ordinary candidate like any other. + +The handler scores each pod cgroup with a CEL expression. The default in Talos v1.14.2, the version Cozystack currently ships for host nodes, is: + +``` +memory_max.hasValue() ? 0.0 : + {Besteffort: 1.0, Burstable: 0.5, Guaranteed: 0.0, Podruntime: 0.0, System: 0.0}[class] * + double(memory_current.orValue(0u)) +``` + +Any cgroup scoring zero is dropped from the candidate set entirely. A pod whose containers all carry a memory limit has `memory.max` set on its pod cgroup, scores `0.0`, and can never be selected. A pod without one stays a candidate no matter how little memory it is using. + +Three consequences follow, and all three are easy to get wrong: + +- Only a **limit** grants immunity. A memory **request** merely moves the pod from BestEffort to Burstable, which under the default `strictCgroupClassOrdering: true` means "killed second" rather than "not killed" — Burstable cgroups are considered only once no BestEffort one is eligible, and the score then breaks ties within the class. That setting arrived in v1.13.4; on v1.13.0 through v1.13.3 the score alone decided, so a large Burstable pod could outrank a small BestEffort one. +- **Every** container in the pod needs a limit, init containers included. Kubelet only sets pod-level `memory.max` when all of them have one, so a single limit-free sidecar puts the whole pod back in the candidate set. +- CPU limits are irrelevant here. The ranking expression is given the cgroup's path, its QoS class, and `memory.max`, `memory.current` and `memory.peak` — no CPU information of any kind. + +Victim selection is also decoupled from the trigger. The cgroup that caused the pressure and the cgroup that gets `SIGKILL`ed are unrelated by design. In practice the pods carrying limits are overwhelmingly tenant workloads — managed applications are sized through resource presets, and a tenant namespace with `resourceQuotas` configured gets a default limit of its own — while system components carried none until the change this page describes. That inverts the intended order: whatever fires the trigger, `metallb-speaker` or `linstor-satellite`, using a few dozen megabytes and entirely uninvolved, is the one killed for it, repeatedly, until the pressure subsides. On Talos v1.13.x a tenant workload thrashing against its own multi-gigabyte limit is enough to fire it, through the node-wide PSI clause described [below](#when-the-trigger-itself-is-the-problem). + +## What Cozystack does + +**A default LimitRange in eligible system namespaces.** The `cozystack-operator` maintains a `LimitRange` named `cozystack-system-defaults` in each namespace it reconciles whose name does not begin with `tenant-`, defaulting container memory for anything that declares none. It covers containers whose upstream chart exposes no `resources` knob. Namespaces with a foreign memory policy or an unresolved safety hold are excluded. + +**Explicit requests and limits on node DaemonSets.** Charts additionally set real values on the DaemonSets that run on every node — the cilium agent and its init containers, metallb speaker, the frr-k8s controller and its sidecars, the linstor satellite along with plunger and drbd-logger, virt-handler, fluent-bit, node-exporter, multus and its init container, velero node-agent, and all three kubevirt-csi-node containers. Requests there are fitted to observed usage, which is a real signal for the scheduler; the LimitRange cannot supply that, because it applies one number to every container in the namespace and so has to keep its default request deliberately tiny. kube-ovn is absent from that list because its vendored chart already sets both. + +The operator's LimitRange stops at the `tenant-` prefix, so tenant namespaces never receive it. A tenant namespace gets a default of its own only from the tenant chart's `tenant-range-limits`, which is rendered only when `resourceQuotas` is configured on the tenant and is empty by default. A tenant workload left without a memory limit therefore stays an eviction candidate, which is the upstream design working as intended and is what restores the ordering described above. + +## Tuning + +Two installer values, both empty by default so the operator's own defaults apply: + +| Value | Operator flag | Default | +|---|---|---| +| `cozystackOperator.systemNamespaceMemoryLimit` | `--system-namespace-memory-limit` | `32Gi` | +| `cozystackOperator.systemNamespaceMemoryRequest` | `--system-namespace-memory-request` | `32Mi` | + +The limit is a ceiling, not a reservation, and the default is deliberately far above any real working set. The handler discards a cgroup for carrying a `memory.max` at all, whatever its value, so what the default buys is immunity rather than a fitted ceiling. That is also why it is set so high: the scan described below sees *declared requests*, never a container's actual resident set, so a ceiling low enough to bind is a ceiling that can convert a rare pressure-driven kill into a deterministic one for any component whose real usage nobody measured. Raising it is close to free; lowering it is where the risk lives, and it is worth doing per component in that component's own chart before doing it fleet-wide here. + +**The limit must stay above the largest memory request in any system namespace.** A defaulted limit below a container's own request is rejected at admission, and the pod simply will not start. List the requests before lowering it — the operator marks exactly the namespaces it treats as system with `cozystack.io/system=true`, the same condition under which it creates the LimitRange: + +```bash +for ns in $(kubectl get ns -l cozystack.io/system=true -o jsonpath='{.items[*].metadata.name}'); do + kubectl get pods -n "$ns" -o json | jq -r --arg ns "$ns" ' + .items[] + | [.spec.containers[], (.spec.initContainers // [])[]][] + | select(.resources.requests.memory != null) + | "\($ns)\t\(.name)\t\(.resources.requests.memory)"' +done | sort -u +``` + +**Keep the request small.** The operator always pairs the default limit with a default request, because a `LimitRange` that sets `default` without `defaultRequest` makes each container's request equal its limit and reserves the whole ceiling at schedule time. Note that leaving `systemNamespaceMemoryRequest` empty does not produce that state: an empty installer value omits the flag, which selects the operator's own `32Mi`. The knob is there to be raised or lowered, and clearing it is not a way to switch the request off. + +The operator also refuses to start when the request exceeds the limit. A `LimitRange` whose `defaultRequest` is above its `default` is rejected by the API server, which would wedge namespace reconciliation for every system package, so the check happens at startup rather than at apply time. + +Setting the limit to `0` disables the feature, so the knob is reversible. Leaving it empty is not the same thing: that selects the operator's `32Gi` default. + +On the next Package reconcile after the feature is disabled, the operator lists LimitRanges cluster-wide by `app.kubernetes.io/managed-by: cozystack-package-controller` and removes every object it owns, including leftovers in namespaces no active Package targets. A same-named LimitRange without that ownership label is left alone. The same ownership boundary applies while the feature is enabled: a foreign object already holding the `cozystack-system-defaults` name is neither overwritten nor adopted, even if it contains only CPU or storage policy, so that namespace receives no Cozystack memory default until the name is free. + +## Withholding and acknowledging a default + +**An unsafe default stays withheld until a cluster administrator acknowledges it.** Before applying, the operator scans Deployments, StatefulSets, DaemonSets, CronJobs, Jobs and Pods, including init containers, for requests above the configured limit without an explicit limit. A visible blocker withdraws the managed LimitRange and records a hold on the Namespace. Removing a Pod or template does not release the hold: the same empty snapshot can occur between deletion and recreation by another controller. + +Lowering a previously applied default also creates a hold, even if no blocker is visible. LimitRanger persists its default on admitted Pods, so a stored limit cannot prove that the workload's source declares one. The operator remembers the previous limit independently of the LimitRange and does not infer permission to tighten from a Pod's `limits` field or `kubernetes.io/limit-ranger` annotation. + +The cluster-scoped Namespace carries `operator.cozystack.io/system-memory-state`, a JSON object with `version`, `lastLimit`, and, when held, `hold`, `target` and `reason`. This is operator-managed safety state. It survives controller restart, removal of the LimitRange, removal of the Package target, and disabling the feature. Do not remove or edit it to clear a hold: doing so discards the evidence protecting future Pod admission. Invalid state blocks new defaults until a cluster administrator repairs it. + +Inspect the hold and the required acknowledgement: + +```bash +kubectl get namespace -o json | jq -r ' + .metadata.annotations["operator.cozystack.io/system-memory-state"] + | fromjson | {reason, target, acknowledgement: (.hold + ":" + .target)}' +``` + +Before acknowledging, inspect the source of every affected workload, including custom resources and static-pod manifests that this scanner cannot read. Give the affected container an explicit limit at least as large as its request, lower its request only when appropriate, or raise the configured default. A currently running Pod's defaulted limit is not that source. Also check actual memory usage: admission safety does not make a small runtime ceiling safe. + +Copy the exact `acknowledgement` value reported for the current target into a separate annotation: + +```bash +kubectl annotate namespace \ + operator.cozystack.io/system-memory-ack=':' --overwrite +``` + +This requires permission to patch the cluster-scoped Namespace; a tenant's namespaced RoleBinding does not grant it. The operator grants no additional permission for acknowledgement. A token authorizes only its hold generation and canonical target quantity. Changing the configured target rotates the hold, even when returning to an earlier value. The operator consumes a matching token once; if a concrete request-above-default blocker still exists, the token is rejected and removed, and the hold stays. Acknowledgement never overrides a foreign LimitRange. + +After successful acknowledgement and a clean scan, the configured default is applied. Existing Pods keep their resources; the new default reaches later admissions. Package reconciles run after ordinary events and are additionally scheduled every five minutes, so acknowledgement and workload changes also progress on a quiet cluster, subject to controller queue and API availability. The reason and required token are visible on the Namespace and in operator logs: + +```bash +kubectl -n cozy-system logs deploy/cozystack-operator | grep "withholding the default container memory limit" +``` + +A hold remains after disable/re-enable, and changing the configured value while held requires acknowledging the new token. The operator also removes its LimitRange when no active Package targets the namespace, considering all Packages sharing it; the Namespace safety state remains for a later reinstall. Setting the limit to zero removes managed defaults even if their safety state is invalid. + +**A namespace another LimitRange already governs is left alone.** If a system namespace — `kube-system` included — already carries a LimitRange that says anything about memory, the operator writes nothing there and withdraws anything it wrote earlier. Any memory-bearing field counts, at either the `Container` or the `Pod` scope: a second `default` leaves the effective ceiling to the order `LimitRanger` happens to iterate them, a `max` below this default or a `min` above the default request rejects the pod outright, and a `maxLimitRequestRatio` rejects it too, since a 32Gi default against a 32Mi default request is a ratio of 1024:1. The `Pod` scope is included because the collision there is not decidable in advance at all — whether a per-container default breaches a pod-wide bound depends on how many containers a not-yet-created pod will have. LimitRanges that bound only cpu or storage say nothing about memory and coexist normally. Note that withdrawing hands the namespace to the administrator's policy in full: if that LimitRange defaults container memory below some container's own request, the resulting admission rejection is theirs to resolve, and it is one that would have occurred without Cozystack in the picture at all. + +A LimitRange only mutates at admission. Existing pods keep running without limits until they restart, so the protection lands progressively as workloads roll rather than the moment the setting is applied. + +## Verifying + +Confirm a pod cgroup actually carries `memory.max` — this is the property that matters, not the QoS class. Talos runs kubelet with the `cgroupfs` driver on a unified cgroup v2 hierarchy, so a pod cgroup is a directory named for the pod UID, dashes and all: + +```bash +POD_UID=$(kubectl get pod -n -o jsonpath='{.metadata.uid}') +NODE=$(kubectl get pod -n -o jsonpath='{.spec.nodeName}') +NODE_IP=$(kubectl get node "$NODE" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') +talosctl -n "$NODE_IP" read "/sys/fs/cgroup/kubepods/burstable/pod$POD_UID/memory.max" +``` + +A byte count rather than the literal `max` means the pod is out of the victim set. Note that `talosctl -n` takes a machine address rather than a Kubernetes node name, which is why the address is looked up instead of being reused from `nodeName`. + +Mind the QoS segment of that path, and do not extrapolate it. BestEffort and Burstable pods live under `kubepods/besteffort/` and `kubepods/burstable/`, but there is no `guaranteed` directory: kubelet creates QoS-level cgroups for those two classes only, so a Guaranteed pod sits directly at `kubepods/pod`. To avoid guessing, list the pod directories two levels down instead: + +```bash +talosctl -n "$NODE_IP" ls -d 2 -t d /sys/fs/cgroup/kubepods +``` + +`talosctl read` wants an `os:admin` talosconfig and a single node per call. Reading the same file through `kubectl debug node/$NODE --profile=sysadmin` under `/host/sys/fs/cgroup` also works, but it depends on the kubelet, on scheduling, and on a namespace that admits a privileged pod — all shakier than the Talos API in precisely the situation that makes you run this check. + +Inspect what the handler has actually killed: + +```bash +talosctl -n get oomactions -o yaml +``` + +Each entry carries the score the victim was selected on, the command lines of the processes in it, and a dump of the trigger context it fired under. Ask for `-o yaml`: the default table shows only a score, and its `Time` column reads a field the spec never sets, so it renders empty. This is a ring buffer of the last 50 actions held in memory, so it does not survive a `machined` restart and is no substitute for an alert — but it is still the better of the two signals here, being structured and complete where the logs are neither. + +Once every eligible pod in a namespace carries a limit, the healthy signature is the controller still triggering under pressure and finding nothing it is allowed to kill: + +```bash +talosctl -n logs controller-runtime | grep OOMController +``` + +Both halves of that signature — `OOM controller triggered`, then `no eligible cgroup to kill` — are emitted at the default log level, so neither needs a debug flag. Note that the structured fields are encoded as JSON rather than logfmt, so the second line reads `no eligible cgroup to kill {"component": "controller-runtime", "controller": "runtime.OOMController", "ranked": 0}`. Grep for `"ranked"` or for the message text; `ranked=` matches nothing and reads as though the handler were not running. The trigger firing is not itself a fault. + +`talosctl dmesg` carries the same lines and is the more common reflex, but it is the weaker source: that path strips log levels and timestamps, truncates at 976 bytes, and suppresses the first four occurrences of each ranking error — precisely the lines worth having if scoring ever misbehaves. + +## What this does not cover + +**Kubernetes static pods.** `kube-apiserver`, `kube-controller-manager` and `kube-scheduler` are managed by Talos rather than by any chart. A `cozystack-system-defaults` LimitRange does reach `kube-system`, since `cozystack-scheduler` installs there, but it cannot touch them: a LimitRange defaults resources at API-server admission, and kubelet builds static pods straight from files on disk without ever passing through it. Sizing those is a Talos machine-config matter. Their mirror pods pass admission. An already persisted mirror Pod with an oversized request and no limit creates a hold, and lowering the previous default requires acknowledgement. A mirror Pod rejected before persistence provides no scan evidence; its absence does not mean the static manifest is safe. + +**Future Pods outside the scanned templates.** A custom controller can change its desired request without producing a Deployment, StatefulSet, DaemonSet, CronJob or Job. If its first replacement Pod is rejected at admission, there is no new Pod for this scanner to inspect. An ownerless ReplicaSet at zero replicas has the same gap. A `RequestsOnly` VPA can likewise introduce an unseen request after defaulting. The hold prevents automatic lowering and automatic recovery after an observed blocker, but does not validate arbitrary future custom-resource changes. Configure explicit resources for those workloads and inspect their admission errors. + +**A handful of vendored containers with no upstream `resources` knob** — the four frr-k8s `cp-*` init containers, the kube-ovn `hostpath-init` and `install-cni` init containers, and `cozy-proxy`, whose chart exposes no resources value at all. These are covered by the namespace LimitRange and nothing else. That is enough for OOM immunity, since the defaulted limit is what puts `memory.max` on the cgroup, but their request is then the generic default rather than a figure fitted to measured usage, so the scheduler gets a weaker signal for them than for the DaemonSets above. Lifting that needs changes upstream. + +**Optional components** `hami` and `kilo` carry no chart-level values, on the grounds that inventing numbers for components with no usage measurements behind them is worse than the blanket default. + +## When the trigger itself is the problem + +Giving system components limits stops them being *victims*. It does not stop the handler *triggering*, and a node under genuine sustained pressure will keep firing. The v1.14.2 default trigger is: + +``` +(multiply_qos_vectors(d_qos_memory_full_total, {System: 8.0, Podruntime: 4.0}) > 3000.0 && + multiply_qos_vectors(qos_memory_full_avg10, {System: 1.0, Podruntime: 1.0}) > 5.0 && + time_since_trigger > duration("5s")) +``` + +It fires only on memory stalls in the `System` and `Podruntime` cgroups, so pressure confined to pods does not trip it. It reached that shape over four changes: [siderolabs/talos#12602](https://github.com/siderolabs/talos/pull/12602) replaced the original global-PSI trigger with the per-QoS form in v1.13.0, [#12632](https://github.com/siderolabs/talos/pull/12632) added the `qos_memory_full_avg10 > 5.0` conjunct to make it less sensitive, [#13725](https://github.com/siderolabs/talos/pull/13725) added the `5s` cooldown in v1.13.6, and [#13895](https://github.com/siderolabs/talos/pull/13895) dropped a second clause, `memory_full_avg10 > 75.0 && time_since_trigger > duration("10s")`, in v1.14.0. That clause was a cause-blind global-PSI backstop: a single workload thrashing inside its own cgroup limit could drive root `memory_full` above 75 while the node had free RAM. Any node still running a v1.13 release carries it, tenant Kubernetes workers included. A tenant worker's machine config is rendered by the `kubernetes-nodes` chart, which accepts no extra documents, so the workaround below is for host nodes only. + +On a v1.13 host node the backstop can be removed without upgrading, through an `OOMConfig` machine-config document that sets the v1.14 trigger — at the cost of the last-resort guard before the kernel OOM killer takes over. The document also accepts `cgroupRankingExpression`, `strictCgroupClassOrdering` and `sampleInterval`; anything left out keeps its default, so overriding the trigger alone is enough here: + +```yaml +apiVersion: v1alpha1 +kind: OOMConfig +triggerExpression: |- + (multiply_qos_vectors(d_qos_memory_full_total, {System: 8.0, Podruntime: 4.0}) > 3000.0 && + multiply_qos_vectors(qos_memory_full_avg10, {System: 1.0, Podruntime: 1.0}) > 5.0 && + time_since_trigger > duration("5s")) +``` + +Prefer fixing the workload that is generating the pressure. Persistent triggering on v1.14 means the system and runtime cgroups keep stalling for memory, and capping the pod that is growing into the node's memory addresses the cause rather than the symptom. On v1.13 it can also mean a pod sitting at its own memory ceiling and reclaiming constantly, which raising that pod's limit addresses. diff --git a/docs/release.md b/docs/release.md index 22366f6be8..bf5aaf044e 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,6 +1,19 @@ # Release Workflow -This document describes Cozystack’s release process. +This document is both the process design (what the release model looks like) and the operational playbook (what to actually do, what breaks, what to verify). Companion to [`agents/changelog.md`](./agents/changelog.md), which is the canonical changelog process. + +## When to use this guide + +You are about to: + +- Cut a pre-release (`vX.Y.0-rc.N`, or `vX.Y.0-alpha.N` / `-beta.N`). +- Cut a regular release (`vX.Y.0`). +- Cut a patch release (`vX.Y.Z` with `Z > 0`). +- Triage cherry-picks before a patch release. +- Investigate why CI is misbehaving on a tag push. +- Review a promote (release) PR. + +If you are about to generate a changelog only, jump straight to [`agents/changelog.md`](./agents/changelog.md). ## Introduction @@ -8,99 +21,129 @@ Cozystack uses a staged release process to ensure stability and flexibility duri There are three types of releases: -- **Release Candidates (RC)** – Preview versions (e.g., `v0.42.0-rc.1`) used for final testing and validation. -- **Regular Releases** – Final versions (e.g., `v0.42.0`) that are feature-complete and thoroughly tested. -- **Patch Releases** – Bugfix-only updates (e.g., `v0.42.1`) made after a stable release, based on a dedicated release branch. +- **Release Candidates (RC)** – Preview versions (e.g., `v1.2.0-rc.1`) used for final testing and validation. +- **Regular Releases** – Final versions (e.g., `v1.2.0`) that are feature-complete and thoroughly tested. +- **Patch Releases** – Bugfix-only updates (e.g., `v1.2.1`) made after a stable release, based on a dedicated release branch. -Each type plays a distinct role in delivering reliable and tested updates while allowing ongoing development to continue smoothly. +All three are matched by `tags.yaml`'s regex `^v\d+\.\d+\.\d+(-(alpha|beta|rc)\.\d+)?$`. The tag push is the trigger for the whole pipeline. **Pre-release tags (`-alpha` / `-beta` / `-rc`) are cut with the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow** (manual dispatch from `main` or `release-X.Y`) — it pushes the tag as the CI app at the dispatch branch's tip, so GitHub fills `base_ref` (`tags.yaml`'s `Get base branch` step refuses tags pushed without a base) and the push triggers `tags.yaml` (a tag pushed with the default `GITHUB_TOKEN` would not). Stable `vX.Y.Z` tags are never pushed by hand at all — they are cut write-once by the promote flow at a PR merge commit. Only `-rc.N` pre-releases feed the promotion flow ([`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) accepts `vX.Y.Z-rc.N` alone); `-alpha.N` / `-beta.N` tags are built and published for download and validated on demand with the [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) button, but are never promoted to a stable release. ## Release Candidates -Release candidates are Cozystack versions that introduce new features and are published before a stable release. -Their purpose is to help validate stability before finalizing a new feature release. -They allow for final rounds of testing and bug fixes without freezing development. +Release candidates are Cozystack versions that introduce new features and are published before a stable release. Their purpose is to help validate stability before finalizing a new feature release. + +Release candidates are given numbers `vX.Y.0-rc.N`, for example, `v1.2.0-rc.1`. The first one is cut from `main`, once all major features for the upcoming release have merged and the release enters its testing phase. + +**Cutting the first rc freezes the line.** [`cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) creates `release-X.Y` at the tagged commit, and from that point the release's content is closed: -Release candidates are given numbers `vX.Y.0-rc.N`, for example, `v0.42.0-rc.1`. -They are created directly in the `main` branch. -An RC is typically tagged when all major features for the upcoming release have been merged into main and the release enters its testing phase. -However, new features and changes can still be added before the regular release `vX.Y.0`. +- Every later cut for that line — `rc.2`, `rc.3`, an `alpha`/`beta`, or a patch-line rc — must be dispatched from `release-X.Y`. A dispatch from `main` is refused. +- Fixes reach the release only by cherry-pick or backport onto `release-X.Y` (see [Backports](#backports)). +- `main` reopens immediately for the next minor, so feature work never has to wait for the release to ship. -Each RC contributes to a cumulative set of release notes that will be finalized when `vX.Y.0` is released. -After testing, if no critical issues remain, the regular release (`vX.Y.0`) is tagged from the last RC or a later commit in main. -This begins the regular release process, creates a dedicated `release-X.Y` branch, and opens the way for patch releases. +This is what makes an rc mean something. Previously `release-X.Y` was created only when the stable release merged, so `rc.2` was cut from `main`'s tip and silently absorbed everything that had landed since `rc.1` — the shipped release could contain code that no rc had ever validated. + +**Un-freezing is manual and deliberate.** If an `rc.1` turns out to be unusable and the line has to be re-cut from `main`, delete `release-X.Y` by hand first; the refuse step blocks every other route back to a `main` dispatch, which is the point of it. Nothing automates the deletion, and nothing should — throwing away a frozen line is a decision worth making explicitly. + +Each RC contributes to a cumulative set of release notes that will be finalized when `vX.Y.0` is released. After testing, if no critical issues remain, the last good rc is **promoted** to the regular release `vX.Y.0` — the exact rc images retagged by digest, never rebuilt and never tagged from a later commit (see [Regular Releases](#regular-releases)). Promotion cuts the write-once `vX.Y.0` tag at the promote PR's merge commit, fast-forwards the `release-X.Y` maintenance branch the freeze already created, and opens the way for patch releases. ## Regular Releases -When making a regular release, we tag the latest RC or a subsequent minimal-change commit as `vX.Y.0`. -In this explanation, we'll use version `v0.42.0` as an example: +A regular release `vX.Y.0` is **promoted from a release-candidate that already passed e2e** — never tagged by hand and never rebuilt. The bytes shipped as `vX.Y.0` are the exact `vX.Y.0-rc.N` images, retagged by digest. We'll use `v1.2.0` as an example. ```mermaid gitGraph commit id: "feature" commit id: "feature 2" - commit id: "feature 3" tag: "v0.42.0" + commit id: "feature 3" tag: "v1.2.0-rc.1" ``` -A regular release sequence starts in the following way: +A regular release sequence runs as follows: -1. Maintainer tags a commit in `main` with `v0.42.0` and pushes it to GitHub. -2. CI workflow triggers on tag push: - 1. Creates a draft page for release `v0.42.0`, if it wasn't created before. - 2. Takes code from tag `v0.42.0`, builds images, and pushes them to ghcr.io. - 3. Makes a new commit `Prepare release v0.42.0` with updated digests, pushes it to the new branch `release-0.42.0`, and opens a PR to `main`. - 4. Builds Cozystack release assets from the new commit `Prepare release v0.42.0` and uploads them to the release draft page. -3. Maintainer reviews PR, tests build artifacts, and edits changelogs on the release draft page. +1. Cut the first release candidate (`v1.2.0-rc.1`) from the last good commit on `main`: run the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow (manual dispatch **from `main`**) with the tag. It tags `main`'s HEAD as the CI app, then **creates `release-1.2` at that commit, freezing the line**; `tags.yaml` then builds the rc images, publishes the rc prerelease, pushes the digest-vendored `release-1.2.0-rc.N` staging branch, and runs its mandatory `rc-e2e` job by calling [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) against that published tag. +2. Watch the rc's mandatory full E2E and wait for it to pass, then complete the other checks in [Pre-release verification checklist](#pre-release-verification-checklist). If the latest `tags.yaml` run cannot provide a green `rc-e2e`, a successful manual dispatch of [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) for the same tag is equivalent gate evidence. `main` is already open for the next minor, so anything landing there is **not** in this release: fixes for `1.2` are cherry-picked onto `release-1.2`, and a later rc is cut by dispatching the workflow **from `release-1.2`**. **Optional:** once the notes are worth reviewing, press the `Generate RC Changelog` button ([`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml)) to generate `docs/changelogs/v1.2.0.md` now and bank it on the `release-1.2.0-rc.N` staging branch — so there is time to review and hand-edit it before promotion. Skipping the button changes nothing; promotion generates the changelog itself. ```mermaid gitGraph commit id: "feature" commit id: "feature 2" - commit id: "feature 3" tag: "v0.42.0" - branch release-0.42.0 - checkout release-0.42.0 - commit id: "Prepare release v0.42.0" + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 checkout main - merge release-0.42.0 id: "Pull Request" + commit id: "feature 4 (next minor)" + checkout release-1.2 + commit id: "cherry-picked fix" tag: "v1.2.0-rc.2" ``` - When testing and editing are completed, the sequence goes on. +3. Once the checks are green, run the [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) workflow (manual dispatch) with that rc tag. Its `parse` job verifies E2E before any staging: it accepts a successful `rc-e2e` job in the latest `tags.yaml` push run for the tag, or any retained successful manual `e2e-tag.yaml` run whose expanded E2E job name targets the tag. The emergency `skip_e2e_gate` input bypasses this check, logs a loud warning, and marks the promote PR with ⚠️. Before its first registry write, promotion also proves that the selected `main` or `release-X.Y` target already contains the candidate-aware PR gate, finalizer and trusted verifier; backport this pipeline to an existing maintenance branch before promoting its next patch. Promotion **creates no stable name before merge**: dispatch does write to the registry — it publishes one `promotion-*` packages candidate, and that tag is real and publicly listable — but it creates no `vX.Y.Z` tag, moves no `latest`, and publishes no stable installer, so nothing a consumer resolves changes until the PR merges. It: + 1. Rewrites the rc version substring to `1.2.0` and commits that exact tree, serializes its `packages/` directory under a run-specific temporary OCI tag, then adds a second commit pinning the candidate's immutable digest in the installer before pushing `release-1.2.0`. Container digests remain exactly the rc's; the two commits give the OCI revision an honest source SHA without pretending the artifact can contain its own digest. + 2. Drafts the `v1.2.0` release and uploads the (restamped) assets. + 3. Generates `docs/changelogs/v1.2.0.md` (in a parallel job, detached at the rc tag) — or, if you pressed the rc-time `Generate RC Changelog` button, copies the file already banked on the `release-1.2.0-rc.N` staging branch instead of regenerating — and commits it onto the `release-1.2.0` branch, so the changelog is reviewed as part of the promotion and finalize can publish the release with it already in place. Non-blocking: if generation fails the PR still opens, carrying a ⚠️ line that says the changelog is missing. + 4. Opens the `cozystack/website` "update managed apps reference" PR (`update-docs-v1.2.0` branch), generating the docs from the `release-1.2.0` staging branch via `FETCH_REF`. Non-blocking, and **do not merge that PR until `v1.2.0` is published** — merging early breaks the live site's version routing (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)). + 5. Opens the `chore(release): promote v1.2.0-rc.N -> v1.2.0` PR into `release-1.2`, labelled `release` only. The PR body records whether E2E was verified or bypassed, plus ✅/⚠️ changelog and website-docs lines. E2E does not run on this promote PR by default because the immutable rc already passed the mandatory full suite; a maintainer may add `full-e2e` to run it again. The stable retag itself is deferred to the merge (step 5), so an abandoned promotion leaves no stable-named images and cannot wedge a re-promotion; retention protects the temporary candidate for as long as this release PR remains open, then may remove it after 30 days. + + The PR targets the maintenance branch, not `main`: `promote-rc.yaml` prefers `release-X.Y` whenever it exists, and since the rc freeze it always does. The digest-vendored `Prepare release` commit therefore stays on the release line instead of landing on `main`. + + ```mermaid + gitGraph + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 + checkout release-1.2 + commit id: "cherry-picked fix" tag: "v1.2.0-rc.2" + branch release-1.2.0 + checkout release-1.2.0 + commit id: "Prepare release v1.2.0" + commit id: "Pin promoted packages artifact" + checkout release-1.2 + merge release-1.2.0 id: "Pull Request" + ``` -4. Maintainer merges the PR. GitHub removes the merged branch `release-0.42.0`. -5. CI workflow triggers on merge: - 1. Moves the tag `v0.42.0` to the newly created merge commit by force-pushing a tag to GitHub. - 2. Publishes the release page (`draft` → `latest`). -6. The maintainer can now announce the release to the community. +4. Maintainer reviews the PR and the draft release against the [Phase 4 checklist](#phase-4--review-and-merge-the-promote-pr): confirm the E2E-verification line is ✅ (or explicitly accept its ⚠️ bypass), the changelog line is ✅, and note the website-docs line — the `cozystack/website` docs PR that promotion parked, which stays unmerged for now. E2E does not run on this promote PR by default. Then merge — **do not squash-merge**, the stable tag must attach to a real merge commit. GitHub removes the merged branch `release-1.2.0` (the repo has auto-delete-on-merge enabled). +5. CI workflow triggers on merge (this is where every consumer-visible or write-once side effect happens, after the rc E2E gate and PR review): + 1. Pulls the pinned packages candidate by digest and verifies that it matches the merged `packages/` tree apart from the artifact's impossible self-reference, carries no rc image tags, and preserves the original rc artifact's container `repo@digest` set. + 2. Creates the tag `v1.2.0` at the newly created merge commit — write-once. The tag is published here for the first time, never moved. + 3. Cuts the write-once `api/apps/v1alpha1/v1.2.0` Go-module tag at the same commit. + 4. Ensures the `release-1.2` maintenance branch exists at the tag commit. Since the rc freeze created it, this is normally a no-op fast-forward: the merge commit is already its tip. + 5. Publishes the release page (`draft` → `latest`), using `docs/changelogs/v1.2.0.md` from the merge commit as the release body. + 6. Retags the verified packages candidate and rc container images by digest to `v1.2.0` (and `:latest` only when `v1.2.0` is the newest published stable), and publishes the stable `cozy-installer` chart — no container rebuild. +6. Once the release is published, merge the parked `cozystack/website` docs PR (opened back at promote time, step 3.iv) so the live site's version routing flips to `v1.2.0` — its versioned docs and `openapi.json` are on GitHub now. The maintainer can then announce the release to the community. ```mermaid gitGraph - commit id: "feature" commit id: "feature 2" - commit id: "feature 3" - branch release-0.42.0 - checkout release-0.42.0 - commit id: "Prepare release v0.42.0" + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 checkout main - merge release-0.42.0 id: "Release v0.42.0" tag: "v0.42.0" + commit id: "feature 4 (next minor)" + checkout release-1.2 + commit id: "cherry-picked fix" tag: "v1.2.0-rc.2" + branch release-1.2.0 + checkout release-1.2.0 + commit id: "Prepare release v1.2.0" + checkout release-1.2 + merge release-1.2.0 id: "Release v1.2.0" tag: "v1.2.0" ``` +Note that `main` never receives the release commit. It forked away at the freeze and carries the next minor's work. + ## Patch Releases -Making a patch release has a lot in common with a regular release, with a couple of differences: +A patch release now follows exactly the same shape as the regular release above — by the time `vX.Y.0` ships, everything is already happening on `release-X.Y`. The only difference is where the line starts: -* A release branch is used instead of `main` -* Patch commits are cherry-picked to the release branch. -* A pull request is opened against the release branch. +* `release-X.Y` already exists, created by the freeze at `vX.Y.0-rc.1`, so there is no branch to create. +* Patch commits are cherry-picked onto it, and the rc is cut by dispatching **from `release-X.Y`** — the same dispatch used for `rc.2` of the original release. -Let's assume that we've released `v0.42.0` and that development is ongoing. +Let's assume that we've released `v1.2.0` and that development is ongoing. We have introduced a couple of new features and some fixes to features that we have released -in `v0.42.0`. +in `v1.2.0`. Once problems were found and fixed, a patch release is due. ```mermaid gitGraph - commit id: "Release v0.42.0" tag: "v0.42.0" + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 + checkout release-1.2 + commit id: "Release v1.2.0" tag: "v1.2.0" checkout main commit id: "feature 4" commit id: "patch 1" @@ -109,58 +152,535 @@ gitGraph ``` -1. The maintainer creates a release branch, `release-0.42,` and cherry-picks patch commits from `main` to `release-0.42`. - These must be only patches to features that were present in version `v0.42.0`. +1. The maintainer cherry-picks patch commits from `main` onto the existing `release-1.2` branch. These must be only patches to features that were present in version `v1.2.0`. + + Cherry-picking can be done as soon as each patch is merged into `main`, or directly before the release. - Cherry-picking can be done as soon as each patch is merged into `main`, - or directly before the release. + The `backport` label automates this: [`backport.yaml`](../.github/workflows/backport.yaml) targets the newest existing `release-X.Y` branch, which during a freeze window is the line being stabilised rather than the last published one. ```mermaid gitGraph - commit id: "Release v0.42.0" tag: "v0.42.0" - branch release-0.42 + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 + checkout release-1.2 + commit id: "Release v1.2.0" tag: "v1.2.0" checkout main commit id: "feature 4" commit id: "patch 1" commit id: "feature 5" commit id: "patch 2" - checkout release-0.42 + checkout release-1.2 cherry-pick id: "patch 1" cherry-pick id: "patch 2" ``` - When all relevant patch commits are cherry-picked, the branch is ready for release. + When all relevant patch commits are cherry-picked, the branch is ready for release. Build with `go build ./cmd/backport-audit`, then confirm with `./backport-audit release-1.2` — it exits non-zero while anything labeled for the line is still missing from it, and prints the URLs. See [Cherry-pick triage before a patch](#cherry-pick-triage-before-a-patch). + +2. The maintainer cuts a release candidate (`v1.2.1-rc.N`) via the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow (manual dispatch **from `release-1.2`**), which tags that branch's `HEAD`. CI builds and publishes the rc, pushes the `release-1.2.1-rc.N` staging branch, and runs the mandatory full `rc-e2e` job against the published tag. +3. Watch `rc-e2e` and wait for it to pass, then complete the other release checks. A successful manual `e2e-tag.yaml` run for `v1.2.1-rc.N` is also valid evidence if the latest `tags.yaml` run does not carry a green `rc-e2e`. **Optional:** press the `Generate RC Changelog` button ([`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml)) to generate `docs/changelogs/v1.2.1.md` at rc time and bank it on the rc staging branch; promotion then reuses it instead of regenerating. +4. Once the checks are green, run [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) (manual dispatch) with that rc tag. It runs the same five jobs as a regular promotion: the dispatch first gates on the latest tag-push `rc-e2e` or a successful tag-correlated manual E2E run (`skip_e2e_gate` overrides it), verifies that `release-1.2` already carries the candidate-aware pipeline, then commits and publishes the rewritten packages tree under a run-specific temporary `promotion-*` tag, adds a second commit pinning the candidate on the `release-1.2.1` branch, drafts the `v1.2.1` release with the restamped assets, generates or picks up `docs/changelogs/v1.2.1.md` and commits it onto that branch, opens the parked `cozystack/website` docs PR (updating the existing `v1.2/` directory in place, since a patch does not promote `next/`), and opens the promote PR into `release-1.2` with only the `release` label. The promote PR body carries the same ✅/⚠️ E2E, changelog, and website-docs lines and runs no E2E by default; add `full-e2e` manually to force a repeat. As with a regular release every stable name is deferred to merge; retention protects the temporary candidate while this PR is open and, after it merges, while `release-1.2` still pins it — until finalize retags it stable. -2. The maintainer tags the `HEAD` commit of branch `release-0.42` as `v0.42.1` and then pushes it to GitHub. -3. CI workflow triggers on tag push: - 1. Creates a draft page for release `v0.42.1`, if it wasn't created before. - 2. Takes code from tag `v0.42.1`, builds images, and pushes them to ghcr.io. - 3. Makes a new commit `Prepare release v0.42.1` with updated digests, pushes it to the new branch `release-0.42.1`, and opens a PR to `release-0.42`. - 4. Builds Cozystack release assets from the new commit `Prepare release v0.42.1` and uploads them to the release draft page. -4. Maintainer reviews PR, tests build artifacts, and edits changelogs on the release draft page. - ```mermaid gitGraph - commit id: "Release v0.42.0" tag: "v0.42.0" - branch release-0.42 + commit id: "feature 3" tag: "v1.2.0-rc.1" + branch release-1.2 + checkout release-1.2 + commit id: "Release v1.2.0" tag: "v1.2.0" checkout main commit id: "feature 4" commit id: "patch 1" commit id: "feature 5" commit id: "patch 2" - checkout release-0.42 + checkout release-1.2 cherry-pick id: "patch 1" - cherry-pick id: "patch 2" tag: "v0.42.1" - branch release-0.42.1 - commit id: "Prepare release v0.42.1" - checkout release-0.42 - merge release-0.42.1 id: "Pull request" + cherry-pick id: "patch 2" tag: "v1.2.1-rc.1" + branch release-1.2.1 + commit id: "Prepare release v1.2.1" + checkout release-1.2 + merge release-1.2.1 id: "Pull request" ``` Finally, when release is confirmed, the release sequence goes on. -5. Maintainer merges the PR. GitHub removes the merged branch `release-0.42.1`. -6. CI workflow triggers on merge: - 1. Moves the tag `v0.42.1` to the newly created merge commit by force-pushing a tag to GitHub. - 2. Publishes the release page (`draft` → `latest`). -7. The maintainer can now announce the release to the community. +5. Maintainer reviews the PR against the [Phase 4 checklist](#phase-4--review-and-merge-the-promote-pr), confirms the E2E-verification and changelog lines and notes the parked website-docs line, and merges it — **do not squash-merge**. GitHub removes the merged branch `release-1.2.1` (auto-delete-on-merge is enabled). +6. CI workflow triggers on merge (all consumer-visible or write-once side effects, after the rc E2E gate and PR review): + 1. Verifies the pinned packages candidate against the merged tree and the original rc artifact's container digests. + 2. Creates the tag `v1.2.1` at the newly created merge commit — write-once. The tag is published here for the first time, never moved. + 3. Cuts the write-once `api/apps/v1alpha1/v1.2.1` Go-module tag at the same commit. + 4. Publishes the release page (`draft` → `latest`; `latest` moves only if `v1.2.1` is the newest published stable). + 5. Retags the verified packages candidate and rc container images by digest to `v1.2.1` and publishes the stable `cozy-installer` chart — no container rebuild. +7. Once the release is published, merge the parked `cozystack/website` docs PR, then announce the release to the community. + +## What CI does during the release process + +The numbered process above is implemented by seven workflows, including the reusable/manual [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml), plus the reusable [`changelog-generate.yaml`](../.github/workflows/changelog-generate.yaml) that two of them share to produce the changelog. Knowing which job does what makes the failure modes much easier to diagnose. + +1. [`cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) — manual dispatch (from `main` or `release-X.Y`): the sole entry point for creating a pre-release tag. Validates the `-alpha`/`-beta`/`-rc` tag, then pushes it (write-once, at the branch tip) as the CI app so `tags.yaml` fires. Cutting the first `vX.Y.0-rc.N` also creates `release-X.Y`, freezing the line; afterwards a `vX.Y.*` dispatch from `main` is refused. Stable tags are never created here. +2. [`tags.yaml`](../.github/workflows/tags.yaml) — fires on any pre-release tag push: `build-amd64` and `build-arm64` build the pre-release side by side, then `prepare-release` publishes it and pushes its digest-pinned staging branch, then — **only for `-rc.N` tags** — the mandatory `rc-e2e` job calls `e2e-tag.yaml` in the same run (`-alpha.N` / `-beta.N` skip it and are validated on demand with `e2e-tag.yaml`'s manual button). `generate-changelog` and `update-website-docs` also live here but are **backstops only** — both are normally produced at promotion time (see 4); on a stable tag push each self-skips or no-ops when the promote flow already did the work. These stable-tag backstops only fire because finalize pushes the stable tag with the CI app token (a `GITHUB_TOKEN` push triggers no run — see 5). +3. [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) — reusable from `tags.yaml::rc-e2e` and manually dispatchable for a published tag. It checks that the tag's release carries its nocloud disk, resolves the digest-pinned tree and runs the full install, OpenAPI, and Chainsaw suite on Talos containers. A tag cut before the container lane existed has to be dispatched from its own ref. Its dispatch run title is the constant workflow name, so the promote gate correlates a manual run through the successful tag-bearing E2E job name rather than the title. +4. [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) — manual dispatch. Five jobs: `parse` validates the rc tag and requires green full-E2E evidence before any staging; `promote` rewrites and commits the rc tags, publishes that packages tree as a run-specific temporary reproducible candidate, commits its digest pin in the `release-X.Y.Z` tree, and drafts the stable release; `changelog` calls the reusable `changelog-generate.yaml` **in parallel** to produce `docs/changelogs/vX.Y.Z.md` (regenerating it, or copying an rc-time changelog already committed to the `release-X.Y.Z-rc.N` staging branch by `changelog-rc.yaml` — see 7); `website-docs` (needs `parse`, `promote`) opens the `cozystack/website` docs PR from the staging branch via `FETCH_REF` (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)); `open-pr` joins them, commits the changelog onto the staging branch, and opens the `release-X.Y.Z` promote PR with the `release` label but no automatic `full-e2e`. So the changelog and website docs are reviewed as part of the promotion, the already-checked rc suite is not repeated by default, and the PR body carries ✅ verified or ⚠️ bypassed plus ✅/⚠️ changelog and website-docs lines. Both `changelog` and `website-docs` are non-blocking. No stable registry name is created at dispatch. +5. [`.github/workflows/pull-requests.yaml`](../.github/workflows/pull-requests.yaml) verifies a promotion candidate against the prospective merge tree and original rc artifact before the required `E2E Tests` status can turn green. [`pull-requests-release.yaml`](../.github/workflows/pull-requests-release.yaml) fires after merge and repeats that verification before any stable tag, then cuts the write-once stable + Go-module tags, publishes the release **with the merged changelog as its body**, retags the candidate and rc images to stable by digest (`:latest` gated on newest-stable), and publishes the stable chart. +6. [`update-releasenotes.yaml`](../.github/workflows/update-releasenotes.yaml) — fires on pushes to `main` that touch `docs/changelogs/v*.md`; syncs that content into the corresponding GitHub Release body. No longer the primary path (finalize sets the body directly, which is race-free and works for maintenance lines too) — this now covers later edits to a published changelog and manual re-syncs via `workflow_dispatch`. +7. [`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml) — manual dispatch, **optional**: generate the changelog at rc time instead of waiting for promotion, while there is still time to review it. It calls the same reusable [`changelog-generate.yaml`](../.github/workflows/changelog-generate.yaml) and commits the result to the `release-X.Y.Z-rc.N` staging branch as `docs/changelogs/vX.Y.Z.md`; the promote-time `changelog` job then copies that file instead of running the AI again. Skipping this workflow changes nothing — promote generates the changelog itself, exactly as before. + +### Phase 1 — `build-amd64`, `build-arm64` and `prepare-release` (hard gate) + +On an rc tag push: builds images for both architectures, commits digest pins, creates the draft release, and pushes the `release-X.Y.Z-rc.N` staging branch. `build-amd64` holds the release-exists no-op and the rejection of a hand-pushed stable tag, and builds the amd64 images and the release assets (its `make build` also pushes the packages artifact and the installer chart, pinned on amd64 digests); `prepare-release` stitches, gates, republishes those two from the repinned tree, and publishes the release. (The stable promote PR is opened later by `promote-rc.yaml`, not here. When the stable tag itself is pushed by finalize, a draft already exists, so this job is a no-op — see [Phase 5](#phase-5--pull-requests-releaseyaml-finalize).) + +The commit (`Prepare release vX.Y.Z`, authored by `cozystack-ci[bot]`) is **digest pins and image tags only**: + +- `packages/core/{installer,platform,testing}/values.yaml` +- `packages/system/*/values.yaml` (cozystack-api, dashboard, kamaji, linstor, kubevirt-csi-node, etc.) +- `packages/apps/kubernetes/images/{kubevirt-csi-driver,ubuntu-container-disk-*}.tag` +- `packages/system/dashboard/templates/configmap.yaml` (the `$tenantText` value) + +Things that look surprising in the diff but are normal: + +- **Zero-diff packages**: when buildx fully caches and upstream base images haven't moved, the digest is byte-identical to the previous release and won't appear. Verify the build step actually ran — don't assume "no diff = nothing built." +- **Sudden churn on `ubuntu-container-disk-*` tags**: `cloud-images.ubuntu.com/noble/current/` is a moving target; these often rebuild even without code changes. +- **Switched registry**: if a self-built image moved from `docker.io` to `ghcr.io`, the registry portion of the digest changes — that's a deliberate move, not a regression. +- **`targetVersion` is NOT touched by the release PR.** Platform migration `targetVersion` is bumped earlier in a feature commit (e.g. `[platform] Bump migration targetVersion to 39 for migration 38`). The release PR only re-pins the `platform-migrations` image digest. + +#### Multi-arch images + +Every pre-release is published for amd64 and arm64. `tags.yaml::build-arm64` builds the images on a native arm64 runner and pushes each as `-arm64`, while `build-amd64` builds amd64, the Talos installer and matchbox for both arches, and the assets on x86 at the same time. `build-amd64` hands its stamped tree, the log of the tags it pushed and the assets to `prepare-release` as workflow artifacts; a missing one fails the upload or the download rather than publishing a partial release. `prepare-release` waits for both builds and runs only if both succeeded: when either fails, the rc is not prepared at all, so a release is never published amd64 only. It applies the stamped tree and runs [`hack/stitch-multiarch.sh`](../hack/stitch-multiarch.sh), which joins each image with its arm64 twin into one index, moves onto it every tag the build pushed for that image (the rc tag and, under `PUBLISH_VERSIONED=1`, its component version, which the build records in `PUSHED_TAGS_LOG`), and repins the tree on the index digests, then [`hack/verify-multiarch.sh`](../hack/verify-multiarch.sh), then republishes the packages artifact and the installer chart from the repinned tree. A stable tag skips `build-arm64` and builds nothing: promotion copies the rc's indexes with every platform, so the stable release is exactly as multi-arch as its rc. Because the two builds run in parallel, an rc that is not prepared (its arm64 build failed, or `prepare-release` failed before the stitch) still leaves behind what the amd64 build pushed: amd64-only images under `:` and under the component-version tags, and the `cozystack-packages:` artifact and the installer chart, both pinned on those amd64 digests. A component tag an earlier rc had moved to an index then points at an amd64-only image again, until a run that succeeds stitches it, a re-run of the same run included. + +`verify-multiarch.sh` is a hard gate. It resolves every ref the tree pins, third-party images included, and fails naming each one that is not an index carrying both `linux/amd64` and `linux/arm64`, or that it cannot resolve at all. The fix is to make the image multi-arch (build it for both, or pin a multi-arch digest of the same version); an image that cannot exist for arm64 goes in [`hack/multiarch-allowlist`](../hack/multiarch-allowlist) by repository, with the reason on the same line. + +That gate sees only what the tree pins by digest. An image a vendored chart renders from its own defaults, or one an operator launches at runtime, is invisible to it. `rc-e2e` closes most of that gap: before tearing the sandbox down, [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) reads every image the three Talos nodes pulled from their image stores (`hack/collect-images.sh`), each by the digest the pull resolved to, and runs `verify-multiarch.sh --refs-file` on that list with the same allowlist. A pre-release fails on any single-arch image, on any ref it cannot read or that is still rate-limited after the retries, and on a list the collection could not produce; a stable tag's run only warns, since it ships its rc's images. A tag whose tree predates the audit (it has no `hack/verify-multiarch.sh`) only warns and passes, so an rc cut before the audit existed can still be re-validated. The list and the audit output are in the run's `image-list-` artifact. Neither check covers a package that e2e never installs and whose images the tree does not pin by digest: an arm64 break there shows up only on an arm64 cluster. + +[`build-arm64-nightly.yaml`](../.github/workflows/build-arm64-nightly.yaml) builds `main` for arm64 every day, so an arm64 build break shows up there before the next rc. Its first step prints `verify-multiarch.sh --report` for `main`'s committed tree: `main` pins amd64 images, so its first-party refs are all listed and are expected to be, since the rc's stitch turns them into indexes. A third-party ref that an in-tree rebuild stamps over at build time, as the Harbor components and shadowbox are, is listed too and goes away on the rc. The lines that matter are the third-party refs no build replaces and refs that cannot be resolved; each of those fails the next rc until it is fixed or allowlisted. matchbox is listed too and does not matter: `build-amd64` rebuilds it as a two-arch index. The stitch names the images it skips in its own summary: an image with no `-arm64` twin, one whose digest is also pinned in a file the rewrite does not reach, and a pair that is not one amd64 and one arm64 image. matchbox (already an index) and the e2e sandbox (never built for arm64, and allowlisted) are skipped on every rc and pass the gate; any other skipped image fails it. + +### RC E2E — mandatory check before promotion + +For an rc, `prepare-release` must finish before `tags.yaml::rc-e2e` calls the published-tag lane, so the full suite tests the digest-pinned staging tree that promotion will reuse, and the release it runs against already carries its disk. `promote-rc.yaml::parse` then checks the latest `tags.yaml` push run whose `head_branch` is the rc tag and accepts it only when the reusable E2E job succeeded. GitHub exposes called-workflow job names as `E2E Release Candidate / E2E vX.Y.Z-rc.N (full suite)`, which the gate matches by the exact tag-bearing suffix. + +The alternate evidence path scans retained successful `workflow_dispatch` runs of `e2e-tag.yaml` and accepts one whose direct E2E job is named `E2E vX.Y.Z-rc.N (full suite)`. The workflow's run title is always `E2E Release Tag` and therefore cannot correlate the input tag; artifact names also carry the tag but expire independently, so the job name is the strongest available correlator. If Actions retention or manual deletion removes the run/job metadata, or that job is renamed without updating the matcher, the gate fails closed and tells the maintainer to run the E2E Release Tag button again. `skip_e2e_gate=true` is an emergency override, not alternate evidence: it writes a loud warning and a ⚠️ PR-body line. + +### Phase 2 — the changelog (generated at rc time or promotion, not here) + +The changelog for `vX.Y.Z` is produced by [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml)'s `changelog` job (a thin call to the reusable [`changelog-generate.yaml`](../.github/workflows/changelog-generate.yaml)) and committed onto the `release-X.Y.Z` staging branch by `open-pr`, so it rides the promote PR and is on the base branch the moment the release publishes. Finalize then uses that file verbatim as the GitHub Release body. Generating it here — on the stable tag push — would be too late by construction: that tag is created by finalize, which has already published the release. It can also be produced earlier, at rc time, via the optional [`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml) button — which runs the same reusable core and commits the file to the `release-X.Y.Z-rc.N` staging branch, from which the promote-time `changelog` job copies it (validated) instead of regenerating. + +The `changelog` job runs **in parallel** with `promote`, detached at the rc tag (the "promotion" configuration in §2 of [`agents/changelog.md`](./agents/changelog.md)), using a separate **read-only** GitHub App token for the AI step so the model cannot mutate the repo even with `--allow-all-tools`. Every write — the commit onto the staging branch and the PR — happens in `open-pr` under a write-scoped token. + +Generation is deliberately **non-blocking**: a Copilot outage or exhausted token quota must never block a release. Only the AI step carries `continue-on-error`, and `open-pr` is gated on `promote` alone (`if: !cancelled() && needs.promote.result == 'success'`) so the promote PR opens regardless. When the changelog is missing the PR body says so explicitly — that ⚠️ line is the signal to act on. + +Recovery, in order of preference: + +1. Commit `docs/changelogs/vX.Y.Z.md` to the `release-X.Y.Z` branch by hand (follow [`agents/changelog.md`](./agents/changelog.md)). `promote` preserves it across a re-dispatch, and `open-pr` will not overwrite it with generated output. +2. Re-dispatch `promote-rc.yaml` — generation runs again; anything already on the branch wins. +3. Merge without one and let `tags.yaml::generate-changelog` (the backstop) open a changelog PR against `main` after the fact. The release is published with the draft's body until that merges. + +Known failure modes: + +- Copilot quota exhausted (`COPILOT_GITHUB_TOKEN` 402). Refill or rotate, then re-dispatch. +- AI step timeout (hard 30-min cap). +- Output truncated or whitespace-only — caught by the `Verify changelog` step (`hack/validate-changelog.sh`), which accepts either header convention (`# Cozystack vX.Y.Z` for a minor or `# vX.Y.Z ()` for a patch), and requires the leading release-link comment pointing at `releases/tag/vX.Y.Z`, a `compare/...vX.Y.Z` link ending in this version, and at least one `## ` section — deliberately with no line-count floor, since a complete short patch changelog (v1.5.1 ships in 19 lines) must not be rejected as a fragment. It downgrades to the missing-changelog path rather than shipping a fragment as release notes. + +**`tags.yaml::generate-changelog` is the backstop.** It self-skips when the changelog is already on `main`. It earns its keep in two cases: generation failed during promotion, or the promote PR targeted `release-X.Y` so the file never reached `main`. Since the rc freeze the second case is **every** release, minor and patch alike — the promote PR always targets the release line (see [Regular Releases](#regular-releases)), which makes this backstop the only route by which a published changelog reaches `main`. In that case it **ports the reviewed file from the tag commit verbatim** rather than regenerating — a second AI pass there would spend quota and then overwrite already-published, already-reviewed release notes. + +### Phase 3 — `update-website-docs` (now the backstop) + +The `cozystack/website` "update managed apps reference" PR is normally opened **earlier, at promote time**, by [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml)'s `website-docs` job — not here. That job fetches the README content from the `release-X.Y.Z` staging branch `promote` just pushed (via the website Makefile's `FETCH_REF` variable), so the docs are ready to review alongside the promote PR while the stable tag is still unborn. It refreshes the trunk version pins from the staging ref before promoting `next/` → `vX.Y/`, so a new minor's docs inherit fresh Talos/cozystack pins instead of a stale `next.yaml`. It is **non-blocking**: if it fails (or trips its `FETCH_REF`-support guard against a website checkout that predates that Makefile variable), the promote PR opens with a ⚠️ website-docs line and this tag-time job backfills the same branch. + +**Merge-timing contract:** the website PR must NOT be merged before the release is published (finalize). Merging it early flips the site's latest-version pointer to a version whose versioned docs and `openapi.json` are not on GitHub yet, so every version-pin and API link for the new version 404s. Both the website PR body and the promote PR body say so; park the website PR until the GitHub release for `vX.Y.Z` is live, then merge. + +This tag-time `update-website-docs` job stays as the **backstop**. It runs with `if: !cancelled() && needs.prepare-release.result == 'success'` (surviving a failed/skipped changelog phase, stopping on cancel), decides whether to promote `next/` → `vX.Y/`, runs `make update-all`, stages `content hugo.yaml data/versions`, and force-pushes the same `update-docs-vX.Y.Z` branch. On the normal path — where `promote-rc.yaml::website-docs` already did the work — it no-ops into "No changes to commit". At tag time the stable tag exists, so its `BRANCH=RELEASE_TAG` fetch resolves without needing `FETCH_REF`. For it to fire at all, finalize must push the stable tag with the **CI app token** (see [Phase 5](#phase-5--pull-requests-releaseyaml-finalize)) — a tag pushed with the default `GITHUB_TOKEN` triggers no workflow run. + +If anyone changes the website Makefile to write somewhere else (e.g. `static/`, `i18n/`), the `git add` list must grow in **both** jobs or the PR silently drops files. + +### Phase 4 — review and merge the promote PR + +Reviewer checklist for the `chore(release): promote vX.Y.Z-rc.N -> vX.Y.Z` PR: + +- [ ] The PR body says `✅ RC full e2e was verified`; if it says the gate was bypassed, obtain explicit maintainer acceptance or add `full-e2e` and wait for that opt-in run. +- [ ] `Verify release packages candidate` is green. It pulls the candidate and original rc artifact by digest, compares the candidate to GitHub's prospective merge tree, and feeds the required `E2E Tests` status even when full E2E is not repeated. Do not edit `packages/` on this PR after generation; if the guard reports drift, re-dispatch promotion to create and pin a new run-specific candidate. Note what a green here does *not* cover: GitHub fires no `pull_request` event when the base branch advances, so a `packages/` change merged to the base afterwards is never re-verified on the PR. Finalize repeats the check against the real merge commit and catches that case before any stable name exists — but it catches it after the promote PR has already merged, so a `packages/` change landing on the base while a promote PR is open is worth holding back. +- [ ] The PR body's **website-docs line** is present: if ✅, the `cozystack/website` docs PR is open and **parked** — do NOT merge it until `vX.Y.Z` is published (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)); if ⚠️, the tag-time backstop will backfill it after finalize. +- [ ] Diff is digest pins + image tags only — nothing else. +- [ ] No accidental config drift (a value file you don't recognize). +- [ ] If any chart bumped, check the new digest pulls — `crane manifest ghcr.io/cozystack/cozystack/@` should return. +- [ ] Build artifacts on the draft release page exist and match the expected count. +- [ ] The PR includes `docs/changelogs/vX.Y.Z.md`, it has content, and the entries match the commit range (see [Changelog pre-publish verification](#changelog-pre-publish-verification)). If the PR body carries the ⚠️ "No changelog in this PR" line, resolve that before merging — finalize publishes this file as the release body. + +### Phase 5 — `pull-requests-release.yaml` (finalize) + +Fires on merge of a PR that is merged, carries the `release` label, and is authored by `cozystack-ci[bot]` (the last guard closes the "name a branch `release-X.Y.Z`, get it labelled, merge it" hole). Head branch must match `release-X.Y.Z[-suffix]`. This is where **every consumer-visible or write-once side effect of a promotion happens** — after the rc's mandatory E2E was checked at dispatch and the PR merged, making promotion transactional at stable names. Steps: + +1. **Verify the packages candidate before creating any stable tag.** [`hack/verify-promoted-packages.sh`](../hack/verify-promoted-packages.sh) pulls the candidate by the digest committed in installer values, proves it matches the merged `packages/` tree after normalizing only the impossible self-reference, rejects surviving rc image tags, then follows the candidate's embedded pre-push `platformSourceRef` back to the original rc artifact and compares normalized container `repo@digest` sets. Any package edit after candidate publication or any changed container digest aborts finalize while all stable names are still absent. +2. **Create the tag at the merge commit** (write-once). The merge commit of `Prepare release vX.Y.Z` did not exist before the PR opened, so there is nothing to move — the tag is created here for the first time. A pre-existing tag at a different commit fails the step loudly rather than being force-moved (see [Tag immutability](#tag-immutability)). The checkout uses `persist-credentials: false` so this push authenticates as the **CI app** (via `git remote set-url`), not the default `GITHUB_TOKEN` — a `GITHUB_TOKEN` push creates no workflow run, so a persisted-credential push would silently skip `tags.yaml` entirely and its stable-tag backstops (`generate-changelog`, `update-website-docs`) would never fire. +3. **Cut the `api/apps/v1alpha1/vX.Y.Z` Go-module tag** (write-once, stable only) at the same commit, so Go consumers of `api/apps/v1alpha1` get the release. Moved here from `tags.yaml`, whose `prepare-release` body is skipped for a promoted stable (the draft already exists). +4. **Ensure the maintenance branch `release-X.Y` exists** at the tag commit. Since the rc freeze creates it, this is normally a no-op fast-forward — the merge commit is already the branch tip. Still created if missing; updated fast-forward-only — a non-fast-forward update warns and is left for a maintainer rather than being force-updated. +5. **Publish the draft release**, with `docs/changelogs/vX.Y.Z.md` from the merge commit as its body (an absent or whitespace-only file warns and leaves the draft's body in place, rather than publishing blank notes). `make_latest` is computed against published-non-prerelease tags: prereleases stay `false`; tags older than the current max stay `false` (and the current max is force-restored to `latest` if necessary, so an older patch tag cut after a newer minor won't downgrade `latest`). +6. **Retag the verified packages candidate and rc container images to stable** by digest (`hack/promote-retag.sh`, no container rebuild) and **publish the stable `cozy-installer` chart**. `:latest` (on both the images and the chart) moves only when this release's `make_latest` was `true` — the same decision as step 5, so the release's `latest` and the images' `:latest` never disagree. The chart is packaged with `platformVersion` stamped into its default values so the documented `helm --install --version X.Y.Z` path reports the stable version. + +### Phase 6 — `update-releasenotes.yaml` (sync GitHub Release body) + +Fires on pushes to `main` that touch `docs/changelogs/v*.md`. Reads each `docs/changelogs/vX.Y.Z.md` and PATCHes the matching GitHub Release's `body` if it differs. Since Phase 5 now sets the body directly from the merge commit, this is no longer on the critical path — it covers later **edits** to a published changelog, the backstop PR landing a changelog that reached `main` after the fact, and manual re-syncs via `workflow_dispatch`. It only watches `main`, which is why finalize (not this job) is what populates a release promoted onto `release-X.Y` — since the freeze, that is every release rather than only a patch. + +## Stable tags come from rc promotion + +A stable `vX.Y.Z` is created only by **promoting an existing release-candidate** that already passed e2e — never by rebuilding and never by a cron. + +- [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) is triggered manually once an rc has gone green. It rewrites the rc version substring in the vendored image tags (the container digests stay the rc's), commits and publishes the resulting declarative packages tree under a unique run-specific `promotion-*` tag, commits that digest pin, and opens the `release-X.Y.Z` staging PR. The PR gate verifies the candidate against the prospective merge and original rc artifact; finalize repeats that check before creating the write-once stable tag at the merge commit, then retags the verified packages artifact and rc's already-built, e2e-passed container digests to the stable tag **by digest** (no container rebuild — see [`hack/promote-retag.sh`](../hack/promote-retag.sh)) and publishes the release. An abandoned promotion leaves no stable-named bytes; its temporary candidate is retention-managed. + +So a commit on a supported `release-X.Y` line ships when a maintainer promotes the next rc for that line — not automatically within 24h. Pushing a stable `vX.Y.Z` tag by hand is **not** a supported path: `tags.yaml` fails fast on a stable tag that has no pre-existing draft ("stable tags come from promote-rc.yaml"), and even so the finalize step would refuse to move a pre-existing tag. Pre-releases (`vX.Y.Z-rc.N`, `-alpha.N`, `-beta.N`) are cut with the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow, which pushes the tag as the CI app; [`tags.yaml`](../.github/workflows/tags.yaml) then fires on that push, builds it, and publishes the pre-release. Cutting via the workflow (rather than a manual `git push`) is what lets repo admins lock `v*` tag creation to the CI app — see the tag-protection note below. + +## Nightly builds + +A nightly is an **installable copy of `main` on GHCR — not a rebuild and not a release**. [`build-main.yaml`](../.github/workflows/build-main.yaml) already builds every push to `main` into the CI registry (OCIR); the nightly promotes that build to the public release registry (GHCR) and proves it installs. + +[`nightly.yaml`](../.github/workflows/nightly.yaml) runs daily (gated by the `NIGHTLY_ENABLED` repo variable) as three jobs; build-disk and e2e both start once mirror has finished: + +1. **mirror** — resolve the exact commit `cozystack-packages:main` was built from, then [`hack/nightly-mirror.sh`](../hack/nightly-mirror.sh) copies every cozystack-owned component image OCIR→GHCR **by digest** (bit-for-bit, no rebuild) and re-publishes the rewritten `cozystack-packages` artifact and the `cozy-installer` chart to GHCR, tagged `0.0.0-nightly...` plus a floating `nightly`. +2. **build-disk** — assemble the Talos `nocloud` disk from the upstream siderolabs imager (the profile references only `ghcr.io/siderolabs/*`, so this needs no cozystack image and no rebuild) and publish it as a GHCR OCI artifact (`cozystack-nocloud`) so a nightly is installable on real hardware. The nightly e2e does not boot it. +3. **e2e** — stage the published GHCR closure (rewritten tree + the `cozy-installer` chart pinned to the GHCR packages artifact) and run the **full** app suite on Talos containers, which do not boot the disk. A nightly has no diff, so Test Impact Analysis does not apply. + +No GitHub release and no `api/apps/v1alpha1/*` Go-module tag are created for a nightly. [`retention.yaml`](../.github/workflows/retention.yaml) prunes old GHCR nightly versions (keeps the newest 14 per package the current nightly publishes, named from the published `cozystack-packages:nightly` tree by [`hack/retention-packages.sh`](../hack/retention-packages.sh), except packages linked to another repository, which are skipped with a warning; the floating `nightly`, release tags and untagged versions are never touched) and abandoned packages candidates. A candidate version is eligible only when all of these hold: it carries at least one tag and **every** tag on that manifest matches the strict `promotion-vX.Y.Z-from-vX.Y.Z-rc.N-run--` form, so a manifest that has been retagged stable is excluded; it was last updated more than 30 days ago; its digest is not pinned by `packages/core/installer/values.yaml` at the head of an **open** pull request that is same-repository, authored by `cozystack-ci[bot]`, on a `release-X.Y.Z` branch and carrying the `release` label; and its digest is not pinned by that same file at the tip of `main` or of any `release-X.Y` branch — which is what keeps a candidate safe between the promote PR merging and finalize retagging it. Any such PR head whose pin is not exactly one `digest=sha256:<64 hex>` stops the candidate sweep for that run and fails the job, without affecting the nightly sweep. + +Install a nightly: + +```bash +helm upgrade --install cozystack \ + oci://ghcr.io/cozystack/cozystack/cozy-installer --version 0.0.0-nightly.20260626.15992304129.1 +# ...or --version nightly for the latest +``` + +The matching Talos node image is `ghcr.io/cozystack/cozystack/cozystack-nocloud:0.0.0-nightly.20260626.15992304129.1` (pull with `oras`). + +## Backports + +### The backport bot + +[`backport.yaml`](../.github/workflows/backport.yaml) wraps [`korthout/backport-action`](https://github.com/korthout/backport-action) and fires on a merged `main`-targeted PR carrying one of: + +| Label | Target branch | +|-------|---------------| +| `kind/backport` | the newest existing `release-X.Y` branch | +| `kind/backport-previous` | the second-newest existing `release-X.Y` branch | + +Resolution is dynamic at run time, and it reads the branches themselves: the job lists the repository's branches, keeps the ones matching `release-.`, and sorts them numerically descending — so `release-1.10` ranks above `release-1.9`, which a lexicographic sort gets backwards. `backport` takes the first, `backport-previous` the second, so both name a branch that exists by construction; asking for `backport-previous` when only one line exists fails the job rather than inventing a target. Nothing is derived arithmetically — an earlier version computed the previous line as `Y-1`, which named a non-existent branch whenever a minor was skipped. + +**During a freeze window the newest line is the one being stabilised, not the last published one.** `release-X.Y` is created when the first `vX.Y.0-rc.1` is cut, before `vX.Y.0` is published, so from the freeze until the release `backport` targets the release being stabilised and `backport-previous` targets the last published stable. That is the intent: the frozen branch is the only way into the upcoming release, which is when backports matter most. The trade-off is that for the length of the window the line one step further back has no label pointing at it, so a fix that has to reach it before the new release publishes must be cherry-picked by hand. + +The bot creates a backport PR with title `[Backport release-X.Y] `. When this PR merges, the title prefix used to re-trigger the bot through `pr-labeler.yaml`, which auto-applied `backport` to any `[Backport release-X.Y]`-titled PR. Combined with the org-level `dosubot` re-applying the label, this caused recursive backports. + +The fix (PR #2584): both job `if:` blocks gate on `github.event.pull_request.base.ref == 'main'`. Backport PRs target `release-X.Y`, so they cannot satisfy this — architectural protection, regardless of which bot relabels them. + +### When the bot fails + +If the run triggered by the merge dies without opening a backport PR, look for a run of this workflow that has not finished before retrying — `gh run list --workflow backport.yaml --branch ` — and clear it, because the retry queues behind it rather than replacing it. Do not filter by status: a run whose jobs are waiting for a runner, or which is held behind the concurrency group, is not `in_progress`, and filtering it out is how you conclude nothing is running. `--branch` narrows the list but does not identify the PR, since head branch names are not unique across forks and `gh run list` has no PR filter, so confirm which PR a run belongs to before cancelling it or you cancel someone else's backport. If cancelling does not clear the run it holds the group until GitHub retires it and the retry keeps queueing; that case has not been exercised here. A run whose jobs have all finished can still sit `in_progress`, which looks the same from the PR as a run that died. Then remove whichever backport label the PR already carries and add it back: a label event is the only other trigger, and it qualifies on the label it carries, so re-applying it is what starts a fresh attempt. + +Conflicting cherry-picks produce a draft PR via `conflict_resolution: draft_commit_conflicts`. Look for the bot's comment with the merge-conflict diff. You either: + +- Resolve in the draft branch and undraft, or +- Drop the bot's branch and cherry-pick manually: + +```bash +git checkout release-X.Y +git cherry-pick -x -s +# resolve conflicts +git commit -s +git push origin release-X.Y # or push to a new branch and open a PR +``` + +To find the bot's failed comments across a batch of PRs: + +```bash +for n in $(gh pr list --search "label:kind/backport label:kind/backport-previous merged:>=2026-01-01" --json number --jq '.[].number'); do + echo "=== #$n ===" + gh pr view $n --json comments --jq '.comments[] | select(.author.login == "github-actions" or (.author.login | contains("backport"))) | .body' | head -20 +done +``` + +Before [PR #3155](https://github.com/cozystack/cozystack/pull/3155), `conflict_resolution` was passed outside `experimental`, so a conflicting cherry-pick used the `fail` fallback and left only a "Backport failed" comment on the source PR. No draft backport PR was opened. When [`backport-audit`](../cmd/backport-audit/README.md) reports `MISSING`, check those comments and the workflow logs to establish why the change did not arrive. + +### Cherry-pick triage before a patch + +A patch release includes bugfixes for code that shipped in the corresponding minor `vX.Y.0`. Use: + +```bash +# 1. Audit what was labeled for this line and has NOT landed on it. +# Exits non-zero when anything is outstanding; see cmd/backport-audit/README.md. +go build ./cmd/backport-audit +./backport-audit release-X.Y + +# 2. List commits on main since the release branch diverged that are NOT yet on release-X.Y +git merge-base origin/main origin/release-X.Y +git log ..origin/main --grep="(#" --oneline + +# 3. Open PRs that may need labeling before the cut +gh pr list --state open --base main --label kind/bug +``` + +Step 1 covers the currently labeled work: [`backport-audit`](../cmd/backport-audit/README.md) assigns each request to the release line selected at merge time, then checks whether its commits reached the branch. Incomplete delivery, open or dropped backports, missing changes and duplicate open claims fail the gate. Late-added labels can disagree with the bot's target, and branch-opening dates are estimated from rc tags; see the audit's documented limits before deciding what to backport. Steps 2 and 3 cover changes that were never labeled. + +**Include rule:** + +- `kind/bug` — especially destructive (data loss, crash-loop, OOM, eviction). +- `area/security` / CVE. +- Narrow chart fixes whose blast radius is contained (e.g. one operator's resources). +- Patch-line-policy-matching dependency bumps (e.g. cilium 1.X.Y → 1.X.(Y+1) on a `release-A.B` shipping cilium 1.X — never a minor bump, only a patch within the line the branch already ships). + +**Skip rule:** + +- `kind/feature` — even with `lgtm`. A patch release is not a delivery vehicle for features. +- CI-only changes (`area/ci`, test infrastructure). +- Docs-only changes (those land via `update-website-docs` automatically). +- Large multi-package dependency churn (Dependabot Go-deps bumps). +- PRs whose conventional-commit type is `!` (breaking) or that mix `feat` with `fix`. + +**Borderline cases:** + +- Large refactors with `lgtm` AND an obvious bug payload — prefer extracting the bug fix into a narrower commit and backporting only that. +- PRs labeled both `kind/bug` and `kind/feature` — split them. + +### Backporting code that touches monotonic counters + +Migrations are the canonical case. `packages/core/platform/images/migrations/run-migrations.sh` is a linear counter — it runs `seq $CURRENT_VERSION $((TARGET_VERSION - 1))` and stops if `CURRENT >= TARGET`. This creates a forward-upgrade trap. Walking through it with `release-1.2` as the maintenance branch: + +- `main` has `targetVersion=10` and migrations 1–9 on disk. +- The `release-1.2` branch was cut when `targetVersion=5` (migrations 1–4 on disk). +- You backport a new migration to `release-1.2` at slot 5, and naively bump `targetVersion` 5 → 6 to enable it. +- Customer running 1.2.x runs the new migration, stamps `cozystack-version=6` in-cluster. +- On upgrade to 1.3 (main's `targetVersion=10`), the runner walks `seq 6 9` and runs main's migrations 6–9 — **main's own migration at slot 5 never runs.** The backport claimed that slot on `release-1.2`; main's slot 5 holds an unrelated migration that gets silently skipped. + +Mitigations, in order of preference: + +1. **Don't bump `targetVersion` in the backport.** Just ship the migration file. Prior precedent in the repo: an earlier maintenance branch's ACME backport did exactly this — the migration shipped but ran on no clusters until a later commit raised the counter. Pair this with relaxing the chart-side `{{ fail }}` guard on the maintenance branch (chart-side coalesce) so the new value works without the migration having executed. +2. **If you must bump,** add an idempotent duplicate of the now-skipped main migration at a higher slot. Bounded cost (one extra file per affected migration) and survives the skip window. +3. **Audit cross-branch counter state before deciding.** Compare migration numbering across `main`, `release-X.Y`, and `release-X.(Y-1)`. If a backport would create a skip window on any forward-upgrade path, choose option 1. + +This rule generalizes to any monotonic-counter state: schema versions, feature-flag generations, anything that gates "have I run X yet?" with a single integer. + +## Changelogs + +### Where the canonical process lives + +[`agents/changelog.md`](./agents/changelog.md) is the source of truth. Read it end-to-end before generating; do not infer the process from past commits or memory. The CI runs it under Copilot via the reusable `changelog-generate.yaml` — called by `promote-rc.yaml::changelog` at promote time and by the optional `changelog-rc.yaml` at rc time — and, as a backstop, `tags.yaml::generate-changelog`; locally the same prompt is replayable by following the doc directly. + +### Common changelog failure modes + +These are mistakes that have shipped in real changelogs. Verify against each before merging the changelog PR. + +**1. Commits from outside the release range.** Caused by running `git log ..HEAD` from `main` while generating a patch changelog — `HEAD` on `main` contains everything that landed since the tag plus backports merged to `release-X.Y` after the tag was cut. Use `git log ..` explicitly. The doc was updated to require this after v1.3.1's changelog shipped with 14 entries for what was actually a 1-commit release. + +**2. Original + backport listed as two separate entries.** Each backport must coalesce with its original into a single entry of the form `(in #orig, backport #bp)`. To verify after generation: + +```bash +# Every entry that mentions "backport" should also reference the original PR number +grep -E 'backport #[0-9]+' docs/changelogs/v.md | grep -v '#[0-9]*, backport' +``` + +Output should be empty. + +**3. Wrong PR author.** The squash-merge commit's author is whoever clicked "Merge" — not the person who wrote the code. **Always** resolve via `gh pr view --json author --jq .author.login`, never `git log --format=%an`. This bites hardest for website-repo entries where the same merger handles most PRs. + +**4. Superseding patch, stale changelog.** Tags are write-once, so a critical fix after `vX.Y.Z` ships as the next patch `vX.Y.(Z+1)` rather than moving the tag. If a changelog PR already exists for a tag you are superseding, the changelog work transfers to the new patch. Compare `git log ..` against what the existing `changelog-vX.Y.Z` branch already documents; only add/remove the deltas. If the only new commits are CI-internal or a revert of a feature that never reached a stable tag, no changelog edit is needed. + +**5. PR numbers swapped inside prose.** The entry-format validator checks bullet entries but ignores Feature Highlights paragraphs and Upgrade Notes. Both have caused wrong PR references in shipped changelogs. Verify every `#NNNN` in prose with `gh pr view `. + +**6. Hallucinated entries.** Every entry must correspond to a commit in `git log ..`. Verify by extracting PR numbers from the file and grep-searching the commit range: + +```bash +grep -oE '#[0-9]+' docs/changelogs/v.md | sort -u | while read pr; do + n=${pr#\#} + if ! git log .. --grep="#${n}" --oneline | grep -q .; then + echo "PR $pr in changelog but not in commit range" + fi +done +``` + +**7. Bot accounts as human contributors.** `cozystack-ci[bot]`, `github-actions`, `dependabot`, `renovate`, and any `app/*` login must NOT appear in the `## Contributors` list. They legitimately appear in **per-entry attribution** (e.g. a Renovate-authored PR) — but not in the human roll-call. + +**8. Title duplicated as description.** `* **fix(foo): X**: fix(foo): X (...)` means you wrote the conventional-commit subject twice and never wrote a user-facing description. The brief and the detail must say different things — the detail explains what the change means for users. + +### Changelog pre-publish verification + +Run these checks before merging the changelog PR. Catches problems in roughly the order they occur: + +```bash +PREV=v1.2.0; NEW=v1.2.1 + +# A. Every chart bump is mentioned somewhere in the changelog +git diff --name-only $PREV..$NEW -- 'packages/*/charts/*/Chart.yaml' 'packages/*/*/charts/*/Chart.yaml' | while read f; do + old=$(git show "$PREV:$f" 2>/dev/null | yq '.version // ""') + new=$(git show "$NEW:$f" 2>/dev/null | yq '.version // ""') + if [ "$old" != "$new" ]; then + component=$(basename "$(dirname "$f")") + if ! grep -qi "$component" docs/changelogs/$NEW.md; then + echo "MISSING: $component $old -> $new ($f)" + fi + fi +done + +# B. Out-of-Chart.yaml pins (Talos, tenant K8s, vendored images) +for path in packages/core/talos/values.yaml packages/core/installer/values.yaml packages/apps/kubernetes/values.yaml; do + git diff $PREV..$NEW -- "$path" | head -40 +done +git diff --name-only $PREV..$NEW -- 'images/*/Dockerfile' + +# C. Every cited #NNNN exists and is a merged PR (not an issue) +grep -oE '#[0-9]+' docs/changelogs/$NEW.md | sort -u | sed 's/#//' | xargs -P 20 -I{} sh -c ' + state=$(gh api /repos/cozystack/cozystack/issues/{} --jq ".pull_request.merged_at // \"ISSUE\"" 2>/dev/null) + echo "#{} $state" +' | grep -E 'ISSUE|null|^#[0-9]+ $' + +# D. URL liveness +grep -oE 'https://[^)" ]+' docs/changelogs/$NEW.md | sort -u | xargs -P 10 -I{} sh -c ' + code=$(curl -s -o /dev/null -w "%{http_code}" {}) + if [ "$code" != "200" ]; then echo "$code {}"; fi +' + +# E. Website PR authors are PR authors, not commit authors +grep -oE 'cozystack/website#[0-9]+' docs/changelogs/$NEW.md | sort -u | sed 's|cozystack/website#||' | while read n; do + pr_author=$(gh pr view $n --repo cozystack/website --json author --jq .author.login) + echo "website#$n -> @$pr_author" +done + +# F. Working tree clean except for the changelog +git status --porcelain | grep -v "docs/changelogs/$NEW.md" +# Should output nothing. +``` + +Ship criterion: every chart bump surfaced, no issue-cited-as-PR, no 404 URLs, every website entry attributed via `gh pr view` (not commit author), `git status` shows only the changelog. + +### Cozystack-specific URL quirks + +`cozystack.io` paths are versioned: `/docs/operations/foo/` and `/docs/next/foo/` both 404. Only `/docs/vX.Y/foo/` and the unversioned canonical `/docs/foo/` serve content. AI-generated changelogs invent the unversioned form. Verify URLs with the `curl` loop above. + +### Platform-component coverage + +For a minor release (`vX.Y.0`), changelog entries should surface what an upstream chart bump actually brought in — features, breaking changes, security fixes. PR commit messages usually don't capture this; the upstream `CHANGELOG.md` or GitHub Release notes do. For each bumped `Chart.yaml`: + +1. Read the `sources:` field — typically a GitHub URL. +2. Fetch release notes strictly inside `old < x <= new` via `gh release view --repo / --json body,name`. +3. Summarize in 2–5 user-impact bullets per component (new features, breaking changes, CVEs, important deprecations). Don't reproduce upstream notes verbatim; link to them. +4. If a PR-level entry already mentions the bump, enrich that entry with bullets — don't duplicate the entry. +5. If 3+ bumps have no PR-level mention, introduce a `## Platform Components` section. + +If you can't find upstream notes for a bumped component, list it with version-only info and no bullets. Hallucinated bullets are worse than missing detail. + +## Release-time fires — what's gone wrong recently + +These are real regressions that escaped to users. The pattern in each: code passed CI, shipped in a stable release, broke customers. Read them as a working list of pre-release verification gaps. + +### v1.3.0 — cert-manager ingressClassName regression (fix #2562) + +**Bug:** A commit migrated ACME HTTP-01 to use the modern `acme.cert-manager.io/http01-ingress-ingressclassname` annotation. The matching ingress-shim code that reads that annotation was added by cert-manager upstream PR #8244 and was **never backported** to the `release-1.19` branch — but cozystack 1.3.0 shipped cert-manager v1.19.3. The annotation was silently dropped at runtime and every solver Ingress fell back to the ClusterIssuer's default class `tenant-root`. Let's Encrypt validation broke for every tenant whose `publishing.ingressName != tenant-root`. + +**Blast radius:** v1.3.0, v1.3.1, and v1.1.7 (the migration was cherry-picked to `release-1.1`). + +**Why CI missed it:** the migration commit was self-contained and `helm template` rendered cleanly. There was no E2E that creates a tenant with a non-default ingress class and watches a real Let's Encrypt cert issue. Verification had been done from the docs side ("the API has been available since cert-manager 1.12"), not against the shipped cert-manager source. + +**Would have caught it:** non-root-tenant E2E with LE-staging cert issuance, asserting the solver Ingress lands on the tenant's class. + +### v1.4.0 — kubevirt-instancetypes upgrade failure (fix #2612) + +**Bug:** `packages/system/kubevirt-instancetypes/Makefile` contained `sed -i '/persistent: true/d' templates/preferences.yaml`, which stripped the only child of `preferredTPM`, leaving `preferredTPM:` (null) in six Windows preferences. KubeVirt v1.6.x silently accepted it. The v1.6.3 → v1.8.2 operator bump (2026-04-27) hardened the CRD's OpenAPI schema and started rejecting null for an object-typed field. `helm upgrade` then failed on every cluster with `kubevirt-instancetypes` enabled. + +**Why CI missed it:** the kubevirt-operator bump and the inert `preferences.yaml` were two weeks apart in commit history. CI only tests fresh installs against the new CRDs; it doesn't run `helm upgrade` from prior versions. Schema regression on a vendored asset that hadn't been touched in 18 months. + +**Would have caught it:** `helm template | kubectl apply --dry-run=server -f -` against the freshly-installed CRDs of every bumped operator. Or an upgrade-from-N-1 E2E lane. + +### v1.4.0 — Flux 2.7 → 2.8 readiness deadlock (#2602) + +**Bug:** helm-controller v1.5.0 (shipped in Flux 2.8) changes the default wait strategy to kstatus polling, which polls **every applied resource including child HelmReleases**. The `packages/apps/kubernetes` umbrella has 19 child HRs that each `dependsOn: parent` (with a `lookup` guard so the dependency only activates after the parent exists). Pre-2.8: parent went Ready as soon as Helm returned. Post-2.8: parent waits for children, children wait for parent. Deadlock. + +**Why CI missed it:** the cycle only manifests on the second reconcile (parent exists, so the `lookup` activates the child's `dependsOn`). Fresh-install E2E dodges this — the parent-already-exists branch isn't taken on the first apply. + +**Would have caught it:** apply-mutate-reapply E2E (any chart bump in the kubernetes umbrella triggers a second reconcile). Plus an explicit PR-template requirement: for any flux/helm-controller bump, paste the upstream breaking-changes section into the PR body. + +### Patterns these share + +1. **Fresh-install CI is insufficient.** All three regressions passed fresh-install lanes. +2. **Upstream breaking-defaults bumps are the common shape.** kubevirt CRD hardening, helm-controller default wait strategy, cert-manager API surface drift. Pre-merge reading of upstream changelogs would have caught two of three. +3. **One-engineer-bitten-once is the only safety net for live verification.** The cert-manager bump PR included an explicit live-cluster verification step — because the same engineer had been bitten the prior week. There is no checklist requiring it. + +## Pre-release verification checklist + +For RCs and final releases, run this before merging the release PR. Each item is here because something has shipped without it and broken users. + +- [ ] **The published rc passed mandatory full E2E.** Confirm the promote PR says `✅ RC full e2e was verified`; the normal promote PR itself runs no E2E. Add `full-e2e` manually only when a repeat is required, and treat a ⚠️ bypass line as an explicit release-risk decision. +- [ ] **Upgrade from previous patch** (`vX.Y.(Z-1)`) on a real cluster. `helm upgrade` succeeds, all HelmReleases reach Ready, no CRD schema rejections. +- [ ] **Upgrade from previous minor** (`v(X-1).Y.0`) for RC and minor releases. Same assertions. +- [ ] **At least one tenant with `publishing.ingressName != tenant-root`** and a real Let's Encrypt-staging cert issuance. +- [ ] **Apply-mutate-reapply** on the kubernetes umbrella HelmRelease (catches second-reconcile bugs). +- [ ] **For every CRD-source bump** (kubevirt, cert-manager, flux, cilium, kamaji): `helm template | kubectl apply --dry-run=server` against the freshly-installed CRDs. +- [ ] **For every flux / helm-controller / kubevirt-operator / cert-manager bump:** paste the upstream breaking-changes section of the bump's CHANGELOG into the PR body before merge. +- [ ] **After cutting the tag,** watch `pull-requests-release.yaml::Finalize Release`. If it fails on `Draft release for v... not found`, undraft the release manually and file the workflow regression — this has been a known recurring failure since the `Publish draft release` step lost its explicit `github-token` in refactor `66a756b6`. +- [ ] **The promote PR carries `docs/changelogs/vX.Y.Z.md` and it was verified** per [Changelog pre-publish verification](#changelog-pre-publish-verification). Finalize publishes this file as the release body, so verifying it before merge is verifying the release notes. +- [ ] **The `cozystack/website` docs PR stays parked until publish.** Promotion opens it early; merging it before the GitHub release for `vX.Y.Z` is live 404s the new version's docs and API links. Merge it right after finalize publishes the release (regular-release step 6, patch step 7). + +## CI failures release engineers commonly hit + +| Symptom | Class | Diagnosis | Block release? | +|---------|-------|-----------|----------------| +| `Draft release for vX.Y.Z not found` on every release-* merge | Workflow regression — `Publish draft release` lost its `github-token` in a past refactor; default `GITHUB_TOKEN` cannot list drafts | At `pull-requests-release.yaml:163`. Undraft manually, file the fix | No | +| Changelog Copilot job returns 402 | Token quota — `COPILOT_GITHUB_TOKEN` premium-request quota empty | Both changelog entry points hit it: the optional rc-time `changelog-rc` dispatch and the promote-time `changelog` job (both call the reusable `changelog-generate.yaml`), plus the `tags.yaml` backstop | No — `continue-on-error: true` on the AI step, generate changelog by hand following [`agents/changelog.md`](./agents/changelog.md) | +| `Prepare Release` skipped and the run red, `Build arm64 images` failed | The rc's arm64 build failed | Read the `build-arm64` log; `build-arm64-nightly.yaml` usually failed the same way first. Fix on the line, cut the next rc | Yes — an rc is not published amd64 only | +| `Stitch multi-arch indexes and republish` fails listing refs | `verify-multiarch.sh` found refs that are not amd64+arm64 indexes, or that it could not check | Each listed ref names its reason. Single-platform manifest or index without one architecture: make the image multi-arch, or allowlist its repository with a reason if it cannot be. Manifest unreadable: correct the ref or digest; the allowlist does not excuse it. Rate-limited or unavailable after 3 tries: re-run once the throttle clears | Yes | +| E2E `kubernetes-test` fails with `CSINode does not contain driver csi.kubevirt.io` | Test ordering bug | Tenant's `kubernetes-${test_name}-csi` HelmRelease still installing when NFS PVC is created. Fix: wait for `csinode/` to advertise the driver before creating the PVC | Yes — flaky and masks real CSI bugs | +| `tenant-root` HR in perpetual upgrade-rollback loop after OIDC patch | Real bug at the intersection of #2602's readiness change and the cozystack reconcile cadence | Look for `rate: Wait(n=1) would exceed context deadline` in operator logs; multiple tenant child HRs stuck `InProgress` | Yes | +| `cozy-dashboard` ImagePullBackOff from `cozystack-ui:latest` | Registry flake | Known noisy on some OCI mirrors; not currently in the prepull set | No — retry | +| PR CI: `docker push` 409 on `:latest` for the same image across two PRs | Concurrent manifest race on a shared floating tag | The `:latest` tag was previously written by every `make image` regardless of PR. Resolved by the image-tag refactor (`IMAGE_TAG=pr--` in PR builds, `IMAGE_TAG=` only at release time) | No — `gh run rerun --failed` | + +**Heuristic:** workflow-token and Copilot-quota issues are out-of-band — release the tag, fix the workflow in a follow-up PR. Test ordering bugs and tenant-root reconcile storms are real and should block. + +If you find yourself doing the same manual fixup on two consecutive releases (e.g. "undraft the release"), open a workflow-regression issue. Workflow bugs with known manual workarounds rot silently for months. + +## Tag immutability + +Published tags are **write-once** — once a `vX.Y.Z` or rc tag is pushed it is never moved or deleted. (Nightlies are not git tags at all — they are GHCR OCI tags, see [Nightly builds](#nightly-builds).) Moving a tag silently poisons the Go module proxy / pkg.go.dev cache for `api/apps/v1alpha1/vX.Y.Z` and drifts SBOM/provenance toolchains, so the release flow is built so a move is impossible by construction: + +| File | Tag / branch handling | +|------|------------------------| +| [`pull-requests-release.yaml`](../.github/workflows/pull-requests-release.yaml) | Verifies the pinned packages candidate before creating a stable name. Creates `vX.Y.Z` at the PR merge commit **write-once** (create if absent, no-op if unchanged, fail loudly if it would move — the merge commit is new, so there is nothing to move). Cuts the `api/apps/v1alpha1/` Go-submodule tag write-once at the same commit, **stable only** (never rc/beta/alpha). The `release-X.Y` maintenance branch is fast-forward-only. Retags the verified packages artifact and rc container digests to the stable image tag (also write-once at the image level). | +| [`tags.yaml`](../.github/workflows/tags.yaml) | The `release-X.Y.Z-rc.N` staging branch is a mutable staging ref (compare-before-force: skipped when unchanged, force+log only when it genuinely moves). Fails fast on a stable `vX.Y.Z` tag with no pre-existing draft — stable tags come from `promote-rc.yaml`, never a hand push. | +| [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) | Mutates no stable tags. Before staging, verifies green full E2E from the latest rc tag-push run or a tag-correlated manual published-tag run (unless loudly bypassed). Commits and publishes the rewritten packages tree under a run-specific temporary `promotion-*` tag, adds a second commit pinning its digest on the `release-X.Y.Z` branch, generates the changelog there, and opens the promote PR with `release` but no automatic `full-e2e`; the PR gate verifies the candidate and finalize repeats that verification before the stable tag and image retag. The staging branch is force-pushed on re-dispatch, but an existing `docs/changelogs/vX.Y.Z.md` is preserved across the rebuild. | + +This is the immutable-tag + rc-promotion model from [#2677](https://github.com/cozystack/cozystack/issues/2677): stable `vX.Y.Z` is created only by promoting an existing rc, rc tags are write-once, and `api/apps/v1alpha1/vX.Y.Z` is created only on a stable release. The old nightly `auto-release.yaml` (which delete-recreated auto-bumped patch tags) has been removed. rc and stable tags accrete permanently; the only churning artifacts are GHCR nightlies and abandoned temporary promotion candidates, which [`retention.yaml`](../.github/workflows/retention.yaml) prunes (see [Nightly builds](#nightly-builds)) — GHCR storage is the cost vector to watch. + +### Repo-side enforcement (tag-protection rulesets) + +The guarantees above are enforced in the workflows. Repo admins back them with two GitHub **tag rulesets** targeting `v*`: + +- **Immutability** — *Restrict updates* + *Restrict deletions*, **no bypass**. No actor (not even the CI app) can move or delete a shipped `v*` tag. Safe to enable at any time: nothing in the release flow ever moves or deletes a `v*` tag, it only creates new ones. +- **Creation control** — *Restrict creations* with the **CI app as the sole bypass actor**, plus *Limit branches/tags updated in a single push* = 1. This turns "don't hand-push tags" from convention into hard enforcement: every `v*` tag — pre-release **and** stable — can then be created only by CI. Enable this **only after** [`cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) is in use, since that workflow is what makes CI the sole creator of pre-release tags; enabling it earlier would lock maintainers out of cutting rc/alpha/beta. (The Go-module tag `api/apps/v1alpha1/*` gets an equivalent pair of rulesets.) + +## Splitting a release-blocking bundle PR + +Sometimes the work that has to land before a release is a 40-commit grab bag (CI stabilization, dependency bumps, chart fixes). Splitting it makes it reviewable and survivable. The strategy that has worked: + +1. **Verify what's already on `main`.** Use `git merge-base --is-ancestor` and subject grep to drop commits that have already landed. +2. **Find existing open PRs that cover the same commits.** Force-push to the existing branch instead of opening a duplicate — preserves review threads. +3. **Identify the "bottleneck" PR.** The one whose changes make previously-silent misconfigurations into hard errors (typically an operator-version bump). Everything else can fan out in parallel. +4. **Squash iterative same-author bug-on-bug fixes** inside each split. Preserve authorship via `--author` on the squashed commit. +5. **Mark the keystone PR as draft** and link in the body to the dependency PRs. Reviewers know it intentionally fails CI until the chain lands. +6. **Beware of concurrent registry pushes.** When fanning out 5+ PR CI runs simultaneously, multiple `make image` jobs can collide on shared floating tags. The `IMAGE_TAG=pr--` refactor solved this; if you see `Conflicted with another upload of the same manifest`, `gh run rerun --failed`. + +## Cleanup after release + +- The `release-X.Y.Z` branch is deleted by GitHub when its PR merges. The `release-X.Y` maintenance branch is created by `cut-prerelease.yaml` at the rc freeze (create-only, never force-moved) and fast-forwarded afterwards by `pull-requests-release.yaml::Ensure maintenance branch`. +- The draft release is published by the same workflow. +- The release body is set by finalize from `docs/changelogs/vX.Y.Z.md` at publication. Afterwards, `update-releasenotes.yaml` re-syncs that file into the GitHub Release body on any push to `main` that touches it — so edits to a published changelog land on the release page the next time `main` moves. +- Local cleanup: remove your worktree (`git worktree remove`) and prune merged release branches. + +## See also + +- [`agents/changelog.md`](./agents/changelog.md) — canonical changelog generation process. +- [`agents/contributing.md`](./agents/contributing.md) — commit/PR conventions, backport label semantics. +- [`agents/releasing.md`](./agents/releasing.md) — pointer file for AI agents handling release tasks. +- [`.github/workflows/cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) — the sole entry point for cutting a pre-release tag as the CI app. +- [`.github/workflows/tags.yaml`](../.github/workflows/tags.yaml) — tag-push pipeline (rc build + mandatory `rc-e2e`; stable-tag changelog/website backstops). +- [`.github/workflows/e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) — mandatory rc and manual published-tag full E2E lane. +- [`.github/workflows/pull-requests-release.yaml`](../.github/workflows/pull-requests-release.yaml) — merge-finalize pipeline. +- [`.github/workflows/promote-rc.yaml`](../.github/workflows/promote-rc.yaml) — rc → stable promotion. +- [`.github/workflows/changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml) — optional rc-time changelog button. +- [`.github/workflows/changelog-generate.yaml`](../.github/workflows/changelog-generate.yaml) — reusable changelog-generation core shared by promote-rc and changelog-rc. +- [`.github/workflows/nightly.yaml`](../.github/workflows/nightly.yaml) — nightly: mirror `main` OCIR→GHCR + full e2e. +- [`hack/nightly-mirror.sh`](../hack/nightly-mirror.sh) — cross-registry image mirror used by the nightly. +- [`agents/image-refs.md`](./agents/image-refs.md) — where image references live, what their tags mean, and the invariants promotion and mirroring rely on. +- [`hack/lib/image-refs.sh`](../hack/lib/image-refs.sh) — the single enumeration of ref-bearing files, shared by the promote, retag and mirror tooling. +- [`.github/workflows/retention.yaml`](../.github/workflows/retention.yaml) — GHCR nightly and abandoned promotion-candidate pruning. +- [`.github/workflows/backport.yaml`](../.github/workflows/backport.yaml) — automatic cherry-pick bot. +- [`.github/workflows/update-releasenotes.yaml`](../.github/workflows/update-releasenotes.yaml) — sync changelog → GitHub Release body. diff --git a/docs/security/incident-response.md b/docs/security/incident-response.md new file mode 100644 index 0000000000..62469519e3 --- /dev/null +++ b/docs/security/incident-response.md @@ -0,0 +1,67 @@ +# Incident Response + +This document is the public summary of how the Cozystack maintainers respond to +a confirmed security vulnerability or incident. Detailed operational procedures +(contact trees, credential inventories, and step-by-step checklists) are +maintained privately by the maintainers; this page describes the process and +the mechanisms it relies on so that reporters and users know what to expect. + +It complements, and does not replace: + +- [`SECURITY.md`](../../SECURITY.md) — how to report a vulnerability, response + commitments, and the disclosure timeline. +- [`CONTRIBUTOR_LADDER.md`](../../CONTRIBUTOR_LADDER.md) — the offboarding + procedure, including access revocation and credential rotation. + +## Roles + +For each incident the maintainers designate: + +- **Incident lead** — a maintainer (typically a core maintainer) who coordinates + triage, the fix, the release, and disclosure, and keeps the reporter informed. +- **Comms owner** — responsible for the public advisory and release-notes + wording, and for any coordination with upstream projects and CNCF. +- **Fix owner(s)** — the maintainer(s) developing and reviewing the fix. + +One person may hold more than one role on a small incident. + +## Lifecycle + +1. **Intake.** A report arrives through GitHub Private Vulnerability Reporting or + a private maintainer contact (see `SECURITY.md`). Receipt is acknowledged + within **3 business days**. +2. **Triage and severity.** The maintainers reproduce the issue and assign a + CVSS v3.1 severity within **7 business days**. This drives the remediation + target and disclosure window (see `SECURITY.md` → *Disclosure timeline*). +3. **Containment.** Where an immediate mitigation or workaround exists (a + configuration change, a policy, or disabling an affected feature), it is + documented and shared with the reporter and, if warranted, with affected + adopters under embargo. +4. **Private fix development.** The fix is developed under a **GitHub Security + Advisory** (private fork) so that code and discussion stay confidential until + release. +5. **Coordinated release.** The fix is shipped through the normal patch-release + and backport process. Which supported lines receive the backport is decided + per fix, weighing severity against backport risk — see the *Supported + Versions* table in `SECURITY.md`, which also records that no documented rule + governs that choice. +6. **Disclosure.** A GHSA — and, when applicable, a CVE — is published. Public + disclosure follows the outbound channels: GitHub Releases and release notes, + the changelog, and the GHSA itself. The reporter is credited unless anonymity + is requested. Disclosure happens no later than the coordinated-disclosure + window committed in `SECURITY.md`. +7. **Access and credential hygiene.** If the incident involved a compromised or + departing maintainer or leaked credential, the access-revocation and + credential-rotation steps in `CONTRIBUTOR_LADDER.md` are executed (GitHub team + access, org roles, Actions secrets, registry tokens, and App keys), completed + within five business days. +8. **Retrospective.** After resolution the maintainers record what happened, the + root cause, and any follow-up hardening, and open tracking issues for + preventative work. + +## Severity and timelines + +Severity classification (CVSS v3.1) and the target remediation and disclosure +timelines are defined once, authoritatively, in +[`SECURITY.md`](../../SECURITY.md) → *Disclosure timeline*. They are not +duplicated here to avoid drift. diff --git a/docs/security/reports/2026-04.md b/docs/security/reports/2026-04.md new file mode 100644 index 0000000000..d4d0864bcb --- /dev/null +++ b/docs/security/reports/2026-04.md @@ -0,0 +1,45 @@ +# Cozystack security report — April 2026 + +Generated 2026-08-11 from the tracker state on that date. Figures are counted from the tracked findings of the organization-wide CVE-scanning pipeline described in [`SECURITY.md`](../../../SECURITY.md). + +**Read this alongside [the backlog reconciliation](reconciliation-2026-08.md).** The state counts below are what the pipeline recorded, and the pipeline has no write-back from the main repository, so it has never marked anything fixed and its outstanding counts overstate real exposure. The reconciliation establishes what was actually remediated and how much of the backlog was never applicable. Do not read the tables here as a measure of exposure without it. + +**Disclosure.** Findings are reported as counts by severity; an individual finding is named once its fix is merged and released. Counts cover findings the pipeline tracks as actionable, after its filters for development-and-build-only dependencies, findings whose upstream has had no fix for over a year, and findings with no severity assigned in any database. Two of those filters are known not to work as documented — see the reconciliation. + +## New findings tracked this month + +**412** — 36 critical, 360 high, 16 medium + +April is the pipeline's first month, and this figure is an initial load across every repository in the organization rather than a month's discovery rate. It is not comparable with later months. + +## Triage decisions recorded this month + +No triage decisions were recorded this month. + +## Remediation activity this month + +**9** version bumps were merged, representing **7** distinct changes once backports of the same change to release branches are collapsed. These are counted from diffs: a pull request that only moves a chart version, edits an upgrade path or changes CI configuration is not counted as a bump, because it cannot carry a fix into an artifact. + +How many tracked findings those bumps carried away is deliberately **not** stated per month. Attributing findings to one month's bumps would cross a severity-banded count of still-open findings with a component list that is publicly enumerable from the commit log, which narrows the search for unpatched exposure more than a bare count does. The remediation disposition of the whole backlog is given once, with an as-of date, in [the reconciliation](reconciliation-2026-08.md). + +## Position at month end + +| State | Count | By severity | +| --- | --- | --- | +| Awaiting a triage decision | 412 | 36 critical, 360 high, 16 medium | +| **Total tracked** | **412** | | + +Nothing appears as fixed in any month because the pipeline cannot record that state at all. Confirmed findings therefore accumulate and never drain, which is a property of the tooling rather than a measure of outstanding work. + +## Where the process stands + +**Triage is behind its target and has no owner.** The rotating Security Champion role that owns the triage clock is defined in the pipeline's governance document and is currently unassigned. That is why decisions arrived in a single bulk pass rather than continuously, and it is the project's real gap — not the bookkeeping problem below. + +**The configured detection schedule** is every six hours for critical findings and weekly for the rest. Whether every scheduled run executed cannot be established from finding data alone, and the record has gaps: no weekly cohort appears for 2026-05-26, and intervals between critical findings reach 26 days. Absence of findings is not proof of a missed run, but neither is it proof of a completed one. + +**Remediation is happening and the tracker cannot see it.** Fixes land in the main repository as ordinary pinned-component bumps and nothing writes that back here. [The reconciliation](reconciliation-2026-08.md) sets out what was actually fixed, which components still need a bump that is already available upstream, and the defects found in the pipeline itself. + +## Reporting a vulnerability + +Please do not use public issues. Use [GitHub Private Vulnerability Reporting](https://github.com/cozystack/cozystack/security/advisories/new) or contact a maintainer privately; the full policy is in [`SECURITY.md`](../../../SECURITY.md). + diff --git a/docs/security/reports/2026-05.md b/docs/security/reports/2026-05.md new file mode 100644 index 0000000000..84667849f8 --- /dev/null +++ b/docs/security/reports/2026-05.md @@ -0,0 +1,49 @@ +# Cozystack security report — May 2026 + +Generated 2026-08-11 from the tracker state on that date. Figures are counted from the tracked findings of the organization-wide CVE-scanning pipeline described in [`SECURITY.md`](../../../SECURITY.md). + +**Read this alongside [the backlog reconciliation](reconciliation-2026-08.md).** The state counts below are what the pipeline recorded, and the pipeline has no write-back from the main repository, so it has never marked anything fixed and its outstanding counts overstate real exposure. The reconciliation establishes what was actually remediated and how much of the backlog was never applicable. Do not read the tables here as a measure of exposure without it. + +**Disclosure.** Findings are reported as counts by severity; an individual finding is named once its fix is merged and released. Counts cover findings the pipeline tracks as actionable, after its filters for development-and-build-only dependencies, findings whose upstream has had no fix for over a year, and findings with no severity assigned in any database. Two of those filters are known not to work as documented — see the reconciliation. + +## New findings tracked this month + +**64** — 5 critical, 59 high + +## Triage decisions recorded this month + +No triage decisions were recorded this month. + +## Remediation activity this month + +**10** version bumps were merged, representing **8** distinct changes once backports of the same change to release branches are collapsed. These are counted from diffs: a pull request that only moves a chart version, edits an upgrade path or changes CI configuration is not counted as a bump, because it cannot carry a fix into an artifact. + +Fixes released with a named advisory: + +| CVE | Component | Upstream fix | Shipped in Cozystack | Pull requests | +| --- | --- | --- | --- | --- | +| [CVE-2026-31431](https://nvd.nist.gov/vuln/detail/CVE-2026-31431) | Talos Linux | v1.12.7 | v1.2.4, v1.3.2, v1.4.0 | [#2545](https://github.com/cozystack/cozystack/pull/2545), [#2547](https://github.com/cozystack/cozystack/pull/2547), [#2548](https://github.com/cozystack/cozystack/pull/2548) | + +How many tracked findings those bumps carried away is deliberately **not** stated per month. Attributing findings to one month's bumps would cross a severity-banded count of still-open findings with a component list that is publicly enumerable from the commit log, which narrows the search for unpatched exposure more than a bare count does. The remediation disposition of the whole backlog is given once, with an as-of date, in [the reconciliation](reconciliation-2026-08.md). + +## Position at month end + +| State | Count | By severity | +| --- | --- | --- | +| Awaiting a triage decision | 476 | 41 critical, 419 high, 16 medium | +| **Total tracked** | **476** | | + +Nothing appears as fixed in any month because the pipeline cannot record that state at all. Confirmed findings therefore accumulate and never drain, which is a property of the tooling rather than a measure of outstanding work. + +## Where the process stands + +**Triage is behind its target and has no owner.** The rotating Security Champion role that owns the triage clock is defined in the pipeline's governance document and is currently unassigned. That is why decisions arrived in a single bulk pass rather than continuously, and it is the project's real gap — not the bookkeeping problem below. + +**The configured detection schedule** is every six hours for critical findings and weekly for the rest. Whether every scheduled run executed cannot be established from finding data alone, and the record has gaps: no weekly cohort appears for 2026-05-26, and intervals between critical findings reach 26 days. Absence of findings is not proof of a missed run, but neither is it proof of a completed one. + +**Remediation is happening and the tracker cannot see it.** Fixes land in the main repository as ordinary pinned-component bumps and nothing writes that back here. [The reconciliation](reconciliation-2026-08.md) sets out what was actually fixed, which components still need a bump that is already available upstream, and the defects found in the pipeline itself. + +## Reporting a vulnerability + +Please do not use public issues. Use [GitHub Private Vulnerability Reporting](https://github.com/cozystack/cozystack/security/advisories/new) or contact a maintainer privately; the full policy is in [`SECURITY.md`](../../../SECURITY.md). + diff --git a/docs/security/reports/2026-06.md b/docs/security/reports/2026-06.md new file mode 100644 index 0000000000..66bb9c2d1d --- /dev/null +++ b/docs/security/reports/2026-06.md @@ -0,0 +1,52 @@ +# Cozystack security report — June 2026 + +Generated 2026-08-11 from the tracker state on that date. Figures are counted from the tracked findings of the organization-wide CVE-scanning pipeline described in [`SECURITY.md`](../../../SECURITY.md). + +**Read this alongside [the backlog reconciliation](reconciliation-2026-08.md).** The state counts below are what the pipeline recorded, and the pipeline has no write-back from the main repository, so it has never marked anything fixed and its outstanding counts overstate real exposure. The reconciliation establishes what was actually remediated and how much of the backlog was never applicable. Do not read the tables here as a measure of exposure without it. + +**Disclosure.** Findings are reported as counts by severity; an individual finding is named once its fix is merged and released. Counts cover findings the pipeline tracks as actionable, after its filters for development-and-build-only dependencies, findings whose upstream has had no fix for over a year, and findings with no severity assigned in any database. Two of those filters are known not to work as documented — see the reconciliation. + +## New findings tracked this month + +**252** — 7 critical, 245 high + +## Triage decisions recorded this month + +| Outcome | Count | By severity | +| --- | --- | --- | +| Confirmed — real, needs a fix | 332 | 30 critical, 292 high, 10 medium | +| Accepted risk — real, not reachable or not worth the change | 61 | 6 critical, 52 high, 3 medium | +| Dismissed — not applicable | 101 | 5 critical, 93 high, 3 medium | + +These decisions were recorded in a **single bulk triage pass on 2026-06-11**, so a per-finding triage latency cannot be recovered from this data: the tracker stores when an issue was closed, not when a decision was reached. What it does show is the age of the backlog when that pass ran — between 2 and 47 business days, with 78% of findings at the upper end. Against a one-business-day target for critical findings, the pass was late by any reading. + +## Remediation activity this month + +**26** version bumps were merged, representing **24** distinct changes once backports of the same change to release branches are collapsed. These are counted from diffs: a pull request that only moves a chart version, edits an upgrade path or changes CI configuration is not counted as a bump, because it cannot carry a fix into an artifact. + +How many tracked findings those bumps carried away is deliberately **not** stated per month. Attributing findings to one month's bumps would cross a severity-banded count of still-open findings with a component list that is publicly enumerable from the commit log, which narrows the search for unpatched exposure more than a bare count does. The remediation disposition of the whole backlog is given once, with an as-of date, in [the reconciliation](reconciliation-2026-08.md). + +## Position at month end + +| State | Count | By severity | +| --- | --- | --- | +| Awaiting a triage decision | 234 | 7 critical, 227 high | +| Confirmed — real, needs a fix | 332 | 30 critical, 292 high, 10 medium | +| Accepted risk — real, not reachable or not worth the change | 61 | 6 critical, 52 high, 3 medium | +| Dismissed — not applicable | 101 | 5 critical, 93 high, 3 medium | +| **Total tracked** | **728** | | + +Nothing appears as fixed in any month because the pipeline cannot record that state at all. Confirmed findings therefore accumulate and never drain, which is a property of the tooling rather than a measure of outstanding work. + +## Where the process stands + +**Triage is behind its target and has no owner.** The rotating Security Champion role that owns the triage clock is defined in the pipeline's governance document and is currently unassigned. That is why decisions arrived in a single bulk pass rather than continuously, and it is the project's real gap — not the bookkeeping problem below. + +**The configured detection schedule** is every six hours for critical findings and weekly for the rest. Whether every scheduled run executed cannot be established from finding data alone, and the record has gaps: no weekly cohort appears for 2026-05-26, and intervals between critical findings reach 26 days. Absence of findings is not proof of a missed run, but neither is it proof of a completed one. + +**Remediation is happening and the tracker cannot see it.** Fixes land in the main repository as ordinary pinned-component bumps and nothing writes that back here. [The reconciliation](reconciliation-2026-08.md) sets out what was actually fixed, which components still need a bump that is already available upstream, and the defects found in the pipeline itself. + +## Reporting a vulnerability + +Please do not use public issues. Use [GitHub Private Vulnerability Reporting](https://github.com/cozystack/cozystack/security/advisories/new) or contact a maintainer privately; the full policy is in [`SECURITY.md`](../../../SECURITY.md). + diff --git a/docs/security/reports/2026-07.md b/docs/security/reports/2026-07.md new file mode 100644 index 0000000000..3d319fef06 --- /dev/null +++ b/docs/security/reports/2026-07.md @@ -0,0 +1,52 @@ +# Cozystack security report — July 2026 + +Generated 2026-08-11 from the tracker state on that date. Figures are counted from the tracked findings of the organization-wide CVE-scanning pipeline described in [`SECURITY.md`](../../../SECURITY.md). + +**Read this alongside [the backlog reconciliation](reconciliation-2026-08.md).** The state counts below are what the pipeline recorded, and the pipeline has no write-back from the main repository, so it has never marked anything fixed and its outstanding counts overstate real exposure. The reconciliation establishes what was actually remediated and how much of the backlog was never applicable. Do not read the tables here as a measure of exposure without it. + +**Disclosure.** Findings are reported as counts by severity; an individual finding is named once its fix is merged and released. Counts cover findings the pipeline tracks as actionable, after its filters for development-and-build-only dependencies, findings whose upstream has had no fix for over a year, and findings with no severity assigned in any database. Two of those filters are known not to work as documented — see the reconciliation. + +## New findings tracked this month + +**80** — 6 critical, 74 high + +## Triage decisions recorded this month + +No triage decisions were recorded this month. + +## Remediation activity this month + +**9** version bumps were merged, representing **8** distinct changes once backports of the same change to release branches are collapsed. These are counted from diffs: a pull request that only moves a chart version, edits an upgrade path or changes CI configuration is not counted as a bump, because it cannot carry a fix into an artifact. + +Fixes released with a named advisory: + +| CVE | Component | Upstream fix | Shipped in Cozystack | Pull requests | +| --- | --- | --- | --- | --- | +| [CVE-2026-53359](https://nvd.nist.gov/vuln/detail/CVE-2026-53359) | Talos Linux | v1.13.6 | v1.6.0, v1.6.1 | [#3240](https://github.com/cozystack/cozystack/pull/3240) | + +How many tracked findings those bumps carried away is deliberately **not** stated per month. Attributing findings to one month's bumps would cross a severity-banded count of still-open findings with a component list that is publicly enumerable from the commit log, which narrows the search for unpatched exposure more than a bare count does. The remediation disposition of the whole backlog is given once, with an as-of date, in [the reconciliation](reconciliation-2026-08.md). + +## Position at month end + +| State | Count | By severity | +| --- | --- | --- | +| Awaiting a triage decision | 314 | 13 critical, 301 high | +| Confirmed — real, needs a fix | 332 | 30 critical, 292 high, 10 medium | +| Accepted risk — real, not reachable or not worth the change | 61 | 6 critical, 52 high, 3 medium | +| Dismissed — not applicable | 101 | 5 critical, 93 high, 3 medium | +| **Total tracked** | **808** | | + +Nothing appears as fixed in any month because the pipeline cannot record that state at all. Confirmed findings therefore accumulate and never drain, which is a property of the tooling rather than a measure of outstanding work. + +## Where the process stands + +**Triage is behind its target and has no owner.** The rotating Security Champion role that owns the triage clock is defined in the pipeline's governance document and is currently unassigned. That is why decisions arrived in a single bulk pass rather than continuously, and it is the project's real gap — not the bookkeeping problem below. + +**The configured detection schedule** is every six hours for critical findings and weekly for the rest. Whether every scheduled run executed cannot be established from finding data alone, and the record has gaps: no weekly cohort appears for 2026-05-26, and intervals between critical findings reach 26 days. Absence of findings is not proof of a missed run, but neither is it proof of a completed one. + +**Remediation is happening and the tracker cannot see it.** Fixes land in the main repository as ordinary pinned-component bumps and nothing writes that back here. [The reconciliation](reconciliation-2026-08.md) sets out what was actually fixed, which components still need a bump that is already available upstream, and the defects found in the pipeline itself. + +## Reporting a vulnerability + +Please do not use public issues. Use [GitHub Private Vulnerability Reporting](https://github.com/cozystack/cozystack/security/advisories/new) or contact a maintainer privately; the full policy is in [`SECURITY.md`](../../../SECURITY.md). + diff --git a/docs/security/reports/2026-08.md b/docs/security/reports/2026-08.md new file mode 100644 index 0000000000..ea0570aa4c --- /dev/null +++ b/docs/security/reports/2026-08.md @@ -0,0 +1,46 @@ +# Cozystack security report — August 2026 + +Generated 2026-08-11 from the tracker state on that date. **This month is incomplete: data is cut off at 2026-08-10.** Figures are counted from the tracked findings of the organization-wide CVE-scanning pipeline described in [`SECURITY.md`](../../../SECURITY.md). + +**Read this alongside [the backlog reconciliation](reconciliation-2026-08.md).** The state counts below are what the pipeline recorded, and the pipeline has no write-back from the main repository, so it has never marked anything fixed and its outstanding counts overstate real exposure. The reconciliation establishes what was actually remediated and how much of the backlog was never applicable. Do not read the tables here as a measure of exposure without it. + +**Disclosure.** Findings are reported as counts by severity; an individual finding is named once its fix is merged and released. Counts cover findings the pipeline tracks as actionable, after its filters for development-and-build-only dependencies, findings whose upstream has had no fix for over a year, and findings with no severity assigned in any database. Two of those filters are known not to work as documented — see the reconciliation. + +## New findings tracked this month + +**17** — 17 high + +## Triage decisions recorded this month + +No triage decisions were recorded this month. + +## Remediation activity this month + +**2** version bumps were merged, representing **1** distinct changes once backports of the same change to release branches are collapsed. These are counted from diffs: a pull request that only moves a chart version, edits an upgrade path or changes CI configuration is not counted as a bump, because it cannot carry a fix into an artifact. + +How many tracked findings those bumps carried away is deliberately **not** stated per month. Attributing findings to one month's bumps would cross a severity-banded count of still-open findings with a component list that is publicly enumerable from the commit log, which narrows the search for unpatched exposure more than a bare count does. The remediation disposition of the whole backlog is given once, with an as-of date, in [the reconciliation](reconciliation-2026-08.md). + +## Position at 2026-08-10 + +| State | Count | By severity | +| --- | --- | --- | +| Awaiting a triage decision | 331 | 13 critical, 318 high | +| Confirmed — real, needs a fix | 332 | 30 critical, 292 high, 10 medium | +| Accepted risk — real, not reachable or not worth the change | 61 | 6 critical, 52 high, 3 medium | +| Dismissed — not applicable | 101 | 5 critical, 93 high, 3 medium | +| **Total tracked** | **825** | | + +Nothing appears as fixed in any month because the pipeline cannot record that state at all. Confirmed findings therefore accumulate and never drain, which is a property of the tooling rather than a measure of outstanding work. + +## Where the process stands + +**Triage is behind its target and has no owner.** The rotating Security Champion role that owns the triage clock is defined in the pipeline's governance document and is currently unassigned. That is why decisions arrived in a single bulk pass rather than continuously, and it is the project's real gap — not the bookkeeping problem below. + +**The configured detection schedule** is every six hours for critical findings and weekly for the rest. Whether every scheduled run executed cannot be established from finding data alone, and the record has gaps: no weekly cohort appears for 2026-05-26, and intervals between critical findings reach 26 days. Absence of findings is not proof of a missed run, but neither is it proof of a completed one. + +**Remediation is happening and the tracker cannot see it.** Fixes land in the main repository as ordinary pinned-component bumps and nothing writes that back here. [The reconciliation](reconciliation-2026-08.md) sets out what was actually fixed, which components still need a bump that is already available upstream, and the defects found in the pipeline itself. + +## Reporting a vulnerability + +Please do not use public issues. Use [GitHub Private Vulnerability Reporting](https://github.com/cozystack/cozystack/security/advisories/new) or contact a maintainer privately; the full policy is in [`SECURITY.md`](../../../SECURITY.md). + diff --git a/docs/security/reports/README.md b/docs/security/reports/README.md new file mode 100644 index 0000000000..8aa4118d05 --- /dev/null +++ b/docs/security/reports/README.md @@ -0,0 +1,32 @@ +# Monthly security reports + +Public reporting on vulnerability handling in the Cozystack project, generated from the tracked findings of the organization-wide CVE-scanning pipeline described in [`SECURITY.md`](../../../SECURITY.md). + +**Start with [the backlog reconciliation](reconciliation-2026-08.md).** The monthly reports state what happened in a month, counted from the pipeline's own records. The pipeline has no write-back from the main repository, so it has never marked a finding as fixed and its outstanding counts overstate real exposure. The reconciliation establishes what was actually remediated, how much of the backlog was never applicable, and what is defective in the tooling. The monthly figures are not interpretable without it. + +| Document | Covers | +| --- | --- | +| [Backlog reconciliation](reconciliation-2026-08.md) | Every finding since the pipeline started, audited against what shipped. As of 2026-08-11 | +| [August 2026](2026-08.md) | Incomplete; data cut off 2026-08-10 | +| [July 2026](2026-07.md) | | +| [June 2026](2026-06.md) | | +| [May 2026](2026-05.md) | | +| [April 2026](2026-04.md) | First month; an initial load rather than a discovery rate | + +## What these reports contain + +Findings are reported as **counts by severity**. An individual finding is named once its fix is merged and released — both such findings to date appear in the reconciliation and in the month they shipped. + +Two deliberate exceptions to that reticence, both arrived at by reviewing the rule rather than applying it mechanically. **Findings with no upstream fix will be named, with mitigation guidance**, because there is nothing to wait for and silence serves nobody: an adopter needs to know when a mitigation is the only available control. And **remediation is not attributed to individual months**, because a count of still-open findings crossed with one month's list of component bumps — which anybody can enumerate from the commit log — narrows the search for unpatched exposure further than a bare count does. + +What stays embargoed is narrow: findings where Cozystack's own packaging, defaults or integration logic introduces or worsens the exposure. There the project is the only source, and the process is the coordinated disclosure in [`SECURITY.md`](../../../SECURITY.md). + +## Scope of the counts + +The counts cover findings from the pipeline, which scans **every non-fork, non-archived repository in the `cozystack` organization** — not this repository alone. They are filtered to what the pipeline treats as actionable: development-and-build-only dependencies, findings whose upstream has had no fix for over a year, findings already carrying a triage decision, and findings with no severity assigned are excluded before a finding is tracked. Two of those filters do not work as documented, which the reconciliation records. + +Vulnerabilities reported privately against Cozystack's own code, packaging, defaults or integration logic follow the separate process in [`SECURITY.md`](../../../SECURITY.md), with its own response targets, and are disclosed through GitHub Security Advisories rather than here. + +## Provenance and its limits + +These figures are self-attested. The tracker is private, because it holds per-finding detail for exposure that is not yet fixed, so a reader cannot currently reproduce the counts. Publishing the pipeline's policy documents and scanning scripts — which contain no finding data — so that the method can be audited independently of the numbers is outstanding work. Until that lands, treat the numbers as a project statement rather than as verified evidence. diff --git a/docs/security/reports/reconciliation-2026-08.md b/docs/security/reports/reconciliation-2026-08.md new file mode 100644 index 0000000000..ba7392759d --- /dev/null +++ b/docs/security/reports/reconciliation-2026-08.md @@ -0,0 +1,139 @@ +# Vulnerability backlog reconciliation — as of 2026-08-11 + +This is a one-off audit of every finding the Cozystack CVE-scanning pipeline has tracked since it started, reconciled against what the project actually shipped. It exists because the pipeline's own state was misleading in both directions, and because a due-diligence reviewer is entitled to the correction rather than the raw counts. + +It supersedes any remediation figure implied by the monthly reports in this directory. The monthly reports state what happened in a month; this document states where the backlog actually stands. + +## Why a reconciliation was needed + +The pipeline records triage decisions by reading its own closed issues. **Nothing writes back to it from the main repository**, so it has never marked a single finding as fixed — `state/triage-overrides.json` holds 477 decisions and not one carries a fix version or a fix pull request. A finding whose component was upgraded two releases ago still sits in the tracker as confirmed and outstanding. + +The consequence is that the tracker's headline numbers were wrong in both directions at once. They **overstated** open exposure, because remediation had happened and was invisible. They also **understated** how much of the backlog was never applicable, because a large class of unactionable findings had never been triaged at all. + +## What the tracker holds + +827 issues, of which **825 are findings** — two (`#386`, `#388`) are quarterly-security-review tracking tasks that the filters should have excluded and did not. + +| State | Count | +| --- | --- | +| Confirmed — triaged as real, needs a fix | 332 | +| Dismissed — not applicable | 101 | +| Accepted risk | 61 | +| Awaiting a triage decision | 333 | +| Marked fixed | **0** | + +All 332 confirmed findings were decided in a single bulk triage pass on 2026-06-11. No triage decisions were recorded in any other month. + +## Applicability: how much of this was ever about what Cozystack ships + +Classified in priority order, no overlaps. The already-dismissed findings supplied the precedents: each of the 162 dismissals carries a written justification, and those justifications establish rules that were then applied to the untriaged backlog. + +| Class | Findings | Share | +| --- | --- | --- | +| Vulnerable code not present in executable form | 291 | 35% | +| Not attributed to any image — reachability cannot be assessed | 131 | 16% | +| Only in components a default installation does not deploy | 104 | 13% | +| Role unreachable — client library against a server-side flaw, interactive tool, build-time packaging | 65 | 8% | +| Platform or version gated — Windows-only, 32-bit-only, installed version below the introduced range | 18 | 2% | +| Already stale against the current tree | 10 | 1% | +| **Patch lag on a component a default installation does run** | **206** | **25%** | + +**466 findings (57%) were never applicable or cannot be assessed.** The largest single block is 291 findings against packages that contain no executable vulnerable code — kernel headers being the dominant case. Cozystack containers never provide a running kernel; Talos does. The project's own maintainers applied exactly this reasoning 64 times to `kernel-headers` and 16 times to `perf`, and dismissed every one — while 194 findings against `linux-libc-dev`, which is the same argument with a different package name, sat untriaged. That is one missing filter rule accounting for a quarter of the tracker. + +A second structural block: **68 findings report operating-system packages inside `system/kamaji`, whose final image is `gcr.io/distroless/static:nonroot`** — an image with no OS package layer at all. Nineteen of those are in the confirmed state, which is a triage error rather than a scanner artifact. + +A third: **51 findings report Node ecosystem packages against `system/dashboard`**, whose shipped image is `nginx-unprivileged` plus a static build directory, with no Node runtime. Thirty-four are confirmed. + +The 206 remaining patch-lag findings are legitimate hygiene: the package is present in a shipped image and a patch exists. None of them asserts a reachable attack path, and they should not be read as one. **Genuine reachable exposure in a stock installation is estimated at 15–40 findings, 2–5% of the tracker.** That is an estimate, not a measurement, and it is sensitive to two questions the tracker cannot answer — whether any shipped artifact carries a `runc` binary, and whether untrusted input reaches specific XML, ASN.1 and archive parsers inside vendored operator images. + +## Remediation that actually happened + +Cross-referencing the tracked findings against merged pull requests, release tags and the pipeline's own 18 weekly scan reports: + +| Disposition of the 332 confirmed findings | Count | +| --- | --- | +| Fixed upstream and already pulled into the tree | 165 | +| Fixed by a change Cozystack authored | 26 | +| Fixed upstream, not yet pulled in | 10 fully, 27 partially | +| No upstream fix exists | 26 | +| Not attributable to an image, so undeterminable | 78 | + +Of the 165, **34 are verified by exact version comparison** against the tree; the remaining 131 rest on the carrier image having been rebuilt, which is strong evidence but not proof without a re-scan. + +The 26 fixed by Cozystack's own work are a distinct and easily missed category: re-vendoring the dashboard console in-tree ([#2963](https://github.com/cozystack/cozystack/pull/2963)) did not upgrade the vulnerable npm dependencies, it **removed them from existence**. Separately, [#3033](https://github.com/cozystack/cozystack/pull/3033) forced module versions during the Kamaji build because no upstream Kamaji tag carried the fixes yet. + +**Remediation was a single wave, not a steady stream.** Twenty-two pull requests between 2026-06-15 and 2026-06-30 account for essentially all of it: 162 confirmed findings and 40 awaiting-triage findings were carried away in that fortnight. Zero in April, seven in May, one in July, none in August. + +The changes that did the most work were largely invisible to identifier-based searching, because they name no CVE at all: + +| Pull request | What it changed | Confirmed findings carried | +| --- | --- | --- | +| [#2963](https://github.com/cozystack/cozystack/pull/2963) | dashboard console vendored in-tree; vulnerable npm dependencies cease to exist | 31 | +| [#2964](https://github.com/cozystack/cozystack/pull/2964) | Keycloak 26.5.2 → 26.6.3 | 25 | +| [#2992](https://github.com/cozystack/cozystack/pull/2992) | fdb-operator v2.30.0 | 17 | +| [#2966](https://github.com/cozystack/cozystack/pull/2966) | Harbor chart 1.19.1 / 2.15.1 | 14 | +| [#3009](https://github.com/cozystack/cozystack/pull/3009) | first-party Alpine bases → 3.24 across six images | 14 | +| [#2834](https://github.com/cozystack/cozystack/pull/2834), [#2916](https://github.com/cozystack/cozystack/pull/2916) | SeaweedFS 4.31 | 13 | +| [#3014](https://github.com/cozystack/cozystack/pull/3014) | managed Kubernetes patch versions | 11 | +| [#2494](https://github.com/cozystack/cozystack/pull/2494) | Go modules and `golang:1.26` in first-party Dockerfiles | 7 | +| [#2972](https://github.com/cozystack/cozystack/pull/2972) | Cilium 1.19.3 → 1.19.5 | 6 | +| [#3019](https://github.com/cozystack/cozystack/pull/3019) | `golang.org/x/crypto` → 0.52.0 | 4 | +| [#2974](https://github.com/cozystack/cozystack/pull/2974) | Velero chart 12.0.3 | 4 | +| [#2941](https://github.com/cozystack/cozystack/pull/2941) | `golang:1.26` across eight remaining builder images | 3 directly | + +The Go toolchain thread deserves separate mention because the weekly reports evidence it independently: findings against ancient Go lines (1.15 through 1.20) appear only in the 2026-04-07 and 2026-05-05 scans and never again. After the toolchain was brought current, the only new `stdlib` findings require a version newer than the current pin. All 62 confirmed `stdlib` findings are formally satisfied by the toolchain now in the tree. + +### Fixes available and not yet taken + +Small, specific and actionable today: + +| Component | In tree | Fix requires | Findings | +| --- | --- | --- | --- | +| `nats-server` | appVersion 2.11.10 | 2.11.14 / 2.11.15 | 6 | +| CoreDNS image | v1.12.4 | 1.14.2 | 1 | +| OpenBao | v2.5.1 | 2.5.4 | 1 | +| `fast-uri` | 3.1.0 | 3.1.1 / 3.1.2 | 2 | + +## Fixes released with a named identifier + +Both are already public in merged commit subjects, so naming them here adds no exposure. + +| CVE | Component | Upstream fix | Shipped in Cozystack | Pull requests | +| --- | --- | --- | --- | --- | +| [CVE-2026-31431](https://nvd.nist.gov/vuln/detail/CVE-2026-31431) | Talos Linux | v1.12.7 | v1.2.4, v1.3.2, v1.4.0 | [#2545](https://github.com/cozystack/cozystack/pull/2545), [#2547](https://github.com/cozystack/cozystack/pull/2547), [#2548](https://github.com/cozystack/cozystack/pull/2548) | +| [CVE-2026-53359](https://nvd.nist.gov/vuln/detail/CVE-2026-53359) | Talos Linux | v1.13.6 | v1.6.0, v1.6.1 | [#3240](https://github.com/cozystack/cozystack/pull/3240), [#3269](https://github.com/cozystack/cozystack/pull/3269) | + +## Defects found in the pipeline itself + +The audit found more wrong with the tooling than with the fixing. All of these are the pipeline's, not the maintainers': + +1. **No write-back.** Nothing observes the main repository, so nothing is ever closed as fixed and the confirmed state grows monotonically and permanently. +2. **Deduplication by CVE identifier alone.** `scan.sh` keys on the CVE, so `package`, `installed_version` and `fixed_version` are taken from whichever scan file sorted first and are never updated afterwards. This is why `stdlib` findings show a single installed version across sixty carriers, and why some show none. The 825 findings reduce to **247 distinct `(package, installed version)` roots**; the five largest roots are 41% of the whole backlog. One stale Go binary produced 62 issues; one `linux-libc-dev` produced 194. +3. **Broken image attribution in the first baseline.** Component matching normalises image references with a substitution that breaks on digest-pinned refs, which is the mechanical cause of the 95 findings recorded with `Unknown` components — all from the 2026-04-07 baseline. One hundred and nine unattributed findings were nonetheless closed as confirmed, permanently recording a problem nobody can locate. +4. **Missing kernel-package filter**, described above: 258 records, a quarter of the tracker. +5. **No distroless or final-layer awareness**, so OS packages are reported against images that have no OS layer and npm packages against images with no Node runtime. +6. **The documented "no severity assigned" filter does not work**, which is how two process-tracking issues entered the finding set. +7. **The documented "dev/test only" filter tests the wrong condition** — it excludes a finding when no component is dev/test, rather than when all components are. + +## What is being done about it + +In priority order, and the order is deliberate: the first item is a person, not a tool. + +1. **Fill the rotating Security Champion role.** It is defined in the pipeline's governance document with a six-month term and a handover overlap, and it is currently unassigned. Nobody owns the triage clock, which is why 494 decisions landed in one day and none since. No amount of tooling substitutes for this. +2. **Triage the genuinely urgent findings**, which the noise has been hiding. The audit surfaced several open findings whose severity and reachability warrant immediate attention ahead of the backlog, including a privilege-escalation path relevant to multi-tenant deployments and an authentication fail-open in a shipped component. These are being handled through the private channel rather than named here, per the disclosure rule below. +3. **Close the write-back loop.** Findings should be keyed on `(vulnerability id, package, purl, target type, artifact reference by digest)` rather than on the CVE alone, and closed when the key is absent from two consecutive scans — distinguishing *no longer shipped* from *fixed*, so that deleting an image is never read as remediation. Fix attribution can then be derived deterministically by searching the main repository's history for the superseded digest, rather than by matching keywords in pull request titles. +4. **Add the input filters** for kernel packages, header and development packages, distroless targets, final-layer-only language ecosystems, and a version-arithmetic gate that rejects a finding whose installed version predates the range the vulnerability was introduced in. +5. **Deduplicate by root rather than by CVE**, which reduces the triage workload by roughly a factor of three without discarding information. +6. **Re-scan against current versions**, which settles the 131 findings whose remediation is evidenced only by a carrier rebuild, and retire the unattributable baseline findings. + +## Disclosure rule + +Aggregate counts by severity are published. An individual finding is named once its fix is merged and released. + +Two revisions to that rule follow from this audit, and both make it more open rather than less. First, **findings with no upstream fix will be named, with mitigation guidance**, because there is nothing to wait for and withholding them serves nobody: adopters need to know that a mitigation is the only available control. Roughly a fifth of the backlog is in that state. Second, per-month attribution of remediation has been removed from the monthly reports, because a severity-banded count of still-open findings crossed with a single month's list of component bumps — which is publicly enumerable from the commit log — narrows the search for unpatched exposure more than a bare count does. That construct was a leak the original rule did not anticipate. + +What stays embargoed is narrow: findings where Cozystack's own packaging, defaults or integration logic introduces or worsens the exposure. Those have no upstream advisory, the project is the only source, and they follow the coordinated-disclosure process in [`SECURITY.md`](../../../SECURITY.md). + +## Provenance + +Generated 2026-08-11 from the tracker state on that date, cross-referenced against 660 pull requests merged into `cozystack/cozystack` between 2026-04-01 and 2026-08-10, the release tags carrying them, and the pipeline's 18 weekly scan reports. The tracker itself is not public, so these counts are self-attested; publishing the pipeline's policy documents and scanning scripts, so that the method can be audited independently of the numbers, is outstanding work. diff --git a/docs/security/self-assessment.md b/docs/security/self-assessment.md new file mode 100644 index 0000000000..cb83609042 --- /dev/null +++ b/docs/security/self-assessment.md @@ -0,0 +1,200 @@ +# Cozystack Self-assessment + +This document is a security self-assessment for [Cozystack](https://github.com/cozystack/cozystack), prepared by the Cozystack maintainers as part of the project's application for CNCF Incubation. + +## Metadata + +| | | +| --- | --- | +| Assessment Stage | Completed | +| Software | | +| Security Provider | No. Cozystack is a cloud platform, not a security product; security is a supporting property, not its primary function. | +| Languages | Go, Helm, shell | +| SBOM | Not published at this time. Image builds can emit a CycloneDX SBOM attestation (`SBOM=1` in `hack/common-envs.mk`), but this is disabled by default; SBOM generation on release is a planned improvement. | + +### Security links + +| Document | URL | +| --- | --- | +| Security policy | | +| Threat model | | +| Governance | | +| Maintainers | | +| Contributor ladder | | +| Code of Conduct | | +| Adopters | | +| OpenSSF Best Practices | | +| OpenSSF Scorecard | | + +## Overview + +Cozystack is an open-source Kubernetes-based platform for building cloud infrastructure and offering managed services (databases, virtual machines, tenant Kubernetes clusters) with multi-tenancy and GitOps delivery. This overview summarizes the security-relevant architecture; the full design-level threat model lives in the repository at [`docs/security/threat-model.md`](https://github.com/cozystack/cozystack/blob/main/docs/security/threat-model.md). + +### Background + +Cozystack is delivered as a **management (root) Kubernetes cluster** onto which tenants, managed services, virtual machines, and tenant Kubernetes clusters are layered. The centre of the security model is the aggregated API server **`cozystack-api`**. Tenants never write privileged Kubernetes objects (HelmRelease, Deployment, Secret, RBAC) directly. Instead they write thin, virtual `apps.cozystack.io/*` **Application** custom resources, and `cozystack-api` translates each one into a Flux `HelmRelease` whose chart reference is fixed server-side. The set of Application kinds is registered dynamically from `ApplicationDefinition` custom resources, so the platform's catalogue of managed services is data-driven rather than hard-coded. + +When OIDC is enabled (opt-in; `authentication.oidc.enabled` is off by default), authentication is centralized through Keycloak/OIDC; in the default OIDC-disabled mode the dashboard is fronted by a token-proxy. Authorization uses Kubernetes RBAC (with roles aggregated per tenant) plus ValidatingAdmissionPolicy. Networking isolation is provided by Cilium. Platform PKI (including the aggregated-API serving certificate) is issued by cert-manager. Tenant Kubernetes clusters run on dedicated Kamaji-hosted control planes with Talos worker nodes running as KubeVirt virtual machines. + +**Usage modes.** Cozystack is deployed in two ways. As a **managed-services backend** (roughly half of deployments), an external provisioning system authenticates with its own ServiceAccount and creates the tenant resources it needs — no OIDC involved. Used **directly**, a human is granted OIDC access to the dashboard and the Cozystack API and operates it, constrained by RBAC to the `apps.cozystack.io` group within their tenant. + +### Multitenancy model + +Cozystack combines hard and soft multitenancy: + +- **Hard multitenancy for tenant-controlled code.** Anything in which a tenant can run arbitrary code — VMs and tenant Kubernetes clusters — is isolated at the hardware-virtualization / dedicated-control-plane boundary, and the tenant has full access inside it. Those VMs and tenant-cluster nodes are themselves ordinary, unprivileged pods in the tenant namespace: governed by Pod Security Standards, denied access to the management-cluster API, and subject to the tenant network and admission policies. +- **Soft multitenancy for managed data services.** Databases and queues run in the management cluster but expose only a fixed, validated set of parameters and never let a tenant execute arbitrary code (unsafe options are disabled). A tenant therefore cannot compromise the management cluster by swapping a container image or enabling a dangerous setting. + +### Actors + +Actors are decomposed by **trust boundary**: two components are distinct actors only when compromise of one does not automatically compromise the other. Cozystack has five trust tiers (T0 highest trust, T4 untrusted). + +| Actor | Trust tier | Privileges | Isolation mechanism | +| --- | --- | --- | --- | +| Platform / cluster administrator (`system:masters`) | T0 — root of trust | Full control of the management cluster; owns GitOps, Talos config, and the Keycloak realm. | Trusted by design; nothing isolates it. | +| GitOps / Flux delivery layer | T1 — platform control plane | Reconciles HelmReleases/Kustomizations into arbitrary privileged manifests; effectively cluster-admin. | Tenants cannot create HelmReleases (RBAC); not tenant-network-reachable. | +| `cozystack-api` aggregated API server | T1 — platform control plane | Converts tenant CRs into HelmReleases; can CRUD HelmReleases, Secrets, and Cilium policies cluster-wide. Acts as the policy-enforcing gateway. | Constrains tenants via fixed chart reference, name/quota/host validation, and admission. | +| System operators / controllers | T1 — platform control plane | Domain-scoped cluster privileges (provision control planes, mint certs/secrets, program CNI). | Distinct ServiceAccounts; isolated from tenants by RBAC and network policy. | +| Tenant administrator (OIDC `-admin` / `-super-admin`) | T2 — tenant boundary | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` in its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) `super-admin` holds the `apps.cozystack.io/*` wildcard (and can create child `Tenant`s); `admin` has an enumerated allowlist excluding some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`). | Namespace-scoped RBAC + admission policies + quota; no HelmRelease or cross-tenant access. | +| Tenant user (OIDC `-view` / `-use`) | T2 — tenant boundary | Read-only view plus operational verbs (VM console/vnc/portforward, start/stop/restart). Cannot create Application CRs. | Strictly dominated by tenant-admin; cannot escalate. | +| Parent vs child (nested) tenant | T2 — tenant boundary | Parent ServiceAccounts bind into descendant namespaces; children do not bind upward. Quota hierarchically capped. | Ancestor→descendant trust is one-way. | +| Tenant workloads / managed services | T3 — workload boundary | Run tenant-controlled code; no cluster-API rights beyond their ServiceAccount. | Namespace-scoped RBAC/identity isolate API objects; on the network, Cilium egress confines each tenant pod to its own subtree + platform system services + `world`, so unrelated tenants cannot reach each other over the pod network (enforced on egress; ingress is open to `cluster` — see Non-goals). | +| Tenant Kubernetes clusters | T3 — workload boundary | Own kube-apiserver (Kamaji hosted control plane, running as pods on the management cluster) and Talos worker nodes as KubeVirt VMs. | Worker nodes carry a VM/kernel boundary; the Kamaji control-plane pods are isolated by namespace/RBAC/NetworkPolicy, not by virtualization. | +| External end user / unauthenticated internet | T4 — untrusted | Reaches published gateway hostnames, the Keycloak login and dashboard oauth2-proxy, and any raw LoadBalancer L4 endpoints a tenant opts to expose (`external: true`, e.g. VPN/MariaDB). | OIDC authentication and hostname policies for the gateway path; Cilium `world` rules. Tenant-published LoadBalancer services are reachable directly, outside the hostname policies and OIDC. | + +### Actions + +| Action | Actors (from → to) | Data crossing boundary | Checks | Result | +| --- | --- | --- | --- | --- | +| Authenticate | End user (T4) → Keycloak → kube-apiserver | OIDC token (`groups`, `preferred_username`) | When OIDC is enabled (off by default): kube-apiserver OIDC verification; UIs fronted by oauth2-proxy (the `gatekeeper` Deployment). In the default OIDC-disabled mode the dashboard uses a token-proxy and the LINSTOR GUI is not externally exposed. | In OIDC mode, an identity whose `groups` map to the `-view/use/admin/super-admin` roles. In the default mode the token-proxy forwards a Kubernetes-issued ServiceAccount JWT as the bearer token, so authorization follows that ServiceAccount's RBAC. The tenant ServiceAccount is bound into the tenant's view/use/admin **and super-admin** RoleBindings, so a holder of the tenant SA token has super-admin-level access over the tenant subtree (equivalent to the `-super-admin` group), not merely the base `cozy:tenant` role. | +| Create an application | Tenant admin (T2) → `cozystack-api` (T1) → Flux (T1) → workload (T3) | Application CR `spec` (Helm values only) | RBAC (delegated to kube-apiserver via `system:auth-delegator`); name and reserved-key validation; admission chain; **server-fixed chart reference**. | Managed service provisioned from a vetted chart; tenant never touched a HelmRelease. | +| Create a tenant / nested tenant | Tenant super-admin (T2) → `cozystack-api` → Flux → new namespace (T2) | Tenant CR (`spec.host`, quotas) | Name checks; **best-effort hierarchical quota cap** (declaration-time snapshot; concurrent writes can briefly overshoot, with a runtime controller backstop); `cozystack-tenant-host-policy` restricts `spec.host` to trusted callers. | New tenant namespace with RBAC, NetworkPolicies, Keycloak groups, quota. | +| Publish a hostname (Gateway/HTTPRoute/TLSRoute/GRPCRoute/Ingress) | Tenant Application spec (T2) → Flux/controllers (T1) → core kube-apiserver | Rendered Gateway/Route/Ingress hostnames; namespace host label | Tenants have **no** RBAC to write Gateways, Routes, Ingresses, or namespace labels — Flux/controllers render them from the tenant's Application spec. The hostname VAPs (Gateway/Route policies, and `cozystack-ingress-hostname-policy` for the default legacy-Ingress path) validate rendered hostnames against the namespace host label (defense-in-depth). The apex is platform-set (`cozystack-tenant-host-policy` restricts `Tenant.spec.host` to trusted callers; the namespace label is immutable). Route/namespace/Ingress policies fail-closed; the Gateway-listener policy is fail-open when the label is absent. | A tenant cannot claim or hijack another tenant's platform-apex hostname, provided its namespace carries the host label. | +| Create a network policy | Tenant admin (T2) → `cozystack-api` | `sdn.cozystack.io/SecurityGroup` spec | RBAC; projected to a `CiliumNetworkPolicy` under the API server's ServiceAccount; tenants cannot write raw `cilium.io` objects. | CNI policy programmed on the tenant's behalf. | +| Delete a platform-critical object | Any actor → core kube-apiserver | DELETE request | `cozystack-no-delete-guardrail` denies DELETE on `platform.cozystack.io/no-delete=true` objects. Operational guidance, not adversarial defense: an actor able to update the object can remove the label first (tenants cannot reach these roots; only T0/T1). | Accidental teardown of platform roots blocked. | +| East-west traffic | Workload (T3) → workload/system | Packets | Cilium enforces policy at both endpoints. Each tenant pod's egress is confined to its own subtree, platform system services, and `world` (out-of-cluster); ingress is open to `world`+`cluster`. A flow needs the source's egress to allow it, so an unrelated tenant has no egress path to another tenant's pod. | Cross-tenant pod-to-pod traffic is **denied** (by source egress). Within a subtree, parent→descendant is allowed; child→parent is denied except for specific ancestor services (vminsert, etcd, ingress). Caveats (see Non-goals): open ingress and LoadBalancer hairpin. | + +### Goals + +- **Tenant isolation (API / identity layer).** A tenant cannot read, modify, or delete an *unrelated* tenant's Application CRs, workloads, or secrets, and a descendant cannot reach its ancestors. Parent→child access is intentional (ancestor ServiceAccounts and OIDC groups are bound one-way into descendant namespaces). +- **Network isolation between unrelated tenants.** Unrelated tenants cannot reach each other over the pod network: each tenant pod's Cilium egress is confined to its own subtree, platform system services, and `world`, and Cilium requires the source's egress to allow a flow — so there is no egress path between unrelated subtrees. Enforced on egress; ingress is permissively open (see Non-goals for caveats). +- **Constrained self-service provisioning.** Tenants may instantiate only vetted `ApplicationDefinition` kinds with server-fixed chart references; they cannot run arbitrary Helm charts or write raw privileged manifests. +- **Hostname and tenancy integrity.** Tenants cannot claim or hijack a platform-apex hostname belonging to another tenant, on both dataplanes. Gateway API path: HTTPRoute/TLSRoute/GRPCRoute/Gateway-listener hostnames must be within the tenant's own apex. Default legacy-Ingress path (`cozystack-ingress-hostname-policy`): a hostname within the tenant's own apex or entirely outside the platform root apex (an external custom domain, e.g. a tenant Kubernetes cluster's Proxied ingress) is allowed, while a hostname under the platform root apex but outside the tenant's own sub-apex — plus hostless rules and `defaultBackend` catch-alls — is denied. Route, namespace and Ingress policies are fail-closed; the Gateway-listener policy is fail-open when the label is absent and relies on the platform-set, immutable `namespace.cozystack.io/host` label. +- **Hierarchical resource-quota enforcement (best-effort).** At declaration time a child tenant's declared quota is checked against its parent's remaining budget; the check is a non-transactional admission snapshot (concurrent writes can briefly overshoot) with the tenant-quota controller as the runtime backstop. +- **Strong isolation for tenant Kubernetes/VM workloads** via Talos worker nodes as KubeVirt VMs (hardware virtualization); the tenant control plane is a Kamaji hosted control plane running as pods on the management cluster, isolated by namespace/RBAC/network rather than by virtualization. +- **Central, token-based authentication** when OIDC is enabled (opt-in; off by default): Keycloak/OIDC with group-to-role mapping into RBAC. The default OIDC-disabled mode fronts the dashboard with a token-proxy instead. +- **Defense in depth at admission**, enforced at both the core kube-apiserver and the aggregated `cozystack-api`, with mostly fail-closed tenancy policies (the Gateway-listener policy is the documented exception noted above). +- **Accidental-deletion protection** for platform-critical roots via the no-delete guardrail (operational guidance, not adversarial defense — a privileged T0/T1 actor can remove the label and delete). + +### Non-goals + +- Defending against a compromised **cluster administrator, node root, or infrastructure administrator** — trusted by design and out of scope (`SECURITY.md`). +- Treating the **T1 platform control plane** (`cozystack-api` ServiceAccount, Flux, system operators) as a sandbox; these hold cluster-admin-equivalent power. +- **Upstream-only vulnerabilities** not introduced or worsened by Cozystack packaging or defaults. +- **Ingress-side isolation of tenant pods.** Tenant-pod ingress is open to `world` and `cluster`, so a tenant pod is reachable by any in-cluster peer whose own egress permits it (platform components) or by a workload not governed by Cozystack's tenant egress policies. Cross-tenant isolation is enforced on the *egress* side, not tenant-pod ingress. +- **Blocking cross-tenant reachability through externally-exposed services.** If a tenant publishes a LoadBalancer external IP, another tenant can reach it via `world` egress. +- **Network isolation between tenants on a hosted cluster without Cilium.** On the `isp-hosted` and `isp-hosted-slim` variants the CNI belongs to the host, and tenant Cilium policies are created only if the host serves the Cilium policy API. Without it, tenants get no network isolation from Cozystack, and isolating them is up to the host. +- **Noisy-neighbour / side-channel** guarantees for co-located tenant pods beyond namespace scoping (and, for tenant Kubernetes/VMs, virtualization). +- Acting as an **egress firewall**: tenant workloads may egress to `world` by default. +- Supporting **pre-release builds and end-of-life versions**. + +## Self-assessment use + +This self-assessment is created by the Cozystack team to perform an internal analysis of the project's security. It is not intended to provide a security audit of Cozystack, or function as an independent assessment or attestation of Cozystack's security health. + +This document serves to provide Cozystack users with an initial understanding of Cozystack's security, where to find existing security documentation, Cozystack's plans for security, and a general overview of Cozystack security practices, both for the development of Cozystack as well as the security of Cozystack. + +This document provides the CNCF TAG-Security with an initial understanding of Cozystack to assist in a joint assessment, necessary for projects under incubation. Taken together, this document and the joint assessment serve as a cornerstone for if and when Cozystack seeks graduation and is preparing for a security audit. + +## Security functions and features + +### Critical + +These components are load-bearing: a failure or compromise directly defeats a tenant-isolation or platform-integrity guarantee. + +- **`cozystack-api` (aggregated API server).** Converts tenant Application CRs into Flux HelmReleases with a server-fixed chart reference, and enforces name/quota/host validation plus in-process admission. The primary control that constrains what tenants can provision. +- **Core kube-apiserver + RBAC + ValidatingAdmissionPolicy.** Per-tenant aggregated ClusterRoles grant no HelmRelease or arbitrary core writes; VAPs enforce hostname/tenancy integrity (mostly fail-closed — see Overview) and deletion protection. +- **Flux (source/helm/kustomize controllers).** Executes releases with cluster-admin-equivalent power; the delivery mechanism for all platform and tenant workloads. +- **Cilium.** The sole enforcement of network isolation between tenants; also the backend for tenant `SecurityGroup` policies. +- **Keycloak** (when OIDC is enabled; opt-in, off by default). The identity source whose OIDC group claims drive RBAC. +- **cert-manager.** Mints the aggregated-API serving CA (the trust anchor for the `apps.cozystack.io` API) and tenant TLS. +- **Kamaji + KubeVirt + Talos** (for tenant Kubernetes clusters). Provide hardware-virtualized isolation for tenant worker nodes; the Kamaji control plane runs as pods on the management cluster. + +### Security relevant + +These support security but are more contained; a compromise is less likely to escalate on its own. + +- **Cozystack controllers/reconcilers** (`ApplicationDefinition`, tenant gateway/ReferenceGrant, quota, workload monitors). +- **oauth2-proxy front doors** (the `gatekeeper` Deployment) for the dashboard and LINSTOR GUI. +- **external-secrets-operator, clustersecret-operator, OpenBao** for secret sync, replication, and vaulting. +- **Curated tenant registries** (`tenantsecret`, `tenantmodule`, `tenantnamespace`) that expose only labelled resources and strip internal labels. +- **reloader, flux-plunger, kube-ovn webhook, lineage-controller webhook.** + +The primary trust-boundary invariant is that tenants cannot influence the chart reference, `valuesFrom`, or write HelmReleases directly. Any change to tenant RBAC or the Application REST layer that weakened this would collapse T2→T1 isolation and should be reviewed accordingly. + +## Project compliance + +Cozystack does not currently claim compliance with, or certification against, any specific security standard or regulatory framework (for example PCI-DSS, ISO 27001, SOC 2, or GDPR). Downstream adopters may operate Cozystack within their own compliance programs, but the project itself makes no formal attestation. + +## Secure development practices + +### Development Pipeline + +- **Source control and review.** All changes land through GitHub pull requests. `.github/CODEOWNERS` requires maintainer review; the project deliberately requires a single approving review rather than two, to keep contribution velocity acceptable (branch protection requiring multiple reviews was considered and intentionally not adopted). +- **Commit sign-off (DCO).** Contributor guidance requires `--signoff` on every commit (`docs/agents/contributing.md`, `.gemini/styleguide.md`), and DCO is **enforced by an automated DCO check on every pull request**. Remaining refinements are to surface the sign-off requirement in the top-level `CONTRIBUTING.md` and to add the DCO check to the branch-protection required-checks list so it is a hard merge gate as well as a reported check. +- **Automated checks on pull requests.** A pre-commit workflow validates generated-code/schema freshness; unit and end-to-end tests run in CI. +- **Static analysis (SAST).** CodeQL runs on pull requests, on push to `main`, and weekly (Go, manual build mode), intended as a required check that blocks merge on new error-severity alerts. It is not yet in the required-checks list on `main` (currently `pre-commit` and `E2E Tests`), so the gate is review convention rather than branch protection. `zizmor` additionally audits the GitHub Actions workflows on pull requests and fails the check on unpinned actions and over-broad `permissions`. Additional Go linters (`gosec`, `govulncheck`, `golangci-lint`) are not yet wired into CI and are a planned improvement. +- **Dependency and image CVE scanning (SCA).** An organization-wide pipeline built on Trivy scans every non-fork, non-archived repository in the `cozystack` organization — Go modules, Dockerfile base images, and container images referenced by the packaged charts — against NVD, vendor advisories and the GitHub Advisory Database. CRITICAL findings are scanned every 6 hours, the remaining severities weekly; findings are noise-filtered and tracked as issues under a triage matrix with per-severity targets (CRITICAL 1 business day to triage, 7-day fix target). Two limitations are worth stating: image references assembled at render time by the `cozy-lib.image` helper are invisible to values-file discovery, and per-CVE findings and triage state are currently held in a private repository, so the control is not yet externally verifiable. Aggregate reporting is now published at [`docs/security/reports/`](reports/): monthly reports reduced to counts by severity, plus a backlog reconciliation auditing every tracked finding against what shipped. Publishing the pipeline policy, the triage matrix, the finding status model and the scanning scripts, while keeping per-finding detail embargoed until fixed, is outstanding. The documented triage target is not being met and the role that owns it is unassigned; remediation is happening but the pipeline cannot record it, and the reconciliation sets out both. +- **Dependency updates.** Renovate manages Go modules, Dockerfiles, and GitHub Actions, plus a small set of regex-pinned references, with OpenSSF OSV vulnerability alerts enabled and automerge disabled. The `helm-values` manager is disabled repo-wide, so the curated component images that make up the shipped platform are bumped by maintainers when preparing a release rather than automatically. There is no Dependabot configuration; Renovate is the dependency-update tool. +- **Supply-chain posture.** OpenSSF Scorecard runs weekly with published results. Most GitHub Actions are pinned by commit SHA; a small number of recently added workflows still use floating tags and are being brought into line. Renovate is configured to pin digests; applying digest pins across container base images is in progress. +- **Signed / reproducible builds.** Honestly noted as gaps: release images are **not** signed (no cosign/sigstore), build provenance is currently disabled (`--provenance=false`), SBOM generation is plumbed but off by default, and reproducible builds are not yet established. These are recognized improvement areas for the incubation timeframe. + +### Communication Channels + +- **Internal.** Maintainers coordinate over Slack, Telegram, and project email (`cncf-cozystack-*` addresses), and in community meetings. +- **Inbound.** Users reach the project through GitHub issues and Discussions; security reports follow the private routing in `SECURITY.md` (see below). +- **Outbound.** The project announces releases and security fixes through GitHub Releases, in-repository changelogs, the documentation site at , and GitHub Security Advisories (GHSA) when applicable. + +### Ecosystem + +Cozystack is a CNCF Sandbox project. It is built on and integrates a broad set of cloud-native components — Kubernetes, Flux, Cilium, KubeVirt, Kamaji, Talos, cert-manager, external-secrets, and Keycloak — and positions itself as a full platform layer for building managed-service clouds on top of Kubernetes. + +## Security issue resolution + +### Responsible Disclosures Process + +The vulnerability reporting process is documented in [`SECURITY.md`](https://github.com/cozystack/cozystack/blob/main/SECURITY.md). Reporters are asked not to use public issues, discussions, pull requests, or chat channels for vulnerabilities. + +The private intake channel is **GitHub Private Vulnerability Reporting** (repository → **Security → Advisories → Report a vulnerability**), the CNCF-recommended path for confidential disclosure; it is enabled on the repository. As a fallback, reporters may contact a maintainer listed in [`MAINTAINERS.md`](https://github.com/cozystack/cozystack/blob/main/MAINTAINERS.md) through an existing private channel, or use a public channel only to request a private contact path without disclosing details. A dedicated security mailing list may be added later but is not required given Private Vulnerability Reporting is available. `SECURITY.md` and the code-ownership model are aligned: it routes reporters to Private Vulnerability Reporting and `MAINTAINERS.md` (the governance roster), while `CODEOWNERS` separately encodes the area-scoped **Reviewer** role from the Contributor Ladder. + +#### Vulnerability Response Process + +Per `SECURITY.md`, the maintainers commit to: + +- Acknowledge receipt within **3 business days**. +- Perform initial triage and severity assessment within **7 business days**. +- Keep the reporter informed as a fix and disclosure plan are developed. + +Resolution follows a coordinated-disclosure model: reporters are asked to keep details private until a fix or mitigation is available and users have had a reasonable opportunity to upgrade. The project commits to a coordinated-disclosure window of **up to 90 days** from acknowledgement, after which the advisory may be published (via GHSA) with available details and workarounds even if a complete fix is not yet ready; extensions apply only by mutual agreement, typically for issues requiring upstream coordination. Target remediation timelines track CVSS v3.1 severity (Critical ~14 days, High ~30, Medium ~90, Low next scheduled release). The project may request or publish a GHSA and/or CVE. Reporters are credited unless anonymity is requested. See [`SECURITY.md`](https://github.com/cozystack/cozystack/blob/main/SECURITY.md) for the authoritative statement. + +### Incident Response + +Detailed operational incident-response procedures are maintained by the maintainers in a private security pipeline, and a public summary is published at [`docs/security/incident-response.md`](https://github.com/cozystack/cozystack/blob/main/docs/security/incident-response.md). The public runbook stitches the existing mechanisms into a single incident lifecycle: intake and triage via `SECURITY.md`, private fix development under a GitHub Security Advisory, coordinated patch releases and backports across supported lines, GHSA/CVE publication, public disclosure through the outbound channels (release notes, changelog, GHSA), and the access-revocation and credential-rotation steps defined in `CONTRIBUTOR_LADDER.md` (completed within five business days). + +## Appendix + +### Known Issues Over Time + +Cozystack uses GitHub Security Advisories (GHSA) to track vulnerabilities and GitHub Private Vulnerability Reporting for confidential intake. No advisories have been published to date; as issues are resolved under coordinated disclosure they will be published as GitHub Security Advisories on the repository, building the public history over time. + +### OpenSSF Best Practices + +Cozystack participates in the OpenSSF Best Practices Badge program as project [#10177](https://www.bestpractices.dev/projects/10177) (the badge is shown in the repository README). The project **achieved the passing-level badge on 2026-04-01** and is now working toward the silver tier; the live status is reflected on the badge page. Cozystack also runs OpenSSF Scorecard weekly with published results. + +### Case Studies + +Cozystack is used in production by the organizations listed in [`ADOPTERS.md`](https://github.com/cozystack/cozystack/blob/main/ADOPTERS.md), including Ænix (managed services for cloud providers), Mediatech (Kubernetes distribution for hosted software), Bootstack (cloud-infrastructure operator), gohost (bare-metal Kubernetes), Urmanac (hosting, including WASM/SpinKube), Hidora / Hikube (sovereign multi-datacenter cloud), QOSI (sovereign GPU clouds for a national AI program), and Cloupard (public IaaS/PaaS on bare metal). These adopters exercise the multi-tenancy and managed-service model described above across several regions. + +### Related Projects / Vendors + +Cozystack occupies a space adjacent to platforms such as Deckhouse, Rancher/RKE, Red Hat OpenShift, and KubeVirt-based virtualization platforms. Its distinguishing security-relevant characteristics are: an aggregated API server that exposes managed services as virtual `apps.cozystack.io` resources (so tenants provision from vetted charts rather than writing Helm/manifests directly), tenant isolation enforced through RBAC plus native ValidatingAdmissionPolicy, and hardware-virtualized tenant Kubernetes clusters built on Talos and KubeVirt. diff --git a/docs/security/threat-model.md b/docs/security/threat-model.md new file mode 100644 index 0000000000..9790d95913 --- /dev/null +++ b/docs/security/threat-model.md @@ -0,0 +1,121 @@ +# Cozystack Threat Model + +This document is a design-level threat model for Cozystack. It describes the actors in the system, the actions that cross trust boundaries, and the security guarantees the platform intends (and does not intend) to provide. It is organized around **trust boundaries**, following the CNCF TAG-Security "Actors / Actions" model, and is intended both as internal security documentation and as the basis for the project's CNCF self-assessment. + +Scope: this is a *design-level* analysis of the Cozystack platform (the management cluster and its multi-tenancy model). It is not a code audit and does not enumerate implementation-specific bugs or misconfigurations. All architectural claims cite source files in this repository so the model can be re-verified as the code evolves. + +## Background + +Cozystack is delivered as a **management (root) Kubernetes cluster** onto which tenants, managed services, virtual machines, and tenant Kubernetes clusters are layered. The linchpin of the security model is the aggregated API server **`cozystack-api`** (`cmd/cozystack-api`, `pkg/apiserver`, `pkg/registry`). Tenants never write privileged Kubernetes objects (HelmRelease, Deployment, Secret, RBAC) directly. Instead they write thin, virtual `apps.cozystack.io/*` **Application** custom resources, and `cozystack-api` translates each one 1:1 into a Flux `HelmRelease` whose chart reference is fixed server-side. + +`cozystack-api` serves three aggregated API groups (`pkg/apiserver/apiserver.go`): `core.cozystack.io/v1alpha1` (`tenantnamespaces`, `tenantsecrets`, `tenantmodules`, `options`), `sdn.cozystack.io/v1alpha1` (`securitygroups`), and `apps.cozystack.io/v1alpha1` (the per-tenant Application kinds). The set of Application kinds is registered **dynamically** at startup from `ApplicationDefinition` custom resources (`pkg/cmd/server/start.go`, `api/v1alpha1/applicationdefinitions_types.go`); when an `ApplicationDefinition` changes, `internal/controller/applicationdefinition_controller.go` rolls the `cozystack-api` Deployment so the new kinds are served. The server is registered with the main kube-apiserver as an `APIService` over TLS whose CA is minted by cert-manager (`packages/system/cozystack-api/templates/`). + +Application storage is a **virtual REST backed by HelmReleases**, not etcd (`pkg/registry/apps/application/rest.go`). On create/update the tenant supplies only `app.Spec` (the Helm values) plus labels and annotations; the chart reference, release-name prefix, and the platform `cozystack-values` Secret mounted as `valuesFrom` all come from server-side configuration, not from the tenant (`pkg/registry/apps/application/rest.go` around the HelmRelease construction; `pkg/config/config.go`). Values keys beginning with `_` are reserved and rejected. + +**Usage modes.** Cozystack is deployed in two ways. As a **managed-services backend** (roughly half of deployments), an external provisioning system authenticates with its own ServiceAccount and creates the tenant resources it needs — no OIDC involved. Used **directly**, a human is granted OIDC access to the dashboard and the Cozystack API and operates it, constrained by RBAC to the `apps.cozystack.io` group within their tenant. + +## Multitenancy model + +Cozystack combines hard and soft multitenancy: + +- **Hard multitenancy for tenant-controlled code.** Anything in which a tenant can run arbitrary code — VMs and tenant Kubernetes clusters — is isolated at the hardware-virtualization / dedicated-control-plane boundary, and the tenant has full access inside it. Those VMs and tenant-cluster nodes are themselves ordinary, unprivileged pods in the tenant namespace: governed by Pod Security Standards, denied access to the management-cluster API, and subject to the tenant network and admission policies. +- **Soft multitenancy for managed data services.** Databases and queues run in the management cluster but expose only a fixed, validated set of parameters and never let a tenant execute arbitrary code (unsafe options are disabled). A tenant therefore cannot compromise the management cluster by swapping a container image or enabling a dangerous setting. + +## Trust boundaries + +The actors below are decomposed **by trust boundary**, not by component: two components are distinct actors only when compromise of one does not automatically compromise the other. Cozystack has five trust tiers. + +- **T0 — Root of trust.** The platform / cluster administrator (`system:masters`), who owns the management cluster, the platform GitOps repository, Talos machine configuration, and the Keycloak realm. Nothing sits above T0. A compromised T0 is explicitly out of the adversary model (see Non-goals and `SECURITY.md`). +- **T1 — Platform control plane (cluster-admin-equivalent).** The GitOps delivery layer (Flux source-, helm-, and kustomize-controllers plus `flux-shard-operator`), the `cozystack-api` aggregated API server's own ServiceAccount, and the system operators (the Cozystack controllers, plus Kamaji, KubeVirt, cert-manager, external-secrets, keycloak-operator, Piraeus/LINSTOR, Cluster API providers, and others). These components hold broad, cluster-admin-equivalent privileges. They *enforce* tenant boundaries but are not themselves sandboxed from one another or from the host. +- **T2 — Tenant boundary.** Tenant administrators and tenant users, scoped to a tenant namespace subtree. This is the primary isolation unit. Tenancy is hierarchical: a parent tenant's ServiceAccounts are bound into every descendant namespace, but child ServiceAccounts are not bound upward, so ancestor→descendant trust is one-way. +- **T3 — Workload boundary.** Tenant workloads, managed-service pods, KubeVirt virtual machines, and tenant Kubernetes clusters (each a dedicated Kamaji-hosted control plane with Talos worker nodes running as VMs). These run tenant-controlled code with no cluster-API rights beyond their own ServiceAccounts. +- **T4 — Untrusted.** External end users and the unauthenticated internet. They reach published ingress/gateway hostnames and the UI front doors (when OIDC is enabled, the Keycloak login and dashboard oauth2-proxy; by default, OIDC-disabled, the dashboard token-proxy) — and, when a tenant opts into external exposure (e.g. `external: true` on apps such as VPN or MariaDB), the raw LoadBalancer L4 endpoints those apps publish, which sit outside the Gateway/OIDC path. + +## Actors + +| Actor | Trust tier | Privileges | Isolation mechanism (separated from neighbours by) | +| --- | --- | --- | --- | +| Platform / cluster admin (`system:masters`) | T0 | Full control of the management cluster; owns GitOps, Talos config, Keycloak realm; may set/change a tenant's `spec.host`. | Root of trust; nothing isolates it (trusted by design). | +| GitOps / Flux delivery layer | T1 | Reconciles HelmReleases/Kustomizations into arbitrary privileged manifests; effectively cluster-admin. | Tenants cannot create HelmReleases/Kustomizations (RBAC); not tenant-network-reachable. | +| `cozystack-api` aggregated API server (ServiceAccount) | T1 | CRUD HelmReleases, Secrets, and CiliumNetworkPolicies cluster-wide; reads Namespaces/RBAC/quotas/nodes. Acts as the policy-enforcing gateway that converts tenant CRs to HelmReleases. | A distinct, network-exposed attack surface from Flux; constrains tenants via fixed chart ref + name/quota/host validation + admission. | +| System operators / controllers | T1 | Domain-scoped cluster privileges (provision control planes, mint certs/secrets, program CNI). | Distinct ServiceAccounts, but all cluster-admin-adjacent; isolated from tenants only by RBAC and network policy. | +| Tenant administrator (OIDC groups `-admin`, `-super-admin`) | T2 | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` within its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) The `-super-admin` group holds the `apps.cozystack.io/*` wildcard (and so can create child `Tenant`s); the `-admin` group has an enumerated allowlist that excludes some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`, `external-dns`, `bootbox`). | Namespace-scoped RoleBindings + admission policies + quota; no access to HelmReleases, arbitrary core writes, or other tenants. | +| Tenant user (OIDC groups `-view`, `-use`) | T2 | Read-only view of Application CRs, pods, and services, plus operational verbs only (VM console/vnc/portforward, start/stop/restart). Cannot create Application CRs. | Strictly dominated by tenant-admin; cannot escalate to create/delete, so it is a distinct actor. | +| Parent tenant vs child (nested) tenant | T2 | Parent ServiceAccounts are bound into descendant namespaces; child ServiceAccounts are not bound upward. Quota is hierarchically capped. | Ancestor→descendant only; compromising a parent compromises its children, not the reverse. | +| Tenant workloads / managed services | T3 | Run tenant-controlled code; no cluster-API rights beyond their ServiceAccount. | Namespace-scoped RBAC/identity isolate their API objects. On the network, Cilium egress confines each tenant pod to its own tenant subtree, platform system services, and `world` (out-of-cluster), so unrelated tenants cannot reach each other over the pod network — enforced on the egress side (tenant-pod ingress is open to `cluster`; see Non-goals for residual caveats). | +| Tenant Kubernetes clusters | T3 | Own kube-apiserver (a Kamaji hosted control plane running as pods on the management cluster) and Talos worker nodes running as hardware-virtualized KubeVirt VMs. | Worker nodes carry a VM/kernel boundary (do not share a kernel with the host or other tenants). The Kamaji control-plane pods run on the management cluster and are isolated by namespace/RBAC/NetworkPolicy, not by virtualization. | +| External end user / unauthenticated internet | T4 | Reaches published gateway hostnames, the UI front door (Keycloak + dashboard oauth2-proxy when OIDC is enabled; a token-proxy by default), and any raw LoadBalancer L4 endpoints a tenant opts to expose (`external: true`). | OIDC authentication (when enabled) and hostname admission policies for the gateway path; Cilium `world`-entity rules. Tenant-published LoadBalancer services are reachable directly and are not behind the hostname policies or OIDC. | + +## Actions + +| Action | Actors (from → to) | Data crossing boundary | Authorization / validation checks | Result | +| --- | --- | --- | --- | --- | +| Authenticate | End user (T4) → Keycloak → kube-apiserver / dashboard | OIDC token with `groups` and `preferred_username` claims | When OIDC is enabled (`authentication.oidc.enabled`, **off by default**): kube-apiserver OIDC verification; dashboard and LINSTOR GUI fronted by oauth2-proxy (the `gatekeeper` Deployment); tenant kubeconfig uses `kubectl oidc-login`. In the default OIDC-disabled mode the dashboard uses a `token-proxy` and the LINSTOR GUI is not externally exposed. | In OIDC mode, an authenticated identity whose `groups` map to RBAC subjects (the `-view/use/admin/super-admin` roles). In the default mode the `token-proxy` forwards a Kubernetes-issued ServiceAccount JWT as the bearer token, so authorization is whatever that ServiceAccount's RBAC grants. The tenant ServiceAccount is bound (via `tenant.yaml`) into the tenant's view/use/admin **and super-admin** RoleBindings, so a holder of the tenant SA token has super-admin-level access over the tenant subtree — equivalent to the `-super-admin` OIDC group, not merely the base `cozy:tenant` role. | +| Create an application (e.g. `apps.cozystack.io/Postgres`) | Tenant admin (T2) → `cozystack-api` (T1) → Flux (T1) → workload (T3) | Application CR `spec` (Helm values only) | RBAC allows `apps.cozystack.io/*` create in the tenant namespace (delegated to kube-apiserver via `system:auth-delegator`); `cozystack-api` validates the name, rejects `_`-prefixed keys, and runs the admission chain; the emitted HelmRelease uses a **server-fixed chart reference**. | Managed service provisioned from a vetted chart; the tenant never touched a HelmRelease or chart. | +| Create a tenant / nested tenant (`apps.cozystack.io/Tenant`) | Tenant super-admin (T2) → `cozystack-api` → Flux → new namespace (T2) | Tenant CR (`spec.host`, quotas, feature toggles) | Namespace-name checks and a **best-effort hierarchical quota cap** (a declaration-time snapshot: a child's declared quota is checked against the parent's remaining budget; concurrent writes can briefly overshoot, with the tenant-quota controller as the runtime backstop); the `cozystack-tenant-host-policy` admission policy allows only trusted callers to set/change `spec.host`. | New tenant namespace with RBAC bindings, NetworkPolicies, Keycloak groups, quota, and gateway inheritance. | +| Publish a hostname (Gateway / HTTPRoute / TLSRoute / GRPCRoute / Ingress) | Tenant Application spec (T2) → Flux / controllers (T1) → core kube-apiserver | Rendered Gateway listener, Route, and Ingress hostnames; namespace `namespace.cozystack.io/host` label | Tenants have **no** RBAC to write Gateways, Routes, Ingresses, or namespace labels; these are rendered by Flux/controllers from the tenant's Application spec. The hostname VAPs then validate the rendered hostnames against the namespace host label as defense-in-depth (so neither a tenant-authored spec nor a buggy chart escapes the apex). The apex itself is platform-set: `cozystack-tenant-host-policy` restricts `Tenant.spec.host` to trusted callers and the namespace-host-label policy keeps the label immutable. The Gateway path requires strict in-apex hostnames; the default legacy-Ingress path (`cozystack-ingress-hostname-policy`) additionally allows external custom domains (outside the platform root apex) while denying platform-apex claims outside the tenant's own sub-apex. Route/namespace/Ingress policies are fail-closed; the Gateway-listener policy is fail-open when the label is absent (see Admission enforcement points). | A tenant cannot publish or hijack another tenant's platform-apex hostname, provided its namespace carries the platform-set host label. | +| Create a network policy (`sdn.cozystack.io/SecurityGroup`) | Tenant admin (T2) → `cozystack-api` | SecurityGroup spec | RBAC on `securitygroups`; `cozystack-api` projects the spec into a `CiliumNetworkPolicy` under its own ServiceAccount; tenants cannot write raw `cilium.io` objects. | CNI policy programmed on the tenant's behalf. | +| Delete a platform-critical object | Any actor → core kube-apiserver | DELETE request | The `cozystack-no-delete-guardrail` policy denies DELETE on objects labelled `platform.cozystack.io/no-delete=true` (e.g. the tenant-root namespace and its HelmRelease). It is operational guidance, not adversarial defense: an actor able to update the object can remove the label and then delete (tenants have no access to these roots; only T0/T1 do). | Accidental teardown of platform roots is blocked. | +| Tenant-to-tenant / tenant-to-platform traffic | Workload (T3) → workload (T3) / system | East-west packets | Cilium enforces policy at **both** endpoints, so a flow needs the source's egress *and* the destination's ingress to allow it. Every tenant pod's egress is confined to its own tenant subtree (namespaces labelled `tenant.cozystack.io/`), platform system services (DNS, apiserver, ingress, dashboard, Keycloak, CDI), and `world` (out-of-cluster). Ingress is permissively open to `world` and `cluster`. | Cross-tenant pod-to-pod traffic is **denied** — an unrelated tenant has no egress path to another tenant's pod, even though that pod's ingress would accept it. Within a tenant subtree, parent→descendant traffic is allowed (descendants carry ancestor namespace labels); child→parent traffic is denied except for specific ancestor services (vminsert, etcd, ingress). See Non-goals for the ingress-side and LoadBalancer caveats. | +| Distribute platform config to tenants | Platform (T1) → tenant namespace (T2) | `cozystack-values` Secret (cluster/namespace host, OIDC toggles, gateway references) | Rendered by the platform charts and merged into every tenant HelmRelease as `valuesFrom`. | Tenants inherit platform config (host apex, etc.) but cannot forge it. | + +## Goals + +Cozystack intends to provide the following security guarantees: + +- **Tenant isolation (API / identity layer).** A tenant cannot read, modify, or delete an *unrelated* tenant's Application CRs, workloads, or secrets, and a descendant tenant cannot reach its ancestors. Parent→child access is intentional and not a violation: an ancestor's ServiceAccounts and OIDC groups are bound into descendant namespaces (one-way), so a parent tenant can act within its children. Isolation is enforced by namespace-scoped RBAC (`packages/system/cozystack-basics/templates/clusterroles.yaml`) and that one-way hierarchical binding (`packages/apps/tenant/templates/tenant.yaml`, helper `cozy-lib.rbac.subjectsForTenantAndAccessLevel`). +- **Network isolation between unrelated tenants.** An unrelated tenant's pods cannot reach a tenant's pods over the cluster pod network: each tenant pod's Cilium egress is confined to its own tenant subtree, platform system services, and `world` (`packages/apps/tenant/templates/networkpolicy.yaml`), and Cilium requires the source's egress to allow a flow — so there is no egress path between unrelated subtrees. Isolation is enforced on the egress side; tenant-pod ingress is permissively open (see Non-goals for the caveats this creates). +- **Constrained self-service provisioning.** Tenants may instantiate only vetted `ApplicationDefinition` kinds with **server-fixed chart references** and cannot run arbitrary Helm charts, create HelmReleases, or write raw privileged manifests. +- **Hostname and tenancy integrity.** Tenants cannot claim or hijack a hostname under the platform apex that belongs to another tenant, on both dataplanes. On the Gateway API path, Gateway-listener and HTTPRoute/TLSRoute/GRPCRoute hostnames must be within the tenant's own apex (`namespace.cozystack.io/host`). On the default legacy-Ingress path, `cozystack-ingress-hostname-policy` allows a hostname within the tenant's own apex or entirely outside the platform root apex (`_cluster.root-host`) — an external custom domain, e.g. a tenant Kubernetes cluster's Proxied ingress — and denies any hostname under the platform root apex but outside the tenant's own sub-apex, plus hostless rules and `spec.defaultBackend` catch-alls. The Tenant-CR host, namespace-host-label, HTTPRoute/TLSRoute/GRPCRoute, and Ingress policies are fail-closed; the Gateway-listener policy is permissive when the host label is absent and so relies on that label being present and immutable (see Admission enforcement points). Policies: `packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml`, `route-hostname-policy.yaml`, `ingress-hostname-policy.yaml`. +- **Hierarchical resource-quota enforcement (best-effort).** At declaration time a child tenant's declared quota is checked against its parent's remaining budget (`pkg/registry/apps/application/quota.go`). The check is a non-transactional admission snapshot: concurrent tenant writes can briefly overshoot the parent budget, and the tenant-quota controller is the runtime backstop. +- **Strong isolation for tenant Kubernetes and VM workloads** via Talos worker nodes running as KubeVirt VMs (hardware virtualization). The tenant control plane itself is a Kamaji hosted control plane running as pods on the management cluster, isolated by namespace/RBAC/network rather than by virtualization. +- **Central, token-based authentication** when OIDC is enabled (opt-in; `authentication.oidc.enabled` is off by default): Keycloak/OIDC with group-to-role mapping into kube-apiserver RBAC and UI access mediated by oauth2-proxy. In the default OIDC-disabled mode the dashboard is fronted by a token-proxy instead. +- **Defense in depth at admission.** ValidatingAdmissionPolicy is enforced at two points — the core kube-apiserver for core and Gateway objects, and inside `cozystack-api` for aggregated `apps.cozystack.io` kinds — and most tenancy policies are fail-closed (the Gateway-listener policy is the documented exception; see Admission enforcement points). +- **Accidental-deletion protection** for platform-critical roots via the no-delete guardrail policy. This is operational guidance, not adversarial defense: a privileged (T0/T1) actor can remove the label and delete. +- **Trustworthy platform PKI**: the aggregated-API serving certificate and tenant TLS are issued and rotated by cert-manager, with CA injection into the `APIService`. + +## Non-goals + +The following are explicitly out of scope for the platform's security guarantees: + +- **Defending against a compromised cluster administrator, node root, or infrastructure administrator.** `system:masters` and the cozystack/Flux service accounts are trusted callers by design (`SECURITY.md`). +- **Treating the T1 platform control plane as a sandbox.** `cozystack-api`'s own ServiceAccount, Flux, and the system operators hold cluster-admin-equivalent power; they enforce tenant boundaries but are not isolated from each other or from the host. +- **Upstream-only vulnerabilities** that Cozystack packaging and defaults do not introduce or worsen. +- **Ingress-side isolation of tenant pods.** Tenant-pod ingress is open to the `world` and `cluster` entities, so a tenant pod is reachable by any in-cluster peer whose own egress permits it (platform components) and by a workload not governed by Cozystack's tenant egress policies (e.g. a privileged or system pod). Cross-tenant isolation is enforced on the *egress* side, not on tenant-pod ingress. +- **Blocking cross-tenant reachability through externally-exposed services.** If a tenant publishes a LoadBalancer with an external IP, another tenant can reach it via `world` egress. +- **Network isolation between tenants on a hosted cluster without Cilium.** On the `isp-hosted` and `isp-hosted-slim` variants the CNI belongs to the host, and the tenant chart creates its Cilium policies only if the host serves the Cilium policy API (`packages/apps/tenant/templates/_helpers.tpl`, helper `tenant.ciliumPolicies`). Without it, tenants get no network isolation from Cozystack, and isolating them is up to the host. +- **Noisy-neighbour and side-channel guarantees** for co-located tenant pods beyond namespace scoping (and, for tenant Kubernetes/VMs, virtualization). Co-located pods are not claimed to be side-channel isolated. +- **Acting as an egress firewall.** Tenant workloads are allowed egress to `world` by default (`packages/apps/tenant/templates/networkpolicy.yaml`). +- **Supporting pre-release builds and end-of-life versions** (`SECURITY.md`). + +## Primary trust-boundary invariant + +The single most load-bearing isolation control is the combination of two facts: + +1. The tenant ClusterRoles (`packages/system/cozystack-basics/templates/clusterroles.yaml`) grant **no** `helm.toolkit.fluxcd.io` access and **no** create/update of arbitrary core resources — only CRUD on `apps.cozystack.io` application kinds (super-admin holds the `*` wildcard; admin an enumerated allowlist) and `sdn.cozystack.io/securitygroups`, read access to pods/services/ingresses, and (for tenant-admins) `delete` on a fixed set of core objects (pods/services/endpoints/events; the `persistentvolumes` entry in the ClusterRole is inert under the namespace-scoped RoleBinding). Tenant isolation is enforced by the *absence* of privileged grants in RBAC, not by an admission policy that whitelists `apps.cozystack.io/*` writes. +2. The Application-to-HelmRelease conversion (`pkg/registry/apps/application/rest.go`) takes the chart reference, release-name prefix, and `valuesFrom` from server-side configuration; the tenant controls only the Helm values (`app.Spec`). + +If a future `ApplicationDefinition`, RBAC change, or API change ever let a tenant influence the chart reference, `valuesFrom`, or write HelmReleases directly, the T2→T1 isolation would collapse. Changes touching tenant RBAC or the Application REST layer should be reviewed against this invariant. + +## Admission enforcement points + +Cozystack has two admission enforcement points, and the distinction matters when reasoning about which policies protect which resources: + +- The **core kube-apiserver** evaluates ValidatingAdmissionPolicy for resources it serves directly — Namespaces, Gateways, HTTPRoutes/TLSRoutes/GRPCRoutes, Ingresses, and the no-delete guardrail. +- **`cozystack-api` itself** evaluates the admission chain (validating webhooks and ValidatingAdmissionPolicy) for aggregated `apps.cozystack.io` resources, because those are served by the aggregated API server and never reach the core apiserver's admission chain. The `cozystack-tenant-host-policy` on `apps.cozystack.io/tenants` is therefore enforced inside `cozystack-api`. So is `cozystack-database-external-tls-policy`, which refuses a `redises` object that asks for a plaintext endpoint outside the cluster: `external: true` without `tls.enabled: true` on create, any update that turns external access on, since an existing instance's spec does not prove its TLS state, and any change of `tls.enabled` by a tenant while external access stays on (the callers trusted by `cozystack-tenant-host-policy` are exempt, to repair such an instance), since before the first reconcile nothing else stops TLS being dropped. A release already external and plaintext before the policy existed stays editable. + +Two hostname-policy details are worth stating precisely. The **HTTPRoute/TLSRoute/GRPCRoute** policies (`route-hostname-policy.yaml`) and the **legacy-Ingress** policy (`ingress-hostname-policy.yaml`) are fail-closed: a route or Ingress in a tenant namespace missing `namespace.cozystack.io/host` is denied, so an absent label cannot open those layers (the route policies also deny a route that declares no hostnames, since such a route inherits the hostnames of the listener it attaches to rather than matching nothing; the Ingress policy likewise denies hostless rules and `spec.defaultBackend`, and permits external custom domains outside the platform root apex). The **namespace host-label** policy is an immutability guard — it stops an untrusted caller from scrubbing or changing an already-set label, but it does not itself reject a namespace that never carried one. The **Gateway-listener** policy (`gateway-hostname-policy.yaml`) applies the same subdomain rule but its CEL short-circuits to allow when the host label is empty, so it is permissive (fail-open) for a tenant namespace that lacks the label. In normal operation the platform sets that label at tenant creation and the namespace host-label policy keeps it immutable, closing the gap in practice — but the Gateway-side guarantee is conditional on the label being present, not unconditional. + +Authentication and authorization for aggregated resources are delegated to the core kube-apiserver via `system:auth-delegator` (TokenReview / SubjectAccessReview), so RBAC decisions remain centralized even though admission runs in `cozystack-api`. + +## References + +The claims in this document can be re-verified against these sources: + +- Aggregated API server: `cmd/cozystack-api/`, `pkg/apiserver/apiserver.go`, `pkg/registry/apps/application/rest.go`, `pkg/config/config.go`. +- Dynamic Application kinds: `pkg/cmd/server/start.go`, `api/v1alpha1/applicationdefinitions_types.go`, `internal/controller/applicationdefinition_controller.go`. +- Tenant RBAC: `packages/system/cozystack-basics/templates/clusterroles.yaml`. +- Admission policies: `packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml`, `route-hostname-policy.yaml`, `ingress-hostname-policy.yaml`, `database-external-tls-policy.yaml`, `packages/core/platform/templates/deletion-protection.yaml`. +- Tenant provisioning and isolation: `packages/apps/tenant/templates/` (`tenant.yaml`, `namespace.yaml`, `networkpolicy.yaml`), `pkg/registry/apps/application/quota.go`. +- UI OIDC front door (oauth2-proxy): `packages/system/dashboard/templates/gatekeeper.yaml`, `packages/system/linstor-gui/templates/gatekeeper.yaml`. +- Reporting and supported versions: `SECURITY.md`. diff --git a/docs/storage-immutability.md b/docs/storage-immutability.md index 9157baefc1..35c03a5151 100644 --- a/docs/storage-immutability.md +++ b/docs/storage-immutability.md @@ -10,7 +10,7 @@ Every stateful Cozystack application that exposes a `storageClass` parameter dec | Consumer | Behavior | | --- | --- | -| Cozystack UI (cozystack/cozystack-ui#6) | Reads the CEL rule from the chart's `openAPISchema` and renders `storageClass` as a disabled, helper-text-annotated field on edit forms. Save-time overlay reinstates the original value before PUT. | +| Cozystack UI (`packages/system/dashboard/images/console`) | Reads the CEL rule from the chart's `openAPISchema` and renders `storageClass` as a disabled, helper-text-annotated field on edit forms. Save-time overlay reinstates the original value before PUT. | | `kubectl edit` / `kubectl patch` against the cozystack aggregated apiserver | **Currently accepted** — the apiserver does not yet evaluate CEL rules embedded in `ApplicationDefinition.openAPISchema`. The change passes through but does not propagate to existing PVCs (see "Why" above), so no data corruption is possible. Apiserver enforcement is tracked in cozystack/cozystack#2657. | | `kubectl edit` / `kubectl patch` against native CRDs | Enforced today by the standard apiextensions apiserver. | @@ -18,9 +18,21 @@ Every stateful Cozystack application that exposes a `storageClass` parameter dec The following charts annotate at least one `storageClass` field as immutable: -- `clickhouse`, `foundationdb`, `harbor`, `http-cache`, `kafka` (Kafka + ZooKeeper), `kubernetes` (top-level only — see below), `mariadb`, `mongodb`, `nats`, `openbao`, `opensearch`, `postgres`, `qdrant`, `rabbitmq`, `redis`, `vm-disk`. +- `clickhouse`, `foundationdb`, `harbor`, `kafka` (Kafka broker + KRaft controller), `kubernetes`, `kubernetes-nodes`, `mariadb`, `mongodb`, `nats`, `openbao`, `opensearch`, `postgres`, `qdrant`, `rabbitmq`, `redis`, `valkey`, `vm-disk`. -The per-node-group `kubernetes.nodeGroups[name].storageClass` field is intentionally NOT annotated immutable: it is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. If a richer ratcheting form (`!has(oldSelf.storageClass) || self.storageClass == oldSelf.storageClass`) becomes available, the annotation should be added back. +The `kubernetes-nodes` chart annotates both its per-pool `storageClass` (the worker node system-disk class, defaulted `replicated`) and its `cluster` field immutable. `storageClass` binds the same PVC-template contract as every other chart in the table, so the plain `self == oldSelf` rule applies for the reason in "Why" above. `cluster` is immutable for a different reason: it wires the pool's CAPI objects to one parent cluster `kubernetes-`, and repointing a live pool at another cluster would orphan its running worker VMs rather than migrate them. + +## `vm-disk` also freezes `source` + +The `vm-disk` chart annotates `source` immutable in addition to `storageClass`, for a mechanism distinct from the PVC-template reasoning above. A `vm-disk` renders a single `DataVolume`, and `DataVolume.spec` is immutable in CDI: once the DataVolume exists, editing `spec.source` (or `spec.storage.storageClassName`) has no effect. The chart therefore looks the existing DataVolume up and reuses its stored `spec`, so an edit to `.Values.source` would otherwise render as a silent no-op while the HelmRelease still reported success — the CR and the cluster diverge with no diagnostic (cozystack/cozystack#2985). + +Because the aggregated apiserver does not yet evaluate the CEL rule (see the table above, cozystack/cozystack#2657), `vm-disk` adds a second, chart-level guard that is the substantive enforcement today: `templates/dv.yaml` fails the render when the requested `storageClass` or `source` differs from what the disk was created with. Both checks compare the operator's input recorded in an annotation at creation — `vm-disk.cozystack.io/storage-class` and `vm-disk.cozystack.io/source` — written only at creation and preserved verbatim afterward, never overwritten with an unapplied edit. The storage-class annotation is written even for the cluster default (`""`), and its key presence — not its value — marks a recorded class, so a disk created with `""` is still caught if later changed to a concrete class. Recording the input rather than reading the live object matters on both axes: the image-clone PVC name carries a chart-version prefix (`vm-image-` was renamed to `vm-default-images-`) that drifts across upgrades with no operator edit, and the live PVC's `storageClassName` on an adopted or foreign DataVolume may legitimately differ from the CR's default without being a change. + +Disks created before these annotations existed (or out-of-band) are handled conservatively so a working release never deadlocks. For `source`, such a disk is compared against its stored `spec.source` directly, but only when that stored source is a shape the chart itself produces (`blank`, `http` with just a `url`, `upload`, or a `pvc` with just `name`/`namespace`). An adopted DataVolume using `spec.sourceRef`, or a source type the chart cannot render (`registry`, `snapshot`, `imageio`, `http` with a `secretRef`, a foreign-namespace `pvc`), has no comparable provenance and is left alone rather than failed on a mismatch. The image-clone exemption fires only when the stored source is a `cozy-public` `pvc` **and** the request is itself an image — the one case where the `vm-image-` → `vm-default-images-` prefix drift is indistinguishable from a real image-name change; a change away from an image source to another type is still caught. For `storageClass` there is no safe live fallback (the CR default need not reflect an adopted disk's real class), so an un-annotated disk is simply not guarded here; apiserver CEL (cozystack/cozystack#2657) is the eventual guard for those. To actually change `storageClass` or `source`, delete and recreate the disk; size growth is the one mutable dimension and is handled separately by `templates/pvc-resize-hook.yaml`. + +The `storageClass` annotation records the effective value applied at creation, including when it came from the chart's `values.yaml` default (`replicated`) rather than an explicit operator value. Applications created through the aggregated apiserver have the default stamped into the CR, so it stays stable; but a direct HelmRelease consumer that omits `storageClass` records the chart default of the day, so changing that default in a future chart version would make every such untouched release start failing the render until its CR pins the old value or the disk is recreated. Treat the `storageClass` default as a frozen contract for the same reason the `source` serialization above is. + +The `source` annotation is a byte-for-byte `toJson` of `.Values.source`, so it is coupled to the exact serialization of the source schema. Adding a subfield with a schema default (which the apiserver would stamp onto every CR) would make the recorded and requested JSON diverge on an untouched disk; such a change needs an annotation-migration story rather than a bare schema bump. If you add a new stateful chart that exposes `storageClass`, annotate it the same way: diff --git a/docs/vm-external-vlan.md b/docs/vm-external-vlan.md new file mode 100644 index 0000000000..81a8fb232d --- /dev/null +++ b/docs/vm-external-vlan.md @@ -0,0 +1,155 @@ +# Attaching a virtual machine to an external VLAN + +This document describes how to attach a Cozystack virtual machine (the `vm-instance` application) directly to an external, physically-routed VLAN — the layer-2 segment a VM needs when it must appear on the same broadcast domain as external hardware (a licensing appliance, a storage box, a gateway managed outside the cluster), with an address from that VLAN's subnet rather than from the cluster overlay. + +The default Cozystack VM networking is overlay-only (the pod network, plus optional KubeOVN VPC subnets). Bridging a VM onto a real VLAN is a different pattern and has one non-obvious constraint: **it works with a Linux bridge and the `bridge` CNI plugin, and it does not work with `macvlan`.** The rest of this guide explains why and gives a working recipe. + +## Why `bridge` and not `macvlan` + +KubeVirt attaches a VM interface to a secondary network using **bridge binding** by default (this is what the `vm-instance` chart emits for every network in `.spec.networks`). With bridge binding the guest's own MAC address is placed on the wire — the launcher pod does not masquerade or translate it. + +A `macvlan` attachment is incompatible with that model. `macvlan` demultiplexes inbound frames strictly by the MAC address of the macvlan child interface. Because KubeVirt puts the *guest's* MAC on the wire — not the macvlan child's — replies from the gateway or other hosts arrive at the parent interface addressed to the guest MAC, do not match any macvlan child, and are silently dropped before they ever reach the VM. The symptom is a guest that can transmit (ARP requests and pings leave, visible in `tcpdump` on the parent interface) but never receives a reply (its neighbor entry for the gateway stays `FAILED`). As a secondary consequence, the host cannot reach macvlan children through the parent interface either, so a host-side service on the parent IP is unreachable from the VMs. + +A **Linux bridge** does not have this limitation: it forwards by learned MAC on all bridged ports, so the guest MAC is reachable, and the host can carry an address on the bridge itself to talk to the VMs. Attach the VLAN sub-interface to a bridge and point a `bridge`-type NAD at it. + +## Overview + +Three pieces cooperate: + +1. A **Linux bridge on each node** that enslaves the tagged VLAN sub-interface. This is node-level networking — it is configured by your node provisioning (netplan / Talos machine config / systemd-networkd), not by a Cozystack chart. +2. A **`NetworkAttachmentDefinition`** of type `bridge` referencing that bridge, created in the VM's tenant namespace. +3. The **`vm-instance`** application referencing the NAD by name in `.networks`, with the guest's static address supplied through cloud-init. + +## Prerequisites + +- The `multus` package is enabled (it provides the `NetworkAttachmentDefinition` CRD and the secondary-network plumbing). +- The `bridge` CNI plugin is present in `/opt/cni/bin` on every node. The `multus` package puts it there itself on every platform, which is the subject of the next section — read it before upgrading a cluster whose `/opt/cni/bin` you provision yourself. Verify with `ls /opt/cni/bin/bridge`; a missing binary makes the NAD fail with `failed to find plugin "bridge" in path [/opt/cni/bin]`. +- There is no IPAM plugin in this path — addresses are assigned inside the guest, not by the CNI. Plan static addresses per VM. + +## Staging the reference CNI plugins + +The `multus` package installs the reference plugins into `/opt/cni/bin` itself, on every platform, because no platform supplies all of them. That is the `stageCniPlugins` chart value, and the platform turns it on for every bundle. What differs is only how much is missing before staging runs. Talos puts six binaries in that directory — `bridge`, `firewall`, `host-local`, `loopback` and `portmap` from the same upstream release, plus `flannel`, which is a different project — so a NAD naming `macvlan`, `ipvlan` or `vlan` fails there too; staging adds the ten it does not carry and rewrites the four that overlap with the same upstream version (`v1.9.1` on both sides), not an older one. Generic Linux offers no guarantee at all — k3s keeps its own copies under `/var/lib/rancher/k3s/data/cni` (`/var/lib/rancher/k3s/data/current/bin` on releases older than October 2024) and Cilium installs only `cilium-cni` and `loopback`, so `/opt/cni/bin` there can lack even `bridge`. + +Carrying the plugins costs roughly 67 MiB uncompressed per architecture in the `multus` image, which every node pulls whether staging is on or off. Size airgap mirrors accordingly. + +Where staging is on, it tries to replace any file of the same name already in that directory, every time the Multus pod is recreated, so a hand-placed or locally patched `bridge` will normally not survive — carry such a change in the `multus` image instead. A plugin it fails to write is reported and skipped rather than removed, so whatever was there stays — an older copy where the node had one, and nothing at all where it did not. It installs 14 of the upstream reference plugins and leaves `loopback`, `dhcp`, `dummy` and `tap` alone. Leaving them alone is not a promise that something else provides them: `loopback` is installed by Cilium and kube-ovn, while `dhcp`, `dummy` and `tap` are simply not staged — `dhcp` needs a host daemon this package does not run, and the other two have no consumer here. If you need one of those, put it on the node yourself. + +**If you provision `/opt/cni/bin` yourself, set the opt-out before upgrading.** The replacement happens as soon as Multus rolls, and turning `stageCniPlugins` off afterwards does not undo it: nothing in this package removes or restores a plugin, so the copies stay until you re-provision the node. Operators pinned to a particular plugin version should decline first and upgrade second. This applies on Talos too, where four of the names staging writes are ones the node image also carries. Declining looks like this, on the `platform` component of the `cozystack.cozystack-platform` Package CR: + +```yaml +apiVersion: cozystack.io/v1alpha1 +kind: Package +metadata: + name: cozystack.cozystack-platform +spec: + components: + platform: + values: + networking: + stageCniPlugins: false +``` + +Editing the multus HelmRelease or the child `Package cozystack.multus` directly does not hold: both are re-rendered by the platform reconcile. An unreadable value there — a spelling that is neither a `true` nor a `false` form — stops the whole platform render rather than resolving to off, so a typo in this key halts reconciliation cluster-wide. + +If `bridge` is missing on a node where staging is on, read the log with `kubectl --namespace cozy-multus logs --container install-cni-plugins`. The line `no /cni-plugins in this image; skipping staging` means the running `multus` image predates the plugin staging and installed nothing: the image reference and the manifest are re-pinned on different schedules, so a tree taken between a Dockerfile change and the next release bake can carry one without the other. Move to a release whose `multus` image carries the plugins. A line beginning `failed to install:` names plugins the container could not write, one of which may be the `bridge` you are looking for. The reason for each is earlier in the same log, printed as that plugin was attempted rather than next to the summary — so with more than one failure the causes are spread above it, not adjacent to it, and the one naming your plugin is the one to read. The container reports these and exits successfully on purpose, so the pod shows Ready and `kubectl get` will not point you here: failing it instead would leave the node unable to start any pod that goes through CNI, since Multus remains the primary CNI with no daemon behind it — host-network pods bypass CNI and still start. That means a NAD failing with `failed to find plugin` is the symptom to trust, and this log is where the reason is. + +If the init container is absent from the Multus pod altogether, staging is off for this cluster. That means `networking.stageCniPlugins` was set to `false` somewhere: no bundle defaults it off, on Talos or anywhere else. Turn it back on with the same block as above and `true` in place of `false`, or stage the plugins through node provisioning instead. + +Changing this value edits the Multus DaemonSet's pod template, so Multus rolls node by node. While a node's `multus-daemon` is down its CNI shim has no daemon behind it, and pod creation and deletion on that node fail for the length of the restart. Flip it during a window where that is acceptable, not while something else is rolling. + +## Step 1 — Linux bridge on the node + +Create a bridge that enslaves the tagged VLAN sub-interface. The VLAN sub-interface itself carries no address; the bridge carries the host's presence on that VLAN (optional, but useful for a gateway-reachability sanity path and for any host-side service the VMs must reach). + +This example uses netplan on an Ubuntu/Debian node; the VLAN id and subnet are illustrative (`203.0.113.0/24`, VLAN 100, gateway `203.0.113.1`). Adapt to your uplink naming and to Talos/`systemd-networkd` if that is your provisioning: + +```yaml +network: + version: 2 + vlans: + # Tagged VLAN sub-interface, no address of its own — enslaved to the bridge. + uplink.100: + id: 100 + link: uplink + bridges: + br100: + interfaces: + - uplink.100 + # Optional host presence on the VLAN. Keep the node's default route on + # its management interface — do not add a default route here. + addresses: + - 203.0.113.2/24 +``` + +Notes: + +- The node's **default route must stay on the management interface.** The bridge address (if any) is only for on-VLAN reachability, not a second default gateway. +- `netplan apply` cannot move an interface into a bridge while a consumer still holds it (for example a `virt-launcher` pod using a previous `macvlan` attachment). Remove the consumer first (delete the VMI so the launcher releases the interface), then reconfigure. +- After a reboot, `systemd-networkd` may briefly report the bridge "routable" while the link is not yet actually up. If your VMs need the VLAN immediately at boot, gate their start on a reachability check, or re-run `netplan apply` until the gateway answers. + +## Step 2 — NetworkAttachmentDefinition + +Create a `bridge`-type NAD in the tenant namespace that will host the VM. The `vm-instance` chart resolves a network by name **in the VM's own namespace**, so one copy of the NAD must exist in every tenant namespace that runs VMs on this VLAN. + +```yaml +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: vlan100 + namespace: tenant-example +spec: + config: | + { + "cniVersion": "0.3.1", + "type": "bridge", + "bridge": "br100", + "ipam": {} + } +``` + +- `bridge` must match the bridge name from step 1 (`br100` here). +- `ipam: {}` — no cluster-side address assignment; the guest configures its address itself (step 3). + +## Step 3 — Attach the VM and assign a static address + +Reference the NAD by name in the `vm-instance` values. This VLAN has no DHCP server, so the static address goes into cloud-init `userData` (`cloudInit`), written by the guest at first boot, and `netplanDhcp` stays off (see [Gotchas](#gotchas)): + +```yaml +# vm-instance values +instanceType: u1.medium +instanceProfile: ubuntu +disks: + - name: example-system +networks: + - name: vlan100 +cloudInit: | + #cloud-config + write_files: + - path: /etc/netplan/60-vlan100.yaml + permissions: "0600" + content: | + network: + version: 2 + ethernets: + # Match the second NIC (the pod-network NIC is the first). Use the + # interface that comes up without a DHCP lease. + enp2s0: + addresses: + - 203.0.113.10/24 + runcmd: + - netplan apply +``` + +The VM ends up with two interfaces: the always-present **pod-network** NIC (`default`, used for cluster-internal traffic and for the `vm-instance` external-access features) and the **VLAN** NIC. The `/24` address above brings up only the connected route for the VLAN subnet — it adds no default route, so the guest's egress stays wherever you want it (typically the pod NIC). If the VLAN is meant to be the guest's default gateway instead, add a default route under the VLAN NIC and remove it from the pod NIC. + +## Gotchas + +- **VMs are dual-homed.** The `vm-instance` chart always adds the pod-network NIC in addition to any `networks` you declare; there is no single-homed (VLAN-only) option today. Address the VLAN NIC inside the guest and leave the pod NIC to the cluster. +- **Leave `netplanDhcp` off on a VM with a static NIC.** `netplanDhcp: true` makes the chart supply cloud-init network-data that asks every `en*` and `eth*` interface for DHCP, under the netplan IDs `cozy-en` and `cozy-eth`. Netplan writes one networkd file per ID, and systemd-networkd applies only the first `.network` file in lexical order that matches a link, so `10-netplan-cozy-en.network` shadows the `10-netplan-enp2s0.network` the recipe above produces and the static address is never applied, whatever number the recipe's netplan file carries. The key is off by default and is meant for guests whose every interface takes its address over DHCP, such as a VM on a VPC subnet. +- **MAC changes on VM re-creation.** KubeVirt generates a fresh guest MAC each time the VM object is re-created, and `vm-instance` exposes no way to pin it, so re-creating a VM changes its MAC. The upstream gateway then holds a stale ARP entry for the old MAC for a few minutes, so "gateway unreachable" immediately after re-creating a VM is expected — wait for the ARP entry to age out (~5 min) rather than treating it as a fault. +- **Host ↔ VM traffic.** If the host must talk to the VMs (a proxy, a health check), give the bridge a host address on the VLAN (step 1) — traffic through a bare VLAN sub-interface to bridge-attached guests will not work the way `macvlan` users expect. + +## See also + +- [`docs/gpu-vgpu.md`](./gpu-vgpu.md) — passing NVIDIA GPUs and vGPU profiles into the same VMs. +- KubeVirt user guide, [Interfaces and Networks](https://kubevirt.io/user-guide/network/interfaces_and_networks/) — bridge binding vs other binding methods. diff --git a/examples/backups/_lib/wait-helpers.sh b/examples/backups/_lib/wait-helpers.sh new file mode 100644 index 0000000000..cec241d138 --- /dev/null +++ b/examples/backups/_lib/wait-helpers.sh @@ -0,0 +1,165 @@ +#!/bin/bash +# Wait helpers shared by every backup walkthrough under examples/backups/. +# Each walkthrough's 00-helpers.sh sources this file after defining the log_* +# functions these call; wait_hr_ready and wait_deleted read $NAMESPACE. +# hack/backup-examples-shared-helpers.bats fails if a walkthrough grows its own +# copy again. + +# Wait until a JSONPath value on a resource matches the desired string. +# +# The optional 7th argument is a TERMINAL value to bail on: a BackupJob or +# RestoreJob that reaches Failed never reaches Succeeded, so polling out the +# remaining budget only delays the report, lets the TTL reaper take the Job's +# Pod log, and under Chainsaw risks the step being SIGKILLed at its op timeout +# instead of failing cleanly. +# +# kubectl's stderr is left attached: a read that fails for RBAC or transport +# reasons says so instead of reading as an empty value. --ignore-not-found +# keeps a resource that does not exist yet, the usual case at the start of a +# wait, from printing an error on every poll. +wait_for_field() { + local resource_type="$1" + local resource_name="$2" + local jsonpath="$3" + local desired="$4" + local namespace="${5:-}" + local timeout="${6:-300}" + local fail_value="${7:-}" + + log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..." + local elapsed=0 + local ns_flag=() + [[ -n "$namespace" ]] && ns_flag=(-n "$namespace") + + while true; do + local current + current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" --ignore-not-found -o jsonpath="$jsonpath" || true) + if [[ "$current" == "$desired" ]]; then + log_success "$resource_type/$resource_name reached '$desired'" + return 0 + fi + if [[ -n "$fail_value" && "$current" == "$fail_value" ]]; then + log_error "$resource_type/$resource_name reached terminal '$current' (expected '$desired')" + return 1 + fi + if [[ $elapsed -ge $timeout ]]; then + log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')" + return 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done +} + +# Wait for a HelmRelease to become Ready, with an existence backstop (the apps +# controller creates the HR asynchronously, so a bare `kubectl wait` right after +# `kubectl apply` races it) and a fail-fast on Stalled=True: a stalled HR has +# exhausted its remediation retries and will never turn Ready, so polling to the +# timeout only hides the real error. +wait_hr_ready() { + local name="$1" + local timeout="${2:-300}" + local elapsed=0 + local lookup state seen=0 + + log_substep "Waiting for HelmRelease/$name to become Ready..." + while true; do + # --ignore-not-found makes an absent release exit 0, so a non-zero exit + # is kubectl failing to answer. Presence is read off the name line, + # because a warning on stderr lands in the same string. + if ! lookup=$(kubectl -n "$NAMESPACE" get hr "$name" --ignore-not-found -o name 2>&1); then + state=error + elif [[ $'\n'"$lookup"$'\n' != *"/$name"$'\n'* ]]; then + state=absent + else + state=present + seen=1 + local ready stalled + ready=$(kubectl -n "$NAMESPACE" get hr "$name" \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' || true) + if [[ "$ready" == "True" ]]; then + log_success "HelmRelease/$name is Ready" + return 0 + fi + stalled=$(kubectl -n "$NAMESPACE" get hr "$name" \ + -o jsonpath='{.status.conditions[?(@.type=="Stalled")].status}' || true) + if [[ "$stalled" == "True" ]]; then + log_error "HelmRelease/$name is Stalled (terminal):" + break + fi + fi + if [[ $elapsed -ge $timeout ]]; then + log_error "Timeout waiting for HelmRelease/$name to become Ready:" + case "$state" in + error) + echo " could not look up HelmRelease/$name: $lookup" >&2 + # Seen earlier: the reads below may answer where this lookup + # did not. + [[ $seen -eq 1 ]] || return 1 + ;; + absent) + if [[ $seen -eq 1 ]]; then + echo " HelmRelease/$name was deleted from $NAMESPACE while waiting" >&2 + else + echo " HelmRelease/$name never appeared in $NAMESPACE" >&2 + fi + return 1 + ;; + esac + break + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + # Reached on Stalled or on timeout. Conditions and history rather than the + # Ready message alone: a failed install is retried on an interval, so Ready + # may describe the retry rather than the failure behind it. + kubectl -n "$NAMESPACE" get hr "$name" \ + -o jsonpath='{range .status.conditions[*]} {.type}={.status} ({.reason}): {.message}{"\n"}{end}' >&2 || true + local hist + if ! hist=$(kubectl -n "$NAMESPACE" get hr "$name" \ + -o jsonpath='{range .status.history[*]} history: {.status} {.chartVersion} {.lastDeployed}{"\n"}{end}'); then + echo " history: kubectl could not read it" >&2 + elif [[ -n "${hist//[[:space:]]/}" ]]; then + printf '%s\n' "$hist" >&2 + else + # A release with no history prints nothing and exits 0; say so, + # because a blank in a failure dump reads as a dump that broke. + echo " history: (none recorded)" >&2 + fi + return 1 +} + +# Wait until a namespaced resource is really gone. +# +# `kubectl wait --for=delete` is not used: it errors out when the resource is +# already absent, which is the normal case on a clean namespace (cleanup.sh is +# idempotent and runs as a pre-clean too), and swallowing that error would also +# swallow a genuine failure. Polling `get --ignore-not-found` treats "already +# gone" and "gone now" as the same success (empty output, exit 0) while a +# retrieval failure — RBAC, API-server, transport — is a non-zero exit that +# must NOT be read as deletion, or cleanup settles on a false success and leaves +# half-uninstalled resources for the next run. Such an error just keeps us +# polling until the resource is confirmed gone or the timeout fires. +wait_deleted() { + local resource_type="$1" + local resource_name="$2" + local timeout="${3:-300}" + local elapsed=0 + local got + + while true; do + if got=$(kubectl -n "$NAMESPACE" get "$resource_type" "$resource_name" --ignore-not-found 2>/dev/null) && [[ -z "$got" ]]; then + [[ $elapsed -gt 0 ]] && log_success "$resource_type/$resource_name is gone" + return 0 + fi + if [[ $elapsed -ge $timeout ]]; then + log_error "Timeout waiting for $resource_type/$resource_name to be deleted; it is still present after ${timeout}s:" + kubectl -n "$NAMESPACE" get "$resource_type" "$resource_name" -o wide >&2 || true + return 1 + fi + [[ $elapsed -eq 0 ]] && log_substep "Waiting for $resource_type/$resource_name to be deleted..." + sleep 5 + elapsed=$((elapsed + 5)) + done +} diff --git a/examples/backups/clickhouse/00-helpers.sh b/examples/backups/clickhouse/00-helpers.sh index 01ce00210d..d3d56b0345 100755 --- a/examples/backups/clickhouse/00-helpers.sh +++ b/examples/backups/clickhouse/00-helpers.sh @@ -13,7 +13,13 @@ export NC='\033[0m' export BOLD='\033[1m' # Default settings (override via environment). -export NAMESPACE="${NAMESPACE:-tenant-test}" +# +# tenant-root, matching the etcd/mariadb/postgres backup demos: the S3 client +# here is the clickhouse-backup sidecar INSIDE the tenant, and cozystack's +# shared seaweedfs lives in tenant-root. An isolated tenant's Cilium egress +# allowlist blocks a Pod in that tenant from reaching tenant-root's seaweedfs, +# so a release outside tenant-root cannot use the in-cluster endpoint below. +export NAMESPACE="${NAMESPACE:-tenant-root}" export CLICKHOUSE_NAME="${CLICKHOUSE_NAME:-clickhouse-test}" export CLICKHOUSE_RESTORE_NAME="${CLICKHOUSE_RESTORE_NAME:-clickhouse-restore}" export BUCKET_NAME="${BUCKET_NAME:-clickhouse-backups}" @@ -22,6 +28,35 @@ export STRATEGY_NAME="${STRATEGY_NAME:-altinity}" export BACKUPJOB_NAME="${BACKUPJOB_NAME:-clickhouse-backup-job}" export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-clickhouse-restore-inplace}" export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-clickhouse-restore-to-copy}" +# S3 endpoint CA. cozystack's default seaweedfs serves its S3 endpoint with a +# self-signed certificate whose CA lives in this Secret; step 03 copies its +# ca.crt into a per-release Secret the clickhouse-backup sidecar trusts via the +# chart's backup.endpointCA (mounted and picked up through SSL_CERT_DIR). The +# name follows the seaweedfs chart's fullnameOverride ("seaweedfs" -> +# "seaweedfs-ca-cert"); step 03 auto-discovers the CA Certificate's actual +# secret when this default is absent, so an upstream fullname change does not +# silently break the copy. On a cluster whose S3 endpoint is signed by a +# publicly-trusted CA, set S3_CA_SECRET="" to skip the copy and leave +# backup.endpointCA unset. +# Whether the CA Secret name is the user's choice or just this default. Step 03 +# treats the two differently: a name the user typed and that does not exist is an +# error worth stopping on, while the default failing to resolve on a cluster with +# no seaweedfs most likely means a publicly-trusted S3 endpoint — a legal input, +# so it warns and leaves backup.endpointCA unset instead of failing. +export S3_CA_SECRET_EXPLICIT="${S3_CA_SECRET:+1}" +export S3_CA_SECRET_EXPLICIT="${S3_CA_SECRET_EXPLICIT:-0}" +export S3_CA_SECRET="${S3_CA_SECRET:-seaweedfs-ca-cert}" +# Independent of NAMESPACE: the shared seaweedfs and its CA live in tenant-root +# even when the demo runs elsewhere. +export S3_CA_NAMESPACE="${S3_CA_NAMESPACE:-tenant-root}" +export S3_CA_KEY="${S3_CA_KEY:-ca.crt}" +# Per-release Secret the CA is copied into; referenced by backup.endpointCA. +# Scoped to CLICKHOUSE_NAME so two instances in one namespace don't share a +# name, and stamped with the ownership label below so step 03 refuses to +# overwrite a foreign Secret and cleanup only deletes one this demo created. +export CH_CA_SECRET_NAME="${CH_CA_SECRET_NAME:-${CLICKHOUSE_NAME}-backup-s3-ca}" +export CH_CA_SECRET_LABEL_KEY="cozystack.io/owned-by-demo" +export CH_CA_SECRET_LABEL_VALUE="clickhouse-backup" # The clickhouse-backup sidecar lives in the application Pod (rendered by the # chart when backup.enabled=true). Tenants don't manage a separate Secret; # the chart projects bucket coordinates into -backup-s3 from the @@ -44,29 +79,39 @@ print_header() { echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2 } -# Wait until a JSONPath value on a resource matches the desired string. -wait_for_field() { - local resource_type="$1" - local resource_name="$2" - local jsonpath="$3" - local desired="$4" - local namespace="${5:-}" - local timeout="${6:-300}" +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" - log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..." +# Wait for a StatefulSet to report a ready replica, tolerating its asynchronous +# creation (the ClickHouse operator creates it after the CHI is reconciled). +wait_sts_ready() { + local name="$1" + local timeout="${2:-300}" local elapsed=0 - local ns_flag=() - [[ -n "$namespace" ]] && ns_flag=(-n "$namespace") + log_substep "Waiting for StatefulSet/$name to report a ready replica..." while true; do - local current - current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true) - if [[ "$current" == "$desired" ]]; then - log_success "$resource_type/$resource_name reached '$desired'" - return 0 + if kubectl -n "$NAMESPACE" get statefulset.apps "$name" >/dev/null 2>&1; then + local ready + ready=$(kubectl -n "$NAMESPACE" get statefulset.apps "$name" \ + -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true) + if [[ "${ready:-0}" -ge 1 ]] 2>/dev/null; then + log_success "StatefulSet/$name has ${ready} ready replica(s)" + return 0 + fi fi if [[ $elapsed -ge $timeout ]]; then - log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')" + log_error "Timeout waiting for StatefulSet/$name to become ready" + # A StatefulSet names its Pods "-N", so ask for the first + # replica by name instead of guessing an operator label — and let + # both streams through, since the point of this branch is to say + # WHY the Pod never came up. Events carry the reason a Pod is + # Pending (unschedulable, missing PVC, image pull) that neither the + # Pod nor the StatefulSet status spells out. + kubectl -n "$NAMESPACE" get statefulset.apps "$name" -o wide >&2 || true + kubectl -n "$NAMESPACE" describe pod "${name}-0" >&2 || true return 1 fi sleep 5 diff --git a/examples/backups/clickhouse/01-create-strategy.sh b/examples/backups/clickhouse/01-create-strategy.sh index c2047f885e..9a4142e6ed 100755 --- a/examples/backups/clickhouse/01-create-strategy.sh +++ b/examples/backups/clickhouse/01-create-strategy.sh @@ -10,6 +10,31 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/00-helpers.sh" print_header "Step 01: Create Altinity backup strategy" + +# Reuse the image the platform's own cozy-default-altinity strategy runs, so the +# demo Pod needs no `apk add` at run time. That matters twice over: the strategy +# runs once per BackupJob and RestoreJob, so fetching packages from the Alpine +# CDN each time makes every backup depend on outbound internet — it fails +# outright on an air-gapped cluster, and it turns a CDN hiccup into a failed +# backup — and this same script is the e2e round-trip's harness, where that +# would surface as an unrelated red E2E run. +# +# Read it off the shipped strategy rather than hardcoding a tag, so the demo +# tracks whatever the platform is pinned to. Falls back to alpine + `apk add` +# only when that strategy is absent (a cluster without the backupstrategy +# controller chart), which is also the one case where the demo has to bootstrap +# its own tooling. +CH_BACKUP_CLIENT_IMAGE="${CH_BACKUP_CLIENT_IMAGE:-$(kubectl get altinity.strategy.backups.cozystack.io cozy-default-altinity \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="ch-backup-client")].image}' 2>/dev/null || true)}" +APK_BOOTSTRAP="" +if [[ -z "$CH_BACKUP_CLIENT_IMAGE" ]]; then + CH_BACKUP_CLIENT_IMAGE="alpine:3.19" + APK_BOOTSTRAP="apk add --no-cache curl jq >/dev/null" + log_warning "cozy-default-altinity not found; falling back to ${CH_BACKUP_CLIENT_IMAGE} + apk add (needs outbound internet)." +else + log_info "Reusing the platform strategy's client image: ${CH_BACKUP_CLIENT_IMAGE}" +fi + log_command "kubectl apply -f - (Altinity strategy: $STRATEGY_NAME)" kubectl apply -f - </dev/null + ${APK_BOOTSTRAP} host="chi-clickhouse-{{ .Release.Name }}-clickhouse-0-0.{{ .Release.Namespace }}.svc" api="http://\${host}:7171" @@ -85,7 +110,7 @@ spec: # source-release prefix. Falls through to "no remote # backup found" if S3_PATH is misconfigured AND none of # the listed names match. - target=\$(curl -fsS \${auth} "\${api}/backup/list/remote" \\ + target=\$(curl -fsS \${auth} --max-time 60 "\${api}/backup/list/remote" \\ | jq -rs --arg prefix "\${prefix}" \\ '[.[] | select((.name // "") | startswith(\$prefix))] | sort_by(.created // "") | last.name // ""') if [ -z "\${target}" ] || [ "\${target}" = "null" ]; then @@ -113,7 +138,7 @@ spec: # vs. "create_remote X"). The strategy Pod runs one operation # at a time, so the most recent matching action is ours. while true; do - row=\$(curl -fsS \${auth} "\${api}/backup/actions" \\ + row=\$(curl -fsS \${auth} --max-time 30 "\${api}/backup/actions" \\ | jq -rs --arg suffix "\${target}" \\ '[.[] | select((.command // "") | endswith(\$suffix))] | sort_by(.start) | last') status=\$(echo "\${row}" | jq -r '.status // ""') @@ -135,11 +160,19 @@ spec: cpu: 200m memory: 128Mi securityContext: - # Pod runs apk add at startup; that needs root. Other fields stay - # locked down (no privilege escalation, no extra caps, default - # seccomp profile). Tenants that enforce PSA "restricted" should - # swap this image for one with curl+jq pre-baked and re-enable - # runAsNonRoot on the strategy CR. + # Same field set as the platform's cozy-default-altinity strategy + # (packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml): + # no privilege escalation, all caps dropped, default seccomp + # profile, and runAsNonRoot deliberately left unset rather than + # false — the Pod only runs curl and jq, and neither the platform + # strategy nor this one pins a UID. + # + # On the default path there is no `apk add` to need root: the image + # above is the platform's, with curl+jq already baked in. Only the + # alpine:3.19 fallback (taken when cozy-default-altinity is absent) + # installs packages at startup, and that is the one case where a + # cluster enforcing PSA "restricted" has to supply its own image + # with the tools pre-baked. allowPrivilegeEscalation: false capabilities: drop: ["ALL"] diff --git a/examples/backups/clickhouse/03-create-bucket.sh b/examples/backups/clickhouse/03-create-bucket.sh index b1075c5a1c..f2828a4166 100755 --- a/examples/backups/clickhouse/03-create-bucket.sh +++ b/examples/backups/clickhouse/03-create-bucket.sh @@ -27,12 +27,18 @@ spec: EOF log_substep "Waiting for bucket HelmRelease to be Ready..." -kubectl -n "$NAMESPACE" wait hr "bucket-${BUCKET_NAME}" --for=condition=ready --timeout=300s -kubectl -n "$NAMESPACE" wait bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" --for=jsonpath='{.status.bucketReady}'=true --timeout=300s +# 660s, above this generated release's 600s install timeout; see "Sizing an +# HR-Ready budget" in docs/agents/e2e-testing.md. +wait_hr_ready "bucket-${BUCKET_NAME}" 660 +kubectl -n "$NAMESPACE" wait bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" --for=jsonpath='{.status.bucketReady}'=true --timeout=180s # Cozystack's bucket app provisions a BucketAccess named "-backup" # (the "-backup" suffix is the BucketAccessClass name); the BucketInfo Secret # carries the same name. -kubectl -n "$NAMESPACE" wait bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-backup" --for=jsonpath='{.status.accessGranted}'=true --timeout=300s +kubectl -n "$NAMESPACE" wait bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-backup" --for=jsonpath='{.status.accessGranted}'=true --timeout=180s +if [[ "${COZY_E2E_BACKUP_PREFLIGHT:-0}" == "1" ]]; then + source "$SCRIPT_DIR/../../../hack/e2e-chainsaw/_lib/backup-access-preflight.sh" + cozy_backup_access_preflight "$NAMESPACE" "bucket-${BUCKET_NAME}-backup" 90 +fi log_substep "Reading bucket coordinates from BucketInfo Secret..." TMP=$(mktemp) @@ -41,7 +47,12 @@ kubectl -n "$NAMESPACE" get secret "bucket-${BUCKET_NAME}-backup" -o jsonpath='{ CH_BACKUP_ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' "$TMP") CH_BACKUP_SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' "$TMP") -CH_BACKUP_ENDPOINT=$(jq -r '.spec.secretS3.endpoint' "$TMP") +# S3_ENDPOINT can be overridden via the environment: BucketInfo advertises the +# EXTERNAL ingress endpoint, which in-cluster Pods cannot always reach or +# TLS-validate (in CI it is the unroutable s3.example.org placeholder); the +# in-cluster alternative is https://seaweedfs-s3.:8333, trusted via the CA +# copied below. +CH_BACKUP_ENDPOINT="${S3_ENDPOINT:-$(jq -r '.spec.secretS3.endpoint' "$TMP")}" CH_BACKUP_REGION=$(jq -r 'if (.spec.secretS3.region // "") == "" then "us-east-1" else .spec.secretS3.region end' "$TMP") CH_BACKUP_BUCKET=$(jq -r '.spec.bucketName' "$TMP") @@ -52,6 +63,82 @@ for v in CH_BACKUP_ACCESS_KEY CH_BACKUP_SECRET_KEY CH_BACKUP_ENDPOINT CH_BACKUP_ [[ -n "${!v}" && "${!v}" != "null" ]] || { log_error "BucketInfo missing required field: ${v}"; exit 1; } done +# Resolve and copy the S3 endpoint CA. The default name tracks the seaweedfs +# chart's fullnameOverride (seaweedfs -> seaweedfs-ca-cert), but a downstream +# fullname change would rename it, so fall back to discovering the cert-manager +# CA Certificate (the seaweedfs-labelled one with spec.isCA=true) and read its +# secretName. Leave S3_CA_SECRET empty to skip the copy on a public-CA endpoint. +CH_BACKUP_CA_SECRET="" +if [[ -n "$S3_CA_SECRET" ]]; then + # Reading the platform's CA lives in S3_CA_NAMESPACE (tenant-root by + # default), which is an admin/platform capability, not a tenant one — see + # the note on step 03 in README.md. So the three outcomes have to be kept + # apart: present, absent, and not-allowed-to-look. Swallowing stderr here + # would report an RBAC denial as "not found" and then advise setting + # S3_CA_SECRET, which cannot fix a permissions problem. + ca_probe_err="" + if ! ca_probe_err=$(kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" -o name 2>&1 >/dev/null); then + if echo "$ca_probe_err" | grep -qiE 'forbidden|cannot (get|list)|Unauthorized'; then + log_error "Cannot read ${S3_CA_NAMESPACE}/${S3_CA_SECRET}: ${ca_probe_err}" + log_error "This is a permissions problem, not a missing Secret: copying the S3 endpoint CA needs read access to ${S3_CA_NAMESPACE}. Ask an admin to copy it into ${NAMESPACE}/${CH_CA_SECRET_NAME}, then re-run with S3_CA_SECRET=\"\" to skip the copy." + exit 1 + fi + log_warning "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} not found; discovering the seaweedfs CA Certificate..." + # List every seaweedfs Certificate as " " and pick the + # CA one in shell — avoids kubectl jsonpath's finicky boolean-literal + # filter. + DISCOVERED_CA=$(kubectl -n "$S3_CA_NAMESPACE" get certificates.cert-manager.io \ + -l app.kubernetes.io/name=seaweedfs \ + -o jsonpath='{range .items[*]}{.spec.isCA}{" "}{.spec.secretName}{"\n"}{end}' 2>/dev/null \ + | awk '$1=="true"{print $2; exit}' || true) + if [[ -n "$DISCOVERED_CA" ]]; then + log_success "Discovered seaweedfs CA secret ${S3_CA_NAMESPACE}/${DISCOVERED_CA}" + S3_CA_SECRET="$DISCOVERED_CA" + elif [[ "${S3_CA_SECRET_EXPLICIT}" == "1" ]]; then + # The user named a specific Secret and it is not there: that is a + # typo or a missing prerequisite, and guessing past it would hide it. + log_error "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} does not exist and no seaweedfs CA Certificate was found in ${S3_CA_NAMESPACE}. Fix the name, or set S3_CA_SECRET=\"\" for a publicly-trusted endpoint." + exit 1 + else + # Nothing was named and there is no seaweedfs here, so this is most + # likely a cluster whose S3 endpoint is publicly trusted — a legal + # input for this demo. Warn and continue with endpointCA unset + # rather than failing on a default the user never chose. + log_warning "No seaweedfs CA found in ${S3_CA_NAMESPACE} and S3_CA_SECRET was not set explicitly." + log_warning "Continuing with backup.endpointCA unset — correct for a publicly-trusted S3 endpoint. If your endpoint uses a private CA, set S3_CA_SECRET to the Secret holding its bundle." + S3_CA_SECRET="" + fi + fi +fi +if [[ -n "$S3_CA_SECRET" ]]; then + log_substep "Copying S3 CA ${S3_CA_NAMESPACE}/${S3_CA_SECRET}[${S3_CA_KEY}] -> ${CH_CA_SECRET_NAME}..." + # The destination name is predictable, so don't clobber a Secret this demo + # does not own. A get that succeeds returns the ownership label (empty if + # absent); a get that fails means the Secret is gone, which is fine. + if existing_owner=$(kubectl -n "$NAMESPACE" get secret "$CH_CA_SECRET_NAME" \ + -o jsonpath="{.metadata.labels.${CH_CA_SECRET_LABEL_KEY//./\\.}}" 2>/dev/null); then + if [[ "$existing_owner" != "$CH_CA_SECRET_LABEL_VALUE" ]]; then + log_error "Secret ${NAMESPACE}/${CH_CA_SECRET_NAME} already exists and is not owned by this demo (missing ${CH_CA_SECRET_LABEL_KEY}=${CH_CA_SECRET_LABEL_VALUE}). Refusing to overwrite it; set CH_CA_SECRET_NAME to a free name." + exit 1 + fi + fi + # The dot in the default key has to be escaped for kubectl jsonpath. + ca_pem=$(kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" \ + -o jsonpath="{.data.${S3_CA_KEY//./\\.}}" | base64 -d) + [[ -n "$ca_pem" ]] || { log_error "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} has no ${S3_CA_KEY}"; exit 1; } + # apply (not create) so a stale copy from an earlier run is corrected; + # kubectl label --local stamps the ownership label into the manifest. + kubectl -n "$NAMESPACE" create secret generic "$CH_CA_SECRET_NAME" \ + --from-literal="ca.crt=${ca_pem}" \ + --dry-run=client -o yaml | \ + kubectl label --local -f - "${CH_CA_SECRET_LABEL_KEY}=${CH_CA_SECRET_LABEL_VALUE}" \ + --dry-run=client -o yaml | \ + kubectl apply -f - + CH_BACKUP_CA_SECRET="$CH_CA_SECRET_NAME" +else + log_warning "S3_CA_SECRET empty: leaving backup.endpointCA unset (assumes a publicly-trusted S3 endpoint)." +fi + # Persist for the next step. 04-create-clickhouse.sh sources this file when it # applies the ClickHouse spec so the chart can render -backup-s3. # The cache stores raw S3 credentials, so apply restrictive perms before @@ -63,6 +150,7 @@ export CH_BACKUP_SECRET_KEY=${CH_BACKUP_SECRET_KEY} export CH_BACKUP_ENDPOINT=${CH_BACKUP_ENDPOINT} export CH_BACKUP_REGION=${CH_BACKUP_REGION} export CH_BACKUP_BUCKET=${CH_BACKUP_BUCKET} +export CH_BACKUP_CA_SECRET=${CH_BACKUP_CA_SECRET} ENV chmod 600 "$SCRIPT_DIR/.bucket-info.env" diff --git a/examples/backups/clickhouse/04-create-clickhouse.sh b/examples/backups/clickhouse/04-create-clickhouse.sh index ee134cd7b7..bf5c508e4e 100755 --- a/examples/backups/clickhouse/04-create-clickhouse.sh +++ b/examples/backups/clickhouse/04-create-clickhouse.sh @@ -14,6 +14,14 @@ source "$SCRIPT_DIR/.bucket-info.env" print_header "Step 04: Provision ClickHouse '${CLICKHOUSE_NAME}'" +# backup.endpointCA is only meaningful when step 03 copied a CA (self-signed S3 +# endpoint); on a publicly-trusted endpoint CH_BACKUP_CA_SECRET is empty and the +# block is omitted entirely so the sidecar stays on the system trust store. +ENDPOINT_CA_BLOCK="" +if [[ -n "${CH_BACKUP_CA_SECRET:-}" ]]; then + ENDPOINT_CA_BLOCK=$(printf '\n endpointCA:\n name: "%s"\n key: ca.crt' "$CH_BACKUP_CA_SECRET") +fi + # backup.enabled=true triggers the chart to emit: # - -backup-s3 Secret (with bucket coordinates from values) # - clickhouse-backup sidecar in the chi-* Pod (HTTP API on :7171) @@ -37,7 +45,7 @@ spec: s3Region: "${CH_BACKUP_REGION}" endpoint: "${CH_BACKUP_ENDPOINT}" s3AccessKey: "${CH_BACKUP_ACCESS_KEY}" - s3SecretKey: "${CH_BACKUP_SECRET_KEY}" + s3SecretKey: "${CH_BACKUP_SECRET_KEY}"${ENDPOINT_CA_BLOCK} clickhouseKeeper: enabled: true replicas: 1 @@ -46,11 +54,12 @@ spec: EOF log_substep "Waiting for ClickHouse HelmRelease..." -kubectl -n "$NAMESPACE" wait hr "clickhouse-${CLICKHOUSE_NAME}" --for=condition=ready --timeout=300s +# 660s, above this generated release's 600s install timeout; see "Sizing an +# HR-Ready budget" in docs/agents/e2e-testing.md. +wait_hr_ready "clickhouse-${CLICKHOUSE_NAME}" 660 log_substep "Waiting for first ClickHouse pod..." -kubectl -n "$NAMESPACE" wait statefulset.apps/"chi-clickhouse-${CLICKHOUSE_NAME}-clickhouse-0-0" \ - --for=jsonpath='{.status.readyReplicas}'=1 --timeout=300s +wait_sts_ready "chi-clickhouse-${CLICKHOUSE_NAME}-clickhouse-0-0" 300 log_substep "Writing sentinel data..." clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id" diff --git a/examples/backups/clickhouse/05-create-backupjob.sh b/examples/backups/clickhouse/05-create-backupjob.sh index ffeb3978b8..bf13e8899d 100755 --- a/examples/backups/clickhouse/05-create-backupjob.sh +++ b/examples/backups/clickhouse/05-create-backupjob.sh @@ -27,7 +27,7 @@ log_substep "Waiting for BackupJob to Succeed (this triggers clickhouse-backup c # `kubectl wait --for=jsonpath='{.status.phase}'=Running` would time out even # though the BackupJob actually finished. wait_for_field already polls until # the terminal phase, which covers both the slow and the fast path. -wait_for_field backupjob "$BACKUPJOB_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 600 +wait_for_field backupjob "$BACKUPJOB_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 480 Failed backup_ref=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}') [[ -n "$backup_ref" ]] || { log_error "BackupJob succeeded but BackupRef is empty"; exit 1; } diff --git a/examples/backups/clickhouse/06-restore-in-place.sh b/examples/backups/clickhouse/06-restore-in-place.sh index 6a7dc13d32..afd3caff3a 100755 --- a/examples/backups/clickhouse/06-restore-in-place.sh +++ b/examples/backups/clickhouse/06-restore-in-place.sh @@ -24,7 +24,7 @@ spec: EOF log_substep "Waiting for in-place RestoreJob to Succeed..." -wait_for_field restorejob "$RESTOREJOB_INPLACE_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 600 +wait_for_field restorejob "$RESTOREJOB_INPLACE_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 480 Failed log_substep "Verifying sentinel data is restored..." count=$(clickhouse_query "$CLICKHOUSE_NAME" "SELECT count() FROM default.sentinel" | tr -d '[:space:]') diff --git a/examples/backups/clickhouse/07-restore-to-copy.sh b/examples/backups/clickhouse/07-restore-to-copy.sh index 122b9df03a..5fcf785fb6 100755 --- a/examples/backups/clickhouse/07-restore-to-copy.sh +++ b/examples/backups/clickhouse/07-restore-to-copy.sh @@ -20,6 +20,14 @@ done print_header "Step 07: To-copy restore into '${CLICKHOUSE_RESTORE_NAME}'" +# The target's sidecar downloads from the same S3 endpoint as the source, so it +# needs the same CA when that endpoint is self-signed. Step 03 already +# materialised the Secret in this namespace; reference it here too. +ENDPOINT_CA_BLOCK="" +if [[ -n "${CH_BACKUP_CA_SECRET:-}" ]]; then + ENDPOINT_CA_BLOCK=$(printf '\n endpointCA:\n name: "%s"\n key: ca.crt' "$CH_BACKUP_CA_SECRET") +fi + # backup.enabled=true so the chart materialises the same sidecar pattern in # the target Pod; the strategy Pod's HTTP call hits the target's sidecar and # pulls from the same S3 backup. @@ -50,7 +58,7 @@ spec: # name with "clickhouse-" (see clickhouse-rd/cozyrds/clickhouse.yaml), # so the actual Helm release name (which the sidecar uses for # S3_PATH) is "clickhouse-${CLICKHOUSE_NAME}". - s3PathOverride: "clickhouse-${CLICKHOUSE_NAME}" + s3PathOverride: "clickhouse-${CLICKHOUSE_NAME}"${ENDPOINT_CA_BLOCK} clickhouseKeeper: enabled: true replicas: 1 @@ -59,9 +67,9 @@ spec: EOF log_substep "Waiting for restore-target ClickHouse HelmRelease..." -kubectl -n "$NAMESPACE" wait hr "clickhouse-${CLICKHOUSE_RESTORE_NAME}" --for=condition=ready --timeout=300s -kubectl -n "$NAMESPACE" wait statefulset.apps/"chi-clickhouse-${CLICKHOUSE_RESTORE_NAME}-clickhouse-0-0" \ - --for=jsonpath='{.status.readyReplicas}'=1 --timeout=300s +# Same 660s as the source instance in 04-create-clickhouse.sh. +wait_hr_ready "clickhouse-${CLICKHOUSE_RESTORE_NAME}" 660 +wait_sts_ready "chi-clickhouse-${CLICKHOUSE_RESTORE_NAME}-clickhouse-0-0" 300 kubectl apply -f - <-backup-s3` Secret in the tenant namespace and an in-pod diff --git a/examples/backups/clickhouse/README.md b/examples/backups/clickhouse/README.md index c70e2b72fd..3b9fc0d3e0 100644 --- a/examples/backups/clickhouse/README.md +++ b/examples/backups/clickhouse/README.md @@ -27,7 +27,7 @@ that POSTs to the sidecar and polls the action log. | `00-helpers.sh` | Shared bash helpers and env defaults; sourced by every step. | n/a | | `01-create-strategy.sh` | Creates the cluster-scoped `Altinity` strategy that wraps `clickhouse-backup`. | admin | | `02-create-backupclass.sh` | Maps `apps.cozystack.io/ClickHouse` to that strategy. | admin | -| `03-create-bucket.sh` | Provisions a `Bucket` and caches its S3 coordinates into `.bucket-info.env` (chmod 600; raw access keys). `cleanup.sh` removes this file. | tenant | +| `03-create-bucket.sh` | Provisions a `Bucket`, caches its S3 coordinates into `.bucket-info.env` (chmod 600; raw access keys), and copies the S3 endpoint CA into a per-release Secret so the sidecar can verify a self-signed endpoint. `cleanup.sh` removes both. | tenant, **plus admin** for the CA copy — see below | | `04-create-clickhouse.sh` | Provisions a `ClickHouse` instance with `backup.enabled=true` (chart emits the backup-s3 Secret + sidecar) and writes a sentinel row. | tenant | | `05-create-backupjob.sh` | Submits a `BackupJob` and waits for Succeeded. | tenant | | `06-restore-in-place.sh` | Drops the sentinel and restores into the same instance via `RestoreJob`. | tenant | @@ -44,7 +44,7 @@ All variables come from `00-helpers.sh`: | Variable | Default | Description | |---|---|---| -| `NAMESPACE` | `tenant-test` | Tenant namespace for the demo. | +| `NAMESPACE` | `tenant-root` | Tenant namespace for the demo. `tenant-root` because the S3 client is the `clickhouse-backup` sidecar inside the tenant and cozystack's shared SeaweedFS lives there; an isolated tenant's Cilium egress allowlist blocks its Pods from reaching `tenant-root`'s SeaweedFS. | | `CLICKHOUSE_NAME` | `clickhouse-test` | Source ClickHouse application name. | | `CLICKHOUSE_RESTORE_NAME` | `clickhouse-restore` | Target ClickHouse for the to-copy restore. | | `BUCKET_NAME` | `clickhouse-backups` | Cozystack `Bucket` to provision. | @@ -53,10 +53,26 @@ All variables come from `00-helpers.sh`: | `BACKUPJOB_NAME` | `clickhouse-backup-job` | BackupJob name. | | `RESTOREJOB_INPLACE_NAME` | `clickhouse-restore-inplace` | In-place RestoreJob name. | | `RESTOREJOB_TOCOPY_NAME` | `clickhouse-restore-to-copy` | To-copy RestoreJob name. | +| `S3_ENDPOINT` | *(from BucketInfo)* | Overrides the S3 endpoint the sidecar uses. `BucketInfo` advertises the **external** ingress URL, which in-cluster Pods cannot always reach or TLS-validate; set `https://seaweedfs-s3.tenant-root:8333` to use the in-cluster endpoint. | +| `S3_CA_SECRET` | `seaweedfs-ca-cert` | Secret holding the S3 endpoint's CA. Step 03 copies its CA into a per-release Secret wired to `backup.endpointCA`, so the sidecar can verify a self-signed endpoint. Auto-discovered from the seaweedfs cert-manager `Certificate` if this name is absent. Set to `""` on a publicly-trusted endpoint to skip the copy and leave `backup.endpointCA` unset. | +| `S3_CA_NAMESPACE` | `tenant-root` | Namespace of `S3_CA_SECRET`. Independent of `NAMESPACE`: the shared SeaweedFS and its CA live in `tenant-root` even when the demo runs elsewhere. | +| `S3_CA_KEY` | `ca.crt` | Key inside `S3_CA_SECRET` holding the PEM bundle. | +| `CH_CA_SECRET_NAME` | `-backup-s3-ca` | Per-release Secret the CA is copied into; referenced by `backup.endpointCA`. Scoped to the instance name so two instances in one namespace don't collide, and stamped with a `cozystack.io/owned-by-demo` label so step 03 refuses to overwrite a Secret it doesn't own and `cleanup.sh` deletes only one this demo created. | +| `CH_BACKUP_CA_SECRET` | *(set by step 03)* | Secret name steps 04 and 07 wire into `backup.endpointCA`. Step 03 sets it to `CH_CA_SECRET_NAME` after copying the CA and leaves it empty when `S3_CA_SECRET=""`. On the admin-copy path, export it yourself to the pre-copied Secret before step 04, since step 03 does no copy and leaves it empty. | ## Prerequisites - Cozystack cluster with the backup-controller and backupstrategy-controller installed. - `kubectl`, `jq`, and (for `04`+) the ClickHouse operator deployed by the chart. +### Who can run step 03 + +Step 03 is otherwise a tenant action, but copying the S3 endpoint CA reads a Secret in `S3_CA_NAMESPACE` (`tenant-root` by default), which is a platform/admin capability rather than a tenant one. Three ways out, in order of preference: + +- run step 03 with credentials that can read that namespace (the admin path this demo assumes); +- have an admin copy the CA into `/` once, then run the demo with `S3_CA_SECRET=""` and `export CH_BACKUP_CA_SECRET=""` before steps 04 and 07, so both ClickHouse flows wire `backup.endpointCA` to that pre-copied Secret; +- skip it entirely with `S3_CA_SECRET=""` when the S3 endpoint's certificate is signed by a publicly-trusted CA — nothing needs a CA bundle then. + +The script distinguishes the three outcomes so a permissions problem does not present as a missing Secret: it stops with an explicit permissions message on `Forbidden`, and on a genuinely absent default (no `seaweedfs-ca-cert`, no seaweedfs `Certificate` to discover, and `S3_CA_SECRET` not set explicitly) it warns and continues with `backup.endpointCA` unset instead of failing — which is the right outcome on a cluster whose endpoint is publicly trusted. Naming a Secret that does not exist is still an error. + [altinity]: https://github.com/Altinity/clickhouse-backup diff --git a/examples/backups/clickhouse/cleanup.sh b/examples/backups/clickhouse/cleanup.sh index 310c1b478e..1cba452185 100755 --- a/examples/backups/clickhouse/cleanup.sh +++ b/examples/backups/clickhouse/cleanup.sh @@ -15,8 +15,38 @@ kubectl -n "$NAMESPACE" delete backup "$BACKUPJOB_NAME" --ignore-not-found kubectl -n "$NAMESPACE" delete clickhouse "$CLICKHOUSE_RESTORE_NAME" --ignore-not-found kubectl -n "$NAMESPACE" delete clickhouse "$CLICKHOUSE_NAME" --ignore-not-found kubectl -n "$NAMESPACE" delete bucket "$BUCKET_NAME" --ignore-not-found + +# `kubectl delete` on these returns as soon as deletionTimestamp is set, while +# the HelmRelease uninstall behind each one (a ClickHouseInstallation, a Keeper +# installation and their PVCs) is still running. Returning here has two costs: +# the next suite starts while helm-controller workers are still draining this +# one, and on the pre-clean path step 04 can re-apply an application whose +# predecessor has not finished uninstalling. So wait for the objects to actually +# go, with teeth — a teardown that silently does not settle is what leaks into +# later tests. +# This script deliberately runs without `set -e` so one absent resource cannot +# abandon the rest of the teardown, which means the waits have to record their +# own failure and surface it at the end. +teardown_rc=0 +wait_deleted clickhouse "$CLICKHOUSE_RESTORE_NAME" 180 || teardown_rc=1 +wait_deleted clickhouse "$CLICKHOUSE_NAME" 180 || teardown_rc=1 +wait_deleted bucket "$BUCKET_NAME" 180 || teardown_rc=1 +# The S3 endpoint CA copied out of the seaweedfs CA Secret by step 03. Delete +# it only when it carries this demo's ownership label, so a foreign Secret that +# happens to share the name survives. The source Secret in S3_CA_NAMESPACE is +# the platform's and is never touched. +if ca_owner=$(kubectl -n "$NAMESPACE" get secret "$CH_CA_SECRET_NAME" \ + -o jsonpath="{.metadata.labels.${CH_CA_SECRET_LABEL_KEY//./\\.}}" 2>/dev/null) \ + && [[ "$ca_owner" == "$CH_CA_SECRET_LABEL_VALUE" ]]; then + kubectl -n "$NAMESPACE" delete secret "$CH_CA_SECRET_NAME" --ignore-not-found +fi rm -f "$SCRIPT_DIR/.bucket-info.env" kubectl delete backupclass "$BACKUPCLASS_NAME" --ignore-not-found kubectl delete altinity.strategy.backups.cozystack.io "$STRATEGY_NAME" --ignore-not-found +if [[ $teardown_rc -ne 0 ]]; then + log_error "Cleanup did not settle: see the resources reported above. Failing rather than leaving a half-uninstalled release for the next test to trip over." + exit 1 +fi + log_success "Cleanup complete." diff --git a/examples/backups/etcd/00-helpers.sh b/examples/backups/etcd/00-helpers.sh index 9a77e910c7..f1f0959448 100755 --- a/examples/backups/etcd/00-helpers.sh +++ b/examples/backups/etcd/00-helpers.sh @@ -15,7 +15,7 @@ export BOLD='\033[1m' # Default settings (override via environment). export NAMESPACE="${NAMESPACE:-tenant-root}" # The chart pins the Helm release name (and the operator-side -# etcd.aenix.io/EtcdCluster name) to "etcd" via +# etcd-operator.cozystack.io/EtcdCluster name) to "etcd" via # templates/check-release-name.yaml, so the apps.cozystack.io/Etcd CR # MUST be named "etcd" per namespace. To-copy in the same namespace is # unsupported by design (chart constraint). @@ -43,35 +43,10 @@ print_header() { echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2 } -# Wait until a JSONPath value on a resource matches the desired string. -wait_for_field() { - local resource_type="$1" - local resource_name="$2" - local jsonpath="$3" - local desired="$4" - local namespace="${5:-}" - local timeout="${6:-300}" - - log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..." - local elapsed=0 - local ns_flag=() - [[ -n "$namespace" ]] && ns_flag=(-n "$namespace") - - while true; do - local current - current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true) - if [[ "$current" == "$desired" ]]; then - log_success "$resource_type/$resource_name reached '$desired'" - return 0 - fi - if [[ $elapsed -ge $timeout ]]; then - log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')" - return 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) - done -} +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" # Ephemeral etcdctl pod name. Created on first etcdctl_exec call; reused # for subsequent calls in the same script run. cleanup.sh removes it. @@ -109,7 +84,7 @@ spec: restartPolicy: Never containers: - name: etcdctl - image: quay.io/coreos/etcd:v3.5.12 + image: quay.io/coreos/etcd:v3.6.11 command: - etcdctl - "--cacert=/etc/etcd/pki/client/ca.crt" diff --git a/examples/backups/etcd/01-create-strategy.sh b/examples/backups/etcd/01-create-strategy.sh index dbc1ca3c29..1c7fbd4713 100755 --- a/examples/backups/etcd/01-create-strategy.sh +++ b/examples/backups/etcd/01-create-strategy.sh @@ -1,7 +1,7 @@ #!/bin/bash # Step 01: Apply the cluster-scoped Etcd strategy. Admin operation - # tenants do not author the strategy CR directly. The strategy carries -# the templated EtcdBackup destination shape; the BackupClass created in +# the templated EtcdSnapshot destination shape; the BackupClass created in # step 02 supplies bucket/endpoint/credentials per tenant via parameters. set -euo pipefail diff --git a/examples/backups/etcd/02-create-bucket.sh b/examples/backups/etcd/02-create-bucket.sh index 63ffbe3448..a6c46a1761 100755 --- a/examples/backups/etcd/02-create-bucket.sh +++ b/examples/backups/etcd/02-create-bucket.sh @@ -45,7 +45,7 @@ for v in ETCD_ACCESS_KEY ETCD_SECRET_KEY ETCD_ENDPOINT ETCD_REGION ETCD_BUCKET; [[ -n "${!v}" && "${!v}" != "null" ]] || { log_error "BucketInfo missing required field: ${v}"; exit 1; } done -log_substep "Materialising the per-app credentials Secret consumed by the rendered EtcdBackup..." +log_substep "Materialising the per-app credentials Secret consumed by the rendered EtcdSnapshot..." # The strategy template references this Secret by name: # "{{ .Application.metadata.name }}-etcd-backup-creds" # The etcd-operator's snapshot Job mounts the Secret and exposes diff --git a/examples/backups/etcd/03-create-etcd-src.sh b/examples/backups/etcd/03-create-etcd-src.sh index 5c4ef42630..d95cabbcf0 100755 --- a/examples/backups/etcd/03-create-etcd-src.sh +++ b/examples/backups/etcd/03-create-etcd-src.sh @@ -1,6 +1,6 @@ #!/bin/bash # Step 03: Apply the source Etcd application, wait for the operator-side -# EtcdCluster to reach Ready, then write a sentinel key that the backup +# EtcdCluster to become Available, then write a sentinel key that the backup # captures and the restore round-trips. The sentinel is the in-cluster # witness that the snapshot actually round-tripped through S3. set -euo pipefail @@ -27,13 +27,13 @@ EOF log_substep "Waiting for HelmRelease 'etcd' to be Ready..." kubectl -n "$NAMESPACE" wait hr "${ETCD_NAME}" --for=condition=ready --timeout=300s -log_substep "Waiting for etcd.aenix.io/EtcdCluster 'etcd' to be Ready..." -kubectl -n "$NAMESPACE" wait etcdcluster.etcd.aenix.io etcd \ - --for=jsonpath='{.status.conditions[?(@.type=="Ready")].status}'=True \ +log_substep "Waiting for etcd-operator.cozystack.io/EtcdCluster 'etcd' to be Available..." +kubectl -n "$NAMESPACE" wait etcdcluster.etcd-operator.cozystack.io etcd \ + --for=jsonpath='{.status.conditions[?(@.type=="Available")].status}'=True \ --timeout=300s -log_substep "Waiting for etcd-0 pod to be Ready..." -kubectl -n "$NAMESPACE" wait pod etcd-0 --for=condition=ready --timeout=300s +log_substep "Waiting for etcd member pods to be Ready..." +kubectl -n "$NAMESPACE" wait pod -l app.kubernetes.io/instance=etcd --for=condition=ready --timeout=300s SENTINEL_KEY="__cozystack_e2e_sentinel" SENTINEL_VAL="pristine-$(date -u +%Y%m%dT%H%M%SZ)" diff --git a/examples/backups/etcd/04-create-backupjob.sh b/examples/backups/etcd/04-create-backupjob.sh index 50958bc385..0613396080 100755 --- a/examples/backups/etcd/04-create-backupjob.sh +++ b/examples/backups/etcd/04-create-backupjob.sh @@ -25,7 +25,7 @@ spec: EOF wait_for_field backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \ - '{.status.phase}' Succeeded "$NAMESPACE" 1200 + '{.status.phase}' Succeeded "$NAMESPACE" 1200 Failed BACKUP_NAME=$(kubectl -n "$NAMESPACE" get backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \ -o jsonpath='{.status.backupRef.name}') diff --git a/examples/backups/etcd/05-restore-in-place.sh b/examples/backups/etcd/05-restore-in-place.sh index 7851ad827a..50ac69ea5f 100755 --- a/examples/backups/etcd/05-restore-in-place.sh +++ b/examples/backups/etcd/05-restore-in-place.sh @@ -43,10 +43,10 @@ spec: EOF wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_INPLACE_NAME" \ - '{.status.phase}' Succeeded "$NAMESPACE" 1800 + '{.status.phase}' Succeeded "$NAMESPACE" 1800 Failed -log_substep "Waiting for etcd-0 to be Ready again after restore..." -kubectl -n "$NAMESPACE" wait pod etcd-0 --for=condition=ready --timeout=300s +log_substep "Waiting for etcd member pods to be Ready again after restore..." +kubectl -n "$NAMESPACE" wait pod -l app.kubernetes.io/instance=etcd --for=condition=ready --timeout=300s # Helm's next apply re-renders the chart's bootstrap-less EtcdCluster # spec, which evicts the bootstrap.restore block the driver stamped on @@ -58,7 +58,7 @@ kubectl -n "$NAMESPACE" wait pod etcd-0 --for=condition=ready --timeout=300s log_substep "Waiting for Helm to evict bootstrap.restore from the live EtcdCluster..." elapsed=0 while true; do - BOOTSTRAP=$(kubectl -n "$NAMESPACE" get etcdcluster.etcd.aenix.io etcd \ + BOOTSTRAP=$(kubectl -n "$NAMESPACE" get etcdcluster.etcd-operator.cozystack.io etcd \ -o jsonpath='{.spec.bootstrap}' 2>/dev/null || true) if [[ -z "$BOOTSTRAP" || "$BOOTSTRAP" == "{}" || "$BOOTSTRAP" == "null" ]]; then log_success "bootstrap.restore evicted by Helm reconcile" diff --git a/examples/backups/etcd/90-scenario-admin-prepare.md b/examples/backups/etcd/90-scenario-admin-prepare.md index ae2701dcf3..b034392442 100644 --- a/examples/backups/etcd/90-scenario-admin-prepare.md +++ b/examples/backups/etcd/90-scenario-admin-prepare.md @@ -7,8 +7,9 @@ prerequisite hand-off ends at "tenants can author `BackupJob` / ## 1. Install the operator + driver -- `packages/system/etcd-operator` ships the upstream `etcd-operator` - (CRDs: `EtcdCluster`, `EtcdBackup`, `EtcdBackupSchedule`). +- `packages/system/etcd-operator` ships the cozystack-authored `etcd-operator` + (`etcd-operator.cozystack.io/v1alpha2`; CRDs: `EtcdCluster`, `EtcdMember`, + `EtcdSnapshot`). - `packages/system/backupstrategy-controller` ships the `strategy.backups.cozystack.io/v1alpha1` API and the controllers that dispatch `BackupJob` / `RestoreJob` to the per-app driver. The diff --git a/examples/backups/etcd/91-scenario-user-backup.md b/examples/backups/etcd/91-scenario-user-backup.md index 2fc9935eea..3f0a4914f6 100644 --- a/examples/backups/etcd/91-scenario-user-backup.md +++ b/examples/backups/etcd/91-scenario-user-backup.md @@ -18,7 +18,7 @@ coordinates baked into `parameters`. ## 2. Deploy the source Etcd (`03-create-etcd-src.sh`) The tenant applies an `apps.cozystack.io/Etcd` named `etcd`. The -HelmRelease materialises an `etcd.aenix.io/EtcdCluster` and three +HelmRelease materialises an `etcd-operator.cozystack.io/EtcdCluster` and three members. The script writes a sentinel key under `etcdctl` so step 05 can witness the round-trip. @@ -27,7 +27,7 @@ can witness the round-trip. A `BackupJob` references the `BackupClass` and the source `Etcd`. The driver resolves the strategy, renders its template against the live application and class parameters, then materialises one -`etcd.aenix.io/EtcdBackup` CR per `BackupJob`. The etcd-operator's +`etcd-operator.cozystack.io/EtcdSnapshot` CR per `BackupJob`. The etcd-operator's controller runs a one-shot Job that snapshots etcd and uploads the file to S3. On `phase=Complete`, the driver creates a Cozystack `Backup` artefact with the S3 coordinates baked into @@ -36,7 +36,7 @@ file to S3. On `phase=Complete`, the driver creates a Cozystack ## What the tenant does NOT do -- Author `EtcdBackup` CRs by hand. The driver owns the lifecycle. +- Author `EtcdSnapshot` CRs by hand. The driver owns the lifecycle. - Manage the underlying S3 credentials. The driver consumes them from the per-app Secret the bucket flow already produced. - Track snapshot retention via cron. A `Plan` resource (not part of diff --git a/examples/backups/etcd/92-scenario-user-restore.md b/examples/backups/etcd/92-scenario-user-restore.md index fedf06a9d0..038a748871 100644 --- a/examples/backups/etcd/92-scenario-user-restore.md +++ b/examples/backups/etcd/92-scenario-user-restore.md @@ -21,7 +21,7 @@ namespace) and `targetApplicationRef` is a same-namespace reference. `helm.toolkit.fluxcd.io/v2 HelmRelease "etcd"` stops Flux from re-rendering the chart while the driver mutates the live cluster. 2. **Snapshot the chart-rendered EtcdCluster spec.** The driver reads - the live `etcd.aenix.io/EtcdCluster "etcd"` spec and stashes it on + the live `etcd-operator.cozystack.io/EtcdCluster "etcd"` spec and stashes it on `RestoreJob.status.conditions[EtcdClusterSpecCaptured].message`. This makes the destructive flow controller-crash-safe: a restart between "snapshot taken" and "cluster recreated" still has the spec @@ -35,9 +35,10 @@ namespace) and `targetApplicationRef` is a same-namespace reference. driver merges the captured spec with a freshly resolved snapshot destination from the `Backup` artefact, then `Create`s the new CR. The etcd-operator picks it up and bootstraps from the S3 snapshot. -6. **Wait for `Ready=True`.** The driver polls the new - `EtcdCluster.status.conditions[Ready]`. -7. **Resume the HelmRelease.** With the cluster Ready, Flux can +6. **Wait for `Available=True`.** The driver polls the new + `EtcdCluster.status.conditions[Available]` (v1alpha2 reports readiness + as `Available`, not `Ready`). +7. **Resume the HelmRelease.** With the cluster Available, Flux can reconcile chart drift safely. Helm's next apply will remove the `bootstrap` block on the live CR; the operator only consults `bootstrap` at first reconcile, so this is a no-op after the fact. diff --git a/examples/backups/etcd/README.md b/examples/backups/etcd/README.md index 7883e39b51..c0775cb0ae 100644 --- a/examples/backups/etcd/README.md +++ b/examples/backups/etcd/README.md @@ -13,7 +13,7 @@ This directory shows how to back up and restore a Cozystack-managed `Etcd` application using the cluster's `Etcd` backup strategy driver. The driver delegates to the [etcd-operator][etcd-operator]: each Cozystack `BackupJob` -materialises an `etcd.aenix.io/v1alpha1 EtcdBackup` CR (a one-shot Job that +materialises an `etcd-operator.cozystack.io/v1alpha2 EtcdSnapshot` CR (a one-shot Job that streams `etcdctl snapshot save` output to S3), and each `RestoreJob` performs an in-place restore by suspending the Etcd HelmRelease, deleting the chart-rendered `EtcdCluster`, and recreating it with @@ -68,7 +68,7 @@ chart change alone is not sufficient. | `00-helpers.sh` | Shared bash helpers and env defaults; sourced by every `01..05` step and by `cleanup.sh`. | n/a | | `01-create-strategy.sh` | Creates the cluster-scoped `Etcd` strategy. | admin | | `02-create-bucket.sh` | Provisions a `Bucket`, mints an `-etcd-backup-creds` Secret in the source namespace, and creates the `BackupClass` bound to the Etcd strategy with resolved S3 coordinates. Caches raw creds in `.bucket-info.env` (chmod 600); `cleanup.sh` removes it. | tenant | -| `03-create-etcd-src.sh` | Provisions the source `Etcd` application, waits for `EtcdCluster.status.conditions[Ready]=True`, and writes a sentinel key under `etcdctl`. | tenant | +| `03-create-etcd-src.sh` | Provisions the source `Etcd` application, waits for `EtcdCluster.status.conditions[Available]=True`, and writes a sentinel key under `etcdctl`. | tenant | | `04-create-backupjob.sh` | Submits a `BackupJob` and waits for `phase=Succeeded`. | tenant | | `05-restore-in-place.sh` | Mutates the sentinel, submits a `RestoreJob` (no `targetApplicationRef`), and verifies the sentinel is round-tripped back to its pre-mutation value. | tenant | | `cleanup.sh` | Best-effort teardown. | admin or tenant | @@ -94,13 +94,12 @@ All variables come from `00-helpers.sh`: running, with the Etcd dispatch wired (see `internal/backupcontroller/etcdstrategy_controller.go`). - `kubectl`, `jq`. -- The `etcd.aenix.io` CRDs installed by `packages/system/etcd-operator` - v0.4.5+. The Cozystack flow specifically depends on `EtcdCluster` - (the driver re-creates it with `spec.bootstrap.restore.source.s3` - during in-place restore) and `EtcdBackup` (one materialised per - `BackupJob`). `EtcdBackupSchedule` is only consumed by the legacy - `backup.enabled=true` chart path. There is no separate `EtcdRestore` - CRD — restore goes through `EtcdCluster.spec.bootstrap`. +- The `etcd-operator.cozystack.io` CRDs installed by + `packages/system/etcd-operator-crds`. The Cozystack flow specifically depends + on `EtcdCluster` (the driver re-creates it with + `spec.bootstrap.restore.source.s3` during in-place restore) and `EtcdSnapshot` + (one materialised per `BackupJob`). There is no separate `EtcdRestore` CRD — + restore goes through `EtcdCluster.spec.bootstrap`. - The Cozystack `etcd-rd` ApplicationDefinition installed so `apps.cozystack.io/Etcd` renders a HelmRelease. @@ -118,11 +117,9 @@ All variables come from `00-helpers.sh`: override it by setting `ETCD_REGION` in your shell before sourcing `00-helpers.sh` if you point the demo at a non-SeaweedFS endpoint whose region differs. -- Step 05 only verifies the happy path. Edge cases (stuck HR resume, - failed bootstrap, mid-restore controller crash) belong in the bats - e2e at `hack/e2e-apps/backup-etcd.bats`. +- Step 05 only verifies the happy path. Edge cases (stuck HR resume, failed bootstrap, mid-restore controller crash) belong in the etcd Chainsaw suite at `hack/e2e-chainsaw/etcd/`. - The destructive in-place flow deletes the `EtcdCluster` (and the operator drops the member PVCs alongside it). All client traffic to etcd is unavailable for the duration of the restore. Plan a window. -[etcd-operator]: https://github.com/aenix-io/etcd-operator +[etcd-operator]: https://github.com/cozystack/etcd-operator diff --git a/examples/backups/foundationdb/00-helpers.sh b/examples/backups/foundationdb/00-helpers.sh index d7c3c204d9..f2707f1318 100755 --- a/examples/backups/foundationdb/00-helpers.sh +++ b/examples/backups/foundationdb/00-helpers.sh @@ -47,35 +47,10 @@ print_header() { echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2 } -# Wait until a JSONPath value on a resource matches the desired string. -wait_for_field() { - local resource_type="$1" - local resource_name="$2" - local jsonpath="$3" - local desired="$4" - local namespace="${5:-}" - local timeout="${6:-300}" - - log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..." - local elapsed=0 - local ns_flag=() - [[ -n "$namespace" ]] && ns_flag=(-n "$namespace") - - while true; do - local current - current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true) - if [[ "$current" == "$desired" ]]; then - log_success "$resource_type/$resource_name reached '$desired'" - return 0 - fi - if [[ $elapsed -ge $timeout ]]; then - log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')" - return 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) - done -} +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" # Run an fdbcli statement against a FoundationDB cluster of the given app # instance. Args: diff --git a/examples/backups/kafka-metadata/00-helpers.sh b/examples/backups/kafka-metadata/00-helpers.sh new file mode 100755 index 0000000000..8d77fba64f --- /dev/null +++ b/examples/backups/kafka-metadata/00-helpers.sh @@ -0,0 +1,268 @@ +#!/bin/bash +# Shared helpers for the Kafka topic-metadata backup/restore demo. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export BLUE='\033[0;34m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export WHITE='\033[1;37m' +export NC='\033[0m' +export BOLD='\033[1m' + +# Default settings (override via environment). The demo derives its OWN Kafka +# strategy + BackupClass from the platform-shipped cozy-default-kafka strategy +# (run-all.sh patches its S3_ENDPOINT and mounts the S3 CA), so the round-trip +# can target an in-cluster, TLS-verifiable S3 endpoint the shipped strategy's +# advertised external ingress is not in CI. The shipped cozy-default-kafka is +# left untouched. +export NAMESPACE="${NAMESPACE:-tenant-root}" +export KAFKA_SRC_NAME="${KAFKA_SRC_NAME:-kafka-meta-src}" +export KAFKA_TARGET_NAME="${KAFKA_TARGET_NAME:-kafka-meta-target}" +export TOPIC="${TOPIC:-orders}" +export PARTITIONS="${PARTITIONS:-3}" +# Colliding topic pair that exercises the --topic regex-vs-literal fix in the +# strategy script: "audit.events" as a Java regex ALSO matches "audit-events" +# (. is any char), so a describe that does not wrap the name in \Q...\E records +# one topic's partition count against the other. Distinct counts make that +# cross-match fail the round-trip verify. +export COLLIDE_DOT="${COLLIDE_DOT:-audit.events}" +export COLLIDE_DOT_PARTS="${COLLIDE_DOT_PARTS:-2}" +export COLLIDE_DASH="${COLLIDE_DASH:-audit-events}" +export COLLIDE_DASH_PARTS="${COLLIDE_DASH_PARTS:-5}" +# The demo's own strategy + BackupClass, derived at run time from the shipped +# cozy-default-kafka so the driver script stays a single source of truth. Named +# distinctly so they never collide with the platform cozy-default class. +export STRATEGY_NAME="${STRATEGY_NAME:-kafka-strategy-default}" +export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-kafka-metadata}" +export BACKUPJOB_NAME="${BACKUPJOB_NAME:-kafka-meta-src-adhoc}" +export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-kafka-meta-src-inplace}" +export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-kafka-meta-src-to-target}" +export PLAN_NAME="${PLAN_NAME:-kafka-meta-src-daily}" +# S3_ENDPOINT overrides the endpoint the backup Pod uses. The shipped strategy +# advertises the EXTERNAL S3 ingress, which in-cluster Pods cannot always +# resolve or TLS-validate (in CI it is an unroutable placeholder). The +# in-cluster alternative is https://seaweedfs-s3..svc:8333 — the .svc FQDN +# is what the seaweedfs serving cert's SAN covers, so curl verifies TLS against +# the copied CA below. CI sets this; a real cluster can leave it unset. +export S3_ENDPOINT="${S3_ENDPOINT:-}" +# Self-signed seaweedfs CA: run-all.sh discovers it and copies ca.crt into +# CA_SECRET, which the demo strategy Pod mounts and points CURL_CA_BUNDLE at. +# Set S3_CA_SECRET="" to skip on a publicly-trusted endpoint. +export CA_SECRET="${CA_SECRET:-kafka-backup-ca}" +export S3_CA_SECRET="${S3_CA_SECRET:-seaweedfs-ca-cert}" +export S3_CA_NAMESPACE="${S3_CA_NAMESPACE:-tenant-root}" +export S3_CA_KEY="${S3_CA_KEY:-ca.crt}" +export CA_MOUNT_DIR="${CA_MOUNT_DIR:-/etc/ssl/kafka-backup-ca}" +# The Cozystack chart names the Strimzi cluster kafka-; its plaintext +# bootstrap Service is kafka--kafka-bootstrap:9092. +# KAFKA_IMAGE runs the long-lived CLI Pod the seed/verify helpers exec into (see +# kafka_run). The backup/restore Jobs no longer use it: the controller resolves +# the target broker's own image at reconcile time and renders it as the +# strategy's .ClientImage. Override this only to match your operator's image if +# the CLI it carries differs. +export KAFKA_IMAGE="${KAFKA_IMAGE:-quay.io/strimzi/kafka:0.45.1-rc1-kafka-3.9.1@sha256:ba52ed046b1dccdbd96f4e68057ce014d862a7c9c1fc670760c023b9aa09f23f}" +export KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}" +# Name of that long-lived CLI Pod; cleanup.sh removes it. +export KAFKA_CLI_POD="${KAFKA_CLI_POD:-kafka-cli}" + +log_info() { echo -e "${BLUE}i${NC} $*" >&2; } +log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; } +log_error() { echo -e "${RED}x${NC} $*" >&2; } +log_step() { echo -e "\n${MAGENTA}${BOLD}> $*${NC}" >&2; } +log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; } + +print_header() { + echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2 +} + +# copy_s3_ca copies the self-signed seaweedfs CA (ca.crt) into CA_SECRET in +# NAMESPACE so the demo strategy Pod can verify TLS against the in-cluster S3 +# endpoint. Echoes "1" when a CA was copied, "0" when skipped (S3_CA_SECRET +# empty → a publicly-trusted endpoint). Mirrors examples/backups/rabbitmq. +copy_s3_ca() { + if [[ -z "$S3_CA_SECRET" ]]; then echo 0; return 0; fi + if ! kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" >/dev/null 2>&1; then + log_warning "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} not found; discovering the seaweedfs CA Certificate..." + local discovered + discovered=$(kubectl -n "$S3_CA_NAMESPACE" get certificates.cert-manager.io \ + -l app.kubernetes.io/name=seaweedfs \ + -o jsonpath='{range .items[*]}{.spec.isCA}{" "}{.spec.secretName}{"\n"}{end}' 2>/dev/null \ + | awk '$1=="true"{print $2; exit}' || true) + [[ -n "$discovered" ]] || { log_error "No seaweedfs CA Certificate found in ${S3_CA_NAMESPACE}; set S3_CA_SECRET explicitly (or empty for a public-CA endpoint)."; return 1; } + log_success "Discovered seaweedfs CA secret ${S3_CA_NAMESPACE}/${discovered}" + S3_CA_SECRET="$discovered" + fi + local ca_pem + ca_pem=$(kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" \ + -o jsonpath="{.data.${S3_CA_KEY//./\\.}}" | base64 -d) + [[ -n "$ca_pem" ]] || { log_error "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} has no ${S3_CA_KEY}"; return 1; } + kubectl -n "$NAMESPACE" create secret generic "$CA_SECRET" \ + --from-literal="ca.crt=${ca_pem}" \ + --dry-run=client -o yaml | kubectl -n "$NAMESPACE" apply -f - >&2 + echo 1 +} + +# provision_demo_strategy derives the demo Kafka strategy from the shipped +# cozy-default-kafka (single source of truth for the driver script), overriding +# S3_ENDPOINT and, when $2==1, mounting CA_SECRET + pointing CURL_CA_BUNDLE at +# it. Then applies the demo BackupClass. Args: . +provision_demo_strategy() { + local endpoint="$1" ca_present="$2" + kubectl get kafka.strategy.backups.cozystack.io cozy-default-kafka -o json \ + | jq \ + --arg ep "$endpoint" --arg name "$STRATEGY_NAME" \ + --arg caDir "$CA_MOUNT_DIR" --arg caPath "${CA_MOUNT_DIR}/ca.crt" \ + --arg caSecret "$CA_SECRET" --argjson ca "$ca_present" ' + .metadata = {name: $name} + | del(.status) + | .spec.template.spec.containers[0].env = ( + ((.spec.template.spec.containers[0].env // []) + | map(if .name == "S3_ENDPOINT" then {name: "S3_ENDPOINT", value: $ep} else . end)) + + (if $ca == 1 then [{name: "CURL_CA_BUNDLE", value: $caPath}] else [] end)) + | if $ca == 1 then + .spec.template.spec.containers[0].volumeMounts = + ((.spec.template.spec.containers[0].volumeMounts // []) + + [{name: "s3-ca", mountPath: $caDir, readOnly: true}]) + | .spec.template.spec.volumes = + ((.spec.template.spec.volumes // []) + [{name: "s3-ca", secret: {secretName: $caSecret}}]) + else . end + ' | kubectl apply -f - >&2 + kubectl apply -f "$SCRIPT_DIR/03-backupclass.yaml" >&2 +} + +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" + +# Wait until the Strimzi Kafka cluster kafka- reports Ready=True — the same +# precondition the driver gates on. +kafka_wait_ready() { + local app="$1" timeout="${2:-600}" + wait_for_field kafka.kafka.strimzi.io "kafka-${app}" \ + '{.status.conditions[?(@.type=="Ready")].status}' True "$NAMESPACE" "$timeout" +} + +# Ensure the long-lived kafka-cli Pod exists and is Ready, so kafka_run can exec +# into it. Idempotent: a Ready Pod this demo owns is reused across calls and +# across the numbered demo scripts; a leftover in a terminal phase (Succeeded / +# Failed) is replaced rather than waited on, and so is one started before the +# Pod ran under the restricted profile, since a Pod's securityContext cannot be +# changed in place; and a same-named Pod this demo does not own is refused +# rather than hijacked or deleted. cleanup.sh removes it. +# Every step returns on failure explicitly: this runs as the left side of +# `|| return 1`, and some callers also wrap it in $(...), so errexit never +# applies inside it and a bare failure would carry on to the next step. +kafka_cli_pod() { + local phase owner nonroot overrides + phase=$(kubectl -n "$NAMESPACE" get pod "$KAFKA_CLI_POD" --ignore-not-found -o jsonpath='{.status.phase}') || return 1 + owner=$(kubectl -n "$NAMESPACE" get pod "$KAFKA_CLI_POD" --ignore-not-found -o jsonpath='{.metadata.labels.cozystack\.io/backup-demo}') || return 1 + if [ -n "$phase" ] && [ "$owner" != "kafka-metadata" ]; then + log_error "Pod $NAMESPACE/$KAFKA_CLI_POD exists but this demo does not own it; refusing to use or delete it" + return 1 + fi + # Only this helper creates the Pod, and it sets every restricted field at + # once, so one of them tells a current Pod from an older one. + nonroot=$(kubectl -n "$NAMESPACE" get pod "$KAFKA_CLI_POD" --ignore-not-found -o jsonpath='{.spec.securityContext.runAsNonRoot}') || return 1 + if { [ "$phase" != "Running" ] && [ "$phase" != "Pending" ]; } || [ "$nonroot" != "true" ]; then + # The Strimzi image runs as non-root UID 1001, so runAsNonRoot and + # seccomp at the Pod level plus allowPrivilegeEscalation=false and + # drop-ALL on the container satisfy PSA "restricted" without pinning a + # runAsUser specific to this image tag. The add-only JSON patch leaves + # the run-generated Pod's image, command and args untouched. + overrides='[{"op":"add","path":"/spec/securityContext","value":{"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}},{"op":"add","path":"/spec/containers/0/securityContext","value":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}}]' + kubectl -n "$NAMESPACE" delete pod "$KAFKA_CLI_POD" --grace-period=1 --ignore-not-found >/dev/null || return 1 + kubectl -n "$NAMESPACE" run "$KAFKA_CLI_POD" --image="$KAFKA_IMAGE" \ + --labels=cozystack.io/backup-demo=kafka-metadata \ + --override-type=json --overrides="$overrides" \ + --restart=Never --command -- sleep infinity >/dev/null || return 1 + fi + kubectl -n "$NAMESPACE" wait --for=condition=Ready "pod/$KAFKA_CLI_POD" \ + --timeout=5m >/dev/null +} + +# Run a bash snippet against the source Kafka, with $BOOT / $BIN / $TOPIC / +# $PARTITIONS pre-set (values injected via printf %q so the snippet needs no +# nested quoting). Host-side analogue used only to seed and verify — the +# backup/restore Jobs are created by the controller from the strategy. +# +# The snippet runs by `kubectl exec` in the long-lived CLI Pod, not a throwaway +# `kubectl run -i` Pod per call. A throwaway Pod's stdout comes back over an +# attach that only carries what the container writes after the attach registers, +# so a CLI that finishes in between returns empty with exit 0. An exec'd process +# owns its pipes, so its output cannot be missed that way, and kubectl's and the +# CLI's stderr stay attached so a failed read says why instead of reading as ''. +kafka_run() { + local app="$1"; shift + local snippet="$1" + local boot="kafka-${app}-kafka-bootstrap.${NAMESPACE}.svc:9092" + kafka_cli_pod || return 1 + kubectl -n "$NAMESPACE" exec -i "$KAFKA_CLI_POD" -- bash -c "set -eu +BOOT=$(printf %q "$boot") +BIN=$(printf %q "$KAFKA_BIN") +TOPIC=$(printf %q "$TOPIC") +PARTITIONS=$(printf %q "$PARTITIONS") +$snippet" +} + +# Create the demo topic with a distinctive retention.ms sentinel so the restore +# can prove the topic config — not just the name — round-tripped. +seed_topic() { + local app="$1" retention="$2" + kafka_run "$app" ' + "$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --create --if-not-exists \ + --topic "$TOPIC" --partitions "$PARTITIONS" --replication-factor 1 \ + --config retention.ms='"$retention"' + ' +} + +# Create a topic with an explicit partition count (no config sentinel). Used for +# the colliding pair, whose distinct partition counts are the round-trip proof. +seed_topic_partitions() { + local app="$1" topic="$2" partitions="$3" + TOPIC="$topic" PARTITIONS="$partitions" kafka_run "$app" ' + "$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --create --if-not-exists \ + --topic "$TOPIC" --partitions "$PARTITIONS" --replication-factor 1 + ' +} + +# Delete a specific topic (literal name via \Q) and wait for it to disappear. +delete_topic() { + local app="$1" topic="$2" + TOPIC="$topic" kafka_run "$app" ' + "$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --delete --topic "\Q$TOPIC\E" || true + for _ in $(seq 1 60); do + list=$("$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --list) || { sleep 2; continue; } + printf "%s\n" "$list" | grep -qxF -- "$TOPIC" || { echo "topic $TOPIC deleted"; exit 0; } + sleep 2 + done + echo "topic $TOPIC still present after wait" >&2; exit 1 + ' +} + +# Print the partition count for a specific topic (literal \Q match), or "" if +# absent. The literal match is what a restore of the colliding pair must honour. +topic_partitions() { + local app="$1" topic="$2" + TOPIC="$topic" kafka_run "$app" ' + line=$("$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --describe --topic "\Q$TOPIC\E" 2>/dev/null | head -1) || exit 0 + [ -n "$line" ] || exit 0 + printf "%s" "$line" | grep -oE "PartitionCount: [0-9]+" | awk "{print \$2}" + ' | tr -d '\r\n' +} + +# Print " " for the demo topic, or "" if absent. +topic_meta() { + local app="$1" + kafka_run "$app" ' + line=$("$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --describe --topic "\Q$TOPIC\E" 2>/dev/null | head -1) || exit 0 + [ -n "$line" ] || exit 0 + parts=$(printf "%s" "$line" | grep -oE "PartitionCount: [0-9]+" | awk "{print \$2}") + ret=$(printf "%s" "$line" | grep -oE "retention.ms=[0-9]+" | head -1 | cut -d= -f2) + printf "%s %s\n" "$parts" "$ret" + ' | tr -d '\r' +} diff --git a/examples/backups/kafka-metadata/03-backupclass.yaml b/examples/backups/kafka-metadata/03-backupclass.yaml new file mode 100644 index 0000000000..81e4341d6a --- /dev/null +++ b/examples/backups/kafka-metadata/03-backupclass.yaml @@ -0,0 +1,20 @@ +# Maps Kind=Kafka to this demo's Kafka strategy (kafka-strategy-default, derived +# at run time from the shipped cozy-default-kafka by run-all.sh). BackupClass is +# cluster-scoped, so a single instance covers every namespace. Named +# kafka-metadata so it does not collide with the platform cozy-default class, +# which routes Kafka to the shipped cozy-default-kafka strategy against the +# system bucket's external endpoint. +--- +apiVersion: backups.cozystack.io/v1alpha1 +kind: BackupClass +metadata: + name: kafka-metadata +spec: + strategies: + - application: + apiGroup: apps.cozystack.io + kind: Kafka + strategyRef: + apiGroup: strategy.backups.cozystack.io + kind: Kafka + name: kafka-strategy-default diff --git a/examples/backups/kafka-metadata/05-kafka-src.yaml b/examples/backups/kafka-metadata/05-kafka-src.yaml new file mode 100644 index 0000000000..041a8dc261 --- /dev/null +++ b/examples/backups/kafka-metadata/05-kafka-src.yaml @@ -0,0 +1,21 @@ +# Source Kafka application. No backup: block — the platform cozy-default flow +# carries the storage coordinates; the driver reads topic metadata through the +# Kafka Admin API and needs nothing from the app spec. The demo topic is seeded +# via the CLI in run-all.sh (with a retention.ms sentinel), so no chart topics +# are declared here. +apiVersion: apps.cozystack.io/v1alpha1 +kind: Kafka +metadata: + name: kafka-meta-src + namespace: tenant-root +spec: + external: false + kafka: + replicas: 1 + size: 2Gi + resourcesPreset: "c1.small" + zookeeper: + replicas: 1 + size: 1Gi + resourcesPreset: "c1.small" + topics: [] diff --git a/examples/backups/kafka-metadata/10-backupjob-adhoc.yaml b/examples/backups/kafka-metadata/10-backupjob-adhoc.yaml new file mode 100644 index 0000000000..0e3e5eef88 --- /dev/null +++ b/examples/backups/kafka-metadata/10-backupjob-adhoc.yaml @@ -0,0 +1,15 @@ +# Ad-hoc BackupJob routed to this demo's kafka-metadata BackupClass, which maps +# apps.cozystack.io/Kafka to the demo kafka-strategy-default (derived from the +# shipped cozy-default-kafka by run-all.sh, targeting an in-cluster S3 endpoint). +# The produced Backup is named after this BackupJob. +apiVersion: backups.cozystack.io/v1alpha1 +kind: BackupJob +metadata: + name: kafka-meta-src-adhoc + namespace: tenant-root +spec: + applicationRef: + apiGroup: apps.cozystack.io + kind: Kafka + name: kafka-meta-src + backupClassName: kafka-metadata diff --git a/examples/backups/kafka-metadata/15-plan.yaml b/examples/backups/kafka-metadata/15-plan.yaml new file mode 100644 index 0000000000..d471496ed6 --- /dev/null +++ b/examples/backups/kafka-metadata/15-plan.yaml @@ -0,0 +1,16 @@ +# A scheduled Plan: the Plan controller stamps a BackupJob per cron tick against +# this demo's kafka-metadata BackupClass. Shown for completeness; run-all.sh +# drives the ad-hoc BackupJob above instead, so it does not wait on the cron. +apiVersion: backups.cozystack.io/v1alpha1 +kind: Plan +metadata: + name: kafka-meta-src-daily + namespace: tenant-root +spec: + applicationRef: + apiGroup: apps.cozystack.io + kind: Kafka + name: kafka-meta-src + backupClassName: kafka-metadata + schedule: + cron: "0 3 * * *" diff --git a/examples/backups/kafka-metadata/20-kafka-target.yaml b/examples/backups/kafka-metadata/20-kafka-target.yaml new file mode 100644 index 0000000000..7642f81bf8 --- /dev/null +++ b/examples/backups/kafka-metadata/20-kafka-target.yaml @@ -0,0 +1,19 @@ +# A separate, empty Kafka application for the to-copy restore — the "bootstrap a +# fresh cluster and apply the metadata onto it" scenario. No topics: the restore +# recreates them from the backup. +apiVersion: apps.cozystack.io/v1alpha1 +kind: Kafka +metadata: + name: kafka-meta-target + namespace: tenant-root +spec: + external: false + kafka: + replicas: 1 + size: 2Gi + resourcesPreset: "c1.small" + zookeeper: + replicas: 1 + size: 1Gi + resourcesPreset: "c1.small" + topics: [] diff --git a/examples/backups/kafka-metadata/25-restorejob-in-place.yaml b/examples/backups/kafka-metadata/25-restorejob-in-place.yaml new file mode 100644 index 0000000000..ef8d5413b9 --- /dev/null +++ b/examples/backups/kafka-metadata/25-restorejob-in-place.yaml @@ -0,0 +1,11 @@ +# In-place restore: re-apply the backed-up topic metadata into the SAME source +# application. Additive — recreates topics that are absent and (re)sets their +# configs; it does not delete topics that exist live but not in the backup. +apiVersion: backups.cozystack.io/v1alpha1 +kind: RestoreJob +metadata: + name: kafka-meta-src-inplace + namespace: tenant-root +spec: + backupRef: + name: kafka-meta-src-adhoc diff --git a/examples/backups/kafka-metadata/30-restorejob-to-copy.yaml b/examples/backups/kafka-metadata/30-restorejob-to-copy.yaml new file mode 100644 index 0000000000..ac2efc23fe --- /dev/null +++ b/examples/backups/kafka-metadata/30-restorejob-to-copy.yaml @@ -0,0 +1,17 @@ +# To-copy restore: apply the source's backed-up topic metadata onto a separate +# target application, leaving the source running. This is the "bootstrap an empty +# cluster, then apply the metadata state" flow. The strategy renders against the +# target (its bootstrap) but reads the exact object the source backup recorded in +# its status.artifact.uri, which the driver injects as ARTIFACT_URI. +apiVersion: backups.cozystack.io/v1alpha1 +kind: RestoreJob +metadata: + name: kafka-meta-src-to-target + namespace: tenant-root +spec: + backupRef: + name: kafka-meta-src-adhoc + targetApplicationRef: + apiGroup: apps.cozystack.io + kind: Kafka + name: kafka-meta-target diff --git a/examples/backups/kafka-metadata/README.md b/examples/backups/kafka-metadata/README.md new file mode 100644 index 0000000000..95fefef8a8 --- /dev/null +++ b/examples/backups/kafka-metadata/README.md @@ -0,0 +1,44 @@ +# Kafka topic-metadata backup/restore demo + +This demo backs up and restores the **topic metadata** of a Cozystack-managed `Kafka` application and proves integrity by round-tripping a per-run `retention.ms` sentinel through object storage. It runs a demo `BackupClass` + strategy that `run-all.sh` derives from the platform-shipped `cozy-default-kafka` (so the driver script stays a single source of truth), pointed at an S3 endpoint the in-cluster backup Pod can actually reach — the shipped strategy's advertised external ingress is not routable from inside every cluster. + +## What is backed up + +Strimzi ships no backup CRD and this strategy deliberately does not touch message data. The `cozy-default-kafka` strategy exports every non-internal topic's **definition** — partition count, replication factor, and non-default topic configs (retention, compaction, `min.insync.replicas`, ...) — through the Kafka Admin API (`kafka-topics --describe`), stores it in the system bucket, and restores it with `kafka-topics --create` + `kafka-configs --alter`. + +Out of scope: **message payloads** (a volume-snapshot strategy, not this one), **consumer-group offsets** (a log position, only meaningful with a byte-identical data restore), **ACLs and quotas**, and **KafkaUsers**. There is no point-in-time recovery. + +Restore is additive at the topic-set level — it recreates absent topics and (re)sets their configs, and never deletes topics that are live but not in the backup — but it does **not** silently accept a divergent topic. For a topic that already exists, restore compares the live shape against the backup: it grows the partition count with `--alter` when the backup asks for more, and **fails loudly** (non-zero, naming the topic and both values) when the backup asks for fewer partitions or a different replication factor, because Kafka cannot shrink partitions or change RF in place and a restore that cannot reach the recorded state must not report `Succeeded`. Restored topics are created directly through the Admin API, so on the target they are **unmanaged** (no `KafkaTopic` CR) — the accepted tradeoff for an Admin-API-only driver. This is the "bootstrap an empty cluster and apply the metadata state onto it" flow. + +**Topics owned by a `KafkaTopic` CR.** If a topic is declared in the Kafka app's `spec.topics`, the Strimzi Topic Operator owns it and reconciles its config from the CR on every pass — it will revert a dynamic config that the Admin-API restore set but the CR does not list. So an in-place config restore is durable only for **out-of-band** topics (created directly, no CR); for CR-declared topics the `KafkaTopic` CR / GitOps is the source of truth and the restore path. A to-copy restore onto a fresh empty cluster is unaffected, because the target has no `KafkaTopic` CRs. + +## Prerequisites + +The platform default backups stack must be installed: the `backupstrategy-controller`, the `cozy-backups` bucket it provisions, and the `cozy-default-kafka` strategy it ships (the demo strategy is derived from it and reuses its bucket coordinates + the controller-projected `cozy-backups-creds`). No demo `Bucket` is created. Set `S3_ENDPOINT` to reach an in-cluster, TLS-verifiable endpoint when the advertised external ingress is not routable (CI uses `https://seaweedfs-s3.tenant-root.svc:8333`, and `run-all.sh` copies the self-signed seaweedfs CA so `curl` verifies it); leave it unset on a cluster whose external S3 endpoint resolves and uses a public CA. + +## Flow + +- `00-helpers.sh` — shared bash helpers (waiters, kafka seed/verify via a long-lived CLI pod it execs into, the demo strategy derivation + S3 CA copy). +- `03-backupclass.yaml` — the demo `BackupClass` (`kafka-metadata`) mapping `Kafka` to the derived `kafka-strategy-default`. +- `05-kafka-src.yaml` — the source application (no `backup:` block; the strategy carries the coordinates). +- `10-backupjob-adhoc.yaml` — an ad-hoc `BackupJob` routed to the demo `kafka-metadata` BackupClass. +- `15-plan.yaml` — a cron `Plan` for scheduled backups. +- `20-kafka-target.yaml` — a separate, empty application for restore-to-copy. +- `25-restorejob-in-place.yaml` / `30-restorejob-to-copy.yaml` — the two restore flows. + +Run the whole round-trip: + +```bash +./run-all.sh # backup, in-place restore, then restore into a fresh copy +./cleanup.sh # remove this demo's objects (idempotent) +``` + +`SKIP_RESTORE=1 ./run-all.sh` stops after a successful backup (backup-only smoke). Override `NAMESPACE` (default `tenant-root`) to run elsewhere. + +## Limitations + +- **RF is the source's, applied as-is.** Restoring into a cluster with fewer brokers than a topic's replication factor fails at `--create`, and a to-copy restore onto a target with a different broker count aborts on the first replication-factor mismatch. The demo uses single-broker clusters (RF 1). +- **Restore is not transactional.** Topics are applied one at a time as the object is read, so a failure part-way through (an RF mismatch, an unreachable broker) leaves the topics already created behind. Re-running the restore is safe — existing topics are reconciled, not recreated. +- **Compacted/transactional specifics.** Only the topic definition and its non-default configs are captured; nothing about message contents or transaction state. +- **"Non-internal" is Kafka's own definition.** The backup excludes Kafka's internal topics (`__consumer_offsets`, `__transaction_state`, `__share_group_state`) but not operator bookkeeping topics that are not `__`-prefixed — `strimzi-store-topic` and friends are listed by the Admin API and captured. They are harmless to recreate (the operator owns them), but a to-copy restore will materialise them on the target. +- **Latest-only per key.** Each backup writes `s3://////kafka-metadata.txt` — distinct backups do not overwrite each other, and a restore reads the exact object its `Backup` recorded. The object is deleted from the bucket when its `Backup` is deleted (best-effort, via a one-shot Job the removal waits on), so a retention-pruned `Plan` does not accumulate objects. diff --git a/examples/backups/kafka-metadata/cleanup.sh b/examples/backups/kafka-metadata/cleanup.sh new file mode 100755 index 0000000000..ae49f71906 --- /dev/null +++ b/examples/backups/kafka-metadata/cleanup.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Remove everything the demo created. Idempotent (--ignore-not-found). +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Cleanup Kafka metadata backup demo" + +kubectl -n "$NAMESPACE" delete restorejob "$RESTOREJOB_TOCOPY_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete restorejob "$RESTOREJOB_INPLACE_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete plan "$PLAN_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete backupjob "$BACKUPJOB_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete backup "$BACKUPJOB_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete kafka.apps.cozystack.io "$KAFKA_TARGET_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete kafka.apps.cozystack.io "$KAFKA_SRC_NAME" --ignore-not-found +# Cluster-scoped demo objects + the copied CA (the shipped cozy-default-kafka is +# never touched). +kubectl delete backupclass.backups.cozystack.io "$BACKUPCLASS_NAME" --ignore-not-found +kubectl delete kafka.strategy.backups.cozystack.io "$STRATEGY_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete secret "$CA_SECRET" --ignore-not-found +kubectl -n "$NAMESPACE" delete pod -l cozystack.io/backup-demo=kafka-metadata --grace-period=1 --ignore-not-found + +log_success "Cleanup complete." diff --git a/examples/backups/kafka-metadata/run-all.sh b/examples/backups/kafka-metadata/run-all.sh new file mode 100755 index 0000000000..cc2d7626ee --- /dev/null +++ b/examples/backups/kafka-metadata/run-all.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# Convenience runner + e2e harness for the Kafka topic-metadata backup/restore +# demo. It applies the same numbered manifests a human reads, so the documented +# flow and the automated test cannot drift. Stops on the first failure. +# +# Flow (demo `kafka-metadata` BackupClass -> `kafka-strategy-default`, derived at +# run time from the shipped `cozy-default-kafka`; no per-demo Bucket): +# source Kafka -> seed topic `orders` with a distinctive retention.ms sentinel, +# plus a colliding pair (`audit.events` / `audit-events`) with distinct +# partition counts -> ad-hoc BackupJob against the demo `kafka-metadata` +# BackupClass (wait Succeeded) +# -> in-place: drop the topics, restore, assert `orders` came back with the +# sentinel and each colliding topic with its own partition count +# -> to-copy: bootstrap an empty target Kafka, restore the metadata onto it, +# assert the topic + sentinel landed there while the source stays intact. +# +# Data integrity is proven at the METADATA layer: the retention.ms sentinel is a +# per-run value, so a restore that recreated a bare `orders` (wrong partitions or +# default config) fails the assertion; the colliding pair proves the strategy +# treats --topic as a literal, not a regex. Message payloads are out of scope. +# +# Runs in tenant-root: the numbered manifests pin that namespace and the system +# bucket credentials live there; see 00-helpers.sh. +# hack/e2e-chainsaw/kafka-metadata/ drives this file as kafka-3-metadata-roundtrip. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +# Per-run sentinel: a unique retention.ms (ms since epoch) so a passing verify +# proves this run's config round-tripped, not a leftover topic. +RETENTION="$(date +%s)000" + +print_header "Step 05: Deploy source Kafka '${KAFKA_SRC_NAME}'" +kubectl apply -f "$SCRIPT_DIR/05-kafka-src.yaml" +wait_hr_ready "kafka-${KAFKA_SRC_NAME}" 300 +kafka_wait_ready "$KAFKA_SRC_NAME" 600 + +print_header "Step 05a: Derive the demo Kafka strategy '${STRATEGY_NAME}' + BackupClass '${BACKUPCLASS_NAME}'" +# Wait for the strategy rather than read it once. The shipped cozy-default-kafka +# is rendered behind a lookup of the platform bucket, so it exists only once a +# Helm upgrade has run after that bucket was provisioned. The controller forces +# that upgrade, but helm-controller holds it while any release in its dependsOn +# is not Ready, so a platform change made just before this runs can keep it +# absent for minutes. Both the endpoint read below and provision_demo_strategy +# derive from it, so wait once here; kubectl wait keeps polling through NotFound +# and fails on any other error. +log_substep "Waiting for the platform's cozy-default-kafka strategy..." +kubectl wait --for=create kafka.strategy.backups.cozystack.io/cozy-default-kafka \ + --timeout=15m >/dev/null \ + || { log_error "cozy-default-kafka strategy did not appear: enable the platform default backups (backupstrategy-controller) before running this demo"; exit 1; } + +# The shipped cozy-default-kafka advertises the platform's EXTERNAL S3 ingress, +# which in-cluster Pods cannot resolve or TLS-validate in CI. Derive a demo +# strategy from it (so the driver script stays a single source of truth) pointed +# at an in-cluster, verifiable endpoint. With an S3_ENDPOINT override, copy the +# self-signed seaweedfs CA and mount it (CURL_CA_BUNDLE); with none, reuse the +# shipped endpoint and skip the CA mount (its cert is publicly trusted, and +# CURL_CA_BUNDLE would REPLACE the system bundle rather than extend it). +if [[ -n "$S3_ENDPOINT" ]]; then + CA_PRESENT=$(copy_s3_ca) +else + S3_ENDPOINT=$(kubectl get kafka.strategy.backups.cozystack.io cozy-default-kafka -o json \ + | jq -r '.spec.template.spec.containers[0].env[] | select(.name=="S3_ENDPOINT") | .value') + [[ -n "$S3_ENDPOINT" ]] || { log_error "shipped cozy-default-kafka has no S3_ENDPOINT; is the platform backup stack installed?"; exit 1; } + CA_PRESENT=0 +fi +provision_demo_strategy "$S3_ENDPOINT" "$CA_PRESENT" +log_success "Demo strategy at endpoint '${S3_ENDPOINT}' (CA mounted: ${CA_PRESENT})" + +print_header "Step 05b: Seed topic '${TOPIC}' (${PARTITIONS} partitions, retention.ms=${RETENTION})" +seed_topic "$KAFKA_SRC_NAME" "$RETENTION" +got=$(topic_meta "$KAFKA_SRC_NAME") +[[ "$got" == "${PARTITIONS} ${RETENTION}" ]] || { log_error "seed verify failed: got '${got}', want '${PARTITIONS} ${RETENTION}'"; exit 1; } +log_success "Seeded '${TOPIC}': ${got}" + +print_header "Step 05c: Seed colliding pair '${COLLIDE_DOT}' (${COLLIDE_DOT_PARTS}p) + '${COLLIDE_DASH}' (${COLLIDE_DASH_PARTS}p)" +# These names collide under Kafka's --topic regex ("audit.events" matches +# "audit-events"). Distinct partition counts make a backup that does not treat +# the name literally record the wrong shape, so the restore verify below fails. +seed_topic_partitions "$KAFKA_SRC_NAME" "$COLLIDE_DOT" "$COLLIDE_DOT_PARTS" +seed_topic_partitions "$KAFKA_SRC_NAME" "$COLLIDE_DASH" "$COLLIDE_DASH_PARTS" +gotdot=$(topic_partitions "$KAFKA_SRC_NAME" "$COLLIDE_DOT") +gotdash=$(topic_partitions "$KAFKA_SRC_NAME" "$COLLIDE_DASH") +[[ "$gotdot" == "$COLLIDE_DOT_PARTS" && "$gotdash" == "$COLLIDE_DASH_PARTS" ]] \ + || { log_error "collision seed verify failed: ${COLLIDE_DOT}=${gotdot} (want ${COLLIDE_DOT_PARTS}), ${COLLIDE_DASH}=${gotdash} (want ${COLLIDE_DASH_PARTS})"; exit 1; } +log_success "Seeded colliding pair: ${COLLIDE_DOT}=${gotdot} ${COLLIDE_DASH}=${gotdash}" + +print_header "Step 10: Submit ad-hoc BackupJob '${BACKUPJOB_NAME}' and wait for Succeeded" +kubectl apply -f "$SCRIPT_DIR/10-backupjob-adhoc.yaml" +wait_for_field backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 600 Failed +BACKUP_NAME=$(kubectl -n "$NAMESPACE" get backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}') +[[ -n "$BACKUP_NAME" ]] || { log_error "BackupJob succeeded but reported no backupRef"; exit 1; } +log_success "Backup artefact: ${BACKUP_NAME}" + +if [[ "${SKIP_RESTORE:-0}" == "1" ]]; then + log_warning "SKIP_RESTORE=1: stopping after a successful backup." + exit 0 +fi + +print_header "Step 25: In-place restore — drop two topics, keep '${COLLIDE_DOT}' live, then restore" +# Drop orders + audit-events but LEAVE audit.events live, so the restore runs +# the existing-topic branch (grep -qxF + --describe/--alter under \Q) for a name +# whose regex would collide with audit-events. A stripped \Q on the restore +# --describe would match the sibling, read the wrong partition count and fail the +# restore here — so this path, not just --create, is exercised end to end. +delete_topic "$KAFKA_SRC_NAME" "$TOPIC" +delete_topic "$KAFKA_SRC_NAME" "$COLLIDE_DASH" +kubectl apply -f "$SCRIPT_DIR/25-restorejob-in-place.yaml" +wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_INPLACE_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 600 Failed +got=$(topic_meta "$KAFKA_SRC_NAME") +[[ "$got" == "${PARTITIONS} ${RETENTION}" ]] || { log_error "in-place restore verify failed: got '${got}', want '${PARTITIONS} ${RETENTION}'"; exit 1; } +# The colliding pair must each come back with their OWN partition count. If the +# backup treated --topic as a regex, one recorded the other's shape and this +# fails. +gotdot=$(topic_partitions "$KAFKA_SRC_NAME" "$COLLIDE_DOT") +gotdash=$(topic_partitions "$KAFKA_SRC_NAME" "$COLLIDE_DASH") +[[ "$gotdot" == "$COLLIDE_DOT_PARTS" && "$gotdash" == "$COLLIDE_DASH_PARTS" ]] \ + || { log_error "collision restore verify failed: ${COLLIDE_DOT}=${gotdot} (want ${COLLIDE_DOT_PARTS}), ${COLLIDE_DASH}=${gotdash} (want ${COLLIDE_DASH_PARTS})"; exit 1; } +log_success "In-place restore verified: '${TOPIC}' back with ${got}; pair ${COLLIDE_DOT}=${gotdot} ${COLLIDE_DASH}=${gotdash}" + +print_header "Step 20/30: To-copy restore into a fresh '${KAFKA_TARGET_NAME}'" +kubectl apply -f "$SCRIPT_DIR/20-kafka-target.yaml" +wait_hr_ready "kafka-${KAFKA_TARGET_NAME}" 300 +kafka_wait_ready "$KAFKA_TARGET_NAME" 600 +kubectl apply -f "$SCRIPT_DIR/30-restorejob-to-copy.yaml" +wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_TOCOPY_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 600 Failed +got=$(topic_meta "$KAFKA_TARGET_NAME") +[[ "$got" == "${PARTITIONS} ${RETENTION}" ]] || { log_error "to-copy restore verify failed on target: got '${got}', want '${PARTITIONS} ${RETENTION}'"; exit 1; } +log_success "To-copy restore verified: '${KAFKA_TARGET_NAME}' has '${TOPIC}' with ${got}" + +# To-copy must not mutate the source. +src=$(topic_meta "$KAFKA_SRC_NAME") +[[ "$src" == "${PARTITIONS} ${RETENTION}" ]] || { log_error "source changed after to-copy restore: got '${src}'"; exit 1; } +log_success "Source '${KAFKA_SRC_NAME}' left intact by the to-copy restore." diff --git a/examples/backups/kafka/00-helpers.sh b/examples/backups/kafka/00-helpers.sh new file mode 100755 index 0000000000..09b03fd6ce --- /dev/null +++ b/examples/backups/kafka/00-helpers.sh @@ -0,0 +1,338 @@ +#!/bin/bash +# Shared helpers for the Kafka topic-data backup/restore demo. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export BLUE='\033[0;34m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export WHITE='\033[1;37m' +export NC='\033[0m' +export BOLD='\033[1m' + +# Default settings (override via environment). +export NAMESPACE="${NAMESPACE:-tenant-root}" +export KAFKA_NAME="${KAFKA_NAME:-kafka-test}" +export KAFKA_RESTORE_NAME="${KAFKA_RESTORE_NAME:-kafka-restore}" +# Two brokers, not one: the Cozystack Kafka chart raises the broker-level +# min.insync.replicas to 2 as soon as kafka.replicas reaches 2, and every +# acks=all write into a topic with fewer replicas than that is rejected. A +# single-broker demo could never show that a restore recreates the topic with +# the replication factor the backup captured; the demo topics are seeded with +# TOPIC_REPLICAS replicas and the restore steps assert it survived the +# round-trip. Both Kafka instances use KAFKA_REPLICAS so the to-copy target can +# host the same replication factor. +export KAFKA_REPLICAS="${KAFKA_REPLICAS:-2}" +export TOPIC_REPLICAS="${TOPIC_REPLICAS:-2}" +# The demo topic carries a "." and a decoy topic differs from it only where +# that "." sits: as a Java regex "orders.v1" also matches "orders-v1", as a +# literal it does not. Every --topic call the CLI treats as a regex is pinned +# with \Q...\E, and this pair is what makes that pinning testable - drop a pin +# and the run fails on its own (the partition-set guard sees the decoy's +# partitions, or the in-place delete takes the decoy with it). +# +# What the decoy reliably proves: the offset listing (the partition-set guard +# counts its partitions), the in-place delete (it disappears), and the two +# record counts (they pick up its records). It does NOT prove the two +# PartitionCount reads: kafka-topics --describe yields topics in the admin +# client result-map order rather than sorted, so which one their head -1 sees +# when a pin is missing is not something to lean on. +export TOPIC="${TOPIC:-orders.v1}" +export DECOY_TOPIC="${DECOY_TOPIC:-orders-v1}" +export DECOY_COUNT="${DECOY_COUNT:-5}" +export PARTITIONS="${PARTITIONS:-3}" +export MESSAGE_COUNT="${MESSAGE_COUNT:-30}" +export BUCKET_NAME="${BUCKET_NAME:-kafka-backups}" +export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-kafka-backup}" +export STRATEGY_NAME="${STRATEGY_NAME:-kafka-job}" +export BACKUPJOB_NAME="${BACKUPJOB_NAME:-kafka-backup-job}" +export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-kafka-restore-inplace}" +export RESTOREJOB_NONEMPTY_NAME="${RESTOREJOB_NONEMPTY_NAME:-kafka-restore-nonempty}" +export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-kafka-restore-to-copy}" +# S3 endpoint CA. Cozystack's default seaweedfs serves its S3 endpoint with a +# self-signed certificate whose CA lives in this Secret; 03-create-bucket.sh +# caches its ca.crt and create_s3_secret projects it into each +# "-backup-s3" Secret, which the strategy Pod mounts and points curl at. +# The name follows the seaweedfs chart's fullnameOverride ("seaweedfs" -> +# "seaweedfs-ca-cert"); 03-create-bucket.sh auto-discovers the CA Certificate's +# actual secret when this default is absent. On a cluster whose S3 endpoint is +# signed by a publicly-trusted CA, set S3_CA_SECRET="" to skip the copy: the +# Pod then verifies against the image's trust store. +export S3_CA_SECRET="${S3_CA_SECRET:-seaweedfs-ca-cert}" +export S3_CA_NAMESPACE="${S3_CA_NAMESPACE:-tenant-root}" +export S3_CA_KEY="${S3_CA_KEY:-ca.crt}" +# The Strimzi Kafka image carries the full kafka-*.sh CLI plus bash, curl and +# tar - everything the generic Job strategy and these host-side helpers need, +# with no purpose-built backup image. +# +# Default to the image the operator already caches on the nodes so the backup +# Pod and these CLI Pods reuse a warm image with no version skew. The operator's +# STRIMZI_KAFKA_IMAGES env is a newline-separated "version=image" map (one entry +# per supported version); split on whitespace, drop the "version=" prefix and +# take the newest (last) entry - the version the operator itself defaults to. +# The image reference is taken as-is: a digest-pinned operator emits +# "=@sha256:..." with no ":", and that is exactly the +# air-gapped deployment the fallback below cannot serve. +# Fall back to a pinned literal only when the operator cannot be read (an +# air-gapped clone with no cluster), so the demo still has a runnable default. +# Setting KAFKA_IMAGE in the environment skips the lookup entirely. +resolve_kafka_image() { + local resolved + resolved=$(kubectl -n cozy-kafka-operator get deploy strimzi-cluster-operator \ + -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="STRIMZI_KAFKA_IMAGES")].value}' 2>/dev/null \ + | tr '[:space:]' '\n' | grep '=' | sed 's/^[^=]*=//' | tail -n1) + if [ -n "$resolved" ]; then + echo "$resolved" + return + fi + # Do not fail open silently: warn on stderr (stdout is captured as the + # image) that the operator could not be read and a public quay.io literal + # is being assumed. On an air-gapped or operator-less cluster - the persona + # the README's override note is for - that literal is unreachable, so the + # run must set KAFKA_IMAGE to an image the nodes can actually pull. + echo "! could not read STRIMZI_KAFKA_IMAGES from deploy/strimzi-cluster-operator in cozy-kafka-operator; assuming quay.io/strimzi/kafka:0.45.1-rc1-kafka-3.9.1. Set KAFKA_IMAGE explicitly if that image is wrong or unreachable (e.g. air-gapped)." >&2 + echo "quay.io/strimzi/kafka:0.45.1-rc1-kafka-3.9.1" +} +export KAFKA_IMAGE="${KAFKA_IMAGE:-$(resolve_kafka_image)}" +export KAFKA_BIN="${KAFKA_BIN:-/opt/kafka/bin}" +# Name of the long-lived CLI Pod kafka_run execs into; cleanup.sh removes it. +# A variable of its own, not the kafka-metadata demo's KAFKA_CLI_POD: the two +# demos may share a namespace, and one setting must not move the other's Pod. +export KAFKA_TOPIC_CLI_POD="${KAFKA_TOPIC_CLI_POD:-kafka-topic-cli}" + +log_info() { echo -e "${BLUE}i${NC} $*" >&2; } +log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; } +log_error() { echo -e "${RED}x${NC} $*" >&2; } +log_step() { echo -e "\n${MAGENTA}${BOLD}> $*${NC}" >&2; } +log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; } +log_command() { echo -e "${WHITE} $ $*${NC}" >&2; } + +separator() { + echo -e "\n${CYAN}------------------------------------------------------------${NC}\n" >&2 +} + +print_header() { + local title="$1" + echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2 +} + +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" + +# In-cluster bootstrap address for a Kafka application instance. The Cozystack +# chart names the Strimzi Kafka cluster "kafka-", and Strimzi names the +# plaintext bootstrap Service "-kafka-bootstrap" - so for an app named +# the Service is "kafka--kafka-bootstrap". The internal plain +# listener on 9092 has no TLS or auth, so a bare host:port is all the CLI needs. +kafka_bootstrap() { + local app="$1" + echo "kafka-${app}-kafka-bootstrap.${NAMESPACE}.svc:9092" +} + +# Ensure the long-lived CLI Pod exists and is Ready, so kafka_run can exec into +# it. A Ready Pod this demo owns is reused across calls and across the numbered +# scripts; a leftover in a terminal phase is replaced; a same-named Pod the demo +# does not own is refused rather than hijacked or deleted. +# Every step returns on failure explicitly: this runs as the left side of +# `|| return 1`, and some callers also wrap it in $(...), so errexit never +# applies inside it and a bare failure would carry on to the next step. +kafka_cli_pod() { + local phase owner overrides + phase=$(kubectl -n "$NAMESPACE" get pod "$KAFKA_TOPIC_CLI_POD" --ignore-not-found -o jsonpath='{.status.phase}') || return 1 + owner=$(kubectl -n "$NAMESPACE" get pod "$KAFKA_TOPIC_CLI_POD" --ignore-not-found -o jsonpath='{.metadata.labels.cozystack\.io/backup-demo}') || return 1 + if [ -n "$phase" ] && [ "$owner" != "kafka" ]; then + log_error "Pod $NAMESPACE/$KAFKA_TOPIC_CLI_POD exists but this demo does not own it; refusing to use or delete it" + return 1 + fi + if [ "$phase" != "Running" ] && [ "$phase" != "Pending" ]; then + # restricted-clean Pod. The Strimzi image runs as a non-root numeric + # UID (1001), so runAsNonRoot/seccomp at the Pod level plus the + # container's allowPrivilegeEscalation=false and drop-ALL satisfy PSA + # "restricted" without pinning runAsUser to a value specific to this + # image tag. An add-only JSON patch layers these onto the run-generated + # Pod's sole container (index 0), leaving image, command and args + # untouched. + overrides='[{"op":"add","path":"/spec/securityContext","value":{"runAsNonRoot":true,"seccompProfile":{"type":"RuntimeDefault"}}},{"op":"add","path":"/spec/containers/0/securityContext","value":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}}]' + kubectl -n "$NAMESPACE" delete pod "$KAFKA_TOPIC_CLI_POD" --grace-period=1 --ignore-not-found >/dev/null || return 1 + kubectl -n "$NAMESPACE" run "$KAFKA_TOPIC_CLI_POD" --image="$KAFKA_IMAGE" \ + --labels=cozystack.io/backup-demo=kafka \ + --override-type=json --overrides="$overrides" \ + --restart=Never --command -- sleep infinity >/dev/null || return 1 + fi + kubectl -n "$NAMESPACE" wait --for=condition=Ready "pod/$KAFKA_TOPIC_CLI_POD" \ + --timeout=5m >/dev/null +} + +# Run a bash snippet in the demo's Strimzi Kafka CLI Pod. The snippet runs with +# these variables pre-set, so it needs no nested shell quoting of its own: +# $BOOT - the target app's plaintext bootstrap (host:port) +# $BIN - the kafka CLI directory +# $TOPIC, $PARTITIONS, $TOPIC_REPLICAS, $MESSAGE_COUNT - the demo knobs +# This is the host-side analogue of the strategy Pod: same stock image, same +# tools, no purpose-built backup container. Pass the snippet single-quoted so +# its own $VAR references reach the Pod's bash unexpanded. +# +# The snippet runs by `kubectl exec`, not in a throwaway `kubectl run -i` Pod. +# That Pod's stdout comes back over an attach carrying only what the container +# writes after the attach registers, so a CLI that finishes first reads as an +# empty reply with exit 0. An exec'd process owns its pipes, so its output +# cannot be missed that way. +kafka_run() { + local app="$1"; shift + local snippet="$1" + local boot + boot="$(kafka_bootstrap "$app")" + kafka_cli_pod || return 1 + kubectl -n "$NAMESPACE" exec -i "$KAFKA_TOPIC_CLI_POD" -- bash -c "set -eu +BOOT=$(printf %q "$boot") +BIN=$(printf %q "$KAFKA_BIN") +TOPIC=$(printf %q "$TOPIC") +PARTITIONS=$(printf %q "$PARTITIONS") +TOPIC_REPLICAS=$(printf %q "$TOPIC_REPLICAS") +MESSAGE_COUNT=$(printf %q "$MESSAGE_COUNT") +DECOY_TOPIC=$(printf %q "$DECOY_TOPIC") +DECOY_COUNT=$(printf %q "$DECOY_COUNT") +$snippet" +} + +# Create the demo topic (idempotent) and publish MESSAGE_COUNT keyed sentinel +# messages spread across PARTITIONS. Keys ("k-") make partition placement +# deterministic on restore: re-producing the same key into a topic with the +# same partition count lands it in the same partition via the default +# (murmur2) partitioner - no per-partition produce needed for keyed records. +seed_topic() { + local app="$1" + kafka_run "$app" ' + "$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --create --if-not-exists \ + --topic "$TOPIC" --partitions "$PARTITIONS" --replication-factor "$TOPIC_REPLICAS" + i=1 + while [ "$i" -le "$MESSAGE_COUNT" ]; do + printf "k-%s\torder-%s\n" "$i" "$i" + i=$((i + 1)) + done | "$BIN"/kafka-console-producer.sh --bootstrap-server "$BOOT" \ + --topic "$TOPIC" --property parse.key=true + ' +} + +# Seed the decoy topic: a single partition and a handful of records, enough to +# tell "still there, untouched" from "deleted" or "backed up by mistake". +seed_decoy_topic() { + local app="$1" + kafka_run "$app" ' + "$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --create --if-not-exists \ + --topic "$DECOY_TOPIC" --partitions 1 --replication-factor "$TOPIC_REPLICAS" + i=1 + while [ "$i" -le "$DECOY_COUNT" ]; do + printf "d-%s\tdecoy-%s\n" "$i" "$i" + i=$((i + 1)) + done | "$BIN"/kafka-console-producer.sh --bootstrap-server "$BOOT" \ + --topic "$DECOY_TOPIC" --property parse.key=true + ' +} + +# Record count of the decoy topic. Prints a bare integer, or "" when the topic +# is gone - which is itself the signal an unpinned --topic regex deleted it. +decoy_message_count() { + topic_message_count "$1" "$DECOY_TOPIC" +} + +# Total number of records currently stored in a topic (the demo topic unless a +# second argument names another), summed over partitions as (end offset - +# begin offset). Prints a bare integer, or "" if the topic does not exist / +# cannot be reached. Offsets, not a consumer, so it is exact and cheap even for +# a compacted or truncated topic. kafka-get-offsets emits +# "topic:partition:offset" lines; pure-shell parsing avoids awk quoting. +topic_message_count() { + local app="$1" + TOPIC="${2:-$TOPIC}" kafka_run "$app" ' + ends=$("$BIN"/kafka-get-offsets.sh --bootstrap-server "$BOOT" --topic "\Q$TOPIC\E" --time -1 2>/dev/null) || exit 0 + begins=$("$BIN"/kafka-get-offsets.sh --bootstrap-server "$BOOT" --topic "\Q$TOPIC\E" --time -2 2>/dev/null) || exit 0 + [ -n "$ends" ] || exit 0 + total=0 + for e in $ends; do + ekey=${e%:*}; eo=${e##*:} + # Default the begin offset to 0 rather than skipping the partition: + # a partition missing from "begins" would otherwise contribute + # nothing and silently undercount the topic. Match on the whole + # topic:partition key so a line from another topic cannot supply + # the begin offset for this partition. Matches the strategy. + bo=0 + for b in $begins; do + if [ "${b%:*}" = "$ekey" ]; then bo=${b##*:}; break; fi + done + total=$((total + eo - bo)) + done + echo "$total" + ' | tr -d '[:space:]' +} + +# Dump every record of the demo topic as "partitionkeyvalue" lines, +# partitions in ascending order and records in offset order within each +# partition. Deterministic, so the same topic dumps byte-identically twice - +# a source dump captured before backup and the restored dump can be diffed to +# prove keys, values, partition placement and per-partition ordering survived +# the round-trip, which a bare offset count cannot. Assumes begin offset 0 +# (every topic in this demo is freshly created); GNU sed renders "\t" as a tab. +topic_dump() { + local app="$1" + kafka_run "$app" ' + ends=$("$BIN"/kafka-get-offsets.sh --bootstrap-server "$BOOT" --topic "\Q$TOPIC\E" --time -1 2>/dev/null) || exit 0 + [ -n "$ends" ] || exit 0 + printf "%s\n" $ends | sort -t: -k2 -n | while IFS=: read -r t p end; do + [ "${end:-0}" -gt 0 ] || continue + "$BIN"/kafka-console-consumer.sh --bootstrap-server "$BOOT" --topic "$TOPIC" \ + --partition "$p" --offset 0 --max-messages "$end" --timeout-ms 60000 \ + --property print.key=true --property print.timestamp=false \ + | sed "s/^/$p\t/" + done + ' +} + +# Replication factor of the demo topic as the broker reports it. Prints a bare +# integer, or "" if the topic does not exist / cannot be reached. The restore +# steps compare it against TOPIC_REPLICAS: the strategy records each topic's +# replication factor in the backup manifest and recreates the topic with it, +# and this is the observable that proves the value survived the round-trip. +topic_replication_factor() { + local app="$1" + kafka_run "$app" ' + desc=$("$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --describe --topic "\Q$TOPIC\E" 2>/dev/null) || exit 0 + printf "%s\n" "$desc" | sed -n "s/.*ReplicationFactor: *\([0-9][0-9]*\).*/\1/p" | head -1 + ' | tr -d '[:space:]' +} + +# Create the "-backup-s3" Secret the Job strategy Pod consumes, from the +# bucket coordinates cached by 03-create-bucket.sh. The generic Job strategy - +# unlike the app-specific drivers - has no chart support to emit this Secret, +# so the demo provides it. Called for the source app (step 04) and the +# restore target (step 07). The S3 endpoint's CA, when 03-create-bucket.sh +# cached one, rides along as ca.crt: the strategy Pod mounts that key and +# verifies the endpoint against it instead of disabling verification. +create_s3_secret() { + local app="$1" + local SCRIPT_DIR + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + [[ -f "$SCRIPT_DIR/.bucket-info.env" ]] || { log_error "missing $SCRIPT_DIR/.bucket-info.env; run 03-create-bucket.sh first"; return 1; } + # shellcheck disable=SC1091 + source "$SCRIPT_DIR/.bucket-info.env" + for v in S3_ACCESS_KEY S3_SECRET_KEY S3_ENDPOINT S3_REGION S3_BUCKET; do + [[ -n "${!v:-}" ]] || { log_error "required variable is missing or empty: ${v}"; return 1; } + done + local ca_args=() + if [[ -n "${S3_CA_B64:-}" ]]; then + ca_args=(--from-literal=ca.crt="$(printf '%s' "$S3_CA_B64" | base64 -d)") + fi + kubectl -n "$NAMESPACE" create secret generic "${app}-backup-s3" \ + --from-literal=accessKey="$S3_ACCESS_KEY" \ + --from-literal=secretKey="$S3_SECRET_KEY" \ + --from-literal=endpoint="$S3_ENDPOINT" \ + --from-literal=region="$S3_REGION" \ + --from-literal=bucket="$S3_BUCKET" \ + ${ca_args[@]+"${ca_args[@]}"} \ + --dry-run=client -o yaml | kubectl apply -f - +} diff --git a/examples/backups/kafka/01-create-strategy.sh b/examples/backups/kafka/01-create-strategy.sh new file mode 100755 index 0000000000..26b8d95d94 --- /dev/null +++ b/examples/backups/kafka/01-create-strategy.sh @@ -0,0 +1,571 @@ +#!/bin/bash +# Step 01: Create the cluster-scoped generic Job backup strategy for Kafka +# topic data. Unlike the app-specific drivers (Altinity, CNPG, ...), the Job +# strategy runs an arbitrary Pod the operator supplies. Here that Pod is a +# stock Strimzi Kafka image (which carries the kafka-*.sh CLI plus bash, curl +# and tar) running a single shell script that branches on `.Mode`: +# backup : freeze each partition's end offset, drain [begin,end) to files, +# tar -> PUT the tarball to S3 +# restore: GET the tarball from S3 -> untar -> recreate the topics -> replay +# every partition file with kafka-console-producer +# +# The consistency model is a frozen-end-offset cut: kafka-get-offsets --time -1 +# is read ONCE per partition, and the consumer drains only up to that offset, +# so everything produced after the read is excluded. The reads happen per +# topic (inside the topic loop), so the cut is atomic across a topic's own +# partitions but NOT across topics - Kafka offers no cross-topic guarantee +# regardless. This is a logical DATA +# backup: topic configs, ACLs, SCRAM users, and consumer-group offsets are NOT +# captured, and restore re-produces records so they receive fresh offsets. +# +# Everything the template injects is funnelled through container env vars so +# the script body stays plain shell. The same PodTemplateSpec serves both +# directions: the engine templates each string leaf independently (it cannot +# add/remove containers per mode), so the one image branches at runtime on +# $MODE. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 01: Create Job backup strategy '${STRATEGY_NAME}'" +log_command "kubectl apply -f - (Job strategy: $STRATEGY_NAME)" + +kubectl apply -f - <", and Strimzi + # exposes its plaintext bootstrap Service as + # "kafka--kafka-bootstrap" on port 9092 (no TLS/auth on the + # internal plain listener). + - name: BOOTSTRAP + value: "kafka-{{ .Release.Name }}-kafka-bootstrap.{{ .Release.Namespace }}.svc:9092" + # S3 object key is scoped by the SOURCE namespace + app name so a + # to-copy restore reads what the source wrote and two same-named + # apps in different namespaces sharing one bucket do not collide on + # /kafka-topics.tar. On backup the source is .Release.Name; on + # restore it is .Backup.ApplicationRef.Name (.Backup is only set on + # restore - the guard keeps backup mode from rendering "" + # into an unused var). The namespace is .Release.Namespace in both + # modes and always names the source: a to-copy target is a + # TypedLocalObjectReference, so it shares the RestoreJob's (source's) + # namespace, and an in-place restore targets the source itself. + - name: SRC_BACKUP + value: "{{ .Release.Name }}" + - name: SRC_RESTORE + value: "{{ if .Backup }}{{ .Backup.ApplicationRef.Name }}{{ end }}" + - name: SRC_NAMESPACE + value: "{{ .Release.Namespace }}" + - name: MODE + value: "{{ .Mode }}" + # S3 coordinates from the demo-provided -backup-s3 Secret + # (created by create_s3_secret in 00-helpers.sh from the Bucket). + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: "{{ .Release.Name }}-backup-s3" + key: accessKey + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: "{{ .Release.Name }}-backup-s3" + key: secretKey + - name: S3_ENDPOINT + valueFrom: + secretKeyRef: + name: "{{ .Release.Name }}-backup-s3" + key: endpoint + - name: S3_REGION + valueFrom: + secretKeyRef: + name: "{{ .Release.Name }}-backup-s3" + key: region + - name: S3_BUCKET + valueFrom: + secretKeyRef: + name: "{{ .Release.Name }}-backup-s3" + key: bucket + # The same Secret's optional ca.crt (the S3 endpoint's CA when it is + # not publicly trusted; create_s3_secret copies the in-cluster + # seaweedfs CA into it) is projected as a file so curl can verify the + # endpoint against it. The key is optional: without it the file is + # absent and curl falls back to the image's trust store. + volumeMounts: + - name: s3-ca + mountPath: /etc/kafka-backup-s3 + readOnly: true + command: ["/usr/bin/bash", "-c"] + args: + - | + set -eu + # Overridable only so hack/kafka-topic-backup-strategy.bats can + # execute this body outside the Pod against stub CLIs; the Pod + # sets none of them. + BIN="\${BIN:-/opt/kafka/bin}" + SCRATCH="\${SCRATCH:-/tmp}" + CA_FILE="\${CA_FILE:-/etc/kafka-backup-s3/ca.crt}" + BOOT="\${BOOTSTRAP}" + + # BucketInfo endpoints are bare host:port; COSI/seaweedfs serves + # S3 over HTTPS, so default to https:// when no scheme is given. + case "\${S3_ENDPOINT}" in + http://*|https://*) BASE="\${S3_ENDPOINT}" ;; + *) BASE="https://\${S3_ENDPOINT}" ;; + esac + + # Split the endpoint into scheme/host/port. The Strimzi image ships + # curl 7.76.1, whose --aws-sigv4 has a bug fixed only in 7.87.0: it + # omits a non-default port from the SigV4 canonical Host even though + # it sends the port in the Host header. seaweedfs recomputes the + # signature from the received "host:port" and so rejects the PUT + # with 403 whenever the endpoint carries a non-default port (the + # e2e harness points S3_ENDPOINT at the :8333 in-cluster Service). + # Strip the port from the signed/sent URL and redirect the + # connection to the real port with --connect-to, so the signed, + # sent and server-side Host all agree on the bare host. This is + # also correct on curl >= 7.87, which drops the default port too. + # Demo-scope parse: handles the "scheme://host[:port]" that + # BucketInfo advertises. An IPv6 literal ([::1]:8333) or an + # endpoint carrying a path prefix would be mis-split - out of + # scope here. + SCHEME="\${BASE%%://*}" + HOSTPORT="\${BASE#*://}"; HOSTPORT="\${HOSTPORT%%/*}" + HOST="\${HOSTPORT%%:*}" + PORT="\${HOSTPORT##*:}" + if [ "\${PORT}" = "\${HOSTPORT}" ]; then PORT=""; fi + if [ "\${SCHEME}" = http ]; then DEFPORT=80; else DEFPORT=443; fi + CONNECT_TO="" + if [ -n "\${PORT}" ] && [ "\${PORT}" != "\${DEFPORT}" ]; then + CONNECT_TO="--connect-to \${HOST}:\${DEFPORT}:\${HOST}:\${PORT}" + fi + + if [ "\${MODE}" = backup ]; then SRC="\${SRC_BACKUP}"; else SRC="\${SRC_RESTORE}"; fi + KEY="\${SRC_NAMESPACE}/\${SRC}/kafka-topics.tar" + OBJ_URL="\${SCHEME}://\${HOST}/\${S3_BUCKET}/\${KEY}" + + # curl --aws-sigv4 signs the request (SigV4) so no separate S3 + # client image is needed. The certificate is verified: against the + # CA projected from the Secret when it carries one, else + # against the image's trust store. \${CA_OPT} and \${CONNECT_TO} + # are deliberately unquoted so an empty value expands to no + # argument. + CA_OPT="" + if [ -s "\${CA_FILE}" ]; then CA_OPT="--cacert \${CA_FILE}"; fi + s3() { curl -fsS \${CA_OPT} \${CONNECT_TO} --aws-sigv4 "aws:amz:\${S3_REGION}:s3" --user "\${AWS_ACCESS_KEY_ID}:\${AWS_SECRET_ACCESS_KEY}" "\$@"; } + + # Records a topic currently holds, summed over its partitions as + # end - begin. kafka-get-offsets prints only the partitions whose + # lookup succeeded and still exits 0, so the listing is held to + # the partition count the caller read off the topic itself: a + # partition lost to a leader election would otherwise drop out of + # the sum and pass a populated topic off as empty. + # + # --topic is a Java regex in kafka-topics and kafka-get-offsets, + # so an unquoted name also matches its siblings ("audit.events" + # matches "audit-events") and would mix another topic's + # partitions into the count. \Q...\E pins it to a literal. + # + # Called as \$(topic_records ...) || exit 1, and a function run + # as the left operand of || has errexit switched off inside it, + # so every command here carries its own || return 1. + topic_records() { + local t=\$1 want_parts=\$2 ends begins seen e ekey eo bo b total + ends=\$("\${BIN}"/kafka-get-offsets.sh --bootstrap-server "\${BOOT}" --topic "\Q\${t}\E" --time -1) || return 1 + begins=\$("\${BIN}"/kafka-get-offsets.sh --bootstrap-server "\${BOOT}" --topic "\Q\${t}\E" --time -2) || return 1 + seen=0 + for e in \${ends}; do seen=\$((seen + 1)); done + if [ "\${seen}" -ne "\${want_parts}" ]; then + echo "offset listing returned \${seen} of \${want_parts} partition(s) for \${t}; refusing to act on an incomplete listing" >&2 + return 1 + fi + total=0 + for e in \${ends}; do + ekey=\${e%:*} + eo=\${e##*:} + # Match the whole topic:partition key, not the partition + # index alone, so a line belonging to another topic cannot + # supply this partition's begin offset. A partition missing + # from "begins" defaults to 0, which can only overstate. + bo=0 + for b in \${begins}; do + if [ "\${b%:*}" = "\${ekey}" ]; then bo=\${b##*:}; break; fi + done + total=\$((total + eo - bo)) + done + echo "\${total}" + } + + WORK="\${SCRATCH}/kafka-dump" + TARBALL="\${SCRATCH}/kafka-topics.tar" + rm -rf "\${WORK}"; mkdir -p "\${WORK}" + + if [ "\${MODE}" = backup ]; then + # Resolve the topic list: explicit TOPICS (comma/space + # separated) or every non-internal topic. Kafka's internal + # topics are double-underscore (__consumer_offsets, + # __transaction_state); single-underscore names such as + # _schemas are user data and must not be filtered. + if [ -n "\${TOPICS}" ]; then + TLIST=\$(printf '%s' "\${TOPICS}" | tr ', ' ' ') + else + # Fail closed: run --list as its own command so a broker + # error (unreachable / auth) aborts the Pod under set -e + # instead of the pipe masking the exit status to grep's and + # || true swallowing it. Only then filter internal topics, so + # an empty TLIST means genuinely no non-internal topics - + # never a silently swallowed failure that would PUT an empty + # tarball and report success. + raw=\$("\${BIN}"/kafka-topics.sh --bootstrap-server "\${BOOT}" --list) + TLIST=\$(printf '%s\n' "\${raw}" | grep -v '^__' || true) + fi + # TOPICS is operator input reaching unquoted expansions. Kafka + # topic names are [a-zA-Z0-9._-], so anything else is refused + # before it can glob (set -f keeps a literal "*" out of the + # loop below) or escape the \Q...\E pins. A name listed twice + # would run the per-topic block twice: the manifest gains a + # second set of lines while the data files overwrite, and the + # restore's count check can then never balance - so a + # duplicate is refused here rather than reported as a + # Succeeded backup nothing can restore. + set -f + UNIQ="" + for t in \${TLIST}; do + case "\${t}" in + *[!A-Za-z0-9._-]*) echo "topic name '\${t}' carries a character outside Kafka's [a-zA-Z0-9._-]; refusing" >&2; exit 1 ;; + esac + case " \${UNIQ} " in + *" \${t} "*) echo "topic \${t} is listed more than once in topics; refusing to back it up twice" >&2; exit 1 ;; + esac + UNIQ="\${UNIQ} \${t}" + done + TLIST=\${UNIQ} + echo "backing up topics [\${TLIST}] from \${BOOT}" + MANIFEST="\${WORK}/manifest.txt" + : > "\${MANIFEST}" + for t in \${TLIST}; do + # Freeze the consistency cut: read each partition's end offset + # (--time -1) ONCE here, and its begin offset (--time -2). + # Draining only up to the frozen end excludes anything + # produced while the backup runs. + ends=\$("\${BIN}"/kafka-get-offsets.sh --bootstrap-server "\${BOOT}" --topic "\Q\${t}\E" --time -1) + begins=\$("\${BIN}"/kafka-get-offsets.sh --bootstrap-server "\${BOOT}" --topic "\Q\${t}\E" --time -2) + # The partition SET is not self-validating (see + # topic_records): re-derive the count from the topic itself + # and fail closed when the listing is short, or a partition + # lost to a leader election would get no manifest line, no + # data file, and never reach the drain check below - a whole + # partition missing from a backup that still reports + # Succeeded. (A short "begins" needs no separate check: its + # per-partition default of 0 can only overstate n, which the + # drain check then catches.) + # + # The same --describe line carries ReplicationFactor. It goes + # into the manifest so restore recreates the topic as the + # source had it: a broker whose min.insync.replicas exceeds + # the recreated topic's replication factor rejects every + # acks=all write, and kafka-console-producer logs the + # rejection without failing, so a replay into an + # under-replicated topic lands nothing. + desc=\$("\${BIN}"/kafka-topics.sh --bootstrap-server "\${BOOT}" --describe --topic "\Q\${t}\E") + pc=\$(printf '%s\n' "\${desc}" | sed -n 's/.*PartitionCount: *\([0-9][0-9]*\).*/\1/p' | head -1) + rf=\$(printf '%s\n' "\${desc}" | sed -n 's/.*ReplicationFactor: *\([0-9][0-9]*\).*/\1/p' | head -1) + case "\${pc}" in + ''|*[!0-9]*) echo "could not read PartitionCount for \${t}" >&2; exit 1 ;; + esac + case "\${rf}" in + ''|*[!0-9]*) echo "could not read ReplicationFactor for \${t}" >&2; exit 1 ;; + esac + seen=0 + for e in \${ends}; do seen=\$((seen + 1)); done + if [ "\${seen}" -ne "\${pc}" ]; then + echo "offset listing returned \${seen} of \${pc} partition(s) for \${t}; refusing to upload an incomplete backup" >&2 + exit 1 + fi + for e in \${ends}; do + ekey=\${e%:*} + p=\${ekey##*:} + end=\${e##*:} + begin=0 + for b in \${begins}; do + # Match the whole topic:partition key, not the partition + # index alone. Keyed on the index, a line belonging to + # another topic supplies this partition's begin offset and + # silently shifts the cut - and the drain check below + # cannot see it, because n is derived from that same wrong + # begin. This holds whatever order the listing arrives in. + if [ "\${b%:*}" = "\${ekey}" ]; then begin=\${b##*:}; break; fi + done + n=\$((end - begin)) + printf '%s %s %s %s %s\n' "\${t}" "\${p}" "\${begin}" "\${end}" "\${rf}" >> "\${MANIFEST}" + if [ "\${n}" -gt 0 ]; then + "\${BIN}"/kafka-console-consumer.sh --bootstrap-server "\${BOOT}" \ + --topic "\${t}" --partition "\${p}" --offset "\${begin}" --max-messages "\${n}" \ + --timeout-ms 120000 --property print.key=true --property print.timestamp=false \ + > "\${WORK}/data-\${t}-\${p}.tsv" + # kafka-console-consumer exits 0 even on a short read: a + # --timeout-ms expiry is swallowed inside ConsoleConsumer, + # so a truncated drain is indistinguishable from a full one + # by exit status. One newline-free record is one line (the + # README's key/value fidelity constraint), so a line-count + # short of n means the [begin,end) cut was not fully + # consumable - transactional control batches or a compacted + # topic, both out of scope - and the backup must fail + # rather than PUT a partial tarball reported as Succeeded. + got=\$(wc -l < "\${WORK}/data-\${t}-\${p}.tsv") + if [ "\${got}" -ne "\${n}" ]; then + echo "partial drain on \${t}:\${p}: captured \${got} of \${n} record(s) in [\${begin},\${end}); refusing to upload an incomplete backup" >&2 + exit 1 + fi + fi + echo " \${t}:\${p} [\${begin},\${end}) -> \${n} record(s)" + done + echo " \${t}: \${pc} partition(s), replication factor \${rf}" + done + tar -C "\${WORK}" -cf "\${TARBALL}" . + s3 -X PUT --upload-file "\${TARBALL}" "\${OBJ_URL}" + echo "uploaded s3://\${S3_BUCKET}/\${KEY}" + else + echo "restoring topics into \${BOOT} from s3://\${S3_BUCKET}/\${KEY}" + s3 -o "\${TARBALL}" "\${OBJ_URL}" + tar -C "\${WORK}" -xf "\${TARBALL}" + [ -f "\${WORK}/manifest.txt" ] || { echo "manifest missing in backup" >&2; exit 1; } + # The manifest arrives inside the downloaded tarball, so treat it + # as untrusted and validate before any field is used as a number. + # A non-numeric partition would otherwise slip through the + # numeric test below: an if condition is exempt from errexit, so + # test exits 2, the term reads false, parts under-counts and the + # topic is created smaller - which the shape check then compares + # against itself and passes. An empty manifest is refused + # too: both verification loops would run zero times and the Pod + # would report a successful restore of nothing. + lines=0 + while read -r mt mp mb me mr; do + lines=\$((lines + 1)) + [ -n "\${mt}" ] || { echo "manifest line \${lines}: missing topic" >&2; exit 1; } + # The topic name reaches unquoted expansions and a case + # pattern below, with globbing on in this branch; hold it to + # Kafka's name charset the way the backup side holds TOPICS. + case "\${mt}" in + *[!A-Za-z0-9._-]*) + echo "manifest line \${lines}: topic name '\${mt}' carries a character outside Kafka's [a-zA-Z0-9._-]; refusing to restore from a malformed backup" >&2 + exit 1 ;; + esac + for v in "\${mp}" "\${mb}" "\${me}" "\${mr}"; do + case "\${v}" in + ''|*[!0-9]*) + echo "manifest line \${lines} (topic \${mt}): non-numeric field '\${v}'; refusing to restore from a malformed backup" >&2 + exit 1 ;; + esac + done + done < "\${WORK}/manifest.txt" + [ "\${lines}" -gt 0 ] || { echo "manifest is empty: the backup recorded no partitions, so there is nothing to restore" >&2; exit 1; } + # Data files and manifest must agree both ways, checked before + # anything is created or replayed. The file names come out of + # the same untrusted tarball as the manifest: a file the + # manifest never listed would be data that skipped the create, + # shape, emptiness and post-replay checks (or, ignored by the + # manifest-driven replay below, silently not restored), and a + # listed partition with records but no file would be found + # only after earlier partitions had already been replayed. + for f in "\${WORK}"/data-*.tsv; do + [ -e "\${f}" ] || continue + base=\${f##*/data-}; base=\${base%.tsv} + ft=\${base%-*} + fp=\${base##*-} + listed=0 + while read -r mt mp mb me mr; do + if [ "\${mt}" = "\${ft}" ] && [ "\${mp}" = "\${fp}" ]; then listed=1; break; fi + done < "\${WORK}/manifest.txt" + if [ "\${listed}" -ne 1 ]; then + echo "backup contains data file \${f##*/}, which the manifest does not list; refusing to restore an inconsistent backup" >&2 + exit 1 + fi + done + while read -r mt mp mb me mr; do + [ "\${me}" -gt "\${mb}" ] || continue + if [ ! -f "\${WORK}/data-\${mt}-\${mp}.tsv" ]; then + echo "backup records \$((me - mb)) record(s) for \${mt}:\${mp} but carries no data file for it; refusing to restore an inconsistent backup" >&2 + exit 1 + fi + done < "\${WORK}/manifest.txt" + # Recreate every backed-up topic with its original partition + # count (needed so keyed records re-produce into the same + # partition) and, unless the replicationFactor parameter + # overrides it, its original replication factor; existing + # topics are left as-is. Topic configs, ACLs and consumer + # offsets are out of scope - this restores DATA only. + SEEN="" + while read -r mt mp mb me mr; do + case " \${SEEN} " in *" \${mt} "*) ;; *) SEEN="\${SEEN} \${mt}" ;; esac + done < "\${WORK}/manifest.txt" + for t in \${SEEN}; do + parts=0 + rf="" + while read -r mt mp mb me mr; do + [ "\${mt}" = "\${t}" ] || continue + if [ "\${mp}" -ge "\${parts}" ]; then parts=\$((mp + 1)); fi + if [ -n "\${rf}" ] && [ "\${rf}" != "\${mr}" ]; then + echo "manifest lists \${t} with replication factors \${rf} and \${mr}; refusing to restore from an inconsistent backup" >&2 + exit 1 + fi + rf=\${mr} + done < "\${WORK}/manifest.txt" + if [ -n "\${REPLICATION_FACTOR}" ]; then rf=\${REPLICATION_FACTOR}; fi + # A replication factor the target cannot host (more replicas + # than brokers) fails right here, in Kafka's own words, rather + # than after a replay; set replicationFactor in the + # BackupClass to restore into a smaller cluster. + "\${BIN}"/kafka-topics.sh --bootstrap-server "\${BOOT}" --create --if-not-exists \ + --topic "\${t}" --partitions "\${parts}" --replication-factor "\${rf}" + # --if-not-exists leaves a pre-existing topic's partition + # count and replication factor alone. The post-replay check + # below compares only the per-topic total, so replaying into + # a topic with a different partition count would silently + # re-place every keyed record and still add up, and a + # pre-existing topic with fewer replicas than the broker's + # min.insync.replicas would take the replay and land nothing. + # Confirm both against what the topic really has. + desc=\$("\${BIN}"/kafka-topics.sh --bootstrap-server "\${BOOT}" --describe --topic "\Q\${t}\E") + have_parts=\$(printf '%s\n' "\${desc}" | sed -n 's/.*PartitionCount: *\([0-9][0-9]*\).*/\1/p' | head -1) + have_rf=\$(printf '%s\n' "\${desc}" | sed -n 's/.*ReplicationFactor: *\([0-9][0-9]*\).*/\1/p' | head -1) + case "\${have_parts}" in + ''|*[!0-9]*) echo "could not read PartitionCount for \${t} after create" >&2; exit 1 ;; + esac + case "\${have_rf}" in + ''|*[!0-9]*) echo "could not read ReplicationFactor for \${t} after create" >&2; exit 1 ;; + esac + if [ "\${have_parts}" != "\${parts}" ]; then + echo "topic \${t} has \${have_parts} partition(s), backup recorded \${parts}; refusing to restore into a differently shaped topic" >&2 + exit 1 + fi + if [ "\${have_rf}" != "\${rf}" ]; then + echo "topic \${t} has replication factor \${have_rf}, restore expects \${rf}; refusing to replay into a pre-existing topic with a different replication factor" >&2 + exit 1 + fi + # kafka-console-producer writes with acks=all, and a broker + # rejects every such write into a topic whose replication + # factor is below its min.insync.replicas - while the + # producer only logs the rejection and exits 0. --describe + # --all prints the effective value, broker default included, + # so the mismatch is named here, with its remedy, instead of + # surfacing as a count shortfall after an empty replay. + misr=\$("\${BIN}"/kafka-configs.sh --bootstrap-server "\${BOOT}" --describe --all --entity-type topics --entity-name "\${t}" \ + | sed -n 's/.*min\.insync\.replicas=\([0-9][0-9]*\).*/\1/p' | head -1) + case "\${misr}" in + ''|*[!0-9]*) echo "could not read min.insync.replicas for \${t}" >&2; exit 1 ;; + esac + if [ "\${have_rf}" -lt "\${misr}" ]; then + echo "topic \${t} has replication factor \${have_rf} but the broker requires min.insync.replicas=\${misr}, so every replayed write would be rejected. Set replicationFactor to at least \${misr} in the BackupClass (or restore into a cluster whose min.insync.replicas fits) and re-run." >&2 + exit 1 + fi + # Refuse a non-empty target BEFORE replaying, not after. The + # replay appends, and the driver retries a Pod that dies + # mid-replay, so checking only afterwards means each attempt + # has already written another full copy into a live topic and + # the count check merely reports the damage. Checking here + # leaves the topic untouched instead. Remedy: empty or delete + # the topic (the in-place step deletes it) and re-run. + held=\$(topic_records "\${t}" "\${have_parts}") || exit 1 + if [ "\${held}" -ne 0 ]; then + echo "topic \${t} already holds \${held} record(s); this restore appends and is supported only against an absent or empty topic. Empty or delete it and re-run." >&2 + exit 1 + fi + echo " \${t}: created or verified empty, \${parts} partition(s), replication factor \${rf}" + done + # Replay topic by topic in manifest order, announcing each one, + # so the Pod log doubles as a resume map: a retry after a Pod + # died mid-replay stops at the emptiness check above on the + # first topic that was already replayed, and this log says which + # topics landed and which still need the target emptied. + # parse.key restores the original key so the default partitioner + # reproduces the original partition placement. + for t in \${SEEN}; do + echo "replaying \${t}" + while read -r mt mp mb me mr; do + [ "\${mt}" = "\${t}" ] || continue + [ "\${me}" -gt "\${mb}" ] || continue + "\${BIN}"/kafka-console-producer.sh --bootstrap-server "\${BOOT}" \ + --topic "\${t}" --property parse.key=true < "\${WORK}/data-\${t}-\${mp}.tsv" + done < "\${WORK}/manifest.txt" + echo " \${t}: replayed" + done + # Verify the replay landed. kafka-console-producer, like the + # consumer on backup, can exit 0 without every record being + # accepted (a broker-side reject prints but does not fail the + # process), so confirm each restored topic now holds exactly the + # record count the manifest recorded (sum of end-begin per + # partition). A shortfall means the replay dropped records; an + # overshoot means the target was not empty - this DATA restore + # appends, so it is supported only against an absent or empty + # topic. Either way, fail rather than report a lossy or + # duplicated restore as complete. + for t in \${SEEN}; do + want=0 + parts=0 + while read -r mt mp mb me mr; do + [ "\${mt}" = "\${t}" ] || continue + want=\$((want + me - mb)) + if [ "\${mp}" -ge "\${parts}" ]; then parts=\$((mp + 1)); fi + done < "\${WORK}/manifest.txt" + have=\$(topic_records "\${t}" "\${parts}") || exit 1 + if [ "\${have}" -ne "\${want}" ]; then + echo "restore verification failed for \${t}: topic holds \${have} record(s), backup recorded \${want}; refusing to report a lossy or duplicated restore as complete" >&2 + exit 1 + fi + echo " \${t}: restored \${have} record(s)" + done + echo "restore complete" + fi + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + securityContext: + # restricted-clean. The Strimzi image's default user is a non-root + # numeric UID (1001), so runAsNonRoot passes without pinning + # runAsUser to a value specific to this image tag. + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + volumes: + - name: s3-ca + secret: + secretName: "{{ .Release.Name }}-backup-s3" + optional: true + items: + - key: ca.crt + path: ca.crt +EOF + +log_success "Job strategy '${STRATEGY_NAME}' created." +echo -e "\n${GREEN}${BOLD}Next:${NC} ./02-create-backupclass.sh" diff --git a/examples/backups/kafka/02-create-backupclass.sh b/examples/backups/kafka/02-create-backupclass.sh new file mode 100755 index 0000000000..6e9d8b081b --- /dev/null +++ b/examples/backups/kafka/02-create-backupclass.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Step 02: Map the Kafka application kind to the Job strategy from step 01. The +# strategy parameters (which topics to back up, an optional replication-factor +# override for the topics restore recreates) travel through the BackupClass and +# are exposed to the strategy template as `.Parameters`. They are also +# snapshotted onto the resulting Backup's driverMetadata, so a later restore +# re-renders with the same values. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 02: Create BackupClass '${BACKUPCLASS_NAME}'" +log_command "kubectl apply -f - (BackupClass: $BACKUPCLASS_NAME -> $STRATEGY_NAME)" + +kubectl apply -f - <-backup-s3" Secret itself. This step only +# caches the coordinates; create_s3_secret (00-helpers.sh) turns them into +# per-app Secrets in steps 04 and 07. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 03: Provision Bucket '${BUCKET_NAME}' in '${NAMESPACE}'" + +log_command "kubectl apply -f - (Bucket: $BUCKET_NAME)" +# spec.users.backup is required for the Bucket app to provision a +# BucketAccess (and credentials Secret) the backup tooling can read. +kubectl apply -f - <-backup" +# (the "-backup" suffix is the BucketAccessClass name); the BucketInfo Secret +# carries the same name. +kubectl -n "$NAMESPACE" wait bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-backup" --for=jsonpath='{.status.accessGranted}'=true --timeout=300s + +# E2E only: accessGranted describes the COSI object, not whether the S3 server +# has reloaded the new IAM identity, so a freshly granted key can still answer +# the strategy Pod's SigV4 PUT with 403 - which reds the run as a backup defect +# rather than the grant-propagation race it is. The shared preflight proves the +# key works before anything depends on it; the other backup round-trips gate it +# the same way. +if [[ "${COZY_E2E_BACKUP_PREFLIGHT:-0}" == "1" ]]; then + # shellcheck source=/dev/null + source "$SCRIPT_DIR/../../../hack/e2e-chainsaw/_lib/backup-access-preflight.sh" + cozy_backup_access_preflight "$NAMESPACE" "bucket-${BUCKET_NAME}-backup" 90 +fi + +log_substep "Reading bucket coordinates from BucketInfo Secret..." +TMP=$(mktemp) +trap 'rm -f "$TMP"' EXIT +kubectl -n "$NAMESPACE" get secret "bucket-${BUCKET_NAME}-backup" -o jsonpath='{.data.BucketInfo}' | base64 -d > "$TMP" + +S3_ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' "$TMP") +S3_SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' "$TMP") +# S3_ENDPOINT can be overridden via the environment: BucketInfo advertises the +# external ingress URL, which in CI is an unroutable placeholder; the e2e +# harness points this at the in-cluster seaweedfs Service instead. +S3_ENDPOINT="${S3_ENDPOINT:-$(jq -r '.spec.secretS3.endpoint' "$TMP")}" +S3_REGION=$(jq -r 'if (.spec.secretS3.region // "") == "" then "us-east-1" else .spec.secretS3.region end' "$TMP") +S3_BUCKET=$(jq -r '.spec.bucketName' "$TMP") + +# `jq -r` returns the literal string "null" for missing JSON paths; fail fast +# here so a missing field surfaces at extraction time instead of as a confusing +# curl/SigV4 error inside the backup Pod. +for v in S3_ACCESS_KEY S3_SECRET_KEY S3_ENDPOINT S3_REGION S3_BUCKET; do + [[ -n "${!v}" && "${!v}" != "null" ]] || { log_error "BucketInfo missing required field: ${v}"; exit 1; } +done + +# The endpoint's CA, so the strategy Pod verifies TLS instead of skipping it. +# The default name tracks the seaweedfs chart's fullnameOverride (seaweedfs -> +# seaweedfs-ca-cert), but a downstream fullname change would rename it, so fall +# back to discovering the cert-manager CA Certificate (the seaweedfs-labelled +# one with spec.isCA=true) and read its secretName. S3_CA_SECRET="" skips the +# copy for a publicly-trusted endpoint (see 00-helpers.sh). +S3_CA_B64="" +if [[ -n "$S3_CA_SECRET" ]]; then + # Only an absent Secret earns a discovery pass. The read crosses into + # another namespace, which a tenant persona is not granted, and a denial + # must say so rather than surface as "not found" and point at a name + # that cannot help. + if ! err=$(kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" 2>&1 >/dev/null); then + case "$err" in + *NotFound*|*"not found"*) ;; + *) + log_error "Cannot read S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET}: ${err}" + log_error "Run this step with a persona that can read Secrets in ${S3_CA_NAMESPACE}, or set S3_CA_SECRET=\"\" for a publicly-trusted endpoint." + exit 1 ;; + esac + log_warning "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} not found; discovering the seaweedfs CA Certificate..." + discovered=$(kubectl -n "$S3_CA_NAMESPACE" get certificates.cert-manager.io \ + -l app.kubernetes.io/name=seaweedfs \ + -o jsonpath='{range .items[*]}{.spec.isCA}{" "}{.spec.secretName}{"\n"}{end}' \ + | awk '$1=="true"{print $2; exit}' || true) + [[ -n "$discovered" ]] || { log_error "No seaweedfs CA Certificate found in ${S3_CA_NAMESPACE} (or the read was denied, see above); set S3_CA_SECRET explicitly (or empty for a publicly-trusted endpoint)."; exit 1; } + log_success "Discovered seaweedfs CA secret ${S3_CA_NAMESPACE}/${discovered}" + S3_CA_SECRET="$discovered" + fi + log_substep "Caching S3 CA ${S3_CA_NAMESPACE}/${S3_CA_SECRET}[${S3_CA_KEY}]..." + S3_CA_B64=$(kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" -o jsonpath="{.data.${S3_CA_KEY//./\\.}}") + [[ -n "$S3_CA_B64" ]] || { log_error "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} has no ${S3_CA_KEY}"; exit 1; } +fi + +# Persist for steps 04 / 07. The cache stores raw S3 credentials, so apply +# restrictive perms before writing the body - umask alone could leave the file +# group/world-readable. The CA stays base64 so the file remains one export per +# line. +umask 077 +cat > "$SCRIPT_DIR/.bucket-info.env" < "$SCRIPT_DIR/.source-dump.txt" +dumped=$(wc -l < "$SCRIPT_DIR/.source-dump.txt" | tr -d '[:space:]') +[[ "$dumped" == "$MESSAGE_COUNT" ]] || { log_error "source dump has ${dumped} records, expected ${MESSAGE_COUNT}"; exit 1; } + +echo -e "\n${GREEN}${BOLD}Next:${NC} ./05-create-backupjob.sh" diff --git a/examples/backups/kafka/05-create-backupjob.sh b/examples/backups/kafka/05-create-backupjob.sh new file mode 100755 index 0000000000..136e260dfa --- /dev/null +++ b/examples/backups/kafka/05-create-backupjob.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Step 05: Create a BackupJob and wait for it to succeed. The controller +# resolves the BackupClass -> Job strategy, renders the Pod template against +# the Kafka app, and runs it as a batch/v1.Job (owned by the BackupJob). The +# Pod freezes each partition's end offset, drains [begin,end) and uploads a +# tarball to S3. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 05: Create BackupJob '${BACKUPJOB_NAME}'" + +kubectl apply -f - < Succeeded between two API observations, in which case +# `kubectl wait --for=jsonpath='{.status.phase}'=Running` would time out even +# though the BackupJob actually finished. wait_for_field already polls until +# the terminal phase, which covers both the slow and the fast path. +wait_for_field backupjob "$BACKUPJOB_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 600 Failed + +backup_ref=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}') +[[ -n "$backup_ref" ]] || { log_error "BackupJob succeeded but BackupRef is empty"; exit 1; } +# The restore steps reference the Backup by this resolved name rather than by +# assuming it equals the BackupJob's. cleanup.sh removes the file. +printf 'export BACKUP_NAME=%q\n' "$backup_ref" > "$SCRIPT_DIR/.backup-name.env" +log_success "Backup '${backup_ref}' is Ready." + +echo -e "\n${GREEN}${BOLD}Next:${NC} ./06-restore-in-place.sh" diff --git a/examples/backups/kafka/06-restore-in-place.sh b/examples/backups/kafka/06-restore-in-place.sh new file mode 100755 index 0000000000..8f7edd2932 --- /dev/null +++ b/examples/backups/kafka/06-restore-in-place.sh @@ -0,0 +1,140 @@ +#!/bin/bash +# Step 06: In-place restore. Delete the topic on the source instance to +# simulate data loss, then ask the Job driver to restore it from S3 back into +# the same Kafka application. The driver recreates the topic (--if-not-exists) +# and replays every partition file, so the topic must be absent first - +# mirroring the ClickHouse demo dropping its table and the NATS demo removing +# its stream. A second RestoreJob against the now-populated topic then has to +# be refused without touching it. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 06: In-place restore from '${BACKUPJOB_NAME}'" + +[[ -f "$SCRIPT_DIR/.backup-name.env" ]] || { log_error "missing $SCRIPT_DIR/.backup-name.env; run 05-create-backupjob.sh first"; exit 1; } +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/.backup-name.env" +[[ -f "$SCRIPT_DIR/.source-dump.txt" ]] || { log_error "missing $SCRIPT_DIR/.source-dump.txt; run 04-create-kafka.sh first"; exit 1; } + +log_substep "Deleting topic '${TOPIC}' to simulate data loss..." +# Delete and wait for the topic to actually disappear in the same Pod: topic +# deletion is asynchronous, and the driver's --create would fail with "topic +# marked for deletion" if the restore raced ahead of the tombstone. +# +# kafka-topics treats --topic as a Java regex, so an unquoted name deletes +# every topic it happens to match ("audit.events" also matches "audit-events"). +# \Q...\E quotes the name back to a literal - the same fix the kafka-metadata +# example carries. --create below takes a literal name and must NOT be quoted. +kafka_run "$KAFKA_NAME" ' + "$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --delete --topic "\Q$TOPIC\E" || true + for _ in $(seq 1 60); do + # Capture --list on its own so a transient broker error does not read + # as empty output and false-positive "deleted"; the pod snippet runs + # under set -eu without pipefail, so the pipe would otherwise mask it. + list=$("$BIN"/kafka-topics.sh --bootstrap-server "$BOOT" --list) || { sleep 2; continue; } + if ! printf "%s\n" "$list" | grep -qxF -- "$TOPIC"; then + echo "topic $TOPIC deleted"; exit 0 + fi + sleep 2 + done + echo "topic still present after wait" >&2; exit 1 +' + +kubectl apply -f - <=: restore replays unconditionally and appends into a + # non-empty topic, so a double-replay (e.g. the strategy Job's backoffLimit + # retry) lands 2x the records and must fail here, not pass. + if (( count != MESSAGE_COUNT )); then + log_error "Record count ${when} is ${count}; expected exactly ${MESSAGE_COUNT}" + return 1 + fi + # Content, not just count: diff the restored topic against the source snapshot + # from step 04. Byte-equality of the per-partition ordered dumps proves keys, + # values, partition placement and ordering survived - the fidelity claims the + # README makes. + if ! diff -u "$SCRIPT_DIR/.source-dump.txt" <(topic_dump "$KAFKA_NAME"); then + log_error "Restored content ${when} does not match the source snapshot" + return 1 + fi + # The replication factor comes from the backup manifest, not from a + # parameter: on this KAFKA_REPLICAS-broker cluster the chart's + # min.insync.replicas rejects every write into a topic recreated with + # fewer replicas, so a replay into an under-replicated topic would have + # landed nothing and failed the count above - but pin the value itself too. + rf=$(topic_replication_factor "$KAFKA_NAME") + if [[ "$rf" != "$TOPIC_REPLICAS" ]]; then + log_error "Replication factor ${when} is '${rf}'; expected ${TOPIC_REPLICAS} as captured from the source" + return 1 + fi + log_success "Verified ${when}: ${count} record(s) in '${TOPIC}' at replication factor ${rf}, content matches source." +} + +log_substep "Verifying topic records are restored..." +verify_restored "after in-place restore" + +# The delete above names ${TOPIC}, which as a Java regex would also match the +# decoy. Its survival is what proves the \Q...\E pin is in place; without it +# this reads back empty. +decoy=$(decoy_message_count "$KAFKA_NAME") +if [[ "$decoy" != "$DECOY_COUNT" ]]; then + log_error "Decoy topic '${DECOY_TOPIC}' holds '${decoy}', expected ${DECOY_COUNT}: the topic-name match is behaving as a regex, not a literal" + exit 1 +fi +log_success "Decoy topic '${DECOY_TOPIC}' untouched: ${decoy} record(s)." + +# Restore appends, so the strategy refuses a target that already holds +# records - before replaying, so a retried Pod cannot stack a second copy onto +# a live topic. Drive that path: a second RestoreJob against the topic just +# restored must settle Failed, name the reason in the strategy Pod's log, and +# leave the topic exactly as the verification above found it. +log_substep "Submitting RestoreJob '${RESTOREJOB_NONEMPTY_NAME}' against the populated topic; it must be refused..." +kubectl apply -f - <-restore"; batch/v1 labels its +# Pods job-name=. At least one attempt must have logged the refusal; +# that nothing was written by any attempt is what verify_restored below +# establishes. +refusals=$(kubectl -n "$NAMESPACE" logs -l "job-name=${RESTOREJOB_NONEMPTY_NAME}-restore" --tail=-1 2>/dev/null \ + | grep -c "already holds" || true) +if [[ "$refusals" -lt 1 ]]; then + log_error "RestoreJob '${RESTOREJOB_NONEMPTY_NAME}' failed, but no attempt logged the non-empty-target refusal" + kubectl -n "$NAMESPACE" logs -l "job-name=${RESTOREJOB_NONEMPTY_NAME}-restore" --tail=50 >&2 || true + exit 1 +fi +verify_restored "after the refused restore" +log_success "Restore into the populated topic was refused (${refusals} attempt(s)) and left it untouched." + +echo -e "\n${GREEN}${BOLD}Next:${NC} ./07-restore-to-copy.sh" diff --git a/examples/backups/kafka/07-restore-to-copy.sh b/examples/backups/kafka/07-restore-to-copy.sh new file mode 100755 index 0000000000..61d79f5ae2 --- /dev/null +++ b/examples/backups/kafka/07-restore-to-copy.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Step 07: To-copy restore. Provision a second Kafka application and restore the +# same backup into it via RestoreJob.spec.targetApplicationRef. The strategy +# connects to the TARGET app (its bootstrap, its -backup-s3 Secret) but +# reads the S3 object keyed by the SOURCE app name, via .Backup.ApplicationRef.Name +# - so the copy lands the source's data. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 07: To-copy restore into '${KAFKA_RESTORE_NAME}'" + +[[ -f "$SCRIPT_DIR/.backup-name.env" ]] || { log_error "missing $SCRIPT_DIR/.backup-name.env; run 05-create-backupjob.sh first"; exit 1; } +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/.backup-name.env" + +# Same broker count as the source: the restore recreates the topic with the +# replication factor the backup captured, which a smaller target could not +# host (see the replicationFactor override in 02-create-backupclass.sh). +kubectl apply -f - <=: see step 06 - a double-replay would inflate this. +if (( count != MESSAGE_COUNT )); then + log_error "Record count on copy is ${count}; expected exactly ${MESSAGE_COUNT}" + exit 1 +fi +# Content, not just count: the copy must match the source snapshot byte-for-byte. +if ! diff -u "$SCRIPT_DIR/.source-dump.txt" <(topic_dump "$KAFKA_RESTORE_NAME"); then + log_error "Restored content on copy does not match the source snapshot" + exit 1 +fi +# Shape too: the copy's topic must carry the replication factor the backup +# captured from the source, not a default (see step 06). +rf=$(topic_replication_factor "$KAFKA_RESTORE_NAME") +if [[ "$rf" != "$TOPIC_REPLICAS" ]]; then + log_error "Replication factor on copy is '${rf}'; expected ${TOPIC_REPLICAS} as captured from the source" + exit 1 +fi +log_success "To-copy restore verified: ${count} record(s) in '${TOPIC}' on '${KAFKA_RESTORE_NAME}' at replication factor ${rf}, content matches source." + +echo -e "\n${GREEN}${BOLD}Next:${NC} ./cleanup.sh" diff --git a/examples/backups/kafka/90-scenario-admin-prepare.md b/examples/backups/kafka/90-scenario-admin-prepare.md new file mode 100644 index 0000000000..42bc1d697e --- /dev/null +++ b/examples/backups/kafka/90-scenario-admin-prepare.md @@ -0,0 +1,43 @@ +# Scenario: Cluster admin prepares Kafka topic-data backups + +A cluster administrator performs the cluster-level preparation once. Tenant admins can then back up and restore the topic data of their Kafka applications with a `BackupJob` or `RestoreJob` that references the BackupClass created here. The demo scripts around those jobs still need admin access, because they create Secrets and work through a CLI Pod; the README's step table says which part of each step a tenant admin can do. + +## Prerequisites + +- A running Cozystack cluster with the backup-controller and + backupstrategy-controller installed. +- `kubectl` access with permissions to create cluster-scoped + `Job.strategy.backups.cozystack.io` and `BackupClass.backups.cozystack.io`. +- Reachable S3-compatible storage. The demo uses the in-cluster `Bucket` app (step 03), and steps 04 and 07 turn its coordinates and the endpoint's CA into a `-backup-s3` Secret that the strategy Pod reads. Creating that Secret needs admin access, since no tenant role grants `create` on `secrets`. + +## Steps + +```bash +./01-create-strategy.sh # Job strategy: stock Strimzi Kafka image + a shell script +./02-create-backupclass.sh # Maps Kind=Kafka -> the strategy, with the topics parameter +``` + +## What gets created + +| Resource | Scope | Purpose | +|---|---|---| +| `Job.strategy.backups.cozystack.io/kafka-job` | Cluster | PodTemplateSpec for a `quay.io/strimzi/kafka` Pod that drains the topics with `kafka-console-consumer`, replays them with `kafka-console-producer`, and moves a tarball to/from S3 with `curl --aws-sigv4` | +| `BackupClass/kafka-backup` | Cluster | Maps `apps.cozystack.io/Kafka` to the strategy. `parameters.topics` selects the topics (empty = every non-internal topic); `parameters.replicationFactor`, when set, overrides the replication factor the backup captured when restore recreates a topic | + +## How the strategy template gets rendered + +The Job driver renders the `Job.spec.template` PodTemplateSpec with this +context for every BackupJob/RestoreJob run: + +| Variable | Source | +|---|---| +| `.Application` | The `Kafka` (`apps.cozystack.io`) object referenced by the BackupJob (or `targetApplicationRef` on restore) — available to templates, though this one reads everything it needs from env | +| `.Release.Name` / `.Release.Namespace` | `metadata.name` / `metadata.namespace` of `.Application` | +| `.Parameters` | `BackupClass.spec.strategies[].parameters` (`topics`, `replicationFactor`); snapshotted onto the Backup's `driverMetadata` so restore re-renders with the same values | +| `.Mode` | `"backup"` or `"restore"` | +| `.Backup` | `{ Name, Namespace, ApplicationRef.{APIGroup,Kind,Name} }` (only for restore runs; `.ApplicationRef` points at the *source* release so to-copy restores read the source's S3 key) | + +The same template renders both backup and restore Pods. The engine templates +each string field independently and cannot restructure the Pod per mode, so +the single Kafka container branches at runtime: `MODE` (from `{{ .Mode }}`) +selects drain + S3 `PUT` vs. S3 `GET` + recreate + replay. diff --git a/examples/backups/kafka/91-scenario-user-backup.md b/examples/backups/kafka/91-scenario-user-backup.md new file mode 100644 index 0000000000..9bff52ee30 --- /dev/null +++ b/examples/backups/kafka/91-scenario-user-backup.md @@ -0,0 +1,55 @@ +# Scenario: User backs up Kafka topic data + +This scenario assumes the admin steps (`01-...`, `02-...`) have already run. + +The user's own action here, at the tenant admin level, is the `BackupJob` in step 05, plus creating the `Bucket` and the `Kafka`. Steps 03 and 04 as scripted need admin access: they read and create Secrets, wait on HelmReleases and operator resources, and seed the topic through a CLI Pod, which no tenant role allows. The README's step table has the details. + +## Prerequisites + +- Tenant namespace (default: `tenant-root`, the tenant whose Pods can reach + the in-cluster seaweedfs; see the README). +- A Cozystack `Bucket` (step 03) and the `-backup-s3` Secret derived from it (step 04). The generic `Job` strategy has no chart support to emit this Secret, so step 04 creates it from the bucket's `BucketInfo` plus the S3 endpoint's CA (see `create_s3_secret` in `00-helpers.sh`). Creating it needs admin access. +- A `Kafka` application with at least one plain (neither compacted nor + transactional) topic holding data. + +## Steps + +```bash +./03-create-bucket.sh # Provision Bucket, cache its credentials and the endpoint CA +./04-create-kafka.sh # Provision Kafka, create the S3 Secret, seed a topic +./05-create-backupjob.sh # Submit BackupJob and wait for Succeeded +``` + +## What happens during backup + +1. The user submits a `BackupJob` referencing the Kafka instance and the + `kafka-backup` BackupClass. +2. The backup-controller resolves the BackupClass → `Job` strategy and renders + the Pod template against the user's Kafka application, with `.Mode="backup"`. +3. A `batch/v1.Job` (owned by the BackupJob via `OwnerReferences`) is created + in the tenant namespace. Its single Strimzi Kafka container: + - resolves the topic list from `parameters.topics` (or every non-internal + topic), refusing a name outside Kafka's charset or listed twice; + - for each topic reads every partition's begin and end offset once, its + `PartitionCount` and `ReplicationFactor`, and refuses the run if the + offset listing is short of the partition count; + - drains `[begin, end)` of each partition with `kafka-console-consumer` + into one `keyvalue` file per partition, refusing the run on a short + read; + - writes a manifest (`topic partition begin end replicationFactor` per + line), `tar`s everything and `PUT`s it to + `s3://///kafka-topics.tar` using + `curl --aws-sigv4`, verified against the CA from the Secret. +4. The controller watches the Job's terminal condition. On `Complete` it + creates a `Backup` CR in the same namespace, recording the BackupClass + parameters under `spec.driverMetadata` (the `parameter/` prefix), so a later + restore re-renders the strategy with the same `topics` and + `replicationFactor`. + +## Result + +| Resource | Where | Purpose | +|---|---|---| +| `BackupJob/` | tenant namespace | Records the run; `status.phase=Succeeded`, `status.backupRef` points at the Backup | +| `Backup/` | tenant namespace | The restorable artifact reference (the tarball lives in S3 under `//kafka-topics.tar`); step 05 caches its name in `.backup-name.env` for the restore steps | +| `batch/v1.Job/-backup` | tenant namespace | The completed strategy Pod | diff --git a/examples/backups/kafka/92-scenario-user-restore.md b/examples/backups/kafka/92-scenario-user-restore.md new file mode 100644 index 0000000000..3215fe056e --- /dev/null +++ b/examples/backups/kafka/92-scenario-user-restore.md @@ -0,0 +1,79 @@ +# Scenario: User restores a Kafka topic-data backup + +Two restore variants are supported. Both append: the strategy recreates a +topic only if it is absent and then replays every record, so it refuses a +target topic that already holds data. + +The user's own action in both variants, at the tenant admin level, is the `RestoreJob`, plus the second `Kafka` in variant B. The scripts as written need admin access: variant A deletes and checks the topic through a CLI Pod and reads the refused restore's Pod log, and variant B checks the topic the same way, creates the target's Secret and waits on its `HelmRelease` and Strimzi `Kafka`, none of which a tenant role allows. The README's step table has the details. + +## A. In-place restore + +Restore back into the same Kafka application. + +```bash +./06-restore-in-place.sh +``` + +The script: + +1. Deletes the topic on the source instance (simulating data loss) and waits + for it to disappear. The topic **must be absent or empty** before the + `RestoreJob` is submitted — the caller is responsible for that. +2. Creates a `RestoreJob` whose `spec.backupRef.name` is the Backup name (from + `.backup-name.env`) and does **not** set `spec.targetApplicationRef`. Per + the API contract, the driver restores into `backup.spec.applicationRef`. +3. The Job driver renders the strategy Pod with `.Mode="restore"`. The Pod + `GET`s `s3://///kafka-topics.tar`, untars it, + validates the manifest, recreates each topic with its original partition + count and replication factor (or the `replicationFactor` override), + confirms the topic's shape and that it is empty, then replays the + partition files topic by topic with `kafka-console-producer` and checks + the record count that landed. In-place, the source and target are the + same app, so the S3 key resolves to the original. +4. After the Pod succeeds, the script verifies the record count, diffs the + content against the snapshot taken in step 04, and checks that the + restored topic carries the replication factor captured from the source. +5. It then submits a second `RestoreJob` against the now-populated topic and + requires it to settle `Failed` — every strategy Pod attempt stops at the + emptiness check — with the topic unchanged. + +## B. To-copy restore + +Restore into a **different** Kafka application instance. + +```bash +./07-restore-to-copy.sh +``` + +The script: + +1. Provisions a second `Kafka/kafka-restore` with the same broker count in the + **same** namespace as the RestoreJob, and creates its `-backup-s3` + Secret pointing at the same bucket. `RestoreJob.spec.targetApplicationRef` + is `corev1.TypedLocalObjectReference` (no `namespace` field), so the driver + always restores into `restoreJob.Namespace`; cross-namespace restore is + intentionally not supported. +2. Creates a `RestoreJob` with + `spec.targetApplicationRef = { kind: Kafka, name: kafka-restore }`. +3. The Job driver fetches the *target* application and renders the Pod with + `.Release.Name=kafka-restore` (so it connects to the target and reads the + target's S3 Secret), but keys the S3 object by the *source* via + `.Backup.ApplicationRef.Name` — so the copy reads what the source wrote. +4. The script verifies the record count, content and replication factor on + the copy. + +## Notes + +- Both flows rely on the strategy template branching on `.Mode`. See + `01-create-strategy.sh`. +- A restore Pod that dies mid-replay is retried by the Job's `backoffLimit`, + and the retry stops at the emptiness check on the first topic the dead Pod + had already replayed. The Pod log names each topic before its replay and + again once it landed; empty or delete every topic it shows as started, not + only the one the refusal names, then re-run. +- The tarball in object storage is not managed by the Cozystack `Backup` + lifecycle: deleting the `Backup` CR removes the reference but leaves the S3 + object in place. Tenants who need to reclaim space should delete the object + (or set a bucket lifecycle policy). +- Consumer-group offsets, topic configs, ACLs and users are not part of the + backup; consumers start from the restored topic's fresh offsets. diff --git a/examples/backups/kafka/README.md b/examples/backups/kafka/README.md new file mode 100644 index 0000000000..0a70558cdf --- /dev/null +++ b/examples/backups/kafka/README.md @@ -0,0 +1,69 @@ +# Kafka topic-data backup/restore example + +This directory shows how to back up and restore the **topic data** of a Cozystack-managed `Kafka` application using the **generic `Job` backup strategy**. Unlike the app-specific drivers (Altinity for ClickHouse, CNPG for Postgres, ...), the `Job` strategy has no built-in knowledge of the application: it runs a Pod the operator supplies. Here that Pod is a stock [Strimzi Kafka image][strimzi-image] (which carries the `kafka-*.sh` CLI plus `bash`, `curl` and `tar`) running one shell script: + +- **backup**: freeze each partition's end offset, drain `[begin, end)` with `kafka-console-consumer` → `tar` → `PUT` the tarball to S3. +- **restore**: `GET` the tarball from S3 → `tar -x` → recreate the topics with their original partition count and replication factor → replay every partition file with `kafka-console-producer`. + +The same `PodTemplateSpec` serves both directions. The strategy engine templates each string field independently — it cannot add or remove containers per mode — so the single image branches at runtime on `{{ .Mode }}` (rendered into the `MODE` env var). Backup pushes to a key scoped by the source app (`{{ .Release.Name }}`); restore reads the key scoped by the backup's source (`{{ .Backup.ApplicationRef.Name }}`), so a to-copy restore lands the source's data into a differently-named target. The upload uses `curl --aws-sigv4`, so no purpose-built backup image is needed — a stock client image plus a shell script is the whole driver. + +## Consistency model + +The backup takes a **frozen-end-offset cut**. For each topic the driver reads every partition's end offset once (`kafka-get-offsets --time -1`) and its begin offset (`--time -2`), then drains only up to the frozen end. Anything produced after that read (offset ≥ the frozen end) is excluded, so a topic's partitions are captured as of the same instant — a coherent point-in-time view of that topic. The reads happen per topic, so the cut is atomic within a topic but not across topics (Kafka offers no cross-topic guarantee regardless). This needs no dependency on record timestamps: it is a cut by log position, not by wall-clock time. + +## Scope and limitations + +This is a **logical DATA backup**, deliberately narrow — it captures topic records (key + value) and nothing else: + +- **Not captured**: topic configs (retention, compaction, `min.insync.replicas`), ACLs, SCRAM users, and **consumer-group offsets**. Restore recreates each topic with its original partition count and its original replication factor: the backup manifest records the `ReplicationFactor` that `kafka-topics --describe` reports for every topic, and restore recreates the topic with it. That is not cosmetic. The Cozystack Kafka chart raises the broker-level `min.insync.replicas` to 2 as soon as `kafka.replicas` reaches 2, the console producer writes with `acks=all`, and a broker rejects every such write into a topic whose replication factor is below its `min.insync.replicas` — while the producer logs the rejection and exits 0. A topic recreated with too few replicas would therefore be restored *empty* and fail the post-replay count with a misleading message. The `replicationFactor` BackupClass parameter overrides the captured value, which is the way to restore into a cluster with fewer brokers than the source; a value the target cannot host fails at `--create` in Kafka's own words, before any replay. The converse is checked too: after the create, the driver reads the topic's effective `min.insync.replicas` with `kafka-configs --describe --all` and refuses, before replaying, a replication factor below it, naming the parameter to raise, since the alternative is an empty replay reported as a count shortfall. Like every parameter it is snapshotted onto the Backup at backup time, so it has to be set on the BackupClass before the BackupJob whose Backup will be restored. +- **Offsets change on restore**: restore re-produces records, so the target assigns fresh offsets. A consumer group's committed position from the source is therefore meaningless on the restored cluster. Preserving consumer offsets requires either offset translation or a byte-identical volume-snapshot restore (Velero/CSI) — neither is in scope here. +- **Topic names are matched literally**: `kafka-topics` and `kafka-get-offsets` treat `--topic` as a Java regex, so a name carrying a metacharacter would otherwise select its siblings (`audit.events` also matches `audit-events`) — deleting, measuring or backing up the wrong topic. Every filtering call here pins the name with `\Q...\E`; `--create` and the console producer/consumer take a literal name and are left unquoted. The demo is set up so this cannot silently regress: the topic is `orders.v1` and a decoy `orders-v1` sits beside it, matching the first as a regex but not as a literal. The decoy is never named in the `BackupClass`, so a correct run leaves it untouched, and removing a pin from the offset listing, from the in-place `--delete` or from either record count reddens the suite: the partition-set guard counts the decoy's partitions, the `--delete` takes the decoy with it, and the pre- and post-replay counts pick up its records. Two pins are *not* covered by it: `kafka-topics --describe` returns topics in the admin client's result-map order rather than sorted, so which topic the two `PartitionCount` reads see through `head -1` is not something a fixture can pin down. The begin-offset lookups need no pin at all — they match on the whole `topic:partition` key, so a line belonging to another topic cannot supply this partition's begin. The `topics` parameter itself is held to Kafka's `[a-zA-Z0-9._-]` name charset and refused on a repeated name: a duplicate would run the per-topic block twice, overwriting its own data files under a manifest that then lists every partition twice, and no restore of that backup could ever balance its count. +- **Fidelity**: records are round-tripped as `keyvalue` text via the console tools, so keys and values must be newline- and tab-free. Message headers and record timestamps are not preserved. A production driver would use a binary-safe client (e.g. a purpose-built image) for arbitrary payloads. +- **Compacted and transactional topics are out of scope**: the drain reads exactly `end - begin` records per partition, which equals the consumable count only on a plain topic. On a compacted topic (sparse offsets) or one written by transactional producers (control markers occupy offsets but are never delivered), fewer records are consumable. `kafka-console-consumer` does not report this — it idles to its `--timeout-ms` and still exits 0 — so the strategy compares the drained line count against `end - begin` and **fails the backup** on any shortfall rather than uploading a partial tarball. The partition *set* is checked the same way, because `kafka-get-offsets` lists only the partitions whose lookup succeeded and still exits 0: the number of partitions it returned is compared against the topic's own `PartitionCount`, so one skipped by a leader election mid-backup fails the run instead of silently vanishing from it. Note the interaction with the parameter default: an empty `topics` backs up *every* non-internal topic, so the all-topics setting is the riskiest, not the safest — one compacted or transactional topic in the namespace aborts the whole run. Restrict `topics` to topics you know are plain, or accept that the default may fail closed. Conversely, if `topics` is empty and the namespace holds only internal topics, the loop backs up nothing: an empty tarball is uploaded and the `Backup` reports `Succeeded`. That is correct for a genuinely empty cluster, but confirm the source actually held user topics before trusting such a run — the restore side refuses such an artifact outright (`manifest is empty`), so an empty backup is not a restorable one. +- **Partition placement**: keyed records re-produce into the same partition because the topic is recreated with the same partition count and the default (murmur2) partitioner is deterministic. Null-key records fare worse: the dump renders a null key as the literal string `null` (the console formatter's default) and the replay reads it back as a four-byte `null` key — so the record not only lands in an unpredictable partition but has its key changed from null to `"null"`, which shifts its compaction identity downstream. +- **Restore appends, it does not replace**: the driver creates the topic only if absent (`--create --if-not-exists`) and then replays every record, so restoring into a topic that still holds data adds to it rather than overwriting. Restore is safe only against an absent or empty topic — the in-place step below deletes the topic first for exactly this reason. An empty topic is not enough on its own: `--if-not-exists` leaves a pre-existing topic's partition count alone, which would silently re-place every keyed record, so the driver also compares the target's `PartitionCount` against the backup and refuses a differently shaped topic. The driver enforces the emptiness too: it reads the target's record count before replaying and refuses a topic that already holds data, so the strategy Job's `backoffLimit: 2` — which retries a Pod that died mid-replay — cannot stack a second copy on top of a partial one. That pre-replay count is held to the topic's own partition count the same way the backup side is, because a listing shortened by a leader election in flight would otherwise sum the surviving empty partitions to zero and let the replay through. The in-place step drives this path on purpose: after the restore it submits a second `RestoreJob` against the populated topic and requires it to settle `Failed` with the topic unchanged. A retry after a Pod died mid-replay stops at that check on the first topic the dead Pod had already replayed and settles `Failed` too; the restore Pod's log names each topic before replaying it and again once it has landed, so the log says which topics still hold a partial copy. The remedy is to empty or delete every topic the log shows as replayed or started — not only the one the refusal names — and re-run. The post-replay count check then still verifies what landed. +- **Each backup overwrites the previous one**: the S3 object key is `//kafka-topics.tar` with no run identifier, so every `Backup` of the same application resolves to the same tarball and restoring an older `Backup` restores the latest data. The generic `Job` driver exposes no unique per-run handle at backup time — the `Backup` artifact is created only after the Job completes — so a per-run key would need a driver change. +- **Local disk usage is unbounded**: the drain stages the whole topic under `/tmp` and then again as a tarball on the Pod's writable layer, and the strategy Pod requests no `ephemeral-storage`. A large topic can fill the node's ephemeral storage and evict other Pods; set an `ephemeral-storage` request and limit sized to your data before backing up anything substantial. +- **TLS is verified against the endpoint's CA**: the in-cluster seaweedfs presents a self-signed certificate, so `03-create-bucket.sh` caches its CA (the `seaweedfs-ca-cert` Secret in `tenant-root`, or the cert-manager CA `Certificate` it discovers when that name is absent) and `create_s3_secret` adds it as `ca.crt` to each `-backup-s3` Secret. The strategy Pod projects that one key as a file and points `curl --cacert` at it; a Secret without the key leaves the file absent and curl falls back to the image's trust store, which is the right behaviour for a publicly-trusted endpoint — set `S3_CA_SECRET=""` to skip the copy there, since `--cacert` replaces the system bundle rather than extending it. The `rabbitmq` example mounts the same CA for the same reason. +- **Pod Security**: the strategy Pod and the demo's CLI Pod set `runAsNonRoot`, `allowPrivilegeEscalation: false`, drop all capabilities and a `RuntimeDefault` seccomp profile, so they satisfy the `restricted` Pod Security Standard on the stock Strimzi image, which runs as non-root UID 1001. Pinning an image that runs as root would need a `runAsUser` suited to that image instead. + +For a faithful, offset-preserving Kafka backup, prefer a volume-snapshot strategy over this logical one; this example exists to show the generic `Job` path end to end. + +## Step order + +| File | Role | Triggered by | +|---|---|---| +| `00-helpers.sh` | Shared bash helpers, env defaults, and the kafka-CLI / S3-secret helpers; sourced by every step. | n/a | +| `01-create-strategy.sh` | Creates the cluster-scoped `Job` strategy (the Kafka backup/restore `PodTemplateSpec`). | admin | +| `02-create-backupclass.sh` | Maps `apps.cozystack.io/Kafka` to that strategy, with the `topics` parameter (and the optional `replicationFactor` override). | admin | +| `03-create-bucket.sh` | Provisions a `Bucket` and caches its S3 coordinates plus the endpoint's CA into `.bucket-info.env` (chmod 600; raw access keys). `cleanup.sh` removes this file. | **admin**. A tenant admin can create the `Bucket` itself, but the script also waits on its `HelmRelease` and COSI objects, reads the `bucket--backup` Secret, and copies the CA from a Secret in `tenant-root` (see the TLS bullet above); no tenant role grants access to `helmreleases` or `objectstorage.k8s.io`, or `get` on those Secrets | +| `04-create-kafka.sh` | Provisions a two-broker `Kafka`, creates the `-backup-s3` Secret, seeds a topic (replication factor 2) with sentinel messages. | **admin**. A tenant admin can create the `Kafka` itself, but the script also waits on its `HelmRelease` and Strimzi `Kafka`, creates the Secret, and seeds the topic through a CLI Pod it execs into; no tenant role grants access to `helmreleases` or `kafkas.kafka.strimzi.io`, `create` on `secrets` or `pods`, or `pods/exec` | +| `05-create-backupjob.sh` | Submits a `BackupJob`, waits for Succeeded and caches the resolved `Backup` name into `.backup-name.env`. | tenant | +| `06-restore-in-place.sh` | Deletes the topic and restores it into the same instance via `RestoreJob`, then submits a second `RestoreJob` that must be refused because the topic is no longer empty. | **admin**. A tenant admin can submit the two `RestoreJob`s, but the script deletes and checks the topic through the CLI Pod and reads the refusal from the restore Pod's log; no tenant role grants `create` on `pods` or `pods/exec`, or `get` on `pods/log` | +| `07-restore-to-copy.sh` | Provisions a second two-broker `Kafka` and restores into it via `RestoreJob.spec.targetApplicationRef`. | **admin**, for the same reasons as step 04. The tenant admin's part is the second `Kafka` and the `RestoreJob` | +| `cleanup.sh` | Removes everything created by the demo. | **admin**: it deletes Secrets, the `Backup`, and the cluster-scoped `BackupClass` and strategy, and no tenant role grants `delete` on any of them | +| `run-all.sh` | Convenience runner that executes 01..07 in order. | demo | + +The scenario walk-throughs `90-scenario-admin-prepare.md`, `91-scenario-user-backup.md` and `92-scenario-user-restore.md` describe the same steps from the admin's and the tenant's side, and say which parts of the tenant's side need admin access. + +## Running + +```sh +# Uses NAMESPACE=tenant-root by default; override any knob via the environment. +./run-all.sh +# ... or step by step, in numeric order. +./cleanup.sh +``` + +The demo runs in `tenant-root` because that is where the in-cluster seaweedfs lives: a tenant created isolated has a Cilium egress allowlist that blocks its Pods from reaching it. Point `NAMESPACE` at another tenant only if its Pods can reach the S3 endpoint the `Bucket` advertises. + +Both Kafka instances run `KAFKA_REPLICAS` (default 2) brokers and the demo topics are created with `TOPIC_REPLICAS` (default 2) replicas. That is deliberate: with two brokers the chart sets `min.insync.replicas` to 2, so a restore that recreated the topic with the wrong replication factor could not land a single record, and the restore steps additionally assert that the restored topic carries the captured replication factor. A single-broker run (`KAFKA_REPLICAS=1 TOPIC_REPLICAS=1`) works too, but proves less. + +By default the scripts resolve `KAFKA_IMAGE` from the running Strimzi operator — the newest image in its `STRIMZI_KAFKA_IMAGES` map, tag- or digest-pinned — so the backup Pod reuses an image already cached on the nodes with no extra pull or version skew. Set `KAFKA_IMAGE` explicitly to pin a different one, for example on an air-gapped clone where the operator cannot be read (the backup Pod only needs the `kafka-*.sh` CLI plus `curl` and `tar`). To see what the operator ships: + +```sh +kubectl -n cozy-kafka-operator get deploy strimzi-cluster-operator \ + -o jsonpath='{range .spec.template.spec.containers[0].env[*]}{.name}={.value}{"\n"}{end}' | grep KAFKA_IMAGES +``` + +[strimzi-image]: https://quay.io/repository/strimzi/kafka diff --git a/examples/backups/kafka/cleanup.sh b/examples/backups/kafka/cleanup.sh new file mode 100755 index 0000000000..0e3340a48d --- /dev/null +++ b/examples/backups/kafka/cleanup.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Cleanup: tear down everything provisioned by the demo so the cluster returns +# to its previous state. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Cleanup Kafka backup demo" + +# Fully qualify every kind: a Cozystack cluster serves four `backups` and two +# `buckets` CRDs, so a bare `kubectl delete backup`/`bucket` may resolve to a +# foreign group whose miss --ignore-not-found then swallows, leaving this demo's +# Cozystack Backup behind for the next run's createJobBackupArtifact to reuse. +kubectl -n "$NAMESPACE" delete restorejob.backups.cozystack.io "$RESTOREJOB_TOCOPY_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete restorejob.backups.cozystack.io "$RESTOREJOB_NONEMPTY_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete restorejob.backups.cozystack.io "$RESTOREJOB_INPLACE_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete backupjob.backups.cozystack.io "$BACKUPJOB_NAME" --ignore-not-found +# The Backup's resolved name is cached by step 05; a run that never got there +# can only have left one named after the BackupJob. +BACKUP_NAME="$BACKUPJOB_NAME" +# shellcheck disable=SC1091 +[[ -f "$SCRIPT_DIR/.backup-name.env" ]] && source "$SCRIPT_DIR/.backup-name.env" +kubectl -n "$NAMESPACE" delete backups.backups.cozystack.io "$BACKUP_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete secret "${KAFKA_RESTORE_NAME}-backup-s3" --ignore-not-found +kubectl -n "$NAMESPACE" delete secret "${KAFKA_NAME}-backup-s3" --ignore-not-found +kubectl -n "$NAMESPACE" delete kafka.apps.cozystack.io "$KAFKA_RESTORE_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete kafka.apps.cozystack.io "$KAFKA_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete bucket.apps.cozystack.io "$BUCKET_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete pod -l cozystack.io/backup-demo=kafka --grace-period=1 --ignore-not-found +rm -f "$SCRIPT_DIR/.bucket-info.env" "$SCRIPT_DIR/.source-dump.txt" "$SCRIPT_DIR/.backup-name.env" +kubectl delete backupclass.backups.cozystack.io "$BACKUPCLASS_NAME" --ignore-not-found +kubectl delete job.strategy.backups.cozystack.io "$STRATEGY_NAME" --ignore-not-found + +# `kubectl delete` returns as soon as deletionTimestamp is set, while the +# HelmRelease uninstall behind each Kafka app (two StatefulSets, ZooKeeper and +# their PVCs) is still draining. Returning here lets the next suite start into a +# namespace whose predecessor is still uninstalling - the neighbouring +# chainsaw/kafka Test's PVC-reclaim step contends with exactly that. Wait for +# the heavy objects to actually go, and with teeth: without `set -e` an absent +# resource cannot abandon the rest of the teardown, so the waits record their +# own failure and the script exits non-zero when the teardown did not settle. +teardown_rc=0 +wait_deleted kafka.apps.cozystack.io "$KAFKA_RESTORE_NAME" 180 || teardown_rc=1 +wait_deleted kafka.apps.cozystack.io "$KAFKA_NAME" 180 || teardown_rc=1 +wait_deleted bucket.apps.cozystack.io "$BUCKET_NAME" 180 || teardown_rc=1 + +if [[ $teardown_rc -ne 0 ]]; then + log_error "Cleanup did not settle: see the resources reported above. Failing rather than leaving a half-uninstalled release for the next test to trip over." + exit 1 +fi + +log_success "Cleanup complete." diff --git a/examples/backups/kafka/run-all.sh b/examples/backups/kafka/run-all.sh new file mode 100755 index 0000000000..9c1aaddcbe --- /dev/null +++ b/examples/backups/kafka/run-all.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Run every numbered step in order. Stops on the first failure. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +for step in 01 02 03 04 05 06 07; do + # find is glob-safe under set -euo pipefail and degrades to empty output + # when no file matches, unlike `ls glob | head` which mixes shell + # expansion semantics with `ls` exit codes. + script="$(find "$SCRIPT_DIR" -maxdepth 1 -type f -name "${step}-*.sh" | sort | head -n1)" + if [[ -z "$script" ]]; then + echo "no script found for step $step" >&2 + exit 1 + fi + echo "=== Running $(basename "$script") ===" + bash "$script" +done diff --git a/examples/backups/mariadb/00-bucket.yaml b/examples/backups/mariadb/00-bucket.yaml index cfcb6b8ec7..05b011087d 100644 --- a/examples/backups/mariadb/00-bucket.yaml +++ b/examples/backups/mariadb/00-bucket.yaml @@ -1,6 +1,7 @@ # Cozystack-managed S3 bucket for storing MariaDB backups. The Bucket -# controller provisions a Secret named "-" per user; the -# MariaDB strategy below references the "backup" user's Secret. +# controller provisions a Secret named "bucket--" per +# user (the bucket app carries the "bucket-" release prefix); the MariaDB +# strategy below references the "backup" user's Secret. --- apiVersion: apps.cozystack.io/v1alpha1 kind: Bucket diff --git a/examples/backups/mariadb/00-helpers.sh b/examples/backups/mariadb/00-helpers.sh new file mode 100755 index 0000000000..32f4c01d97 --- /dev/null +++ b/examples/backups/mariadb/00-helpers.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# Shared helpers for the MariaDB backup/restore demo. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export BLUE='\033[0;34m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export WHITE='\033[1;37m' +export NC='\033[0m' +export BOLD='\033[1m' + +# Default settings (override via environment). +export NAMESPACE="${NAMESPACE:-tenant-root}" +export BUCKET_NAME="${BUCKET_NAME:-mariadb-backups}" +# Bucket user declared in 00-bucket.yaml; the COSI flow materialises the +# credentials Secret as "bucket--". +export BUCKET_USER="${BUCKET_USER:-backup}" +export MARIADB_SRC_NAME="${MARIADB_SRC_NAME:-mariadb-src}" +export MARIADB_TARGET_NAME="${MARIADB_TARGET_NAME:-mariadb-target}" +# The mariadb-rd ApplicationDefinition renders the HelmRelease with +# releaseName = "mariadb-" + appName (release.prefix in +# packages/system/mariadb-rd/cozyrds/mariadb.yaml), so the operator-side +# k8s.mariadb.com/MariaDB CR (and its primary Service) is mariadb-. +export MARIADB_SRC_CR="mariadb-${MARIADB_SRC_NAME}" +export MARIADB_TARGET_CR="mariadb-${MARIADB_TARGET_NAME}" +export STRATEGY_NAME="${STRATEGY_NAME:-mariadb-strategy-default}" +export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-mariadb-default}" +export BACKUPJOB_NAME="${BACKUPJOB_NAME:-mariadb-src-adhoc}" +export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-mariadb-src-to-mariadb-target}" +export PLAN_NAME="${PLAN_NAME:-mariadb-src-daily}" +# App user the demo writes and reads the sentinel row as. Its password is +# chart-generated (the charts no longer accept a plaintext password) and read +# from the release's -credentials Secret by mysql_exec at call time. +export MARIADB_APP_USER="${MARIADB_APP_USER:-app}" + +# S3 endpoint CA. cozystack's default seaweedfs serves its S3 endpoint with a +# self-signed certificate whose CA lives in this Secret; the demo copies its +# ca.crt into a per-app Secret the mariadb-operator Backup CR trusts via +# storage.s3.tls.caSecretKeyRef. The name follows the seaweedfs chart's +# fullnameOverride ("seaweedfs" -> "seaweedfs-ca-cert"); run-all.sh +# auto-discovers the CA Certificate's actual secret when this default is +# absent, so an upstream fullname change does not silently break the copy. On +# a cluster whose S3 endpoint is signed by a publicly-trusted CA, set +# S3_CA_SECRET="" to skip the copy and drop the tls block from the manifests. +export S3_CA_SECRET="${S3_CA_SECRET:-seaweedfs-ca-cert}" +export S3_CA_NAMESPACE="${S3_CA_NAMESPACE:-tenant-root}" +export S3_CA_KEY="${S3_CA_KEY:-ca.crt}" + +log_info() { echo -e "${BLUE}i${NC} $*" >&2; } +log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; } +log_error() { echo -e "${RED}x${NC} $*" >&2; } +log_step() { echo -e "\n${MAGENTA}${BOLD}> $*${NC}" >&2; } +log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; } + +print_header() { + echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2 +} + +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" + +# Name of the primary pod of a k8s.mariadb.com/MariaDB CR (the writer). +# mariadb-operator 25.10.x reports the primary identity on +# MariaDB.status.currentPrimary rather than a pod label, so read it there. +mariadb_primary_pod() { + local cr="$1" + kubectl -n "$NAMESPACE" get mariadbs.k8s.mariadb.com "$cr" \ + -o jsonpath='{.status.currentPrimary}' 2>/dev/null +} + +# Run a single SQL statement against a MariaDB CR's primary as the app user, +# routing through the operator-managed primary Service (mariadb--primary). +# Args: +mysql_exec() { + local cr="$1" sql="$2" + local pod pass + pod=$(mariadb_primary_pod "$cr") + [[ -n "$pod" ]] || { log_error "no primary pod for MariaDB CR '$cr'"; return 1; } + pass=$(kubectl -n "$NAMESPACE" get secret "${cr}-credentials" -o json \ + | jq -r --arg u "$MARIADB_APP_USER" '.data[$u] // empty' | base64 -d) + [[ -n "$pass" ]] || { log_error "no password for '$MARIADB_APP_USER' in ${cr}-credentials"; return 1; } + kubectl -n "$NAMESPACE" exec "$pod" -c mariadb -- \ + mariadb -u"$MARIADB_APP_USER" -p"$pass" -h "${cr}-primary" -e "$sql" +} + +# Authenticate as root against a MariaDB CR's primary over TCP, using the +# chart-generated root password in -credentials. root is the sharp edge on a +# restore into a copy: the operator only applies rootPasswordSecretKeyRef at +# datadir bootstrap, so if a restore overwrote the grant table with the source's +# root hash the target's advertised root would be wrong for good. Args: +mysql_root_login() { + local cr="$1" + local pod pass + pod=$(mariadb_primary_pod "$cr") + [[ -n "$pod" ]] || { log_error "no primary pod for MariaDB CR '$cr'"; return 1; } + pass=$(kubectl -n "$NAMESPACE" get secret "${cr}-credentials" \ + -o "jsonpath={.data['root']}" | base64 -d) + [[ -n "$pass" ]] || { log_error "no root password in ${cr}-credentials"; return 1; } + kubectl -n "$NAMESPACE" exec "$pod" -c mariadb -- \ + mariadb -uroot -p"$pass" -h "${cr}-primary" -e "SELECT 1;" +} + +# Run a single SQL statement against a MariaDB CR's primary as root over TCP. +# Used for statements the app user is not privileged for (CREATE USER / GRANT), +# e.g. seeding an out-of-band account the chart does not declare. root has no dot, +# so a plain jsonpath read of its key is safe here. Args: +mysql_root_exec() { + local cr="$1" sql="$2" + local pod pass + pod=$(mariadb_primary_pod "$cr") + [[ -n "$pod" ]] || { log_error "no primary pod for MariaDB CR '$cr'"; return 1; } + pass=$(kubectl -n "$NAMESPACE" get secret "${cr}-credentials" \ + -o "jsonpath={.data['root']}" | base64 -d) + [[ -n "$pass" ]] || { log_error "no root password in ${cr}-credentials"; return 1; } + kubectl -n "$NAMESPACE" exec "$pod" -c mariadb -- \ + mariadb -uroot -p"$pass" -h "${cr}-primary" -e "$sql" +} diff --git a/examples/backups/mariadb/05-mariadb-src.yaml b/examples/backups/mariadb/05-mariadb-src.yaml index 6c03d89942..4f9986cf74 100644 --- a/examples/backups/mariadb/05-mariadb-src.yaml +++ b/examples/backups/mariadb/05-mariadb-src.yaml @@ -24,5 +24,4 @@ spec: - app users: app: - password: REPLACE_WITH_PASSWORD maxUserConnections: 100 diff --git a/examples/backups/mariadb/10-mariadb-strategy.yaml b/examples/backups/mariadb/10-mariadb-strategy.yaml index 7435141c69..995793848d 100644 --- a/examples/backups/mariadb/10-mariadb-strategy.yaml +++ b/examples/backups/mariadb/10-mariadb-strategy.yaml @@ -32,8 +32,8 @@ spec: # so deploy a per-app Secret named "-mariadb-backup-creds" # holding AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY extracted from # BucketInfo (extract the access key id / secret access key from - # the BucketInfo JSON and project them under those two keys; a - # bats e2e harness lands in a follow-up PR). + # the BucketInfo JSON and project them under those two keys; + # run-all.sh in this directory does this automatically). accessKeyIdSecretKeyRef: name: "{{ .Application.metadata.name }}-mariadb-backup-creds" key: "AWS_ACCESS_KEY_ID" diff --git a/examples/backups/mariadb/30-mariadb-target.yaml b/examples/backups/mariadb/30-mariadb-target.yaml index fb5da0152d..d859ae5a0b 100644 --- a/examples/backups/mariadb/30-mariadb-target.yaml +++ b/examples/backups/mariadb/30-mariadb-target.yaml @@ -12,11 +12,11 @@ # mariadb-src; replicate this whenever you want to verify a restore # under non-root credentials. # -# Same-namespace to-copy flows are smoke-testable manually by applying -# this manifest alongside mariadb-src. Cross-tenant restores (target in -# tenant-test, source's seaweedfs in tenant-root) are blocked by the -# per-tenant Cilium egress policy and stay a manual / dev-cluster flow. -# A dedicated bats e2e harness will land in a follow-up PR. +# Same-namespace to-copy flows are exercised by run-all.sh in this +# directory (and by the mariadb-2-backup-roundtrip Chainsaw suite). +# Cross-tenant restores (target in tenant-test, source's seaweedfs in +# tenant-root) are blocked by the per-tenant Cilium egress policy and +# stay a manual / dev-cluster flow. --- apiVersion: apps.cozystack.io/v1alpha1 kind: MariaDB @@ -32,5 +32,4 @@ spec: - app users: app: - password: REPLACE_WITH_PASSWORD maxUserConnections: 100 diff --git a/examples/backups/mariadb/40-restorejob-to-copy.yaml b/examples/backups/mariadb/40-restorejob-to-copy.yaml index d29d6f294e..93c9b5659c 100644 --- a/examples/backups/mariadb/40-restorejob-to-copy.yaml +++ b/examples/backups/mariadb/40-restorejob-to-copy.yaml @@ -4,10 +4,10 @@ # dump into the running target while mariadb-src stays running and # untouched. # -# Smoke-testable manually with kubectl apply -f for the same-namespace -# happy path. The cross-tenant variant stays manual; see -# 30-mariadb-target.yaml for the network-policy reason. A dedicated bats -# e2e harness will land in a follow-up PR. +# Driven for the same-namespace happy path by run-all.sh in this +# directory (and the mariadb-2-backup-roundtrip Chainsaw suite); also +# applicable manually with kubectl apply -f. The cross-tenant variant +# stays manual; see 30-mariadb-target.yaml for the network-policy reason. --- apiVersion: backups.cozystack.io/v1alpha1 kind: RestoreJob diff --git a/examples/backups/mariadb/README.md b/examples/backups/mariadb/README.md index 1a548d9ba7..97b2aa17e2 100644 --- a/examples/backups/mariadb/README.md +++ b/examples/backups/mariadb/README.md @@ -27,18 +27,29 @@ MariaDB application via the `strategy.backups.cozystack.io/v1alpha1` | 7 | `35-restorejob-in-place.yaml` | namespaced | Destructive restore back into the source. | | 8 | `40-restorejob-to-copy.yaml` | namespaced | Non-destructive restore into the target. | -Replace `REPLACE_WITH_*` placeholders in `10-mariadb-strategy.yaml` -(bucket, S3 endpoint) and `05-mariadb-src.yaml` (password) before -applying. - -The MariaDB driver delegates execution to the open-source -[mariadb-operator](https://github.com/mariadb-operator/mariadb-operator) -(`k8s.mariadb.com` CRDs, already shipped with Cozystack). Backups -materialise as `Backup` CRs and restores as `Restore` CRs in the -application's namespace; the operator handles the actual `mariadb-dump` -/ `mariadb-import` invocations. - -Same-namespace flows are smoke-testable manually with `kubectl apply -f`. -Cross-tenant restores are blocked by the per-tenant Cilium egress policy -and stay a manual flow. A dedicated bats e2e harness will land in a -follow-up PR. +The MariaDB driver delegates execution to the open-source [mariadb-operator](https://github.com/mariadb-operator/mariadb-operator) (`k8s.mariadb.com` CRDs, already shipped with Cozystack). Backups materialise as `Backup` CRs and restores as `Restore` CRs in the application's namespace; the operator handles the actual `mariadb-dump` / `mariadb-import` invocations. + +## Placeholders and derived Secrets + +`10-mariadb-strategy.yaml` carries `REPLACE_WITH_COSI_BUCKET_NAME` and `REPLACE_WITH_S3_ENDPOINT` (a path-style endpoint without scheme); the demo `app` user declared by `05-mariadb-src.yaml` / `30-mariadb-target.yaml` has a chart-generated password, not a substituted one. The strategy also references two Secrets, `-mariadb-backup-creds` (the S3 credentials the mariadb-operator Backup reads) and `-mariadb-backup-ca` (the CA it trusts for a self-signed endpoint), where `` is the Cozystack application name. The strategy template renders those names against whichever application it drives, so the restore target needs its own pair too — a restore into `mariadb-target` looks up `mariadb-target-mariadb-backup-*`. `run-all.sh` resolves the placeholders from the provisioned `Bucket`'s `BucketInfo` Secret and materialises the pair for both the source and the target; editing by hand, you copy the coordinates from `kubectl -n get secret bucket--backup -o jsonpath='{.data.BucketInfo}'` and the CA from the seaweedfs CA secret in `tenant-root` (`seaweedfs-ca-cert` by default; `run-all.sh` auto-discovers it from the cert-manager Certificate if the name differs). + +Drop the `tls` block from `10-mariadb-strategy.yaml` (and set `S3_CA_SECRET=""` for `run-all.sh`) when the S3 endpoint is signed by a publicly-trusted CA. + +## Run it + +```sh +# Defaults to NAMESPACE=tenant-root; override for a tenant namespace. +NAMESPACE=tenant-root examples/backups/mariadb/run-all.sh +# Tear everything down afterwards (idempotent). +NAMESPACE=tenant-root examples/backups/mariadb/cleanup.sh +``` + +`S3_CA_NAMESPACE` defaults to `tenant-root` (where the shared seaweedfs and its CA live) independently of `NAMESPACE`. Running against a different tenant whose own seaweedfs CA lives elsewhere means overriding `S3_CA_NAMESPACE` (and `S3_CA_SECRET`) too, alongside `NAMESPACE`. + +`run-all.sh` writes a sentinel row into the source, waits for the `BackupJob` to reach `Succeeded`, restores to a copy with a to-copy `RestoreJob`, and asserts the sentinel round-tripped through S3 into the restored copy while the source is left untouched. Set `SKIP_RESTORE=1` to stop after a successful backup. + +Same-namespace flows are the supported path. Cross-tenant restores (target in `tenant-test`, source's seaweedfs in `tenant-root`) are blocked by the per-tenant Cilium egress policy and stay a manual / dev-cluster flow. + +## Automated e2e + +The Chainsaw suite at `hack/e2e-chainsaw/mariadb/` drives this same `run-all.sh` as a second test (`mariadb-2-backup-roundtrip`), selected by Test-Impact Analysis whenever the mariadb app or the suite changes, and on every release cut. It runs in `tenant-root` against the in-cluster seaweedfs endpoint — the isolated e2e tenant cannot reach it across the Cilium egress policy, and the external ingress endpoint is an unroutable placeholder in the sandbox. diff --git a/examples/backups/mariadb/cleanup.sh b/examples/backups/mariadb/cleanup.sh new file mode 100755 index 0000000000..4774ae151a --- /dev/null +++ b/examples/backups/mariadb/cleanup.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Best-effort teardown of everything the demo creates. Idempotent; +# missing resources are skipped silently. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Cleanup" + +log_substep "Deleting RestoreJob..." +kubectl -n "$NAMESPACE" delete restorejob.backups.cozystack.io "$RESTOREJOB_TOCOPY_NAME" --ignore-not-found + +log_substep "Deleting Plan + BackupJob + derived Backup artefacts..." +kubectl -n "$NAMESPACE" delete plan.backups.cozystack.io "$PLAN_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete backupjob.backups.cozystack.io "$BACKUPJOB_NAME" --ignore-not-found +# Only the Backups of THIS demo's apps — never `--all`: on a shared cluster the +# namespace can hold Backup artefacts of other applications and flows. +for b in $(kubectl -n "$NAMESPACE" get backups.backups.cozystack.io \ + -o jsonpath='{range .items[*]}{.metadata.name} {.spec.applicationRef.name}{"\n"}{end}' 2>/dev/null | + awk -v src="$MARIADB_SRC_NAME" -v tgt="$MARIADB_TARGET_NAME" '$2 == src || $2 == tgt {print $1}'); do + kubectl -n "$NAMESPACE" delete backups.backups.cozystack.io "$b" --ignore-not-found +done + +# The Cozystack BackupJob/RestoreJob own their operator-side k8s.mariadb.com +# Backup/Restore CRs only by the OwningJob labels (not OwnerReferences), so +# deleting the Cozystack jobs does not cascade to them. Left behind, a stale +# operator-side Backup would be reused by findMariaDBBackupForJob on the next +# run against a bucket path that no longer exists. Prune them by owner. +log_substep "Deleting operator-side k8s.mariadb.com Backup/Restore CRs..." +for owner in "$BACKUPJOB_NAME" "$RESTOREJOB_TOCOPY_NAME"; do + kubectl -n "$NAMESPACE" delete backups.k8s.mariadb.com,restores.k8s.mariadb.com \ + -l "backups.cozystack.io/owned-by.BackupJobName=${owner}" --ignore-not-found +done + +log_substep "Deleting MariaDB apps..." +kubectl -n "$NAMESPACE" delete mariadb.apps.cozystack.io "$MARIADB_SRC_NAME" "$MARIADB_TARGET_NAME" --ignore-not-found + +log_substep "Deleting per-app backup Secrets..." +kubectl -n "$NAMESPACE" delete secret \ + "${MARIADB_SRC_NAME}-mariadb-backup-creds" "${MARIADB_SRC_NAME}-mariadb-backup-ca" \ + "${MARIADB_TARGET_NAME}-mariadb-backup-creds" "${MARIADB_TARGET_NAME}-mariadb-backup-ca" --ignore-not-found + +log_substep "Deleting Bucket..." +kubectl -n "$NAMESPACE" delete bucket.apps.cozystack.io "$BUCKET_NAME" --ignore-not-found + +log_substep "Deleting BackupClass + MariaDB strategy (cluster-scoped)..." +kubectl delete backupclass.backups.cozystack.io "$BACKUPCLASS_NAME" --ignore-not-found +kubectl delete mariadb.strategy.backups.cozystack.io "$STRATEGY_NAME" --ignore-not-found + +log_success "Cleanup complete." diff --git a/examples/backups/mariadb/run-all.sh b/examples/backups/mariadb/run-all.sh new file mode 100755 index 0000000000..dbc845ce83 --- /dev/null +++ b/examples/backups/mariadb/run-all.sh @@ -0,0 +1,254 @@ +#!/bin/bash +# Convenience runner + e2e harness for the MariaDB backup/restore demo. +# +# It applies the SAME numbered manifests a human reads (00-bucket.yaml .. +# 40-restorejob-to-copy.yaml), filling the REPLACE_WITH_* placeholders from the +# provisioned Bucket and deriving the two Secrets the manifests reference +# (-mariadb-backup-creds, -mariadb-backup-ca) so the documented flow +# and the automated test can never drift. Stops on the first failure. +# +# Flow: Bucket -> source MariaDB (+ a sentinel row) -> MariaDB strategy + +# BackupClass -> ad-hoc BackupJob (wait Succeeded) -> empty target MariaDB + +# to-copy RestoreJob (wait Succeeded) -> assert the sentinel round-tripped +# through S3 into the restored copy while the source stays untouched. +# +# Why to-copy and not in-place: in-place replays the dump straight into the +# live source MariaDB (dropping and re-creating the restored databases there), +# so it cannot witness the restore on a separate instance. To-copy leaves the +# source running and lands the marker row on a distinct target, a stronger +# restore proof; the in-place dispatch is covered at the unit level by the +# controller tests. +# +# Override NAMESPACE / endpoint / CA via the environment; see 00-helpers.sh. +# hack/e2e-chainsaw/mariadb/ drives this file as mariadb-2-backup-roundtrip. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +# Substitute the manifest placeholders. $BUCKET / $S3_HOST are resolved from the +# Bucket below. The app user's password is chart-generated, not substituted. +subst() { + sed \ + -e "s|REPLACE_WITH_COSI_BUCKET_NAME|${BUCKET}|g" \ + -e "s|REPLACE_WITH_S3_ENDPOINT|${S3_HOST}|g" \ + "$SCRIPT_DIR/$1" +} + +# The mariadb-operator provisions the demo database + app user + grant from the +# chart as k8s.mariadb.com CRs named after the release (mariadb-). The +# grant is the last of the three to reconcile, so its readiness is the signal +# that the app user can actually write into demo. Both "demo" and "app" are +# already DNS-safe, so the grant CR is -demo-app. +wait_app_grant_ready() { + local cr="$1" timeout="${2:-180}" + wait_for_field grants.k8s.mariadb.com "${cr}-demo-app" \ + '{.status.conditions[?(@.type=="Ready")].status}' True "$NAMESPACE" "$timeout" +} + +print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml" +# 660s, above this generated release's 600s install timeout; see "Sizing an +# HR-Ready budget" in docs/agents/e2e-testing.md. +wait_hr_ready "bucket-${BUCKET_NAME}" 660 +wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \ + '{.status.bucketReady}' true "$NAMESPACE" 300 +wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \ + '{.status.accessGranted}' true "$NAMESPACE" 300 +if [[ "${COZY_E2E_BACKUP_PREFLIGHT:-0}" == "1" ]]; then + source "$SCRIPT_DIR/../../../hack/e2e-chainsaw/_lib/backup-access-preflight.sh" + cozy_backup_access_preflight "$NAMESPACE" "bucket-${BUCKET_NAME}-${BUCKET_USER}" 90 +fi + +log_substep "Reading bucket coordinates from BucketInfo Secret..." +TMP=$(mktemp) +trap 'rm -f "$TMP"' EXIT +kubectl -n "$NAMESPACE" get secret "bucket-${BUCKET_NAME}-${BUCKET_USER}" \ + -o jsonpath='{.data.BucketInfo}' | base64 -d > "$TMP" +S3_ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' "$TMP") +S3_SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' "$TMP") +# S3_ENDPOINT can be overridden via the environment: BucketInfo advertises the +# EXTERNAL ingress endpoint, which in-cluster Pods cannot always reach or +# TLS-validate; the in-cluster alternative is https://seaweedfs-s3.:8333 +# (trusted via the copied CA below). +S3_ENDPOINT="${S3_ENDPOINT:-$(jq -r '.spec.secretS3.endpoint' "$TMP")}" +BUCKET=$(jq -r '.spec.bucketName' "$TMP") +for v in S3_ACCESS_KEY S3_SECRET_KEY S3_ENDPOINT BUCKET; do + [[ -n "${!v}" && "${!v}" != "null" ]] || { log_error "BucketInfo missing required field: ${v}"; exit 1; } +done +# The strategy manifest takes a path-style endpoint WITHOUT scheme, so strip it. +S3_HOST=${S3_ENDPOINT#https://}; S3_HOST=${S3_HOST#http://} +log_success "Bucket '${BUCKET}' at endpoint '${S3_ENDPOINT}'." + +print_header "Step 05a: Materialise the Secrets the MariaDB strategy references" +# Resolve the S3 endpoint CA secret. The default name tracks the seaweedfs +# chart's fullnameOverride (seaweedfs -> seaweedfs-ca-cert), but a downstream +# fullname change would rename it, so fall back to discovering the cert-manager +# CA Certificate (the seaweedfs-labelled one with spec.isCA=true) and read its +# secretName. Leave S3_CA_SECRET empty to skip the copy on a public-CA endpoint. +if [[ -n "$S3_CA_SECRET" ]] \ + && ! kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" >/dev/null 2>&1; then + log_warning "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} not found; discovering the seaweedfs CA Certificate..." + # List every seaweedfs Certificate as " " and pick the CA + # one in shell — avoids kubectl jsonpath's finicky boolean-literal filter. + DISCOVERED_CA=$(kubectl -n "$S3_CA_NAMESPACE" get certificates.cert-manager.io \ + -l app.kubernetes.io/name=seaweedfs \ + -o jsonpath='{range .items[*]}{.spec.isCA}{" "}{.spec.secretName}{"\n"}{end}' 2>/dev/null \ + | awk '$1=="true"{print $2; exit}' || true) + if [[ -n "$DISCOVERED_CA" ]]; then + log_success "Discovered seaweedfs CA secret ${S3_CA_NAMESPACE}/${DISCOVERED_CA}" + S3_CA_SECRET="$DISCOVERED_CA" + else + log_error "No seaweedfs CA Certificate found in ${S3_CA_NAMESPACE}; set S3_CA_SECRET explicitly (or empty for a public-CA endpoint)." + exit 1 + fi +fi +# The strategy (10) renders -mariadb-backup-creds / -mariadb-backup-ca +# against whichever application it drives. .Application is the cozystack +# apps.cozystack.io/MariaDB object, so .Application.metadata.name is the app +# name (mariadb-src), NOT the prefixed operator-side CR (mariadb-mariadb-src). +# mariadb-operator reads AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY from the +# creds Secret and trusts ca.crt from the CA Secret when talking to a +# self-signed S3 endpoint, so a restore TARGET needs its own pair too +# (materialised again in step 30). kubectl apply (not create) so a stale pair +# from an earlier run is corrected. +materialise_backup_secrets() { + local app="$1" + kubectl -n "$NAMESPACE" create secret generic "${app}-mariadb-backup-creds" \ + --from-literal="AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY}" \ + --from-literal="AWS_SECRET_ACCESS_KEY=${S3_SECRET_KEY}" \ + --dry-run=client -o yaml | kubectl apply -f - + + if [[ -n "$S3_CA_SECRET" ]]; then + log_substep "Copying S3 CA ${S3_CA_NAMESPACE}/${S3_CA_SECRET}[${S3_CA_KEY}] -> ${app}-mariadb-backup-ca..." + local ca_pem + ca_pem=$(kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" \ + -o jsonpath="{.data.${S3_CA_KEY//./\\.}}" | base64 -d) + [[ -n "$ca_pem" ]] || { log_error "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} has no ${S3_CA_KEY}"; exit 1; } + kubectl -n "$NAMESPACE" create secret generic "${app}-mariadb-backup-ca" \ + --from-literal="ca.crt=${ca_pem}" \ + --dry-run=client -o yaml | kubectl apply -f - + else + log_warning "S3_CA_SECRET empty: the strategy still references ${app}-mariadb-backup-ca." + log_warning "Remove the tls block from 10-mariadb-strategy.yaml by hand when the S3 endpoint uses a public CA." + fi +} +materialise_backup_secrets "$MARIADB_SRC_NAME" + +print_header "Step 05b: Deploy source MariaDB '${MARIADB_SRC_NAME}' and wait for it to be healthy" +subst 05-mariadb-src.yaml | kubectl -n "$NAMESPACE" apply -f - +# 660s, above this generated release's 600s install timeout; see "Sizing an +# HR-Ready budget" in docs/agents/e2e-testing.md. hack/mariadb-first-boot-waits.bats +# derives the floor and holds the Chainsaw op in +# hack/e2e-chainsaw/mariadb/chainsaw-test.yaml against this wait: raise one, +# raise both. +wait_hr_ready "mariadb-${MARIADB_SRC_NAME}" 660 +wait_for_field mariadbs.k8s.mariadb.com "$MARIADB_SRC_CR" \ + '{.status.conditions[?(@.type=="Ready")].status}' True "$NAMESPACE" 600 +wait_app_grant_ready "$MARIADB_SRC_CR" + +print_header "Step 05c: Write a sentinel row so the backup has something to prove" +SENTINEL_TOKEN="e2e-$(date +%s)-$$" +mysql_exec "$MARIADB_SRC_CR" " + CREATE TABLE IF NOT EXISTS demo.e2e_sentinel (id INT PRIMARY KEY, token VARCHAR(64)); + REPLACE INTO demo.e2e_sentinel (id, token) VALUES (1, '${SENTINEL_TOKEN}');" +log_success "Sentinel token: ${SENTINEL_TOKEN}" + +print_header "Step 05d: Create an out-of-band account the chart does not declare" +# Its GRANTs live only in mysql.global_priv, which the strategy excludes from the +# backup (IgnoreGlobalPriv=true). A restore must therefore bring its DATA back but +# NOT the account itself — the documented ignoreGlobalPriv limitation. Step 40 +# asserts both halves, so the unconditionally-set flag has a live test behind the +# behaviour the docs describe, not only a unit assertion on the CR field. +OOB_USER="oob_${SENTINEL_TOKEN//[^a-zA-Z0-9_]/_}" +mysql_root_exec "$MARIADB_SRC_CR" " + CREATE USER IF NOT EXISTS '${OOB_USER}'@'%' IDENTIFIED BY 'oob-pw'; + GRANT SELECT ON demo.* TO '${OOB_USER}'@'%'; + CREATE TABLE IF NOT EXISTS demo.oob_marker (id INT PRIMARY KEY, who VARCHAR(128)); + REPLACE INTO demo.oob_marker (id, who) VALUES (1, '${OOB_USER}');" +log_success "Out-of-band account '${OOB_USER}' + demo.oob_marker created directly in SQL." + +print_header "Step 10/15: Create the MariaDB strategy + BackupClass" +subst 10-mariadb-strategy.yaml | kubectl apply -f - +kubectl apply -f "$SCRIPT_DIR/15-backupclass.yaml" + +print_header "Step 25: Submit ad-hoc BackupJob '${BACKUPJOB_NAME}' and wait for Succeeded" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/25-backupjob-adhoc.yaml" +wait_for_field backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 600 Failed +BACKUP_NAME=$(kubectl -n "$NAMESPACE" get backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \ + -o jsonpath='{.status.backupRef.name}') +[[ -n "$BACKUP_NAME" ]] || { log_error "BackupJob succeeded but reported no backupRef"; exit 1; } +log_success "Backup artefact: ${BACKUP_NAME}" + +if [[ "${SKIP_RESTORE:-0}" == "1" ]]; then + log_warning "SKIP_RESTORE=1: stopping after a successful backup." + exit 0 +fi + +print_header "Step 30/40: Restore to a copy '${MARIADB_TARGET_NAME}' and wait for Succeeded" +# The strategy renders -mariadb-backup-creds / -ca against the TARGET +# during a restore, so the target needs its own Secret pair (same S3 coords). +materialise_backup_secrets "$MARIADB_TARGET_NAME" +subst 30-mariadb-target.yaml | kubectl -n "$NAMESPACE" apply -f - +# Same 660s as the source above, held against the same Chainsaw op. +wait_hr_ready "mariadb-${MARIADB_TARGET_NAME}" 660 +wait_for_field mariadbs.k8s.mariadb.com "$MARIADB_TARGET_CR" \ + '{.status.conditions[?(@.type=="Ready")].status}' True "$NAMESPACE" 600 +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/40-restorejob-to-copy.yaml" +wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_TOCOPY_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed +# The restore replays the logical dump (schema + rows), but the app user is NOT +# in the dump — it comes from the target chart. Wait for its grant before the +# read below so the SELECT authenticates. +wait_app_grant_ready "$MARIADB_TARGET_CR" + +print_header "Step 40 verify: the sentinel round-tripped through S3 into the copy" +GOT=$(mysql_exec "$MARIADB_TARGET_CR" "SELECT token FROM demo.e2e_sentinel WHERE id = 1;" \ + | awk 'NR==2{print}' | tr -d '[:space:]') +if [[ "$GOT" != "$SENTINEL_TOKEN" ]]; then + log_error "sentinel mismatch: target has '${GOT}', expected '${SENTINEL_TOKEN}'" + exit 1 +fi +log_success "Round-trip verified: '${MARIADB_TARGET_NAME}' restored sentinel '${GOT}' from S3." + +# The app-user read above already proves the restore did not overwrite the +# target's grant table with the source's. root is the one the operator can never +# repair after bootstrap, so assert it too: its login must still match the +# chart-generated password in the target Secret. +print_header "Step 40 verify: root still authenticates against the restored copy" +if ! mysql_root_login "$MARIADB_TARGET_CR" >/dev/null; then + log_error "root cannot authenticate against '${MARIADB_TARGET_NAME}': the restore overwrote the grant table and the target Secret's root password no longer matches" + exit 1 +fi +log_success "root login verified against '${MARIADB_TARGET_NAME}'." + +print_header "Step 40 verify: ignoreGlobalPriv dropped the out-of-band account but kept its data" +# demo.oob_marker rode the logical dump, so it must be present in the copy; the +# account lived only in the excluded grant table, so it must NOT. This is the DR +# direction the docs describe and the strategy's unconditional IgnoreGlobalPriv=true +# relies on. +OOB_DATA=$(mysql_exec "$MARIADB_TARGET_CR" "SELECT who FROM demo.oob_marker WHERE id = 1;" \ + | awk 'NR==2{print}' | tr -d '[:space:]') +if [[ "$OOB_DATA" != "$OOB_USER" ]]; then + log_error "out-of-band data did not round-trip: target has '${OOB_DATA}', expected '${OOB_USER}'" + exit 1 +fi +OOB_ACCT=$(mysql_root_exec "$MARIADB_TARGET_CR" \ + "SELECT COUNT(*) FROM mysql.user WHERE User = '${OOB_USER}';" \ + | awk 'NR==2{print}' | tr -d '[:space:]') +if [[ "$OOB_ACCT" != "0" ]]; then + log_error "out-of-band account '${OOB_USER}' reappeared in the copy (count=${OOB_ACCT}): the grant table was NOT excluded from the backup" + exit 1 +fi +log_success "ignoreGlobalPriv verified: data restored, out-of-band account absent (only chart-declared accounts are reconstructed)." + +# To-copy must not mutate the source. Regressing into a source-touching restore +# would corrupt the running instance, so assert the source still reads back. +SRC_GOT=$(mysql_exec "$MARIADB_SRC_CR" "SELECT token FROM demo.e2e_sentinel WHERE id = 1;" \ + | awk 'NR==2{print}' | tr -d '[:space:]') +if [[ "$SRC_GOT" != "$SENTINEL_TOKEN" ]]; then + log_error "source sentinel changed after to-copy restore: source has '${SRC_GOT}', expected '${SENTINEL_TOKEN}'" + exit 1 +fi +log_success "Source '${MARIADB_SRC_NAME}' left untouched by the to-copy restore." diff --git a/examples/backups/mongodb/00-bucket.yaml b/examples/backups/mongodb/00-bucket.yaml new file mode 100644 index 0000000000..c8402c64b4 --- /dev/null +++ b/examples/backups/mongodb/00-bucket.yaml @@ -0,0 +1,14 @@ +# Cozystack-managed S3 bucket for storing MongoDB backups. The Bucket +# controller provisions a Secret named "bucket--" per user +# (the bucket app carries the "bucket-" release prefix); run-all.sh reads the +# "backup" user's BucketInfo and projects the access keys into the MongoDB +# application's backup config. +--- +apiVersion: apps.cozystack.io/v1alpha1 +kind: Bucket +metadata: + name: mongodb-backups +spec: + users: + backup: + readonly: false diff --git a/examples/backups/mongodb/00-helpers.sh b/examples/backups/mongodb/00-helpers.sh new file mode 100755 index 0000000000..2f8b82cc4d --- /dev/null +++ b/examples/backups/mongodb/00-helpers.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# Shared helpers for the MongoDB backup/restore demo. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export BLUE='\033[0;34m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export WHITE='\033[1;37m' +export NC='\033[0m' +export BOLD='\033[1m' + +# Default settings (override via environment). +export NAMESPACE="${NAMESPACE:-tenant-root}" +export BUCKET_NAME="${BUCKET_NAME:-mongodb-backups}" +# Bucket user declared in 00-bucket.yaml; the COSI flow materialises the +# credentials Secret as "bucket--". +export BUCKET_USER="${BUCKET_USER:-backup}" +export MONGODB_SRC_NAME="${MONGODB_SRC_NAME:-mongodb-src}" +export MONGODB_TARGET_NAME="${MONGODB_TARGET_NAME:-mongodb-target}" +# The mongodb-rd ApplicationDefinition renders the HelmRelease with +# releaseName = "mongodb-" + appName (release.prefix in +# packages/system/mongodb-rd/cozyrds/mongodb.yaml), and the chart sets the +# psmdb.percona.com/PerconaServerMongoDB metadata.name to .Release.Name, so the +# operator-side CR (and its rs0 Service) is mongodb-. +export MONGODB_SRC_CR="mongodb-${MONGODB_SRC_NAME}" +export MONGODB_TARGET_CR="mongodb-${MONGODB_TARGET_NAME}" +export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-cozy-default}" +export BACKUPJOB_NAME="${BACKUPJOB_NAME:-mongodb-src-adhoc}" +export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-mongodb-src-to-mongodb-target}" +export PLAN_NAME="${PLAN_NAME:-mongodb-src-daily}" + +log_info() { echo -e "${BLUE}i${NC} $*" >&2; } +log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; } +log_error() { echo -e "${RED}x${NC} $*" >&2; } +log_step() { echo -e "\n${MAGENTA}${BOLD}> $*${NC}" >&2; } +log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; } + +print_header() { + echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2 +} + +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" + +# Wait until the psmdb.percona.com/PerconaServerMongoDB CR reports state=ready +# (the operator's own readiness signal — the rs0 members are up and initialised +# and, when backup.enabled, the pbm agents are running). No fail-fast value: the +# cluster state is recomputed every operator reconcile and can pass through +# "error" transiently during replset init before it converges to "ready", so a +# terminal-on-error gate would fail the whole run on one blip. Poll for the +# positive value only, bounded by the timeout. +wait_psmdb_ready() { + local cr="$1" timeout="${2:-600}" + wait_for_field perconaservermongodbs.psmdb.percona.com "$cr" \ + '{.status.state}' ready "$NAMESPACE" "$timeout" +} + +# Name of the first data-bearing pod of a PerconaServerMongoDB CR's rs0 replica +# set. For the single-replica instances this demo uses, rs0-0 is the sole member +# and therefore the primary, so writes land there directly. +mongodb_primary_pod() { + local cr="$1" + echo "${cr}-rs0-0" +} + +# Run a mongosh script against a PerconaServerMongoDB CR's rs0 primary as the +# operator-managed databaseAdmin system user (read from the internal users +# Secret the psmdb operator materialises), so writes and reads authenticate +# regardless of app-user grant timing. Args: +mongosh_exec() { + local cr="$1" js="$2" + local pod secret admin_user admin_pw + pod=$(mongodb_primary_pod "$cr") + secret="internal-${cr}-users" + admin_user=$(kubectl -n "$NAMESPACE" get secret "$secret" \ + -o jsonpath='{.data.MONGODB_DATABASE_ADMIN_USER}' 2>/dev/null | base64 -d) + admin_pw=$(kubectl -n "$NAMESPACE" get secret "$secret" \ + -o jsonpath='{.data.MONGODB_DATABASE_ADMIN_PASSWORD}' 2>/dev/null | base64 -d) + [[ -n "$admin_user" && -n "$admin_pw" ]] || { log_error "no databaseAdmin credentials in ${NAMESPACE}/${secret}"; return 1; } + kubectl -n "$NAMESPACE" exec "$pod" -c mongod -- \ + mongosh --quiet -u "$admin_user" -p "$admin_pw" --authenticationDatabase admin \ + --eval "$js" +} diff --git a/examples/backups/mongodb/05-mongodb-src.yaml b/examples/backups/mongodb/05-mongodb-src.yaml new file mode 100644 index 0000000000..d828d44bcd --- /dev/null +++ b/examples/backups/mongodb/05-mongodb-src.yaml @@ -0,0 +1,44 @@ +# Source MongoDB instance. +# +# The psmdb operator can only service on-demand PerconaServerMongoDBBackup CRs +# when the cluster has the percona-backup-mongodb agents running and a storage +# declared — that is what `backup.enabled: true` wires up. The cozystack +# backup-strategy driver names that storage ("s3-storage", the one the chart +# declares) via the cozy-default-mongodb Strategy. This example uses the default +# flow, where the application owns the storage and the driver does not inject it +# — so, unlike MariaDB, the application must opt into backups. (Set +# backup.useSystemBucket: true instead to have the driver inject the platform +# system-bucket storage; see docs/operations/backup-classes.md.) +# +# The bucket coordinates (destinationPath / endpointURL / s3AccessKey / +# s3SecretKey) point the storage at the Bucket provisioned in 00-bucket.yaml. +# insecureSkipTLSVerify is true because Cozystack's default in-cluster +# seaweedfs serves a self-signed cert and the psmdb s3 storage has no CA-bundle +# field. run-all.sh fills the REPLACE_WITH_* placeholders from the Bucket's +# BucketInfo Secret. +# +# The "app" user + "demo" database let a smoke harness (or a human) write a +# sentinel document before the BackupJob fires. +--- +apiVersion: apps.cozystack.io/v1alpha1 +kind: MongoDB +metadata: + name: mongodb-src +spec: + replicas: 1 + size: 1Gi + databases: + demo: + roles: + admin: + - app + users: + app: + password: REPLACE_WITH_PASSWORD + backup: + enabled: true + destinationPath: "s3://REPLACE_WITH_COSI_BUCKET_NAME/mongodb-src/" + endpointURL: "REPLACE_WITH_S3_ENDPOINT" + insecureSkipTLSVerify: true + s3AccessKey: "REPLACE_WITH_S3_ACCESS_KEY" + s3SecretKey: "REPLACE_WITH_S3_SECRET_KEY" diff --git a/examples/backups/mongodb/10-backupjob-adhoc.yaml b/examples/backups/mongodb/10-backupjob-adhoc.yaml new file mode 100644 index 0000000000..78d81a812d --- /dev/null +++ b/examples/backups/mongodb/10-backupjob-adhoc.yaml @@ -0,0 +1,15 @@ +# Ad-hoc BackupJob — useful for smoke-testing the strategy or running a +# point-in-time backup outside the Plan schedule. Uses the platform-managed +# cozy-default BackupClass, whose MongoDB entry routes to the +# cozy-default-mongodb Strategy (storageName=s3-storage, type=logical). +--- +apiVersion: backups.cozystack.io/v1alpha1 +kind: BackupJob +metadata: + name: mongodb-src-adhoc +spec: + applicationRef: + apiGroup: apps.cozystack.io + kind: MongoDB + name: mongodb-src + backupClassName: cozy-default diff --git a/examples/backups/mongodb/15-plan.yaml b/examples/backups/mongodb/15-plan.yaml new file mode 100644 index 0000000000..89333009ef --- /dev/null +++ b/examples/backups/mongodb/15-plan.yaml @@ -0,0 +1,19 @@ +# Scheduled backups via a Plan (every 6h). The core Plan controller stamps a +# BackupJob per tick; each is dispatched through the same cozy-default MongoDB +# strategy as the ad-hoc job. Applied for documentation parity with the other +# backup examples; run-all.sh drives the ad-hoc BackupJob instead so the smoke +# test does not wait for a cron tick. +--- +apiVersion: backups.cozystack.io/v1alpha1 +kind: Plan +metadata: + name: mongodb-src-daily +spec: + applicationRef: + apiGroup: apps.cozystack.io + kind: MongoDB + name: mongodb-src + backupClassName: cozy-default + schedule: + type: cron + cron: "0 */6 * * *" diff --git a/examples/backups/mongodb/20-mongodb-target.yaml b/examples/backups/mongodb/20-mongodb-target.yaml new file mode 100644 index 0000000000..dc36bc9d38 --- /dev/null +++ b/examples/backups/mongodb/20-mongodb-target.yaml @@ -0,0 +1,24 @@ +# Empty MongoDB target for the to-copy restore. The psmdb operator replays the +# logical dump into this running cluster, so it needs the percona-backup-mongodb +# agents too — hence backup.enabled: true. It uses the SAME bucket as the source +# but its OWN prefix (mongodb-target/): the restore reads the source archive via +# backupSource, so the target never needs to write into the source's prefix, and +# a shared prefix would interleave the two clusters' oplog/backup metadata (both +# run replset rs0) — a source PITR could then resolve oplog chunks the target +# wrote. No "app" user or database is declared: the restore recreates the demo +# database from the dump. +--- +apiVersion: apps.cozystack.io/v1alpha1 +kind: MongoDB +metadata: + name: mongodb-target +spec: + replicas: 1 + size: 1Gi + backup: + enabled: true + destinationPath: "s3://REPLACE_WITH_COSI_BUCKET_NAME/mongodb-target/" + endpointURL: "REPLACE_WITH_S3_ENDPOINT" + insecureSkipTLSVerify: true + s3AccessKey: "REPLACE_WITH_S3_ACCESS_KEY" + s3SecretKey: "REPLACE_WITH_S3_SECRET_KEY" diff --git a/examples/backups/mongodb/25-restorejob-in-place.yaml b/examples/backups/mongodb/25-restorejob-in-place.yaml new file mode 100644 index 0000000000..49a1c42b17 --- /dev/null +++ b/examples/backups/mongodb/25-restorejob-in-place.yaml @@ -0,0 +1,14 @@ +# In-place restore. Omitting targetApplicationRef restores back into the source +# MongoDB (clusterName = the source's PerconaServerMongoDB). This is +# DESTRUCTIVE — the psmdb operator replays the logical dump into the live source +# cluster. Applied for documentation parity; run-all.sh drives the +# non-destructive to-copy restore (30) instead. Optionally add a PITR target via +# spec.options.recoveryTime (RFC3339) — see README. +--- +apiVersion: backups.cozystack.io/v1alpha1 +kind: RestoreJob +metadata: + name: mongodb-src-in-place +spec: + backupRef: + name: mongodb-src-adhoc diff --git a/examples/backups/mongodb/30-restorejob-to-copy.yaml b/examples/backups/mongodb/30-restorejob-to-copy.yaml new file mode 100644 index 0000000000..0f52525e62 --- /dev/null +++ b/examples/backups/mongodb/30-restorejob-to-copy.yaml @@ -0,0 +1,23 @@ +# To-copy restore. Targets the empty mongodb-target MongoDB app from +# 20-mongodb-target.yaml; the driver creates a PerconaServerMongoDBRestore CR +# with clusterName=mongodb-mongodb-target and a backupSource pointing at the +# source backup's S3 destination, so the operator replays the logical dump into +# the target while mongodb-src stays running and untouched. +# +# Driven for the same-namespace happy path by run-all.sh in this directory; also +# applicable manually with kubectl apply -f. The mongodb-2-backup-roundtrip +# Chainsaw suite stops before this step while #3832 is open, so nothing in CI +# exercises it. The cross-tenant variant stays manual; see the README for the +# network-policy reason. +--- +apiVersion: backups.cozystack.io/v1alpha1 +kind: RestoreJob +metadata: + name: mongodb-src-to-mongodb-target +spec: + backupRef: + name: mongodb-src-adhoc + targetApplicationRef: + apiGroup: apps.cozystack.io + kind: MongoDB + name: mongodb-target diff --git a/examples/backups/mongodb/README.md b/examples/backups/mongodb/README.md new file mode 100644 index 0000000000..42d44737c9 --- /dev/null +++ b/examples/backups/mongodb/README.md @@ -0,0 +1,91 @@ +# MongoDB backup / restore example + +End-to-end example for backing up and restoring a Cozystack-managed MongoDB +application via the `strategy.backups.cozystack.io/v1alpha1` `MongoDB` strategy +driver. The driver delegates execution to the +[Percona Server for MongoDB operator](https://docs.percona.com/percona-operator-for-mongodb/) +(`psmdb.percona.com` CRDs, already shipped with Cozystack): backups materialise +as `PerconaServerMongoDBBackup` CRs and restores as `PerconaServerMongoDBRestore` +CRs in the application's namespace; the operator handles the actual +percona-backup-mongodb (pbm) logical dump / restore. + +Files are numbered so that `kubectl apply -f` order matches the dependency graph. + +| Step | File | Scope | Purpose | +| --- | --- | --- | --- | +| 0 | `00-bucket.yaml` | namespaced | COSI-backed S3 bucket + `backup` user. | +| 1 | `05-mongodb-src.yaml` | namespaced | Source MongoDB application (backups enabled). | +| 2 | `10-backupjob-adhoc.yaml` | namespaced | One-shot BackupJob for smoke testing. | +| 3 | `15-plan.yaml` | namespaced | Cron schedule (every 6h). | +| 4 | `20-mongodb-target.yaml` | namespaced | Empty MongoDB target for to-copy restore. | +| 5 | `25-restorejob-in-place.yaml` | namespaced | Destructive restore back into the source. | +| 6 | `30-restorejob-to-copy.yaml` | namespaced | Non-destructive restore into the target. | + +## How MongoDB backups differ from the other drivers + +The psmdb operator runs the pbm agents when the cluster has `backup.enabled: true`, and services an on-demand `PerconaServerMongoDBBackup` only once the storage the CR names is declared on the cluster. Where that storage comes from depends on the flow the application picked. On the default flow (this example) the chart declares it from the application's own `backup.*` values, and the `cozy-default-mongodb` Strategy only names it (`storageName: s3-storage`); the driver injects nothing. With `backup.useSystemBucket: true` the chart leaves the storage out and the driver injects the platform bucket's coordinates onto the cluster before each backup. Either way the driver surfaces a clear `Ready=False` precondition (rather than a silent hang) when the target cluster has backups disabled; see `docs/operations/backup-classes.md` for both flows. + +Restores use `PerconaServerMongoDBRestore.spec.backupSource`, carrying the S3 +destination read back from the source backup, so the same artifact restores +in-place **or** into a differently-named instance, and point-in-time recovery is +available via `spec.options.recoveryTime`. + +## Placeholders + +`05-mongodb-src.yaml` / `20-mongodb-target.yaml` carry `REPLACE_WITH_PASSWORD`, +`REPLACE_WITH_COSI_BUCKET_NAME`, `REPLACE_WITH_S3_ENDPOINT` (a URL *with* scheme, +e.g. `https://seaweedfs-s3.tenant-root:8333`), and `REPLACE_WITH_S3_ACCESS_KEY` / +`REPLACE_WITH_S3_SECRET_KEY`. `run-all.sh` resolves them from the provisioned +`Bucket`'s `BucketInfo` Secret. `insecureSkipTLSVerify: true` lets pbm reach +Cozystack's default self-signed in-cluster seaweedfs — the psmdb s3 storage has +no CA-bundle field (unlike the MariaDB driver's `tls.caSecretKeyRef`), so +skipping verification is the supported way to trust it. Drop it (default +`false`) when the S3 endpoint is signed by a publicly-trusted CA. + +## Point-in-time recovery + +psmdb records an oplog stream between logical backups (pitr). To restore to a +timestamp instead of the plain snapshot, add `recoveryTime` (RFC3339, the same +option the CNPG/Postgres driver uses) to a RestoreJob: + +```yaml +spec: + options: + recoveryTime: "2026-08-05T12:34:56Z" # RFC3339 (UTC) +``` + +An empty/omitted `recoveryTime` restores the backup snapshot as taken. An +unrecognised key under `spec.options` is ignored but reported as a +`UnknownRestoreOption` Warning event on the RestoreJob. + +## Run it + +```sh +# Defaults to NAMESPACE=tenant-root; override for a tenant namespace. +NAMESPACE=tenant-root examples/backups/mongodb/run-all.sh +# Tear everything down afterwards (idempotent). +NAMESPACE=tenant-root examples/backups/mongodb/cleanup.sh +``` + +`run-all.sh` writes a sentinel document into the source, waits for the +`BackupJob` to reach `Succeeded`, restores to a copy with a to-copy `RestoreJob`, +and asserts the sentinel round-tripped through S3 into the restored copy while +the source is left untouched. Set `SKIP_RESTORE=1` to stop after a successful +backup. + +The to-copy restore currently ends with the target holding the source cluster's system users, so the target's own credentials stop authenticating and that last assertion fails; #3832 tracks it, and `SKIP_RESTORE=1` stops before that step. + +Same-namespace flows are the supported path. Cross-tenant restores (target in +`tenant-test`, source's seaweedfs in `tenant-root`) are blocked by the per-tenant +Cilium egress policy and stay a manual / dev-cluster flow. + +## Automated e2e + +The Chainsaw suite at `hack/e2e-chainsaw/mongodb/` drives this same `run-all.sh` +as a second test (`mongodb-2-backup-roundtrip`), selected by Test-Impact +Analysis whenever the mongodb app or the suite changes, and on every release +cut. It runs in `tenant-root` against the in-cluster seaweedfs endpoint — the +isolated e2e tenant cannot reach it across the Cilium egress policy, and the +external ingress endpoint is an unroutable placeholder in the sandbox. + +It drives `run-all.sh` with `SKIP_RESTORE=1` and stops once the BackupJob succeeds, so the restore steps are not covered by CI while #3832 is open: a to-copy restore replays the source cluster's system users into the target, after which the target's own credentials no longer authenticate and the sentinel cannot be read back. diff --git a/examples/backups/mongodb/cleanup.sh b/examples/backups/mongodb/cleanup.sh new file mode 100755 index 0000000000..4db47906af --- /dev/null +++ b/examples/backups/mongodb/cleanup.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Best-effort teardown of everything the demo creates. Idempotent; +# missing resources are skipped silently. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Cleanup" + +log_substep "Deleting RestoreJobs..." +kubectl -n "$NAMESPACE" delete restorejob.backups.cozystack.io \ + "$RESTOREJOB_TOCOPY_NAME" mongodb-src-in-place --ignore-not-found + +log_substep "Deleting Plan + BackupJob + derived Backup artefacts..." +kubectl -n "$NAMESPACE" delete plan.backups.cozystack.io "$PLAN_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete backupjob.backups.cozystack.io "$BACKUPJOB_NAME" --ignore-not-found +# Only the Backups of THIS demo's apps — never `--all`: on a shared cluster the +# namespace can hold Backup artefacts of other applications and flows. +for b in $(kubectl -n "$NAMESPACE" get backups.backups.cozystack.io \ + -o jsonpath='{range .items[*]}{.metadata.name} {.spec.applicationRef.name}{"\n"}{end}' 2>/dev/null | + awk -v src="$MONGODB_SRC_NAME" -v tgt="$MONGODB_TARGET_NAME" '$2 == src || $2 == tgt {print $1}'); do + kubectl -n "$NAMESPACE" delete backups.backups.cozystack.io "$b" --ignore-not-found +done + +# The Cozystack BackupJob/RestoreJob own their operator-side psmdb.percona.com +# Backup/Restore CRs only by the OwningJob labels (not OwnerReferences), so +# deleting the Cozystack jobs does not cascade to them. Left behind, a stale +# operator-side Backup would be reused by findMongoDBBackupForJob on the next +# run against a bucket path that no longer exists. Prune them by owner. +log_substep "Deleting operator-side psmdb.percona.com Backup/Restore CRs..." +for owner in "$BACKUPJOB_NAME" "$RESTOREJOB_TOCOPY_NAME" mongodb-src-in-place; do + kubectl -n "$NAMESPACE" delete \ + perconaservermongodbbackups.psmdb.percona.com,perconaservermongodbrestores.psmdb.percona.com \ + -l "backups.cozystack.io/owned-by.BackupJobName=${owner}" --ignore-not-found +done + +log_substep "Deleting MongoDB apps..." +kubectl -n "$NAMESPACE" delete mongodb.apps.cozystack.io "$MONGODB_SRC_NAME" "$MONGODB_TARGET_NAME" --ignore-not-found + +log_substep "Deleting Bucket..." +kubectl -n "$NAMESPACE" delete bucket.apps.cozystack.io "$BUCKET_NAME" --ignore-not-found + +log_success "Cleanup complete." diff --git a/examples/backups/mongodb/run-all.sh b/examples/backups/mongodb/run-all.sh new file mode 100755 index 0000000000..9793bea3b5 --- /dev/null +++ b/examples/backups/mongodb/run-all.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# Convenience runner + e2e harness for the MongoDB backup/restore demo. +# +# It applies the SAME numbered manifests a human reads (00-bucket.yaml .. +# 30-restorejob-to-copy.yaml), filling the REPLACE_WITH_* placeholders from the +# provisioned Bucket so the documented flow and the automated test can never +# drift. Stops on the first failure. +# +# Flow: Bucket -> source MongoDB with backup.enabled (+ a sentinel document) -> +# ad-hoc BackupJob against the cozy-default BackupClass (wait Succeeded) -> +# empty target MongoDB + to-copy RestoreJob (wait Succeeded) -> assert the +# sentinel round-tripped through S3 into the restored copy while the source +# stays untouched. +# +# Why to-copy and not in-place: in-place replays the dump straight into the live +# source cluster, so it cannot witness the restore on a separate instance. +# To-copy leaves the source running and lands the marker document on a distinct +# target, a stronger restore proof; the in-place dispatch is covered at the unit +# level by the controller tests. +# +# Override NAMESPACE / endpoint via the environment; see 00-helpers.sh. +# hack/e2e-chainsaw/mongodb/ drives this file as mongodb-2-backup-roundtrip. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +# App user password baked into the source spec; the demo writes the sentinel as +# the operator databaseAdmin (see mongosh_exec), so this is just to satisfy the +# chart's user declaration. +MONGODB_PASSWORD="${MONGODB_PASSWORD:-Xai7Wepo0aeThie8}" + +# Substitute the manifest placeholders. $BUCKET / $S3_ENDPOINT / the access keys +# are resolved from the Bucket below. +subst() { + sed \ + -e "s|REPLACE_WITH_COSI_BUCKET_NAME|${BUCKET}|g" \ + -e "s|REPLACE_WITH_S3_ENDPOINT|${S3_ENDPOINT}|g" \ + -e "s|REPLACE_WITH_S3_ACCESS_KEY|${S3_ACCESS_KEY}|g" \ + -e "s|REPLACE_WITH_S3_SECRET_KEY|${S3_SECRET_KEY}|g" \ + -e "s|REPLACE_WITH_PASSWORD|${MONGODB_PASSWORD}|g" \ + "$SCRIPT_DIR/$1" +} + +print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml" +# 660s, above this generated release's 600s install timeout; see "Sizing an +# HR-Ready budget" in docs/agents/e2e-testing.md. +wait_hr_ready "bucket-${BUCKET_NAME}" 660 +wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \ + '{.status.bucketReady}' true "$NAMESPACE" 300 +wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \ + '{.status.accessGranted}' true "$NAMESPACE" 300 +if [[ "${COZY_E2E_BACKUP_PREFLIGHT:-0}" == "1" ]]; then + source "$SCRIPT_DIR/../../../hack/e2e-chainsaw/_lib/backup-access-preflight.sh" + cozy_backup_access_preflight "$NAMESPACE" "bucket-${BUCKET_NAME}-${BUCKET_USER}" 90 +fi + +log_substep "Reading bucket coordinates from BucketInfo Secret..." +TMP=$(mktemp) +trap 'rm -f "$TMP"' EXIT +kubectl -n "$NAMESPACE" get secret "bucket-${BUCKET_NAME}-${BUCKET_USER}" \ + -o jsonpath='{.data.BucketInfo}' | base64 -d > "$TMP" +S3_ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' "$TMP") +S3_SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' "$TMP") +# S3_ENDPOINT can be overridden via the environment: BucketInfo advertises the +# EXTERNAL ingress endpoint, which in-cluster Pods (the pbm agent runs inside +# the mongod pod) cannot always reach or TLS-validate; the in-cluster +# alternative is https://seaweedfs-s3.:8333, reached with +# insecureSkipTLSVerify=true (psmdb s3 storage has no CA-bundle field). +S3_ENDPOINT="${S3_ENDPOINT:-$(jq -r '.spec.secretS3.endpoint' "$TMP")}" +BUCKET=$(jq -r '.spec.bucketName' "$TMP") +for v in S3_ACCESS_KEY S3_SECRET_KEY S3_ENDPOINT BUCKET; do + [[ -n "${!v}" && "${!v}" != "null" ]] || { log_error "BucketInfo missing required field: ${v}"; exit 1; } +done +log_success "Bucket '${BUCKET}' at endpoint '${S3_ENDPOINT}'." + +print_header "Step 05: Deploy source MongoDB '${MONGODB_SRC_NAME}' with backups enabled" +subst 05-mongodb-src.yaml | kubectl -n "$NAMESPACE" apply -f - +# 660s, above this generated release's 600s install timeout; see "Sizing an +# HR-Ready budget" in docs/agents/e2e-testing.md. +wait_hr_ready "$MONGODB_SRC_CR" 660 +wait_psmdb_ready "$MONGODB_SRC_CR" 600 + +print_header "Step 05b: Write a sentinel document so the backup has something to prove" +SENTINEL_TOKEN="e2e-$(date +%s)-$$" +mongosh_exec "$MONGODB_SRC_CR" \ + "db.getSiblingDB('demo').e2e_sentinel.replaceOne({_id:1},{_id:1,token:'${SENTINEL_TOKEN}'},{upsert:true})" +log_success "Sentinel token: ${SENTINEL_TOKEN}" + +print_header "Step 10: Submit ad-hoc BackupJob '${BACKUPJOB_NAME}' and wait for Succeeded" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/10-backupjob-adhoc.yaml" +wait_for_field backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed +BACKUP_NAME=$(kubectl -n "$NAMESPACE" get backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \ + -o jsonpath='{.status.backupRef.name}') +[[ -n "$BACKUP_NAME" ]] || { log_error "BackupJob succeeded but reported no backupRef"; exit 1; } +log_success "Backup artefact: ${BACKUP_NAME}" + +if [[ "${SKIP_RESTORE:-0}" == "1" ]]; then + # hack/e2e-chainsaw/mongodb/ asserts this line to prove the skip really + # happened, so rewording it reds that suite with a bare match failure. + log_warning "SKIP_RESTORE=1: stopping after a successful backup." + exit 0 +fi + +print_header "Step 20/30: Restore to a copy '${MONGODB_TARGET_NAME}' and wait for Succeeded" +subst 20-mongodb-target.yaml | kubectl -n "$NAMESPACE" apply -f - +wait_hr_ready "$MONGODB_TARGET_CR" 660 +wait_psmdb_ready "$MONGODB_TARGET_CR" 600 +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/30-restorejob-to-copy.yaml" +wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_TOCOPY_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed + +print_header "Step 30 verify: the sentinel round-tripped through S3 into the copy" +GOT=$(mongosh_exec "$MONGODB_TARGET_CR" \ + "print(db.getSiblingDB('demo').e2e_sentinel.findOne({_id:1}).token)" | tr -d '[:space:]') +if [[ "$GOT" != "$SENTINEL_TOKEN" ]]; then + log_error "sentinel mismatch: target has '${GOT}', expected '${SENTINEL_TOKEN}'" + exit 1 +fi +log_success "Round-trip verified: '${MONGODB_TARGET_NAME}' restored sentinel '${GOT}' from S3." + +# To-copy must not mutate the source. Regressing into a source-touching restore +# would corrupt the running instance, so assert the source still reads back. +SRC_GOT=$(mongosh_exec "$MONGODB_SRC_CR" \ + "print(db.getSiblingDB('demo').e2e_sentinel.findOne({_id:1}).token)" | tr -d '[:space:]') +if [[ "$SRC_GOT" != "$SENTINEL_TOKEN" ]]; then + log_error "source sentinel changed after to-copy restore: source has '${SRC_GOT}', expected '${SENTINEL_TOKEN}'" + exit 1 +fi +log_success "Source '${MONGODB_SRC_NAME}' left untouched by the to-copy restore." diff --git a/examples/backups/nats/00-helpers.sh b/examples/backups/nats/00-helpers.sh index 2908526a01..e717f172e6 100755 --- a/examples/backups/nats/00-helpers.sh +++ b/examples/backups/nats/00-helpers.sh @@ -28,8 +28,10 @@ export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-nats-restore-inplace} export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-nats-restore-to-copy}" # natsio/nats-box ships the `nats` CLI plus curl + tar + sh - everything the # generic Job strategy needs, with no purpose-built backup image. The seed / -# verify helpers below run it as a throwaway Pod via `kubectl run`. +# verify helpers below run it as a long-lived Pod they `kubectl exec` into. export NATS_BOX_IMAGE="${NATS_BOX_IMAGE:-natsio/nats-box:0.14.5}" +# Name of that long-lived CLI Pod; cleanup.sh removes it. +export NATS_CLI_POD="${NATS_CLI_POD:-nats-cli}" log_info() { echo -e "${BLUE}i${NC} $*" >&2; } log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } @@ -48,69 +50,88 @@ print_header() { echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2 } -# Wait until a JSONPath value on a resource matches the desired string. -wait_for_field() { - local resource_type="$1" - local resource_name="$2" - local jsonpath="$3" - local desired="$4" - local namespace="${5:-}" - local timeout="${6:-300}" - - log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..." - local elapsed=0 - local ns_flag=() - [[ -n "$namespace" ]] && ns_flag=(-n "$namespace") - - while true; do - local current - current=$(kubectl get "$resource_type" "$resource_name" "${ns_flag[@]}" -o jsonpath="$jsonpath" 2>/dev/null || true) - if [[ "$current" == "$desired" ]]; then - log_success "$resource_type/$resource_name reached '$desired'" - return 0 - fi - if [[ $elapsed -ge $timeout ]]; then - log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')" - return 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) - done -} +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" # In-cluster NATS client URL for the given application instance. The NATS app # names its client Service after the application (fullnameOverride=) and # stores per-user passwords in the "-credentials" Secret. The demo sets a # fixed password (NATS_PASSWORD) on both the source and restore-target apps, so -# a single URL shape works everywhere. +# a single URL shape works everywhere. The password is not in the URL: nats_cli +# passes it to the CLI in NATS_PASSWORD on stdin, to keep it out of the exec +# request URL the apiserver records in its audit log. nats_url() { local app="$1" - echo "nats://${NATS_USER}:${NATS_PASSWORD}@${app}.${NAMESPACE}.svc:4222" + echo "nats://${NATS_USER}@${app}.${NAMESPACE}.svc:4222" +} + +# Ensure the long-lived nats-box CLI Pod exists and is Ready, so nats_cli can +# exec into it. Idempotent: a Ready Pod this demo owns is reused across calls and +# across the numbered demo scripts; a leftover in a terminal phase (Succeeded / +# Failed) is replaced rather than waited on; and a same-named Pod this demo does +# not own is refused rather than hijacked or deleted. cleanup.sh removes it. +# Every step returns on failure explicitly: this runs as the left side of +# `|| return 1`, and some callers also wrap it in $(...), so errexit never +# applies inside it and a bare failure would carry on to the next step. +nats_cli_pod() { + local phase owner + phase=$(kubectl -n "$NAMESPACE" get pod "$NATS_CLI_POD" --ignore-not-found -o jsonpath='{.status.phase}') || return 1 + owner=$(kubectl -n "$NAMESPACE" get pod "$NATS_CLI_POD" --ignore-not-found -o jsonpath='{.metadata.labels.cozystack\.io/backup-demo}') || return 1 + if [ -n "$phase" ] && [ "$owner" != "nats" ]; then + log_error "Pod $NAMESPACE/$NATS_CLI_POD exists but this demo does not own it; refusing to use or delete it" + return 1 + fi + if [ "$phase" != "Running" ] && [ "$phase" != "Pending" ]; then + kubectl -n "$NAMESPACE" delete pod "$NATS_CLI_POD" --grace-period=1 --ignore-not-found >/dev/null || return 1 + kubectl -n "$NAMESPACE" run "$NATS_CLI_POD" --image="$NATS_BOX_IMAGE" \ + --labels=cozystack.io/backup-demo=nats \ + --restart=Never --command -- sleep infinity >/dev/null || return 1 + fi + kubectl -n "$NAMESPACE" wait --for=condition=Ready "pod/$NATS_CLI_POD" \ + --timeout=5m >/dev/null } -# Run a `nats` CLI invocation against an application instance from a throwaway -# nats-box Pod. Args after the app name are passed verbatim to `nats`. -# Example: nats_cli "$NATS_NAME" stream ls +# Run a `nats` CLI invocation against an application instance. Args after the app +# name are passed verbatim to `nats`. Example: nats_cli "$NATS_NAME" stream ls +# +# The command runs by `kubectl exec` in the long-lived nats-box Pod, not a +# throwaway `kubectl run -i` Pod per call. A throwaway Pod's stdout comes back +# over an attach that only carries what the container writes after the attach +# registers, so a CLI that finishes in between returns empty with exit 0. An +# exec'd process owns its pipes, so its output cannot be missed that way, and +# kubectl's and the CLI's stderr stay attached so a failed read says why. +# +# The password travels in NATS_PASSWORD on stdin rather than in the --server URL, +# which the exec request carries into the apiserver's audit log. The nats CLI +# reads NATS_PASSWORD from its environment. nats_cli() { local app="$1"; shift - kubectl -n "$NAMESPACE" run "nats-cli-$RANDOM" \ - --image="$NATS_BOX_IMAGE" --restart=Never --rm -i --quiet \ - --command -- nats --server "$(nats_url "$app")" "$@" + nats_cli_pod || return 1 + printf '%s\n' "$NATS_PASSWORD" | kubectl -n "$NAMESPACE" exec -i "$NATS_CLI_POD" -- sh -c ' + IFS= read -r NATS_PASSWORD || true + export NATS_PASSWORD + server=$1; shift + exec nats --server "$server" "$@" + ' sh "$(nats_url "$app")" "$@" } # Number of messages currently stored in a JetStream stream, or "" if the -# stream does not exist. +# stream does not exist. Only jq's stderr is dropped (a missing stream prints a +# harmless "stream not found" there); nats_cli's own stderr stays, so an exec +# that fails says why instead of surfacing as an empty count. stream_message_count() { local app="$1" local stream="$2" - nats_cli "$app" stream info "$stream" --json 2>/dev/null \ + nats_cli "$app" stream info "$stream" --json \ | jq -r '.state.messages // empty' 2>/dev/null | tr -d '[:space:]' } # Create the "-backup-s3" Secret the Job strategy Pod consumes, from the # bucket coordinates cached by 03-create-bucket.sh. The generic Job strategy - # unlike the app-specific drivers - has no chart support to emit this Secret, -# so the tenant provides it. Called for the source app (step 04) and the +# so the demo provides it. Called for the source app (step 04) and the # restore target (step 07). create_s3_secret() { local app="$1" diff --git a/examples/backups/nats/01-create-strategy.sh b/examples/backups/nats/01-create-strategy.sh index da5f84b395..1e3b803be0 100755 --- a/examples/backups/nats/01-create-strategy.sh +++ b/examples/backups/nats/01-create-strategy.sh @@ -64,7 +64,7 @@ spec: value: "{{ if .Backup }}{{ .Backup.ApplicationRef.Name }}{{ end }}" - name: MODE value: "{{ .Mode }}" - # S3 coordinates from the tenant-provided -backup-s3 Secret + # S3 coordinates from the demo-provided -backup-s3 Secret # (created by create_s3_secret in 00-helpers.sh from the Bucket). - name: AWS_ACCESS_KEY_ID valueFrom: diff --git a/examples/backups/nats/03-create-bucket.sh b/examples/backups/nats/03-create-bucket.sh index 10bf3cf328..b12ae50b3f 100755 --- a/examples/backups/nats/03-create-bucket.sh +++ b/examples/backups/nats/03-create-bucket.sh @@ -1,7 +1,7 @@ #!/bin/bash # Step 03: Provision an in-cluster Bucket and cache its S3 coordinates. Unlike # the ClickHouse demo - where the chart emits the backup Secret from -# spec.backup.* - the generic Job strategy has no chart support, so the tenant +# spec.backup.* - the generic Job strategy has no chart support, so the demo # materialises the "-backup-s3" Secret itself. This step only caches the # coordinates; create_s3_secret (00-helpers.sh) turns them into per-app Secrets # in steps 04 and 07. diff --git a/examples/backups/nats/90-scenario-admin-prepare.md b/examples/backups/nats/90-scenario-admin-prepare.md index 2314212e69..e5357443c2 100644 --- a/examples/backups/nats/90-scenario-admin-prepare.md +++ b/examples/backups/nats/90-scenario-admin-prepare.md @@ -1,8 +1,6 @@ # Scenario: Cluster admin prepares NATS backups -A cluster administrator performs the cluster-level preparation once. Tenants -will then be able to back up and restore their NATS applications by referencing -the BackupClass created here, with no further admin action. +A cluster administrator performs the cluster-level preparation once. Tenant admins can then back up and restore their NATS applications with a `BackupJob` or `RestoreJob` that references the BackupClass created here. The demo scripts around those jobs still need admin access, because they create Secrets and work through a CLI Pod; the README's step table says which part of each step a tenant admin can do. ## Prerequisites @@ -10,9 +8,7 @@ the BackupClass created here, with no further admin action. backupstrategy-controller installed. - `kubectl` access with permissions to create cluster-scoped `Job.strategy.backups.cozystack.io` and `BackupClass.backups.cozystack.io`. -- Reachable S3-compatible storage. The demo uses the in-cluster `Bucket` app - (step 03); the tenant turns its coordinates into a `-backup-s3` Secret - that the strategy Pod reads (step 04 / 07). +- Reachable S3-compatible storage. The demo uses the in-cluster `Bucket` app (step 03), and steps 04 and 07 turn its coordinates into a `-backup-s3` Secret that the strategy Pod reads. Creating that Secret needs admin access, since no tenant role grants `create` on `secrets`. ## Steps diff --git a/examples/backups/nats/91-scenario-user-backup.md b/examples/backups/nats/91-scenario-user-backup.md index 5af74c5a91..adc5c983d7 100644 --- a/examples/backups/nats/91-scenario-user-backup.md +++ b/examples/backups/nats/91-scenario-user-backup.md @@ -2,13 +2,12 @@ This scenario assumes the admin steps (`01-...`, `02-...`) have already run. +The user's own action here, at the tenant admin level, is the `BackupJob` in step 05, plus creating the `Bucket` and the `NATS`. Steps 03 and 04 as scripted need admin access: they read and create Secrets, wait on HelmReleases and the `StatefulSet`, and seed the stream through a CLI Pod, which no tenant role allows. The README's step table has the details. + ## Prerequisites - Tenant namespace (default: `tenant-test`). -- A Cozystack `Bucket` (step 03) and the `-backup-s3` Secret derived from - it (step 04). The generic `Job` strategy has no chart support to emit this - Secret, so the tenant creates it from the bucket's `BucketInfo` (see - `create_s3_secret` in `00-helpers.sh`). +- A Cozystack `Bucket` (step 03) and the `-backup-s3` Secret derived from it (step 04). The generic `Job` strategy has no chart support to emit this Secret, so step 04 creates it from the bucket's `BucketInfo` (see `create_s3_secret` in `00-helpers.sh`). Creating it needs admin access. - A `NATS` application with JetStream enabled and a stream holding data. ## Steps diff --git a/examples/backups/nats/92-scenario-user-restore.md b/examples/backups/nats/92-scenario-user-restore.md index 1dbd1cc356..9f0e7c02ae 100644 --- a/examples/backups/nats/92-scenario-user-restore.md +++ b/examples/backups/nats/92-scenario-user-restore.md @@ -2,6 +2,8 @@ Two restore variants are supported. +The user's own action in both variants, at the tenant admin level, is the `RestoreJob`, plus the second `NATS` in variant B. The scripts as written need admin access: variant A deletes and checks the stream through a CLI Pod, and variant B checks the stream the same way, creates the target's Secret and waits on its `HelmRelease` and `StatefulSet`, none of which a tenant role allows. The README's step table has the details. + ## A. In-place restore Restore back into the same NATS application. diff --git a/examples/backups/nats/README.md b/examples/backups/nats/README.md index f8adecbf30..30948e65e1 100644 --- a/examples/backups/nats/README.md +++ b/examples/backups/nats/README.md @@ -30,12 +30,12 @@ driver, with tools you already have. | `00-helpers.sh` | Shared bash helpers, env defaults, and the `nats`/S3-secret helpers; sourced by every step. | n/a | | `01-create-strategy.sh` | Creates the cluster-scoped `Job` strategy (the nats-box `PodTemplateSpec`). | admin | | `02-create-backupclass.sh` | Maps `apps.cozystack.io/NATS` to that strategy, with `stream`/`natsUser` parameters. | admin | -| `03-create-bucket.sh` | Provisions a `Bucket` and caches its S3 coordinates into `.bucket-info.env` (chmod 600; raw access keys). `cleanup.sh` removes this file. | tenant | -| `04-create-nats.sh` | Provisions a `NATS` instance, creates the `-backup-s3` Secret, seeds a JetStream stream with sentinel messages. | tenant | +| `03-create-bucket.sh` | Provisions a `Bucket` and caches its S3 coordinates into `.bucket-info.env` (chmod 600; raw access keys). `cleanup.sh` removes this file. | **admin**. A tenant admin can create the `Bucket` itself, but the script also waits on its `HelmRelease` and COSI objects and reads the `bucket--backup` Secret; no tenant role grants access to `helmreleases` or `objectstorage.k8s.io`, or `get` on that Secret | +| `04-create-nats.sh` | Provisions a `NATS` instance, creates the `-backup-s3` Secret, seeds a JetStream stream with sentinel messages. | **admin**. A tenant admin can create the `NATS` itself, but the script also waits on its `HelmRelease` and `StatefulSet`, creates the Secret, and seeds the stream through a CLI Pod it execs into; no tenant role grants access to `helmreleases` or `statefulsets`, `create` on `secrets` or `pods`, or `pods/exec` | | `05-create-backupjob.sh` | Submits a `BackupJob` and waits for Succeeded. | tenant | -| `06-restore-in-place.sh` | Deletes the stream and restores it into the same instance via `RestoreJob`. | tenant | -| `07-restore-to-copy.sh` | Provisions a second `NATS` and restores into it via `RestoreJob.spec.targetApplicationRef`. | tenant | -| `cleanup.sh` | Removes everything created by the demo. | admin or tenant | +| `06-restore-in-place.sh` | Deletes the stream and restores it into the same instance via `RestoreJob`. | **admin**. A tenant admin can submit the `RestoreJob`, but the script deletes and checks the stream through the CLI Pod; no tenant role grants `create` on `pods` or `pods/exec` | +| `07-restore-to-copy.sh` | Provisions a second `NATS` and restores into it via `RestoreJob.spec.targetApplicationRef`. | **admin**, for the same reasons as step 04. The tenant admin's part is the second `NATS` and the `RestoreJob` | +| `cleanup.sh` | Removes everything created by the demo. | **admin**: it deletes Secrets, the `Backup`, and the cluster-scoped `BackupClass` and strategy, and no tenant role grants `delete` on any of them | | `run-all.sh` | Convenience runner that executes 01..07 in order. | demo | | `90-scenario-admin-prepare.md` | Narrative for the admin preparation steps. | docs | | `91-scenario-user-backup.md` | Narrative for the user backup flow. | docs | @@ -72,7 +72,7 @@ All variables come from `00-helpers.sh`: - **Not run in CI.** Like every other directory under `examples/backups/`, this demo is a runnable reference, not an automated regression test — the e2e - harness (`hack/e2e-apps/*.bats`) does not exercise backup examples. Run it by + harness (`hack/e2e-chainsaw/`) does not exercise backup examples. Run it by hand against a cluster. - **`nats` CLI specifics.** The strategy invokes `nats stream backup`/`nats stream restore`; flag names can vary across `nats` CLI versions. Pin diff --git a/examples/backups/nats/cleanup.sh b/examples/backups/nats/cleanup.sh index 58d29d5527..40788695be 100755 --- a/examples/backups/nats/cleanup.sh +++ b/examples/backups/nats/cleanup.sh @@ -16,6 +16,7 @@ kubectl -n "$NAMESPACE" delete secret "${NATS_RESTORE_NAME}-backup-s3" --ignore- kubectl -n "$NAMESPACE" delete secret "${NATS_NAME}-backup-s3" --ignore-not-found kubectl -n "$NAMESPACE" delete nats "$NATS_RESTORE_NAME" --ignore-not-found kubectl -n "$NAMESPACE" delete nats "$NATS_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete pod -l cozystack.io/backup-demo=nats --grace-period=1 --ignore-not-found kubectl -n "$NAMESPACE" delete bucket "$BUCKET_NAME" --ignore-not-found rm -f "$SCRIPT_DIR/.bucket-info.env" kubectl delete backupclass "$BACKUPCLASS_NAME" --ignore-not-found diff --git a/examples/backups/postgres/00-helpers.sh b/examples/backups/postgres/00-helpers.sh new file mode 100755 index 0000000000..ac004e504a --- /dev/null +++ b/examples/backups/postgres/00-helpers.sh @@ -0,0 +1,144 @@ +#!/bin/bash +# Shared helpers for the PostgreSQL backup/restore demo. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export BLUE='\033[0;34m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export WHITE='\033[1;37m' +export NC='\033[0m' +export BOLD='\033[1m' + +# Default settings (override via environment). +export NAMESPACE="${NAMESPACE:-tenant-root}" +export BUCKET_NAME="${BUCKET_NAME:-pg-backups}" +export PG_SRC_NAME="${PG_SRC_NAME:-pg-src}" +export PG_TARGET_NAME="${PG_TARGET_NAME:-pg-target}" +# The apps/postgres chart names the cnpg.io Cluster (and its Pods' +# cnpg.io/cluster label) after the Helm release, which is postgres-. +export PG_SRC_CLUSTER="postgres-${PG_SRC_NAME}" +export PG_TARGET_CLUSTER="postgres-${PG_TARGET_NAME}" +export STRATEGY_NAME="${STRATEGY_NAME:-cnpg-strategy-default}" +export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-postgres-cnpg}" +export BACKUPJOB_NAME="${BACKUPJOB_NAME:-pg-src-adhoc}" +# A second ad-hoc BackupJob taken after the PITR marker writes; its completion +# is the deterministic "WAL past the recovery target is archived" gate for the +# point-in-time restore (see run-all.sh step 45). +export BACKUPJOB_POSTMARKER_NAME="${BACKUPJOB_POSTMARKER_NAME:-pg-src-postmarker}" +export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-pg-src-to-pg-target}" +export RESTOREJOB_PITR_NAME="${RESTOREJOB_PITR_NAME:-pg-src-to-pg-target-pitr}" +export RESTOREJOB_UNREACHABLE_NAME="${RESTOREJOB_UNREACHABLE_NAME:-pg-src-to-pg-target-unreachable}" +export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-pg-src-in-place}" +export PLAN_NAME="${PLAN_NAME:-pg-src-daily}" + +# S3 endpoint CA. cozystack's default seaweedfs serves its S3 endpoint with a +# self-signed certificate whose CA lives in this Secret; the demo copies its +# ca.crt into a per-app Secret the barman-cloud ObjectStore trusts via +# endpointCA. The name follows the seaweedfs chart's fullnameOverride +# ("seaweedfs" -> "seaweedfs-ca-cert"); run-all.sh auto-discovers the CA +# Certificate's actual secret when this default is absent, so an upstream +# fullname change does not silently break the copy. On a cluster whose S3 +# endpoint is signed by a publicly-trusted CA, set S3_CA_SECRET="" to skip the +# copy and drop endpointCA from the manifests. +export S3_CA_SECRET="${S3_CA_SECRET:-seaweedfs-ca-cert}" +export S3_CA_NAMESPACE="${S3_CA_NAMESPACE:-tenant-root}" +export S3_CA_KEY="${S3_CA_KEY:-ca.crt}" + +log_info() { echo -e "${BLUE}i${NC} $*" >&2; } +log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; } +log_error() { echo -e "${RED}x${NC} $*" >&2; } +log_step() { echo -e "\n${MAGENTA}${BOLD}> $*${NC}" >&2; } +log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; } + +print_header() { + echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2 +} + +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" + +# Name of the primary pod of a cnpg.io Cluster (the one accepting writes). +cnpg_primary_pod() { + local cluster="$1" + kubectl -n "$NAMESPACE" get pod \ + -l "cnpg.io/cluster=${cluster},cnpg.io/instanceRole=primary" \ + -o name 2>/dev/null | head -n1 +} + +# Run a psql statement on a cnpg.io Cluster's primary. Args: +psql_exec() { + local cluster="$1" db="$2" sql="$3" + local pod + pod=$(cnpg_primary_pod "$cluster") + [[ -n "$pod" ]] || { log_error "no primary pod for cnpg cluster '$cluster'"; return 1; } + kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- \ + psql -U postgres -d "$db" -tAc "$sql" +} + +# Run a psql statement AS the application user over TCP (via the cluster's -rw +# Service), forcing password authentication. psql_exec above connects as the +# in-pod postgres superuser through the local socket (peer auth) and so never +# exercises a user's password; only this path proves the password the +# -credentials Secret advertises actually authenticates. The password is +# read from that chart-managed Secret with jq, not a jsonpath: a username may +# contain a '.', and jsonpath's `{.data['a.b']}` treats the dot as a path step +# and returns nothing, so a dotted user would read an empty password and time out +# on a credential that is fine. jq indexes the key as a literal string. The chart +# names the CNPG Cluster, the -rw Service and the credentials Secret all after the +# release, so the cluster IS the release here - derive the Secret from it rather +# than taking a separate name that a caller can get wrong (a release/app-name +# mix-up read a Secret that never exists and made every login time out). +# Args: +psql_app_exec() { + local cluster="$1" user="$2" db="$3" sql="$4" + local pod pw + pod=$(cnpg_primary_pod "$cluster") + [[ -n "$pod" ]] || { log_error "no primary pod for cnpg cluster '$cluster'"; return 1; } + pw=$(kubectl -n "$NAMESPACE" get secret "${cluster}-credentials" -o json \ + | jq -r --arg u "$user" '.data[$u] // empty' | base64 -d) + [[ -n "$pw" ]] || { log_error "no password for user '${user}' in ${cluster}-credentials"; return 1; } + kubectl -n "$NAMESPACE" exec "$pod" -c postgres -- \ + env PGPASSWORD="$pw" psql -h "${cluster}-rw" -U "$user" -d "$db" -tAc "$sql" +} + +# Block until the application user can authenticate against a cluster, or fail +# after s. Needed after a to-copy restore: the driver clears +# bootstrap.enabled once recovery converges so the chart's init-job re-runs +# ALTER ROLE ... WITH PASSWORD, converging the recovered roles (which carry the +# source's password hashes) onto the freshly generated -credentials +# Secret. That convergence is asynchronous - a HelmRelease upgrade plus its +# post-upgrade init-job hook - so this is a readiness wait on an eventually +# consistent property, not a retry over a deterministic step. +# Args: +wait_for_app_login() { + local cluster="$1" user="$2" db="$3" timeout="${4:-300}" + local elapsed=0 errfile + errfile=$(mktemp) + log_substep "Waiting for user '${user}' to authenticate against ${cluster}..." + while true; do + # Success is decided on stdout alone (the '1' from SELECT 1), so a stray + # notice on stderr cannot read as a failure. stderr is kept in errfile, + # not discarded: a transient not-yet-converged attempt stays quiet, but + # the LAST attempt's stderr is what names the cause on timeout (a missing + # password, or the wrong Secret) - without it a real failure reads as a + # generic timeout. + if [[ "$(psql_app_exec "$cluster" "$user" "$db" 'SELECT 1;' 2>"$errfile" | tr -d '[:space:]')" == "1" ]]; then + rm -f "$errfile" + log_success "user '${user}' authenticated against ${cluster}" + return 0 + fi + if [[ $elapsed -ge $timeout ]]; then + log_error "Timeout waiting for user '${user}' to authenticate against ${cluster}: the credentials Secret advertises a password the roles never received (bootstrap not cleared, or the init-job did not reconcile). Last attempt: $(cat "$errfile")" + rm -f "$errfile" + return 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done +} diff --git a/examples/backups/postgres/05-postgres-src.yaml b/examples/backups/postgres/05-postgres-src.yaml index 409d649ddb..35685d40e3 100644 --- a/examples/backups/postgres/05-postgres-src.yaml +++ b/examples/backups/postgres/05-postgres-src.yaml @@ -28,8 +28,7 @@ spec: admin: - app users: - app: - password: REPLACE_WITH_PASSWORD + app: {} backup: enabled: true # schedule intentionally empty: we do NOT use the chart's legacy diff --git a/examples/backups/postgres/10-cnpg-strategy.yaml b/examples/backups/postgres/10-cnpg-strategy.yaml index 91acb68fb3..13eba0c3f6 100644 --- a/examples/backups/postgres/10-cnpg-strategy.yaml +++ b/examples/backups/postgres/10-cnpg-strategy.yaml @@ -9,7 +9,12 @@ metadata: name: cnpg-strategy-default spec: template: - serverName: "{{ .Application.metadata.name }}" + # serverName (the WAL-archive S3 prefix) is intentionally NOT set: the + # driver preserves whatever prefix the live cnpg.io Cluster already + # archives under (the chart pins it to the Helm release name, + # postgres-), and only a cluster with no plugin attached would take a + # strategy-supplied value. Setting a different name here would be ignored + # for chart-managed clusters and would only cause confusion. barmanObjectStore: # destinationPath uses the COSI BucketClaim's actual bucket name (look # it up via `kubectl -n get bucketclaim bucket- -o diff --git a/examples/backups/postgres/30-postgres-target.yaml b/examples/backups/postgres/30-postgres-target.yaml index 4989c1da43..d1bec7f03b 100644 --- a/examples/backups/postgres/30-postgres-target.yaml +++ b/examples/backups/postgres/30-postgres-target.yaml @@ -4,11 +4,12 @@ # those values so the new cluster's restore_command can fetch WALs from # the source's S3 archive. # -# The CI e2e (hack/e2e-apps/backup-postgres.bats) deploys this manifest -# in the same namespace as pg-src, exercising the same-namespace to-copy -# happy path. The cross-tenant variant (target in tenant-test, source's -# seaweedfs in tenant-root) is blocked by the per-tenant Cilium egress -# policy and stays a manual / dev-cluster flow. +# This manifest is the reference for the same-namespace to-copy happy path: +# deploy it alongside pg-src and the restore leaves the source running. The +# postgres backup e2e (hack/e2e-chainsaw/postgres, postgres-2-backup-roundtrip) +# drives exactly this flow via run-all.sh in CI. The +# cross-tenant variant (target in tenant-test, source's seaweedfs in +# tenant-root) is blocked by the per-tenant Cilium egress policy. --- apiVersion: apps.cozystack.io/v1alpha1 kind: Postgres diff --git a/examples/backups/postgres/35-restorejob-in-place.yaml b/examples/backups/postgres/35-restorejob-in-place.yaml index 2c04abc5a5..16d7253e74 100644 --- a/examples/backups/postgres/35-restorejob-in-place.yaml +++ b/examples/backups/postgres/35-restorejob-in-place.yaml @@ -2,7 +2,12 @@ # back into pg-src. WARNING: deletes the live cnpg.io Cluster and its PVCs # before re-bootstrapping from S3 - any data not yet streamed to WAL is lost. # -# Optional: spec.options.recoveryTime (RFC3339) for point-in-time recovery. +# Optional: spec.options.recoveryTime (RFC3339) turns this into a point-in-time +# recovery, same as the to-copy variant in 45-restorejob-pitr.yaml (which the +# e2e drives with a real, in-window timestamp). Uncomment and set a timestamp +# inside your recoverable window - see 45-restorejob-pitr.yaml and +# docs/operations/backup-classes.md ("Point-in-time recovery") for how to find +# the earliest/latest restorable time. --- apiVersion: backups.cozystack.io/v1alpha1 kind: RestoreJob diff --git a/examples/backups/postgres/40-restorejob-to-copy.yaml b/examples/backups/postgres/40-restorejob-to-copy.yaml index 27b2aa7d4c..00fc6d4520 100644 --- a/examples/backups/postgres/40-restorejob-to-copy.yaml +++ b/examples/backups/postgres/40-restorejob-to-copy.yaml @@ -3,9 +3,10 @@ # from the source's Barman archive while pg-src stays running and # untouched. # -# Driven by the CI e2e (hack/e2e-apps/backup-postgres.bats) for the -# same-namespace happy path. The cross-tenant variant stays manual; see -# 30-postgres-target.yaml for the network-policy reason. +# The reference for the same-namespace happy path, and the final step the +# postgres backup e2e (hack/e2e-chainsaw/postgres, postgres-2-backup-roundtrip) +# asserts via run-all.sh in CI. The cross-tenant variant +# stays manual; see 30-postgres-target.yaml for the network-policy reason. --- apiVersion: backups.cozystack.io/v1alpha1 kind: RestoreJob diff --git a/examples/backups/postgres/45-restorejob-pitr.yaml b/examples/backups/postgres/45-restorejob-pitr.yaml new file mode 100644 index 0000000000..7c9434f0c5 --- /dev/null +++ b/examples/backups/postgres/45-restorejob-pitr.yaml @@ -0,0 +1,34 @@ +# Point-in-time recovery (PITR) to a copy. +# +# Restores the source's Barman archive into pg-target as of a specific instant +# via spec.options.recoveryTime (RFC3339). The chart maps it onto CNPG's +# bootstrap.recovery.recoveryTarget.targetTime: PostgreSQL restores the newest +# base backup taken at/before that instant and replays archived WAL up to it, +# so the copy reflects the database exactly as of recoveryTime - later writes +# are absent. +# +# recoveryTime MUST fall inside the recoverable window: at/after the base +# backup's completion and at/before the latest WAL shipped to object storage. +# A target past the latest archived WAL cannot converge - the RestoreJob fails +# with condition Ready=False/RecoveryTargetUnreachable instead of hanging. See +# docs/operations/backup-classes.md ("Point-in-time recovery") for how to +# discover the earliest/latest restorable time. +# +# run-all.sh substitutes REPLACE_WITH_RECOVERY_TIME with a timestamp it captures +# between two marker writes, then asserts the pre-marker row is present and the +# post-marker row is absent. Editing by hand, replace it with an RFC3339 +# timestamp inside your window, e.g. "2026-05-01T12:00:00Z". +--- +apiVersion: backups.cozystack.io/v1alpha1 +kind: RestoreJob +metadata: + name: pg-src-to-pg-target-pitr +spec: + backupRef: + name: pg-src-adhoc + targetApplicationRef: + apiGroup: apps.cozystack.io + kind: Postgres + name: pg-target + options: + recoveryTime: "REPLACE_WITH_RECOVERY_TIME" diff --git a/examples/backups/postgres/README.md b/examples/backups/postgres/README.md new file mode 100644 index 0000000000..998149101f --- /dev/null +++ b/examples/backups/postgres/README.md @@ -0,0 +1,38 @@ +# PostgreSQL backup & restore (barman-cloud plugin) + +End-to-end demo of backing up a cozystack `Postgres` application to S3 and restoring it, using the CloudNativePG **barman-cloud plugin** (the native `spec.backup.barmanObjectStore` path is deprecated in CNPG 1.27 and removed in 1.29). Backups flow through the platform `BackupClass` + `CNPG` strategy so one strategy CR serves every `Postgres` instance in the tenant. + +## What it provisions + +The numbered manifests are the documented flow; a human can read and apply them by hand, and `run-all.sh` drives the same files for a smoke test. + +- `00-bucket.yaml` — a cozystack-managed S3 `Bucket` for the backups, with a `backup` user. +- `05-postgres-src.yaml` — the source `Postgres`, with `backup.enabled: true` so `archive_command` streams WALs from helm-install onwards. +- `10-cnpg-strategy.yaml` — the `CNPG` strategy templated against the live application. +- `15-backupclass.yaml` — maps `Kind=Postgres` to that strategy. +- `20-plan.yaml` — an optional cron `Plan` for recurring backups. +- `25-backupjob-adhoc.yaml` — an ad-hoc `BackupJob`. +- `30-postgres-target.yaml` + `40-restorejob-to-copy.yaml` — restore into a fresh copy, leaving the source running. +- `35-restorejob-in-place.yaml` — the destructive in-place restore variant. +- `45-restorejob-pitr.yaml` — point-in-time recovery: restore into the copy as of a specific `spec.options.recoveryTime` (RFC3339). + +## Placeholders and derived Secrets + +`05-postgres-src.yaml` and `10-cnpg-strategy.yaml` carry `REPLACE_WITH_COSI_BUCKET_NAME` and `REPLACE_WITH_S3_ENDPOINT` (the demo `app` user's password is chart-generated, not substituted). They also reference two Secrets, `-cnpg-backup-creds` (the S3 credentials the barman-cloud sidecar reads) and `-cnpg-backup-ca` (the CA it trusts for a self-signed endpoint). The strategy template renders those names against whichever application it drives, so the restore target needs its own pair too — a restore against `pg-target` looks up `pg-target-cnpg-backup-*`. `run-all.sh` resolves the placeholders from the provisioned `Bucket`'s `BucketInfo` Secret and materialises the pair for both the source and the target; editing by hand, you copy the coordinates from `kubectl -n get secret bucket--backup -o jsonpath='{.data.BucketInfo}'` and the CA from the seaweedfs CA secret in `tenant-root` (`seaweedfs-ca-cert` by default; `run-all.sh` auto-discovers it from the cert-manager Certificate if the name differs). + +Drop the `endpointCA` block from `05`/`10` (and set `S3_CA_SECRET=""` for `run-all.sh`) when the S3 endpoint is signed by a publicly-trusted CA. + +## Run it + +```sh +# Defaults to NAMESPACE=tenant-root; override for a tenant namespace. +NAMESPACE=tenant-root examples/backups/postgres/run-all.sh +# Tear everything down afterwards (idempotent). +NAMESPACE=tenant-root examples/backups/postgres/cleanup.sh +``` + +`run-all.sh` writes a sentinel row into the source, waits for the `BackupJob` to reach `Succeeded`, restores to a copy, and asserts the sentinel round-tripped through S3 into the restored copy. It then runs a **point-in-time recovery**: it writes a `before` marker, captures the server timestamp, writes an `after` marker, and restores the copy to that timestamp (`45-restorejob-pitr.yaml`), asserting the `before` row survived and the `after` row did not. Finally it submits a RestoreJob with a `recoveryTime` an hour in the future and asserts it **fails fast** with `status.phase: Failed` and reason `RecoveryTargetUnreachable` (rather than hanging to the restore deadline). Set `SKIP_RESTORE=1` to stop after a successful backup, or `SKIP_PITR=1` to stop after the latest-point restore. + +## Automated e2e + +The Chainsaw suite at `hack/e2e-chainsaw/postgres/` drives this same `run-all.sh` as a second test (`postgres-2-backup-roundtrip`), selected by CI whenever the postgres app (or these scripts) change and on every release cut. It runs in `tenant-root` against the in-cluster seaweedfs endpoint — the isolated e2e tenant cannot reach it across the Cilium egress policy, and the external ingress endpoint is an unroutable placeholder in the sandbox. diff --git a/examples/backups/postgres/cleanup.sh b/examples/backups/postgres/cleanup.sh new file mode 100755 index 0000000000..52bab728e4 --- /dev/null +++ b/examples/backups/postgres/cleanup.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Best-effort teardown of everything the demo creates. Idempotent; +# missing resources are skipped silently. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Cleanup" + +# The CNPG driver suspends the source/target Postgres HelmRelease while a +# RestoreJob runs. If the demo was aborted mid-restore the HR is left +# suspended; deleting the RestoreJob without resuming it first strands the app. +# Resume both best-effort BEFORE deleting the RestoreJobs. +for app in "$PG_SRC_NAME" "$PG_TARGET_NAME"; do + if kubectl -n "$NAMESPACE" get hr "postgres-${app}" >/dev/null 2>&1; then + log_substep "Resuming HelmRelease postgres-${app} (no-op if not suspended)..." + kubectl -n "$NAMESPACE" patch hr "postgres-${app}" --type=merge \ + -p '{"spec":{"suspend":false}}' >/dev/null || true + fi +done + +log_substep "Deleting RestoreJobs..." +kubectl -n "$NAMESPACE" delete restorejob.backups.cozystack.io \ + "$RESTOREJOB_TOCOPY_NAME" "$RESTOREJOB_PITR_NAME" "$RESTOREJOB_UNREACHABLE_NAME" \ + "$RESTOREJOB_INPLACE_NAME" --ignore-not-found + +log_substep "Deleting Plan + BackupJob + derived Backup artefacts..." +kubectl -n "$NAMESPACE" delete plan.backups.cozystack.io "$PLAN_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete backupjob.backups.cozystack.io \ + "$BACKUPJOB_NAME" "$BACKUPJOB_POSTMARKER_NAME" --ignore-not-found +# Only the Backups of THIS demo's apps — never `--all`: on a shared cluster the +# namespace can hold Backup artefacts of other applications and flows. +for b in $(kubectl -n "$NAMESPACE" get backups.backups.cozystack.io \ + -o jsonpath='{range .items[*]}{.metadata.name} {.spec.applicationRef.name}{"\n"}{end}' 2>/dev/null | + awk -v src="$PG_SRC_NAME" -v tgt="$PG_TARGET_NAME" '$2 == src || $2 == tgt {print $1}'); do + kubectl -n "$NAMESPACE" delete backups.backups.cozystack.io "$b" --ignore-not-found +done + +log_substep "Deleting Postgres apps..." +kubectl -n "$NAMESPACE" delete postgres.apps.cozystack.io "$PG_SRC_NAME" "$PG_TARGET_NAME" --ignore-not-found + +log_substep "Deleting per-app backup Secrets..." +kubectl -n "$NAMESPACE" delete secret \ + "${PG_SRC_NAME}-cnpg-backup-creds" "${PG_SRC_NAME}-cnpg-backup-ca" \ + "${PG_TARGET_NAME}-cnpg-backup-creds" "${PG_TARGET_NAME}-cnpg-backup-ca" --ignore-not-found + +log_substep "Deleting Bucket..." +kubectl -n "$NAMESPACE" delete bucket.apps.cozystack.io "$BUCKET_NAME" --ignore-not-found + +log_substep "Deleting BackupClass + CNPG strategy (cluster-scoped)..." +kubectl delete backupclass.backups.cozystack.io "$BACKUPCLASS_NAME" --ignore-not-found +kubectl delete cnpg.strategy.backups.cozystack.io "$STRATEGY_NAME" --ignore-not-found + +log_success "Cleanup complete." diff --git a/examples/backups/postgres/run-all.sh b/examples/backups/postgres/run-all.sh new file mode 100755 index 0000000000..83e7cfb7aa --- /dev/null +++ b/examples/backups/postgres/run-all.sh @@ -0,0 +1,362 @@ +#!/bin/bash +# Convenience runner + e2e harness for the PostgreSQL barman-cloud backup demo. +# +# It applies the SAME numbered manifests a human reads (00-bucket.yaml .. +# 40-restorejob-to-copy.yaml), filling the REPLACE_WITH_* placeholders from the +# provisioned Bucket and deriving the two Secrets the manifests reference +# (-cnpg-backup-creds, -cnpg-backup-ca) so the documented flow and the +# automated test can never drift. Stops on the first failure. +# +# Flow: Bucket -> source Postgres (+ a sentinel row) -> CNPG strategy + +# BackupClass -> ad-hoc BackupJob (wait Succeeded) -> empty target Postgres + +# to-copy RestoreJob (wait Succeeded) -> assert the sentinel round-tripped +# through S3 into the restored copy. +# +# Override NAMESPACE / endpoint / CA via the environment; see 00-helpers.sh. +# hack/e2e-chainsaw/postgres/ drives this file as postgres-2-backup-roundtrip. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +# Recovery target for the PITR phase; captured between two marker writes far +# below. Defaulted here so subst() can reference it under `set -u` before the +# PITR phase sets it (the placeholder only appears in 45-restorejob-pitr.yaml). +RECOVERY_TIME="${RECOVERY_TIME:-}" + +# Substitute the manifest placeholders. $BUCKET / $S3_HOST are resolved from the +# Bucket below; $RECOVERY_TIME is the PITR target (empty for every manifest +# except 45-restorejob-pitr.yaml). The app user's password is chart-generated, +# not substituted. +subst() { + sed \ + -e "s|REPLACE_WITH_COSI_BUCKET_NAME|${BUCKET}|g" \ + -e "s|REPLACE_WITH_S3_ENDPOINT|${S3_HOST}|g" \ + -e "s|REPLACE_WITH_RECOVERY_TIME|${RECOVERY_TIME}|g" \ + "$SCRIPT_DIR/$1" +} + +print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml" +# 660s, above this generated release's 600s install timeout; see "Sizing an +# HR-Ready budget" in docs/agents/e2e-testing.md. +wait_hr_ready "bucket-${BUCKET_NAME}" 660 +wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \ + '{.status.bucketReady}' true "$NAMESPACE" 300 +wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-backup" \ + '{.status.accessGranted}' true "$NAMESPACE" 300 +if [[ "${COZY_E2E_BACKUP_PREFLIGHT:-0}" == "1" ]]; then + source "$SCRIPT_DIR/../../../hack/e2e-chainsaw/_lib/backup-access-preflight.sh" + cozy_backup_access_preflight "$NAMESPACE" "bucket-${BUCKET_NAME}-backup" 90 +fi + +log_substep "Reading bucket coordinates from BucketInfo Secret..." +TMP=$(mktemp) +trap 'rm -f "$TMP"' EXIT +kubectl -n "$NAMESPACE" get secret "bucket-${BUCKET_NAME}-backup" \ + -o jsonpath='{.data.BucketInfo}' | base64 -d > "$TMP" +S3_ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' "$TMP") +S3_SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' "$TMP") +# S3_ENDPOINT can be overridden via the environment: BucketInfo advertises the +# EXTERNAL ingress endpoint, which in-cluster Pods cannot always reach or +# TLS-validate; the in-cluster alternative is https://seaweedfs-s3.:8333 +# (trusted via the copied CA below). +S3_ENDPOINT="${S3_ENDPOINT:-$(jq -r '.spec.secretS3.endpoint' "$TMP")}" +BUCKET=$(jq -r '.spec.bucketName' "$TMP") +for v in S3_ACCESS_KEY S3_SECRET_KEY S3_ENDPOINT BUCKET; do + [[ -n "${!v}" && "${!v}" != "null" ]] || { log_error "BucketInfo missing required field: ${v}"; exit 1; } +done +# The manifests template "https://REPLACE_WITH_S3_ENDPOINT", so strip the scheme. +S3_HOST=${S3_ENDPOINT#https://}; S3_HOST=${S3_HOST#http://} +log_success "Bucket '${BUCKET}' at endpoint '${S3_ENDPOINT}'." + +print_header "Step 05a: Materialise the Secrets the barman-cloud ObjectStore references" +# Resolve the S3 endpoint CA secret. The default name tracks the seaweedfs +# chart's fullnameOverride (seaweedfs -> seaweedfs-ca-cert), but a downstream +# fullname change would rename it, so fall back to discovering the cert-manager +# CA Certificate (the seaweedfs-labelled one with spec.isCA=true) and read its +# secretName. Leave S3_CA_SECRET empty to skip the copy on a public-CA endpoint. +if [[ -n "$S3_CA_SECRET" ]] \ + && ! kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" >/dev/null 2>&1; then + log_warning "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} not found; discovering the seaweedfs CA Certificate..." + # List every seaweedfs Certificate as " " and pick the CA + # one in shell — avoids kubectl jsonpath's finicky boolean-literal filter. + DISCOVERED_CA=$(kubectl -n "$S3_CA_NAMESPACE" get certificates.cert-manager.io \ + -l app.kubernetes.io/name=seaweedfs \ + -o jsonpath='{range .items[*]}{.spec.isCA}{" "}{.spec.secretName}{"\n"}{end}' 2>/dev/null \ + | awk '$1=="true"{print $2; exit}' || true) + if [[ -n "$DISCOVERED_CA" ]]; then + log_success "Discovered seaweedfs CA secret ${S3_CA_NAMESPACE}/${DISCOVERED_CA}" + S3_CA_SECRET="$DISCOVERED_CA" + else + log_error "No seaweedfs CA Certificate found in ${S3_CA_NAMESPACE}; set S3_CA_SECRET explicitly (or empty for a public-CA endpoint)." + exit 1 + fi +fi +# The chart (05) and the strategy (10) both reference -cnpg-backup-creds +# and -cnpg-backup-ca. The barman-cloud sidecar reads AWS_ACCESS_KEY_ID / +# AWS_SECRET_ACCESS_KEY from the creds Secret and trusts ca.crt from the CA +# Secret when talking to a self-signed S3 endpoint. The strategy template +# renders the names against whichever application it drives, so a restore +# TARGET needs its own pair too (materialised again in step 30 below). +# kubectl apply (not create) so a stale pair from an earlier run is corrected. +materialise_backup_secrets() { + local app="$1" + kubectl -n "$NAMESPACE" create secret generic "${app}-cnpg-backup-creds" \ + --from-literal="AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY}" \ + --from-literal="AWS_SECRET_ACCESS_KEY=${S3_SECRET_KEY}" \ + --dry-run=client -o yaml | kubectl apply -f - + + if [[ -n "$S3_CA_SECRET" ]]; then + log_substep "Copying S3 CA ${S3_CA_NAMESPACE}/${S3_CA_SECRET}[${S3_CA_KEY}] -> ${app}-cnpg-backup-ca..." + local ca_pem + ca_pem=$(kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" \ + -o jsonpath="{.data.${S3_CA_KEY//./\\.}}" | base64 -d) + [[ -n "$ca_pem" ]] || { log_error "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} has no ${S3_CA_KEY}"; exit 1; } + kubectl -n "$NAMESPACE" create secret generic "${app}-cnpg-backup-ca" \ + --from-literal="ca.crt=${ca_pem}" \ + --dry-run=client -o yaml | kubectl apply -f - + else + log_warning "S3_CA_SECRET empty: the manifests still reference ${app}-cnpg-backup-ca." + log_warning "Remove endpointCA from 05/10 by hand when the S3 endpoint uses a public CA." + fi +} +materialise_backup_secrets "$PG_SRC_NAME" + +print_header "Step 05b: Deploy source Postgres '${PG_SRC_NAME}' and wait for it to be healthy" +subst 05-postgres-src.yaml | kubectl -n "$NAMESPACE" apply -f - +# 660s, above this generated release's 600s install timeout; see "Sizing an +# HR-Ready budget" in docs/agents/e2e-testing.md. +wait_hr_ready "postgres-${PG_SRC_NAME}" 660 +wait_for_field clusters.postgresql.cnpg.io "$PG_SRC_CLUSTER" \ + '{.status.phase}' 'Cluster in healthy state' "$NAMESPACE" 360 +# The 'demo' database and 'app' user are created by the chart's init Job, not +# by cluster readiness — wait for it before writing into demo. +wait_for_field jobs.batch "postgres-${PG_SRC_NAME}-init-job" \ + '{.status.succeeded}' 1 "$NAMESPACE" 180 + +print_header "Step 05c: Write a sentinel row so the backup has something to prove" +SENTINEL_TOKEN="e2e-$(date +%s)-$$" +psql_exec "$PG_SRC_CLUSTER" demo " + CREATE TABLE IF NOT EXISTS e2e_sentinel (id int PRIMARY KEY, token text); + INSERT INTO e2e_sentinel (id, token) VALUES (1, '${SENTINEL_TOKEN}') + ON CONFLICT (id) DO UPDATE SET token = EXCLUDED.token; + CHECKPOINT; + SELECT pg_switch_wal();" +log_success "Sentinel token: ${SENTINEL_TOKEN}" + +print_header "Step 10/15: Create the CNPG strategy + BackupClass" +subst 10-cnpg-strategy.yaml | kubectl apply -f - +kubectl apply -f "$SCRIPT_DIR/15-backupclass.yaml" + +print_header "Step 25: Submit ad-hoc BackupJob '${BACKUPJOB_NAME}' and wait for Succeeded" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/25-backupjob-adhoc.yaml" +wait_for_field backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 1200 Failed +BACKUP_NAME=$(kubectl -n "$NAMESPACE" get backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \ + -o jsonpath='{.status.backupRef.name}') +[[ -n "$BACKUP_NAME" ]] || { log_error "BackupJob succeeded but reported no backupRef"; exit 1; } +log_success "Backup artefact: ${BACKUP_NAME}" + +if [[ "${SKIP_RESTORE:-0}" == "1" ]]; then + log_warning "SKIP_RESTORE=1: stopping after a successful backup." + exit 0 +fi + +print_header "Step 30/40: Restore to a copy '${PG_TARGET_NAME}' and wait for Succeeded" +# The strategy template renders -cnpg-backup-creds / -ca against the +# TARGET application during a restore, so the target needs its own Secret pair +# (same S3 coordinates as the source's archive). +materialise_backup_secrets "$PG_TARGET_NAME" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/30-postgres-target.yaml" +# Let the target's first install settle before the RestoreJob driver suspends +# its HelmRelease — suspending an HR mid-install races helm-controller. +# 660s, as for the source above. +wait_hr_ready "postgres-${PG_TARGET_NAME}" 660 +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/40-restorejob-to-copy.yaml" +wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_TOCOPY_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 1200 Failed +# The RestoreJob succeeding means the driver finished re-bootstrapping the +# target from S3; wait for the copy to actually come up before reading it. +wait_for_field clusters.postgresql.cnpg.io "$PG_TARGET_CLUSTER" \ + '{.status.phase}' 'Cluster in healthy state' "$NAMESPACE" 600 + +print_header "Step 40 verify: the sentinel round-tripped through S3 into the copy" +GOT=$(psql_exec "$PG_TARGET_CLUSTER" demo "SELECT token FROM e2e_sentinel WHERE id = 1;" | tr -d '[:space:]') +if [[ "$GOT" != "$SENTINEL_TOKEN" ]]; then + log_error "sentinel mismatch: target has '${GOT}', expected '${SENTINEL_TOKEN}'" + exit 1 +fi +log_success "Round-trip verified: '${PG_TARGET_NAME}' restored sentinel '${GOT}' from S3." + +# The passwords in -credentials are chart-generated and do not match the +# password hashes recovery brought back with the roles; the driver clears +# bootstrap.enabled so the init-job reconciles them. Prove the app user can +# actually log in against the restored copy - psql_exec above only ever used the +# in-pod postgres superuser, so a broken credential would otherwise pass silently. +print_header "Step 40 verify: the app user authenticates against the restored copy" +wait_for_app_login "$PG_TARGET_CLUSTER" app demo 360 +APP_GOT=$(psql_app_exec "$PG_TARGET_CLUSTER" app demo \ + "SELECT token FROM e2e_sentinel WHERE id = 1;" | tr -d '[:space:]') +if [[ "$APP_GOT" != "$SENTINEL_TOKEN" ]]; then + log_error "app-user read mismatch: target has '${APP_GOT}', expected '${SENTINEL_TOKEN}'" + exit 1 +fi +log_success "App-user login verified: 'app' authenticated and read sentinel '${APP_GOT}' from '${PG_TARGET_NAME}'." + +if [[ "${SKIP_PITR:-0}" == "1" ]]; then + log_warning "SKIP_PITR=1: stopping after the latest-point restore." + exit 0 +fi + +print_header "Step 45: point-in-time recovery (restore as of a timestamp between two writes)" +# The base backup (pg-src-adhoc) predates the two markers below; pg-src has +# archived WAL continuously since install, so a PITR to a time BETWEEN the +# markers must replay the 'before' write and stop short of the 'after' write. +log_substep "Writing the 'before' marker into the source..." +psql_exec "$PG_SRC_CLUSTER" demo " + CREATE TABLE IF NOT EXISTS pitr_marker (id int PRIMARY KEY, label text); + INSERT INTO pitr_marker (id, label) VALUES (1, 'before') + ON CONFLICT (id) DO UPDATE SET label = EXCLUDED.label;" + +# Capture the recovery target from the SERVER clock, right after 'before' +# committed. WAL record timestamps (which recovery_target_time compares +# against) come from the server, so the client clock must not be trusted. +# Sub-second (.US) precision is REQUIRED: whole-second to_char floors, and the +# 'before' write and this capture can land in the same wall-clock second, so a +# truncated target would sort *before* the 'before' commit and drop it. +RECOVERY_TIME=$(psql_exec "$PG_SRC_CLUSTER" demo \ + "SELECT replace(to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), ' ', 'T') || 'Z';" \ + | tr -d '[:space:]') +[[ -n "$RECOVERY_TIME" ]] || { log_error "failed to capture recovery target time"; exit 1; } +log_success "Recovery target: ${RECOVERY_TIME}" + +# Open a clear gap so the 'after' commit timestamp is strictly past the target, +# then close the WAL segment holding both markers. +sleep 5 +log_substep "Writing the 'after' marker (must NOT survive the PITR)..." +psql_exec "$PG_SRC_CLUSTER" demo " + INSERT INTO pitr_marker (id, label) VALUES (2, 'after') + ON CONFLICT (id) DO UPDATE SET label = EXCLUDED.label; + CHECKPOINT; + SELECT pg_switch_wal();" + +# Gate the restore on the WAL covering the target actually being in object +# storage, deterministically. A fixed sleep races the barman-cloud archiver +# (a slow CI cluster was seen to lag minutes, which then tripped the restore's +# unreachable-target fail-fast on a target that WAS recoverable). A completed +# CNPG BackupJob guarantees WAL up to its stop point - past 'after', hence past +# the target - is archived (the same invariant the restore driver's WAL-archive +# gate keys off), and WAL is archived in order, so the whole chain up to the +# target is present. The point-in-time restore below still recovers from the +# ORIGINAL base backup (pg-src-adhoc); this one only advances the archive. +print_header "Step 45: complete a post-marker backup so the target's WAL is archived" +kubectl -n "$NAMESPACE" apply -f - <reason mapping is covered deterministically by the controller unit tests +# (logIndicatesRecoveryTargetUnreachable / recoveryTargetUnreachable). So assert +# the two things that always hold: the restore terminates Failed, and the failure +# message attributes it to the PITR target (both reason paths name the +# "recoverable window"). +UNREACHABLE_TIME=$(psql_exec "$PG_SRC_CLUSTER" demo \ + "SELECT replace(to_char((now() + interval '1 hour') AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS'), ' ', 'T') || 'Z';" \ + | tr -d '[:space:]') +[[ -n "$UNREACHABLE_TIME" ]] || { log_error "failed to capture unreachable target time"; exit 1; } +log_success "Unreachable target: ${UNREACHABLE_TIME}" +kubectl -n "$NAMESPACE" apply -f - < RestoreJob terminated Failed, attributed to the recovery target." + +# In-place restore last: it deletes pg-src and re-bootstraps from S3, so nothing +# after it may depend on the source. Regression guard for the archive-vs-recovery +# serverName collision - before bootstrap.newServerName the restored cluster +# archived onto the source's own WAL prefix and wedged on "Expected empty archive". +print_header "Step 50: in-place restore of '${PG_SRC_NAME}' from '${BACKUP_NAME}' and wait for Succeeded" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/35-restorejob-in-place.yaml" +wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_INPLACE_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 1200 Failed +wait_for_field clusters.postgresql.cnpg.io "$PG_SRC_CLUSTER" \ + '{.status.phase}' 'Cluster in healthy state' "$NAMESPACE" 600 + +print_header "Step 50 verify: the sentinel survived the in-place restore" +GOT=$(psql_exec "$PG_SRC_CLUSTER" demo "SELECT token FROM e2e_sentinel WHERE id = 1;" | tr -d '[:space:]') +if [[ "$GOT" != "$SENTINEL_TOKEN" ]]; then + log_error "in-place restore sentinel mismatch: source has '${GOT}', expected '${SENTINEL_TOKEN}'" + exit 1 +fi +log_success "In-place restore verified: '${PG_SRC_NAME}' re-bootstrapped and archives to a fresh prefix (sentinel '${GOT}')." diff --git a/examples/backups/rabbitmq/00-bucket.yaml b/examples/backups/rabbitmq/00-bucket.yaml new file mode 100644 index 0000000000..3461179297 --- /dev/null +++ b/examples/backups/rabbitmq/00-bucket.yaml @@ -0,0 +1,19 @@ +# Cozystack-managed S3 bucket for storing this demo's RabbitMQ definitions. +# The Bucket controller provisions a Secret named "bucket--" +# per user (the bucket app carries the "bucket-" release prefix); run-all.sh +# reads the "backup" user's BucketInfo and projects its access keys into the +# rabbitmq-backup-creds Secret the strategy below references. +# +# The platform ships a cozy-default-rabbitmq strategy that writes to the shared +# cozy-backups system bucket; this demo instead provisions its own Bucket and a +# matching strategy so the round-trip is self-contained and its objects are +# torn down with the demo (see cleanup.sh). +--- +apiVersion: apps.cozystack.io/v1alpha1 +kind: Bucket +metadata: + name: rabbitmq-backups +spec: + users: + backup: + readonly: false diff --git a/examples/backups/rabbitmq/00-helpers.sh b/examples/backups/rabbitmq/00-helpers.sh new file mode 100755 index 0000000000..e283b9bac0 --- /dev/null +++ b/examples/backups/rabbitmq/00-helpers.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# Shared helpers for the RabbitMQ definitions backup/restore demo. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export BLUE='\033[0;34m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export WHITE='\033[1;37m' +export NC='\033[0m' +export BOLD='\033[1m' + +# Default settings (override via environment). +export NAMESPACE="${NAMESPACE:-tenant-root}" +export RABBITMQ_SRC_NAME="${RABBITMQ_SRC_NAME:-rabbitmq-src}" +export RABBITMQ_TARGET_NAME="${RABBITMQ_TARGET_NAME:-rabbitmq-target}" +# The rabbitmq-rd ApplicationDefinition renders the HelmRelease with +# releaseName = "rabbitmq-" + appName (release.prefix in +# packages/system/rabbitmq-rd/cozyrds/rabbitmq.yaml), and the chart sets the +# rabbitmq.com/RabbitmqCluster metadata.name to .Release.Name, so the +# operator-side CR (its Service, default-user Secret and server pods) is +# rabbitmq-. +export RABBITMQ_SRC_CR="rabbitmq-${RABBITMQ_SRC_NAME}" +export RABBITMQ_TARGET_CR="rabbitmq-${RABBITMQ_TARGET_NAME}" +# This demo provisions its own Bucket + Rabbitmq strategy + BackupClass rather +# than reusing the platform cozy-default flow, so the round-trip is +# self-contained and its S3 objects are torn down with the demo. The Bucket +# controller materialises the credentials Secret as "bucket--". +export BUCKET_NAME="${BUCKET_NAME:-rabbitmq-backups}" +export BUCKET_USER="${BUCKET_USER:-backup}" +export STRATEGY_NAME="${STRATEGY_NAME:-rabbitmq-strategy-default}" +export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-rabbitmq-default}" +export BACKUPJOB_NAME="${BACKUPJOB_NAME:-rabbitmq-src-adhoc}" +export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-rabbitmq-src-in-place}" +export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-rabbitmq-src-to-rabbitmq-target}" +export PLAN_NAME="${PLAN_NAME:-rabbitmq-src-daily}" +# The strategy Pod's curl reads AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY from +# this Secret and trusts the S3 endpoint's CA from ca.crt in the CA Secret; +# run-all.sh materialises both from the provisioned Bucket before dispatch. +export CREDS_SECRET="${CREDS_SECRET:-rabbitmq-backup-creds}" +export CA_SECRET="${CA_SECRET:-rabbitmq-backup-ca}" +# S3 endpoint CA. cozystack's default seaweedfs serves its S3 endpoint with a +# self-signed certificate whose CA lives in this Secret; run-all.sh copies its +# ca.crt into CA_SECRET, which the strategy Pod mounts and points CURL_CA_BUNDLE +# at. The name follows the seaweedfs chart's fullnameOverride +# ("seaweedfs" -> "seaweedfs-ca-cert"); run-all.sh auto-discovers the CA +# Certificate's actual secret when this default is absent. On a cluster whose S3 +# endpoint is signed by a publicly-trusted CA, set S3_CA_SECRET="" to skip the +# copy and drop the CURL_CA_BUNDLE env + CA volume from 03-rabbitmq-strategy.yaml. +export S3_CA_SECRET="${S3_CA_SECRET:-seaweedfs-ca-cert}" +export S3_CA_NAMESPACE="${S3_CA_NAMESPACE:-tenant-root}" +export S3_CA_KEY="${S3_CA_KEY:-ca.crt}" + +log_info() { echo -e "${BLUE}i${NC} $*" >&2; } +log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; } +log_error() { echo -e "${RED}x${NC} $*" >&2; } +log_step() { echo -e "\n${MAGENTA}${BOLD}> $*${NC}" >&2; } +log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; } + +print_header() { + echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2 +} + +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" + +# Wait until the rabbitmq.com/RabbitmqCluster CR reports AllReplicasReady=True +# (the cluster-operator's own readiness signal — every server pod is up and the +# management API is serving). No fail-fast value: the condition can be briefly +# absent or False during rolling startup before it converges. +wait_rabbitmq_ready() { + local cr="$1" timeout="${2:-600}" + wait_for_field rabbitmqclusters.rabbitmq.com "$cr" \ + '{.status.conditions[?(@.type=="AllReplicasReady")].status}' True "$NAMESPACE" "$timeout" +} + +# Run rabbitmqctl inside a RabbitmqCluster's first server pod. rabbitmqctl talks +# to the local node over the Erlang dist port and needs no HTTP credentials, so +# it is the simplest way to mutate and read the DEFINITIONS (vhosts, policies) +# this demo uses as its round-trip sentinel. Args: +rabbitmqctl_exec() { + local cr="$1"; shift + kubectl -n "$NAMESPACE" exec "${cr}-server-0" -c rabbitmq -- rabbitmqctl "$@" +} + +# Seed the sentinel: a vhost plus a policy on it. Both are broker DEFINITIONS +# captured by /api/definitions, so their survival through S3 proves the backup +# round-tripped the definitions (not merely that a Job ran). Args: +rabbitmq_seed_sentinel() { + local cr="$1" token="$2" + rabbitmqctl_exec "$cr" add_vhost "$token" >/dev/null + rabbitmqctl_exec "$cr" set_policy -p "$token" sentinel-policy '.*' \ + '{"max-length":1000}' --apply-to queues >/dev/null +} + +# True (0) when the sentinel vhost exists, false (1) when it is absent. The +# exec is captured separately and a FAILED exec aborts rather than being folded +# into "absent" - otherwise a transient exec failure would read as "the sentinel +# is not there", silently passing the negative pre-checks in run-all.sh. +# Args: +rabbitmq_has_sentinel() { + local cr="$1" token="$2" out + if ! out=$(rabbitmqctl_exec "$cr" list_vhosts name 2>/dev/null); then + log_error "rabbitmqctl list_vhosts failed on $cr; cannot determine sentinel presence" + exit 1 + fi + printf '%s\n' "$out" | grep -qxF "$token" +} diff --git a/examples/backups/rabbitmq/03-rabbitmq-strategy.yaml b/examples/backups/rabbitmq/03-rabbitmq-strategy.yaml new file mode 100644 index 0000000000..1533ee0926 --- /dev/null +++ b/examples/backups/rabbitmq/03-rabbitmq-strategy.yaml @@ -0,0 +1,185 @@ +# RabbitMQ definitions backup strategy for this demo. It mirrors the shipped +# cozy-default-rabbitmq strategy +# (packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml) +# but points at this demo's own Bucket instead of the platform cozy-backups +# system bucket, so the round-trip is self-contained. The in-Pod curl script is +# kept identical to the shipped one; only the S3 coordinates differ. +# +# The controller re-renders the `{{ .Mode | .Release | .ArtifactURI | .BackupName }}` +# templates below per BackupJob/RestoreJob against the live application object. +# The REPLACE_WITH_* markers are filled by run-all.sh from the provisioned +# Bucket (kubectl apply happens through run-all's subst, not directly). +--- +apiVersion: strategy.backups.cozystack.io/v1alpha1 +kind: Rabbitmq +metadata: + name: rabbitmq-strategy-default +spec: + # Record the stored object's location on the produced Backup so it is + # introspectable. The key is scoped by the SOURCE namespace/name/backup so + # distinct backups of one application never overwrite each other. + artifactURITemplate: "s3://REPLACE_WITH_COSI_BUCKET_NAME/{{ .Release.Namespace }}/{{ .Release.Name }}/{{ .BackupName }}/definitions.json" + template: + spec: + restartPolicy: Never + containers: + - name: rabbitmq-backup + # Reuse whatever image the platform strategy runs so the demo Pod + # needs no package install at run time. Read it off the shipped + # strategy (run-all.sh substitutes it here) rather than hardcoding a + # tag. + image: "REPLACE_WITH_BACKUP_CLIENT_IMAGE" + imagePullPolicy: IfNotPresent + env: + - name: MODE + value: "{{ .Mode }}" + # The Cozystack RabbitMQ release name is prefixed with "rabbitmq-", + # so the operator's client Service and default-user Secret are + # rabbitmq-. On restore .Release is the TARGET, so host and + # credentials point at the broker being restored into. + - name: RABBITMQ_HOST + value: "rabbitmq-{{ .Release.Name }}.{{ .Release.Namespace }}.svc" + # optional: the cleanup run (on Backup deletion) only touches S3 and + # may execute after the source broker - and this Secret - are gone. + - name: RABBITMQ_USER + valueFrom: + secretKeyRef: + name: "rabbitmq-{{ .Release.Name }}-default-user" + key: username + optional: true + - name: RABBITMQ_PASSWORD + valueFrom: + secretKeyRef: + name: "rabbitmq-{{ .Release.Name }}-default-user" + key: password + optional: true + # ARTIFACT_URI is the single source of truth for the object location: + # the driver renders it from spec.artifactURITemplate, records it on + # the Backup's status.artifact.uri, and injects it here. On restore + # it is that recorded value read back from the Backup. + - name: ARTIFACT_URI + value: "{{ .ArtifactURI }}" + # S3 coordinates come from this demo's Bucket, projected by + # run-all.sh into rabbitmq-backup-creds. Unlike the shipped strategy + # (external ACME-valid ingress, verified against the image's system + # CAs) the in-cluster seaweedfs-s3 endpoint is self-signed, so the + # seaweedfs CA is mounted and pointed at via CURL_CA_BUNDLE, which + # libcurl honours transparently - the curl script stays byte-for-byte + # the shipped one, with no -k. + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: rabbitmq-backup-creds + key: AWS_ACCESS_KEY_ID + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: rabbitmq-backup-creds + key: AWS_SECRET_ACCESS_KEY + - name: S3_ENDPOINT + value: "REPLACE_WITH_S3_ENDPOINT" + - name: S3_REGION + value: "us-east-1" + - name: S3_FORCE_PATH_STYLE + value: "true" + # Self-signed seaweedfs CA copied into rabbitmq-backup-ca by + # run-all.sh and mounted below. On a publicly-trusted S3 endpoint + # drop this env plus the volume/volumeMount and set S3_CA_SECRET="" + # (see 00-helpers.sh). + - name: CURL_CA_BUNDLE + value: /etc/ssl/rabbitmq-backup-ca/ca.crt + volumeMounts: + - name: s3-ca + mountPath: /etc/ssl/rabbitmq-backup-ca + readOnly: true + command: ["/bin/sh", "-c"] + args: + - | + set -eu + + case "${S3_ENDPOINT}" in + http://*|https://*) base="${S3_ENDPOINT}" ;; + *) base="https://${S3_ENDPOINT}" ;; + esac + + # ARTIFACT_URI is s3:///. Split it and build the HTTP + # object URL, honouring path-style vs virtual-hosted addressing so + # this works against both SeaweedFS (path-style) and an external S3 + # configured for virtual-hosted-style. + case "${ARTIFACT_URI}" in + s3://*) rest="${ARTIFACT_URI#s3://}"; bucket="${rest%%/*}"; key="${rest#*/}" ;; + *) echo "expected an s3:// artifact URI, got '${ARTIFACT_URI}' (set spec.artifactURITemplate)" >&2; exit 1 ;; + esac + if [ "${S3_FORCE_PATH_STYLE}" = "false" ]; then + scheme="${base%%://*}"; host="${base#*://}" + obj_url="${scheme}://${bucket}.${host}/${key}" + else + obj_url="${base}/${bucket}/${key}" + fi + + s3() { curl -fsS --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" "$@"; } + + # Cleanup runs when the owning Backup is deleted. It only touches + # S3 - the source broker may already be gone - so it deletes the + # object and exits before any management-API access. A missing + # object is not an error. + if [ "${MODE}" = cleanup ]; then + echo "deleting ${ARTIFACT_URI}" + # A missing object is success (delete is idempotent); any other + # non-2xx is a real failure, so this exits non-zero and the + # controller retries rather than orphaning the object. + code=$(curl -sS -o /dev/null -w '%{http_code}' \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + -X DELETE "${obj_url}" || true) + case "${code}" in + 2*|404) echo "deleted (HTTP ${code})"; exit 0 ;; + *) echo "delete failed: HTTP ${code}" >&2; exit 1 ;; + esac + fi + + api="http://${RABBITMQ_HOST}:15672/api/definitions" + + # Precondition gates on the application being Ready; the + # management API can still be warming up. Wait for it before the + # export/import so a cold API is not read as a backup failure. + for _ in $(seq 1 60); do + if curl -fsS -u "${RABBITMQ_USER}:${RABBITMQ_PASSWORD}" --max-time 5 \ + "http://${RABBITMQ_HOST}:15672/api/overview" >/dev/null 2>&1; then + break + fi + sleep 5 + done + + if [ "${MODE}" = backup ]; then + echo "exporting RabbitMQ definitions from ${RABBITMQ_HOST} to ${ARTIFACT_URI}" + curl -fsS -u "${RABBITMQ_USER}:${RABBITMQ_PASSWORD}" "${api}" -o /tmp/definitions.json + s3 -X PUT --upload-file /tmp/definitions.json "${obj_url}" + echo "uploaded ${ARTIFACT_URI}" + else + echo "importing RabbitMQ definitions into ${RABBITMQ_HOST} from ${ARTIFACT_URI}" + echo "note: import is additive (POST /api/definitions merges); it does not delete objects created after the backup" + s3 -o /tmp/definitions.json "${obj_url}" + curl -fsS -u "${RABBITMQ_USER}:${RABBITMQ_PASSWORD}" \ + -H "content-type: application/json" \ + -X POST "${api}" --data-binary @/tmp/definitions.json + echo "import complete" + fi + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + volumes: + - name: s3-ca + secret: + secretName: rabbitmq-backup-ca diff --git a/examples/backups/rabbitmq/04-backupclass.yaml b/examples/backups/rabbitmq/04-backupclass.yaml new file mode 100644 index 0000000000..b141cdee21 --- /dev/null +++ b/examples/backups/rabbitmq/04-backupclass.yaml @@ -0,0 +1,18 @@ +# Maps Kind=RabbitMQ to this demo's Rabbitmq strategy. BackupClass is +# cluster-scoped, so a single instance covers every namespace. Named +# rabbitmq-default so it does not collide with the platform cozy-default class, +# which routes RabbitMQ to the shipped cozy-default-rabbitmq strategy. +--- +apiVersion: backups.cozystack.io/v1alpha1 +kind: BackupClass +metadata: + name: rabbitmq-default +spec: + strategies: + - application: + apiGroup: apps.cozystack.io + kind: RabbitMQ + strategyRef: + apiGroup: strategy.backups.cozystack.io + kind: Rabbitmq + name: rabbitmq-strategy-default diff --git a/examples/backups/rabbitmq/05-rabbitmq-src.yaml b/examples/backups/rabbitmq/05-rabbitmq-src.yaml new file mode 100644 index 0000000000..dc65ce3af5 --- /dev/null +++ b/examples/backups/rabbitmq/05-rabbitmq-src.yaml @@ -0,0 +1,13 @@ +# Source RabbitMQ application. Note there is no `backup:` block: the S3 +# coordinates live on the Rabbitmq strategy (03-rabbitmq-strategy.yaml), whose +# Pod does the definitions export, not on the application. The application chart +# is unmodified. +apiVersion: apps.cozystack.io/v1alpha1 +kind: RabbitMQ +metadata: + name: rabbitmq-src +spec: + replicas: 1 + size: 1Gi + resourcesPreset: c1.small + version: v4.2 diff --git a/examples/backups/rabbitmq/10-backupjob-adhoc.yaml b/examples/backups/rabbitmq/10-backupjob-adhoc.yaml new file mode 100644 index 0000000000..93071cdf24 --- /dev/null +++ b/examples/backups/rabbitmq/10-backupjob-adhoc.yaml @@ -0,0 +1,14 @@ +# Ad-hoc backup of the source RabbitMQ, routed through this demo's rabbitmq-default +# BackupClass. The driver renders the rabbitmq-strategy-default strategy, which +# exports broker definitions to this demo's Bucket. On success the BackupJob +# reports status.backupRef.name — the Backup this demo restores from. +apiVersion: backups.cozystack.io/v1alpha1 +kind: BackupJob +metadata: + name: rabbitmq-src-adhoc +spec: + applicationRef: + apiGroup: apps.cozystack.io + kind: RabbitMQ + name: rabbitmq-src + backupClassName: rabbitmq-default diff --git a/examples/backups/rabbitmq/15-plan.yaml b/examples/backups/rabbitmq/15-plan.yaml new file mode 100644 index 0000000000..6a551b395c --- /dev/null +++ b/examples/backups/rabbitmq/15-plan.yaml @@ -0,0 +1,18 @@ +# Scheduled backups of the source RabbitMQ. The Plan controller creates a +# BackupJob on each cron fire, using the same rabbitmq-default routing as the +# ad-hoc job above. Applied by run-all.sh for completeness; the round-trip below +# drives the ad-hoc BackupJob directly. +apiVersion: backups.cozystack.io/v1alpha1 +kind: Plan +metadata: + name: rabbitmq-src-daily +spec: + applicationRef: + apiGroup: apps.cozystack.io + kind: RabbitMQ + name: rabbitmq-src + backupClassName: rabbitmq-default + schedule: + type: cron + # Daily at midnight, matching the Plan name. + cron: "0 0 * * *" diff --git a/examples/backups/rabbitmq/20-rabbitmq-target.yaml b/examples/backups/rabbitmq/20-rabbitmq-target.yaml new file mode 100644 index 0000000000..d010068e6a --- /dev/null +++ b/examples/backups/rabbitmq/20-rabbitmq-target.yaml @@ -0,0 +1,11 @@ +# Separate target application for restore-to-copy. It runs alongside the source +# and receives the source's definitions without the source being touched. +apiVersion: apps.cozystack.io/v1alpha1 +kind: RabbitMQ +metadata: + name: rabbitmq-target +spec: + replicas: 1 + size: 1Gi + resourcesPreset: c1.small + version: v4.2 diff --git a/examples/backups/rabbitmq/25-restorejob-in-place.yaml b/examples/backups/rabbitmq/25-restorejob-in-place.yaml new file mode 100644 index 0000000000..c632beff31 --- /dev/null +++ b/examples/backups/rabbitmq/25-restorejob-in-place.yaml @@ -0,0 +1,11 @@ +# In-place restore: import the backed-up definitions back into the SOURCE broker. +# Omitting targetApplicationRef makes the driver restore into the Backup's own +# application (rabbitmq-src). backupRef names the Backup, which the Job driver +# names after the BackupJob that produced it (rabbitmq-src-adhoc). +apiVersion: backups.cozystack.io/v1alpha1 +kind: RestoreJob +metadata: + name: rabbitmq-src-in-place +spec: + backupRef: + name: rabbitmq-src-adhoc diff --git a/examples/backups/rabbitmq/30-restorejob-to-copy.yaml b/examples/backups/rabbitmq/30-restorejob-to-copy.yaml new file mode 100644 index 0000000000..da3b1a869c --- /dev/null +++ b/examples/backups/rabbitmq/30-restorejob-to-copy.yaml @@ -0,0 +1,15 @@ +# Restore-to-copy: import the source's definitions into a DIFFERENT application +# (rabbitmq-target) in the same namespace, leaving the source running. The +# driver reads the object under the SOURCE's key and authenticates to the TARGET +# broker with the target's own default-user credentials. +apiVersion: backups.cozystack.io/v1alpha1 +kind: RestoreJob +metadata: + name: rabbitmq-src-to-rabbitmq-target +spec: + backupRef: + name: rabbitmq-src-adhoc + targetApplicationRef: + apiGroup: apps.cozystack.io + kind: RabbitMQ + name: rabbitmq-target diff --git a/examples/backups/rabbitmq/README.md b/examples/backups/rabbitmq/README.md new file mode 100644 index 0000000000..268c24399b --- /dev/null +++ b/examples/backups/rabbitmq/README.md @@ -0,0 +1,38 @@ +# RabbitMQ backup/restore demo + +This demo backs up and restores a Cozystack-managed RabbitMQ instance, proving data integrity by round-tripping a sentinel definition through object storage. It provisions its own `Bucket`, `Rabbitmq` strategy and `BackupClass` so the round-trip is self-contained; the platform ships an equivalent `cozy-default-rabbitmq` strategy that writes to the shared `cozy-backups` system bucket, which you can route to from your own `BackupClass` instead. + +## What is backed up + +RabbitMQ's cluster operator ships no backup CRD, and RabbitMQ has no consistent online export of message payloads. This strategy therefore backs up broker **definitions** — vhosts, users, permissions, queues, exchanges, bindings, policies and parameters — via the management HTTP API (`GET /api/definitions`), and restores them with `POST /api/definitions`. **Message payloads are out of scope**, and there is **no point-in-time recovery**: a full-data backup would be a Velero volume snapshot, which is a different strategy. + +Definitions import is additive: an in-place restore re-creates dropped definitions but does not delete objects that exist live but not in the backup. + +## Prerequisites + +The `backupstrategy-controller` and the backup CRDs must be installed (the platform default backups stack). `run-all.sh` provisions its own `Bucket`, reads the S3 coordinates from its `BucketInfo`, projects them into a `rabbitmq-backup-creds` Secret, copies the endpoint's CA into `rabbitmq-backup-ca`, and reuses the container image the shipped `cozy-default-rabbitmq` strategy runs — so that strategy must be present, but this demo does not write to the system bucket. + +To-copy restore imports the source's definitions into a separate broker via `POST /api/definitions`, which merges the source's users, permissions and vhosts into the target. It does not clobber the target's own operator-generated default user, because the cluster-operator gives each RabbitmqCluster a distinct random default username. It does, however, import the source's users (with their password hashes), so after a to-copy restore those source credentials are valid logins on the target — grant access accordingly. + +Each backup is stored under a per-run key (`///definitions.json`), so distinct backups of one application do not overwrite each other and a restore reads the exact object its Backup recorded. + +## Flow + +- `00-helpers.sh` — shared bash helpers (waiters, sentinel seed/check via `rabbitmqctl`). +- `00-bucket.yaml` — the demo `Bucket` backing the round-trip. +- `03-rabbitmq-strategy.yaml` — the `Rabbitmq` strategy (S3 coordinates filled from the Bucket by `run-all.sh`). +- `04-backupclass.yaml` — the `rabbitmq-default` `BackupClass` mapping `RabbitMQ` to that strategy. +- `05-rabbitmq-src.yaml` — the source application (no `backup:` block; the strategy carries the S3 coordinates). +- `10-backupjob-adhoc.yaml` — an ad-hoc `BackupJob` routed to `rabbitmq-default`. +- `15-plan.yaml` — a cron `Plan` for scheduled backups. +- `20-rabbitmq-target.yaml` — a separate application for restore-to-copy. +- `25-restorejob-in-place.yaml` / `30-restorejob-to-copy.yaml` — the two restore flows. + +Run the whole round-trip: + +```bash +./run-all.sh # backup, drop the sentinel, restore in place, then restore to a copy +./cleanup.sh # remove this demo's objects (idempotent) +``` + +`SKIP_RESTORE=1 ./run-all.sh` stops after a successful backup (backup-only smoke). Override `NAMESPACE` (default `tenant-root`) to run elsewhere. `BucketInfo` advertises the external S3 ingress endpoint, which in-cluster Pods cannot always reach or TLS-validate; set `S3_ENDPOINT=https://seaweedfs-s3..svc:8333` to target the in-cluster SeaweedFS instead — the `.svc` FQDN is the name the SeaweedFS serving cert's SAN covers, so `curl` can verify TLS against the copied CA (the e2e harness does this). diff --git a/examples/backups/rabbitmq/cleanup.sh b/examples/backups/rabbitmq/cleanup.sh new file mode 100755 index 0000000000..95d608cf53 --- /dev/null +++ b/examples/backups/rabbitmq/cleanup.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Idempotent teardown of the RabbitMQ backup demo. Removes only this demo's +# objects — never a blanket --all — so it is safe to run before a re-run and in +# a chainsaw finally step. Order: restore/backup jobs and Plan, then this demo's +# Backup artifacts, then the applications, then the demo's Bucket + strategy + +# BackupClass + Secrets. The platform cozy-backups bucket and cozy-default-rabbitmq +# strategy are platform-owned and left alone. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Cleaning up RabbitMQ backup demo (namespace: $NAMESPACE)" + +kubectl -n "$NAMESPACE" delete restorejob.backups.cozystack.io \ + "$RESTOREJOB_TOCOPY_NAME" "$RESTOREJOB_INPLACE_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete plan.backups.cozystack.io "$PLAN_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete backupjob.backups.cozystack.io "$BACKUPJOB_NAME" --ignore-not-found + +# Prune only the Backup artifacts belonging to this demo's applications. +for app in "$RABBITMQ_SRC_NAME" "$RABBITMQ_TARGET_NAME"; do + kubectl -n "$NAMESPACE" get backups.backups.cozystack.io \ + -o jsonpath='{range .items[?(@.spec.applicationRef.name=="'"$app"'")]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ + | while read -r b; do + [[ -n "$b" ]] && kubectl -n "$NAMESPACE" delete backups.backups.cozystack.io "$b" --ignore-not-found + done +done + +kubectl -n "$NAMESPACE" delete rabbitmq.apps.cozystack.io \ + "$RABBITMQ_SRC_NAME" "$RABBITMQ_TARGET_NAME" --ignore-not-found + +# The Backup deletions above run their S3 cleanup Job through the strategy and +# the creds/CA Secrets, so tear those down only now that the Backups are gone. +kubectl -n "$NAMESPACE" delete secret "$CREDS_SECRET" "$CA_SECRET" --ignore-not-found +kubectl -n "$NAMESPACE" delete bucket.apps.cozystack.io "$BUCKET_NAME" --ignore-not-found +kubectl delete backupclass.backups.cozystack.io "$BACKUPCLASS_NAME" --ignore-not-found +kubectl delete rabbitmqs.strategy.backups.cozystack.io "$STRATEGY_NAME" --ignore-not-found + +log_success "cleanup complete" diff --git a/examples/backups/rabbitmq/run-all.sh b/examples/backups/rabbitmq/run-all.sh new file mode 100755 index 0000000000..d947b3f48c --- /dev/null +++ b/examples/backups/rabbitmq/run-all.sh @@ -0,0 +1,207 @@ +#!/bin/bash +# End-to-end RabbitMQ definitions backup/restore demo. It proves DATA +# INTEGRITY, not liveness: a sentinel vhost (a broker definition) is seeded, +# backed up, dropped, and must reappear after a restore — first in place, then +# into a separate copy. +# +# The demo provisions its OWN Bucket + Rabbitmq strategy + BackupClass, filling +# the REPLACE_WITH_* markers in 03-rabbitmq-strategy.yaml from the Bucket, so the +# round-trip is self-contained and its S3 objects tear down with the demo. The +# platform-shipped cozy-default-rabbitmq strategy (which writes to the shared +# cozy-backups system bucket) is left untouched; this demo only reads the client +# image off it. +# +# Env knobs: +# NAMESPACE (default tenant-root) +# SKIP_RESTORE=1 stop after a successful backup (backup-only smoke) +# S3_ENDPOINT override the S3 endpoint the backup Pod uses. BucketInfo +# advertises the EXTERNAL ingress endpoint, which in-cluster +# Pods cannot always reach or TLS-validate; the in-cluster +# alternative is https://seaweedfs-s3..svc:8333 — the .svc +# FQDN is the name the seaweedfs serving cert's SAN covers, so +# curl verifies TLS against the copied CA below (the 2-label +# seaweedfs-s3. is not in the SAN). CI sets this; a real +# cluster can leave it unset to use the advertised endpoint. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +# Fill the strategy manifest's placeholders from the provisioned Bucket and the +# platform client image. $BUCKET / $S3_ENDPOINT / $CLIENT_IMAGE are resolved +# below; using '|' as the sed delimiter keeps the image ref's slashes intact. +subst() { + sed \ + -e "s|REPLACE_WITH_COSI_BUCKET_NAME|${BUCKET}|g" \ + -e "s|REPLACE_WITH_S3_ENDPOINT|${S3_ENDPOINT}|g" \ + -e "s|REPLACE_WITH_BACKUP_CLIENT_IMAGE|${CLIENT_IMAGE}|g" \ + "$SCRIPT_DIR/$1" +} + +print_header "RabbitMQ definitions backup/restore demo (namespace: $NAMESPACE)" + +# --- Bucket ------------------------------------------------------------------ +print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/00-bucket.yaml" +# 660s, above this generated release's 600s install timeout; see "Sizing an +# HR-Ready budget" in docs/agents/e2e-testing.md. +wait_hr_ready "bucket-${BUCKET_NAME}" 660 +wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \ + '{.status.bucketReady}' true "$NAMESPACE" 300 +wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \ + '{.status.accessGranted}' true "$NAMESPACE" 300 + +log_substep "Reading bucket coordinates from BucketInfo Secret..." +TMP=$(mktemp) +trap 'rm -f "$TMP"' EXIT +kubectl -n "$NAMESPACE" get secret "bucket-${BUCKET_NAME}-${BUCKET_USER}" \ + -o jsonpath='{.data.BucketInfo}' | base64 -d > "$TMP" +S3_ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' "$TMP") +S3_SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' "$TMP") +S3_ENDPOINT="${S3_ENDPOINT:-$(jq -r '.spec.secretS3.endpoint' "$TMP")}" +BUCKET=$(jq -r '.spec.bucketName' "$TMP") +for v in S3_ACCESS_KEY S3_SECRET_KEY S3_ENDPOINT BUCKET; do + [[ -n "${!v}" && "${!v}" != "null" ]] || { log_error "BucketInfo missing required field: ${v}"; exit 1; } +done +# The strategy's curl accepts a full URL (http:// or https://), so keep the +# scheme rather than stripping it as the mariadb-operator path does. +case "$S3_ENDPOINT" in http://*|https://*) : ;; *) S3_ENDPOINT="https://${S3_ENDPOINT}" ;; esac +log_success "Bucket '${BUCKET}' at endpoint '${S3_ENDPOINT}'." + +# --- Secrets the strategy references ----------------------------------------- +print_header "Step 01: Materialise the Secrets the Rabbitmq strategy references" +# Resolve the S3 endpoint CA secret. The default name tracks the seaweedfs +# chart's fullnameOverride (seaweedfs -> seaweedfs-ca-cert), but a downstream +# fullname change would rename it, so fall back to discovering the cert-manager +# CA Certificate (the seaweedfs-labelled one with spec.isCA=true) and read its +# secretName. Leave S3_CA_SECRET empty to skip the copy on a public-CA endpoint. +if [[ -n "$S3_CA_SECRET" ]] \ + && ! kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" >/dev/null 2>&1; then + log_warning "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} not found; discovering the seaweedfs CA Certificate..." + DISCOVERED_CA=$(kubectl -n "$S3_CA_NAMESPACE" get certificates.cert-manager.io \ + -l app.kubernetes.io/name=seaweedfs \ + -o jsonpath='{range .items[*]}{.spec.isCA}{" "}{.spec.secretName}{"\n"}{end}' 2>/dev/null \ + | awk '$1=="true"{print $2; exit}' || true) + if [[ -n "$DISCOVERED_CA" ]]; then + log_success "Discovered seaweedfs CA secret ${S3_CA_NAMESPACE}/${DISCOVERED_CA}" + S3_CA_SECRET="$DISCOVERED_CA" + else + log_error "No seaweedfs CA Certificate found in ${S3_CA_NAMESPACE}; set S3_CA_SECRET explicitly (or empty for a public-CA endpoint)." + exit 1 + fi +fi + +log_substep "Projecting bucket access keys into ${CREDS_SECRET}..." +kubectl -n "$NAMESPACE" create secret generic "$CREDS_SECRET" \ + --from-literal="AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY}" \ + --from-literal="AWS_SECRET_ACCESS_KEY=${S3_SECRET_KEY}" \ + --dry-run=client -o yaml | kubectl -n "$NAMESPACE" apply -f - + +if [[ -n "$S3_CA_SECRET" ]]; then + log_substep "Copying S3 CA ${S3_CA_NAMESPACE}/${S3_CA_SECRET}[${S3_CA_KEY}] -> ${CA_SECRET}..." + CA_PEM=$(kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" \ + -o jsonpath="{.data.${S3_CA_KEY//./\\.}}" | base64 -d) + [[ -n "$CA_PEM" ]] || { log_error "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} has no ${S3_CA_KEY}"; exit 1; } + kubectl -n "$NAMESPACE" create secret generic "$CA_SECRET" \ + --from-literal="ca.crt=${CA_PEM}" \ + --dry-run=client -o yaml | kubectl -n "$NAMESPACE" apply -f - +else + # The strategy still mounts CA_SECRET; create an empty one so the Pod + # schedules, and warn: a public-CA endpoint should have the CURL_CA_BUNDLE + # env + CA volume dropped from 03-rabbitmq-strategy.yaml by hand instead. + log_warning "S3_CA_SECRET empty: creating an empty ${CA_SECRET} so the Pod schedules." + log_warning "Remove the CURL_CA_BUNDLE env + s3-ca volume from 03-rabbitmq-strategy.yaml when the S3 endpoint uses a public CA." + kubectl -n "$NAMESPACE" create secret generic "$CA_SECRET" \ + --from-literal="ca.crt=" \ + --dry-run=client -o yaml | kubectl -n "$NAMESPACE" apply -f - +fi + +# --- Strategy + BackupClass -------------------------------------------------- +print_header "Step 02: Create the Rabbitmq strategy + BackupClass" +# Reuse the image the platform's cozy-default-rabbitmq strategy runs, so the demo +# Pod needs no package install at run time. Its presence also confirms the +# platform default backups stack (backupstrategy-controller + CRDs) is installed, +# without which the BackupJob below cannot reconcile at all. +# Wait for it rather than read it once. The strategy is rendered behind a lookup +# of the platform bucket, so it exists only once a Helm upgrade has run after +# that bucket was provisioned. The controller forces that upgrade, but helm- +# controller holds it while any release in its dependsOn is not Ready, so a +# platform change made just before this runs can keep it absent for minutes. +# kubectl wait keeps polling through NotFound and fails on any other error. +log_substep "Waiting for the platform's cozy-default-rabbitmq strategy..." +kubectl wait --for=create rabbitmqs.strategy.backups.cozystack.io/cozy-default-rabbitmq \ + --timeout=15m >/dev/null \ + || { log_error "cozy-default-rabbitmq strategy did not appear: enable the platform default backups (backupstrategy-controller) before running this demo"; exit 1; } +CLIENT_IMAGE=$(kubectl get rabbitmqs.strategy.backups.cozystack.io cozy-default-rabbitmq \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="rabbitmq-backup")].image}') +[[ -n "$CLIENT_IMAGE" ]] || { log_error "cozy-default-rabbitmq strategy has no rabbitmq-backup container image"; exit 1; } +log_substep "Reusing the platform strategy's client image: ${CLIENT_IMAGE}" +subst 03-rabbitmq-strategy.yaml | kubectl apply -f - +kubectl apply -f "$SCRIPT_DIR/04-backupclass.yaml" + +# --- Source application + sentinel ------------------------------------------- +log_step "Provisioning source RabbitMQ '$RABBITMQ_SRC_NAME'" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/05-rabbitmq-src.yaml" +# 660s, above this generated release's 600s install timeout; see "Sizing an +# HR-Ready budget" in docs/agents/e2e-testing.md. +wait_hr_ready "$RABBITMQ_SRC_CR" 660 +wait_rabbitmq_ready "$RABBITMQ_SRC_CR" 600 + +SENTINEL="sentinel-$(date +%s)-$$" +log_step "Seeding sentinel definition (vhost + policy): $SENTINEL" +rabbitmq_seed_sentinel "$RABBITMQ_SRC_CR" "$SENTINEL" +rabbitmq_has_sentinel "$RABBITMQ_SRC_CR" "$SENTINEL" \ + || { log_error "sentinel vhost did not seed on the source"; exit 1; } +log_success "sentinel present on the source" + +# --- Backup ------------------------------------------------------------------ +log_step "Creating ad-hoc BackupJob (backupClassName=$BACKUPCLASS_NAME)" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/10-backupjob-adhoc.yaml" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/15-plan.yaml" +wait_for_field backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed + +BACKUP_NAME=$(kubectl -n "$NAMESPACE" get backupjobs.backups.cozystack.io "$BACKUPJOB_NAME" \ + -o jsonpath='{.status.backupRef.name}') +[[ -n "$BACKUP_NAME" ]] || { log_error "BackupJob succeeded but reported no backupRef"; exit 1; } +ARTIFACT_URI=$(kubectl -n "$NAMESPACE" get backups.backups.cozystack.io "$BACKUP_NAME" \ + -o jsonpath='{.status.artifact.uri}' 2>/dev/null || true) +log_success "backup complete: Backup/$BACKUP_NAME (artifact: ${ARTIFACT_URI:-})" + +if [[ "${SKIP_RESTORE:-0}" == "1" ]]; then + log_warning "SKIP_RESTORE=1: stopping after a successful backup." + exit 0 +fi + +# --- In-place restore -------------------------------------------------------- +log_step "Dropping the sentinel, then restoring in place" +rabbitmqctl_exec "$RABBITMQ_SRC_CR" delete_vhost "$SENTINEL" >/dev/null +rabbitmq_has_sentinel "$RABBITMQ_SRC_CR" "$SENTINEL" \ + && { log_error "sentinel vhost still present after delete_vhost"; exit 1; } +log_substep "sentinel dropped from the source" + +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/25-restorejob-in-place.yaml" +wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_INPLACE_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed +rabbitmq_has_sentinel "$RABBITMQ_SRC_CR" "$SENTINEL" \ + || { log_error "in-place restore did not bring back the sentinel vhost"; exit 1; } +log_success "in-place restore round-tripped the sentinel" + +# --- Restore-to-copy --------------------------------------------------------- +log_step "Provisioning target RabbitMQ '$RABBITMQ_TARGET_NAME' and restoring the source's definitions into it" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/20-rabbitmq-target.yaml" +wait_hr_ready "$RABBITMQ_TARGET_CR" 660 +wait_rabbitmq_ready "$RABBITMQ_TARGET_CR" 600 +rabbitmq_has_sentinel "$RABBITMQ_TARGET_CR" "$SENTINEL" \ + && { log_error "target already has the sentinel before restore — cannot prove the copy"; exit 1; } + +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/30-restorejob-to-copy.yaml" +wait_for_field restorejobs.backups.cozystack.io "$RESTOREJOB_TOCOPY_NAME" \ + '{.status.phase}' Succeeded "$NAMESPACE" 900 Failed +rabbitmq_has_sentinel "$RABBITMQ_TARGET_CR" "$SENTINEL" \ + || { log_error "to-copy restore did not put the sentinel on the target"; exit 1; } +rabbitmq_has_sentinel "$RABBITMQ_SRC_CR" "$SENTINEL" \ + || { log_error "source lost the sentinel during to-copy restore (source must stay untouched)"; exit 1; } +log_success "to-copy restore placed the sentinel on the target, source untouched" + +print_header "Demo complete — definitions round-tripped through object storage" diff --git a/examples/backups/redis/00-helpers.sh b/examples/backups/redis/00-helpers.sh new file mode 100755 index 0000000000..8c7272a57a --- /dev/null +++ b/examples/backups/redis/00-helpers.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# Shared helpers for the Redis (spotahome RedisFailover) backup/restore demo. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export BLUE='\033[0;34m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export WHITE='\033[1;37m' +export NC='\033[0m' +export BOLD='\033[1m' + +# Default settings (override via environment). NAMESPACE is tenant-root because +# the demo provisions its Bucket there (the in-cluster seaweedfs the backup Job +# reaches lives in tenant-root, and the default e2e tenant-test is egress- +# isolated from it). +export NAMESPACE="${NAMESPACE:-tenant-root}" +# APP_KIND=Valkey runs the same round-trip against a Valkey app: it runs on the +# same RedisFailover operator, and its chart names every resource valkey- +# where the redis chart uses redis-. +export APP_KIND="${APP_KIND:-Redis}" +case "$APP_KIND" in + Redis) APP_PREFIX=redis-; APP_RESOURCE=redis ;; + Valkey) APP_PREFIX=valkey-; APP_RESOURCE=valkeys.apps.cozystack.io ;; + *) echo "APP_KIND must be Redis or Valkey, got '$APP_KIND'" >&2; exit 1 ;; +esac +export APP_PREFIX APP_RESOURCE +export REDIS_NAME="${REDIS_NAME:-redis-test}" +export REDIS_RESTORE_NAME="${REDIS_RESTORE_NAME:-redis-restore}" +export BACKUPJOB_NAME="${BACKUPJOB_NAME:-redis-backup-job}" +export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-redis-restore-inplace}" +export RESTOREJOB_TOCOPY_NAME="${RESTOREJOB_TOCOPY_NAME:-redis-restore-to-copy}" +# This demo provisions its own Bucket + Redis strategy + BackupClass rather than +# reusing the platform cozy-default flow, so the round-trip is self-contained +# (its S3 objects tear down with the demo) and works in CI, where the shared +# system bucket's advertised external endpoint is not routable. The Bucket +# controller materialises the credentials Secret as "bucket--". +export BUCKET_NAME="${BUCKET_NAME:-redis-backups}" +export BUCKET_USER="${BUCKET_USER:-backup}" +export STRATEGY_NAME="${STRATEGY_NAME:-redis-strategy-default}" +export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-redis-default}" +# The strategy Pod's curl reads AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY from +# this Secret and verifies the S3 endpoint's CA from ca.crt in the CA Secret; +# run-all.sh materialises both from the provisioned Bucket before dispatch. +export CREDS_SECRET="${CREDS_SECRET:-redis-backup-creds}" +export CA_SECRET="${CA_SECRET:-redis-backup-ca}" +# S3 endpoint CA. cozystack's default seaweedfs serves its S3 endpoint with a +# self-signed certificate whose CA lives in this Secret; run-all.sh copies its +# ca.crt into CA_SECRET, which the strategy Pod mounts at /etc/s3-ca. The name +# follows the seaweedfs chart's fullnameOverride ("seaweedfs" -> +# "seaweedfs-ca-cert"); run-all.sh auto-discovers the CA Certificate's actual +# secret when this default is absent. On a cluster whose S3 endpoint is signed +# by a publicly-trusted CA, set S3_CA_SECRET="" to skip the copy. +export S3_CA_SECRET="${S3_CA_SECRET:-seaweedfs-ca-cert}" +export S3_CA_NAMESPACE="${S3_CA_NAMESPACE:-tenant-root}" +export S3_CA_KEY="${S3_CA_KEY:-ca.crt}" +export MARKER_KEY="${MARKER_KEY:-sentinel:marker}" +# Single-token value: redis_cmd strips all whitespace from the reply, so a +# value with spaces would not compare equal. A UUID-shaped token is enough to +# prove the exact bytes round-tripped through object storage. +export MARKER_VALUE="${MARKER_VALUE:-roundtrip-4f1c9a2b}" + +log_info() { echo -e "${BLUE}i${NC} $*" >&2; } +log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; } +log_error() { echo -e "${RED}x${NC} $*" >&2; } +log_step() { echo -e "\n${MAGENTA}${BOLD}> $*${NC}" >&2; } +log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; } + +print_header() { + local title="$1" + echo -e "\n${MAGENTA}${BOLD}== $title ==${NC}\n" >&2 +} + +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" + +# The cozystack redis chart names its RedisFailover (hence the operator's rfr-/ +# rfs- Services and the -auth Secret) redis-, the valkey chart valkey-, +# mirroring the strategy driver that derives the same base from .Release.Prefix +# and .Release.Name. So app X's password lives in X-auth. The chart +# renders that Secret only with authEnabled, so its absence reads as no +# password; any other error still fails the read. +redis_pw() { + kubectl -n "$NAMESPACE" get secret "${APP_PREFIX}$1-auth" --ignore-not-found -o jsonpath='{.data.password}' | base64 -d +} + +# Run a redis-cli command against an application's CURRENT master. Master +# discovery asks a sentinel of the operator's rfs- Deployment +# (master name "mymaster"), so a write always lands on the writable node even +# after a failover. Args after the app name are passed verbatim to redis-cli. +# redis_cmd redis-test SET sentinel:marker hello +# +# The command runs by `kubectl exec` in a sentinel Pod, whose image carries +# redis-cli (the valkey image ships it as a link to valkey-cli). A throwaway `kubectl run -i` Pod is not usable here: its stdout +# comes back over an attach, which only carries what the container writes after +# the attach is established, and kubectl falls back to the Pod log only when +# the attach fails. A redis-cli that finishes in between comes back as an +# empty reply with exit 0. An exec'd process writes into pipes of its own, so +# its output cannot be missed that way. +# +# The password travels on stdin rather than in argv, which the exec request +# URL carries into the API server's audit log. kubectl's and redis-cli's +# stderr stay attached, so a failed read says why instead of reading as ''. +redis_cmd() { + local app="$1" pw; shift + pw=$(redis_pw "$app") + printf '%s\n' "$pw" | kubectl -n "$NAMESPACE" exec -i "deploy/rfs-${APP_PREFIX}$app" -c sentinel -- sh -c ' + IFS= read -r pw || true + addr=$(redis-cli -p 26379 sentinel get-master-addr-by-name mymaster) || exit 1 + h=$(echo "$addr" | sed -n 1p); p=$(echo "$addr" | sed -n 2p) + [ -n "$h" ] && [ -n "$p" ] || { echo "sentinel knows no master for mymaster" >&2; exit 1; } + [ -z "$pw" ] || export REDISCLI_AUTH="$pw" + exec redis-cli -h "$h" -p "$p" "$@" + ' sh "$@" | tr -d '[:space:]' +} + +# Block until the RedisFailover has an elected master reachable through sentinel. +wait_redis_master() { + local app="$1" timeout="${2:-300}" elapsed=0 + log_substep "Waiting for '$app' master election via sentinel..." + while true; do + local h + h=$(redis_cmd "$app" PING || true) + [[ "$h" == "PONG" ]] && { log_success "'$app' master is reachable"; return 0; } + (( elapsed >= timeout )) && { log_error "Timeout waiting for '$app' master"; return 1; } + sleep 5 + elapsed=$((elapsed + 5)) + done +} diff --git a/examples/backups/redis/01-create-redis.sh b/examples/backups/redis/01-create-redis.sh new file mode 100755 index 0000000000..ecc4c6cb1c --- /dev/null +++ b/examples/backups/redis/01-create-redis.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Step 01: Provision a Redis (RedisFailover) instance and seed a sentinel marker +# key used to prove the backup/restore round-trip carries the exact bytes. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 01: Provision Redis '${REDIS_NAME}'" + +kubectl apply -f - </dev/null +got=$(redis_cmd "$REDIS_NAME" GET "$MARKER_KEY") +[[ "$got" == "$MARKER_VALUE" ]] || { log_error "seed failed: read back '${got}'"; exit 1; } +log_success "Marker seeded on '${REDIS_NAME}'." + +echo -e "\n${GREEN}${BOLD}Next:${NC} ./02-create-backupjob.sh" diff --git a/examples/backups/redis/02-create-backupjob.sh b/examples/backups/redis/02-create-backupjob.sh new file mode 100755 index 0000000000..56158a174a --- /dev/null +++ b/examples/backups/redis/02-create-backupjob.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Step 02: Create a BackupJob against the platform cozy-default BackupClass. The +# controller resolves it to the cozy-default-redis strategy, projects +# cozy-backups-creds into the namespace, renders the Pod template against the +# Redis app, and runs it as a batch/v1.Job that dumps an RDB snapshot to S3. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 02: Create BackupJob '${BACKUPJOB_NAME}'" + +kubectl apply -f - < Succeeded between two observations, so an explicit "wait Running" +# gate would flake. +wait_for_field backupjob "$BACKUPJOB_NAME" '{.status.phase}' Succeeded "$NAMESPACE" 600 Failed + +backup_ref=$(kubectl -n "$NAMESPACE" get backupjob "$BACKUPJOB_NAME" -o jsonpath='{.status.backupRef.name}') +[[ -n "$backup_ref" ]] || { log_error "BackupJob succeeded but BackupRef is empty"; exit 1; } +log_success "Backup '${backup_ref}' is Ready." + +echo -e "\n${GREEN}${BOLD}Next:${NC} ./03-restore-in-place.sh" diff --git a/examples/backups/redis/03-restore-in-place.sh b/examples/backups/redis/03-restore-in-place.sh new file mode 100755 index 0000000000..6839a4137d --- /dev/null +++ b/examples/backups/redis/03-restore-in-place.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Step 03: In-place restore. Overwrite the marker on the source to simulate data +# drift, then restore the RDB snapshot back into the SAME Redis app. The strategy +# FLUSHALLs the master and replays the RDB, so the marker returns to its backed-up +# value. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 03: In-place restore from '${BACKUPJOB_NAME}'" + +log_substep "Corrupting the marker to simulate data loss..." +redis_cmd "$REDIS_NAME" SET "$MARKER_KEY" "corrupted-do-not-keep" >/dev/null +# Read the corruption back and assert it: a SET that did not land would leave +# the marker at its backed-up value, and the final check would then pass +# without the restore having done anything. +corrupted=$(redis_cmd "$REDIS_NAME" GET "$MARKER_KEY") +[[ "$corrupted" == "corrupted-do-not-keep" ]] || { log_error "corruption step did not take (got '${corrupted}'); aborting so the restore assertion cannot false-pass"; exit 1; } + +kubectl apply -f - <//redis-test/dump.rdb`. When `status.phase` is `Succeeded`, `status.backupRef.name` names the `Backup` artifact the restore flow consumes. `02-create-backupjob.sh` automates this. diff --git a/examples/backups/redis/92-scenario-user-restore.md b/examples/backups/redis/92-scenario-user-restore.md new file mode 100644 index 0000000000..c3c36d6183 --- /dev/null +++ b/examples/backups/redis/92-scenario-user-restore.md @@ -0,0 +1,39 @@ +# Scenario: tenant restores a Redis backup + +Both flows drive from the same `Backup` (named after the BackupJob). + +## In-place + +Restore into the source app — the driver FLUSHALLs the master and replays the RDB: + +```yaml +apiVersion: backups.cozystack.io/v1alpha1 +kind: RestoreJob +metadata: + name: redis-restore-inplace + namespace: tenant-test +spec: + backupRef: + name: redis-backup-job +``` + +## To-copy + +Deploy a second, differently-named `Redis` first, then restore into it with `targetApplicationRef`. The Job connects to the target's master but reads the object keyed by the SOURCE app name (`.Backup.ApplicationRef.Name`), so the source keeps running untouched: + +```yaml +apiVersion: backups.cozystack.io/v1alpha1 +kind: RestoreJob +metadata: + name: redis-restore-to-copy + namespace: tenant-test +spec: + backupRef: + name: redis-backup-job + targetApplicationRef: + apiGroup: apps.cozystack.io + kind: Redis + name: redis-restore +``` + +`03-restore-in-place.sh` and `04-restore-to-copy.sh` automate these and assert the seeded marker round-tripped. There is no PITR: RDB is a discrete snapshot, so `spec.options.recoveryTime` is unsupported. diff --git a/examples/backups/redis/README.md b/examples/backups/redis/README.md new file mode 100644 index 0000000000..4572b69aa5 --- /dev/null +++ b/examples/backups/redis/README.md @@ -0,0 +1,35 @@ +# Redis backup / restore demo + +Proves an honest data round-trip for a Cozystack-managed `Redis` (spotahome `RedisFailover`) through the platform system bucket: a sentinel marker key is written, backed up, and asserted to return byte-for-byte after both restore flows. + +The spotahome operator ships no backup CRD, so the `strategy.backups.cozystack.io/Redis` driver runs a one-shot Job that dumps an RDB snapshot to S3 and replays it on restore (RDB → RESP via `redis-cli --pipe`). This is the `cozy-default-redis` strategy the `cozy-default` BackupClass binds `Redis` to. + +## Prerequisites + +- A Cozystack cluster with the `backupstrategy-controller` running and its system bucket provisioned, so `cozy-default` (and the `cozy-default-redis` strategy) are present and the controller can project `cozy-backups-creds` into the app namespace. +- `kubectl` pointed at the cluster; the tenant namespace defaults to `tenant-test` (override with `NAMESPACE`). + +## Run + +```bash +./run-all.sh # steps 01-04: provision, back up, restore in-place, restore to-copy +SKIP_RESTORE=1 ./run-all.sh # stop after a successful backup +./cleanup.sh # remove everything the demo created +``` + +Or step through them: + +| Step | Script | What it does | +|------|--------|--------------| +| 01 | `01-create-redis.sh` | Provision `Redis`, seed the marker on the master | +| 02 | `02-create-backupjob.sh` | `BackupJob` (`backupClassName: cozy-default`) → RDB dump to S3 | +| 03 | `03-restore-in-place.sh` | Corrupt the marker, `RestoreJob` back into the source, assert it returns | +| 04 | `04-restore-to-copy.sh` | `RestoreJob` with `targetApplicationRef` into a fresh copy, assert the marker landed | + +## Notes + +- **Valkey.** `APP_KIND=Valkey` runs the same walkthrough against a Valkey app, which runs on the same RedisFailover operator; every `redis-` resource below is then `valkey-`. +- **Master discovery** goes through the operator's sentinel Service `rfs-redis-` (master name `mymaster`), so writes and the dump/replay always target the current master even after a failover. +- **Faithful restore.** The restore loads the RDB into a throwaway local redis and replays what a real engine emits, so all logical databases (`SELECT` per db), stream last-ids and consumer groups survive — a merge onto db0 does not happen. +- **Endpoint TLS.** The projected `endpoint` in `cozy-backups-creds` is always scheme-stripped, so the strategy prepends `https://` and the transport is always TLS (there is no `http://` path). It verifies against the image's system CA store by default — the platform bucket's ACME cert needs nothing more — honours a private CA from `backupStorage.endpointCASecretName` when set, and skips verification only when `backupStorage.insecureSkipTLSVerify: true` is set explicitly. +- **No PITR.** RDB is a discrete snapshot; the engine ships no continuous log to replay to an arbitrary time, so `spec.options.recoveryTime` is not supported (the etcd-like case). diff --git a/examples/backups/redis/backupclass.yaml b/examples/backups/redis/backupclass.yaml new file mode 100644 index 0000000000..33dee84c49 --- /dev/null +++ b/examples/backups/redis/backupclass.yaml @@ -0,0 +1,25 @@ +# Maps Kind=Redis and Kind=Valkey to this demo's Redis strategy. BackupClass is cluster-scoped, +# so a single instance covers every namespace. Named redis-default so it does +# not collide with the platform cozy-default class, which routes Redis to the +# shipped cozy-default-redis strategy. +--- +apiVersion: backups.cozystack.io/v1alpha1 +kind: BackupClass +metadata: + name: redis-default +spec: + strategies: + - application: + apiGroup: apps.cozystack.io + kind: Redis + strategyRef: + apiGroup: strategy.backups.cozystack.io + kind: Redis + name: redis-strategy-default + - application: + apiGroup: apps.cozystack.io + kind: Valkey + strategyRef: + apiGroup: strategy.backups.cozystack.io + kind: Redis + name: redis-strategy-default diff --git a/examples/backups/redis/bucket.yaml b/examples/backups/redis/bucket.yaml new file mode 100644 index 0000000000..75cca89ed2 --- /dev/null +++ b/examples/backups/redis/bucket.yaml @@ -0,0 +1,19 @@ +# Cozystack-managed S3 bucket for storing this demo's Redis RDB snapshots. +# The Bucket controller provisions a Secret named "bucket--" per +# user (the bucket app carries the "bucket-" release prefix); run-all.sh reads +# the "backup" user's BucketInfo and projects its access keys into the +# redis-backup-creds Secret the strategy references. +# +# The platform ships a cozy-default-redis strategy that writes to the shared +# cozy-backups system bucket; this demo instead provisions its own Bucket and a +# matching strategy so the round-trip is self-contained and its objects are torn +# down with the demo (see cleanup.sh). +--- +apiVersion: apps.cozystack.io/v1alpha1 +kind: Bucket +metadata: + name: redis-backups +spec: + users: + backup: + readonly: false diff --git a/examples/backups/redis/cleanup.sh b/examples/backups/redis/cleanup.sh new file mode 100755 index 0000000000..81fc7ec68c --- /dev/null +++ b/examples/backups/redis/cleanup.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Cleanup: tear down everything provisioned by the demo so the cluster returns +# to its previous state. Idempotent — safe to run before a fresh round (a stale +# Succeeded BackupJob would otherwise falsely satisfy the wait). +# -e so a real API/permission failure aborts loudly instead of printing "Cleanup +# complete." over a half-torn-down demo; every delete below is +# --ignore-not-found, so an already-gone resource is not an error. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Cleanup Redis backup demo" + +kubectl -n "$NAMESPACE" delete restorejob "$RESTOREJOB_TOCOPY_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete restorejob "$RESTOREJOB_INPLACE_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete backupjob "$BACKUPJOB_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete backup "$BACKUPJOB_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete "$APP_RESOURCE" "$REDIS_RESTORE_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete "$APP_RESOURCE" "$REDIS_NAME" --ignore-not-found + +# Demo-owned backups plumbing: the BackupClass and strategy CR are cluster-scoped +# (no -n). Deleting the Bucket releases its S3 objects and the projected keys. +kubectl delete backupclass "$BACKUPCLASS_NAME" --ignore-not-found +kubectl delete redis.strategy.backups.cozystack.io "$STRATEGY_NAME" --ignore-not-found +kubectl -n "$NAMESPACE" delete secret "$CREDS_SECRET" --ignore-not-found +kubectl -n "$NAMESPACE" delete secret "$CA_SECRET" --ignore-not-found +kubectl -n "$NAMESPACE" delete bucket "$BUCKET_NAME" --ignore-not-found + +log_success "Cleanup complete." diff --git a/examples/backups/redis/redis-strategy.yaml b/examples/backups/redis/redis-strategy.yaml new file mode 100644 index 0000000000..6533b0e381 --- /dev/null +++ b/examples/backups/redis/redis-strategy.yaml @@ -0,0 +1,223 @@ +# Redis backup strategy for this demo. It mirrors the shipped cozy-default-redis +# strategy +# (packages/system/backupstrategy-controller/templates/strategy-redis-default.yaml) +# but points at this demo's own Bucket instead of the platform cozy-backups +# system bucket, so the round-trip is self-contained and tears down with the +# demo. The in-Pod script is kept identical to the shipped one; only the S3 +# coordinates differ: they come from this demo's redis-backup-creds Secret and +# the endpoint is injected literally, and the self-signed seaweedfs CA is mounted +# at /etc/s3-ca (the script already prefers it over the system store). +# +# The controller re-renders the `{{ .Mode | .Release | .ObjectKey }}` templates +# below per BackupJob/RestoreJob against the live application object. The +# REPLACE_WITH_* markers are filled by run-all.sh from the provisioned Bucket +# (kubectl apply happens through run-all's subst, not directly). +--- +apiVersion: strategy.backups.cozystack.io/v1alpha1 +kind: Redis +metadata: + name: redis-strategy-default +spec: + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: 1800 + volumes: + # The Redis app's own CA, mounted optionally: ca.crt lands only for a + # tls.enabled app, and the script keys its server-TLS path off its + # presence — mirrors the shipped strategy. + - name: redis-ca + secret: + secretName: '{{ .Release.Prefix | default "redis-" }}{{ .Release.Name }}.ca-cert' + optional: true + - name: s3-ca + secret: + secretName: redis-backup-ca + containers: + - name: redis-backup + # Reuse whatever image the platform strategy runs so the demo Pod needs + # no package install at run time; run-all.sh substitutes it here off the + # shipped cozy-default-redis strategy rather than hardcoding a tag. + image: "REPLACE_WITH_BACKUP_CLIENT_IMAGE" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: redis-ca + mountPath: /etc/redis-ca + readOnly: true + - name: s3-ca + mountPath: /etc/s3-ca + readOnly: true + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: '{{ .Release.Prefix | default "redis-" }}{{ .Release.Name }}-auth' + key: password + optional: true + # S3 coordinates come from this demo's Bucket: the access keys are + # projected by run-all.sh into redis-backup-creds, and the endpoint + # and bucket are injected literally from the provisioned Bucket. + - name: S3_ENDPOINT + value: "REPLACE_WITH_S3_ENDPOINT" + - name: S3_BUCKET + value: "REPLACE_WITH_COSI_BUCKET_NAME" + - name: S3_REGION + value: "us-east-1" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: redis-backup-creds + key: AWS_ACCESS_KEY_ID + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: redis-backup-creds + key: AWS_SECRET_ACCESS_KEY + optional: true + # Off: the seaweedfs CA is mounted at /etc/s3-ca and the script + # verifies against it. Skip-verify stays an explicit opt-in. + - name: S3_INSECURE_SKIP_VERIFY + value: "false" + command: ["/bin/sh", "-c"] + args: + - | + set -euo pipefail + + : "${S3_ENDPOINT:?missing endpoint}" + : "${S3_BUCKET:?missing bucket}" + : "${S3_REGION:?missing region}" + : "${AWS_ACCESS_KEY_ID:?missing AWS_ACCESS_KEY_ID in redis-backup-creds}" + : "${AWS_SECRET_ACCESS_KEY:?missing AWS_SECRET_ACCESS_KEY in redis-backup-creds}" + + MODE="{{ .Mode }}" + KEY="{{ .ObjectKey }}" + + # S3 transport, needed by every mode (backup, restore, cleanup). + case "${S3_ENDPOINT}" in + http://*|https://*) BASE="${S3_ENDPOINT}" ;; + *) BASE="https://${S3_ENDPOINT}" ;; + esac + TLS="" + case "${BASE}" in + https://*) + if [ -f /etc/s3-ca/ca.crt ]; then TLS="--cacert /etc/s3-ca/ca.crt"; + elif [ -f /etc/s3-ca/tls.crt ]; then TLS="--cacert /etc/s3-ca/tls.crt"; + elif [ "${S3_INSECURE_SKIP_VERIFY:-false}" = "true" ]; then TLS="-k"; + fi + ;; + esac + s3() { + curl -fsS ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" "$@" + } + OBJ="${BASE}/${S3_BUCKET}/${KEY}" + + # Cleanup deletes only this backup's object and never touches Redis, + # so a deleted Backup must NOT fall into the restore path (which + # FLUSHALLs the master). Idempotent — a 404 is success. Mirrors the + # shipped strategy. + if [ "${MODE}" = "cleanup" ]; then + echo "deleting s3://${S3_BUCKET}/${KEY}" + code=$(curl -sS -o /dev/null -w '%{http_code}' ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + -X DELETE "${OBJ}") + case "${code}" in + 2??|404) echo "deleted (HTTP ${code})"; exit 0 ;; + *) echo "delete failed: HTTP ${code}" >&2; exit 1 ;; + esac + fi + + # Server-side TLS: present iff the app has tls.enabled (see the + # optional redis-ca mount) — mirrors the shipped strategy. + RTLS="" + if [ -f /etc/redis-ca/ca.crt ]; then + RTLS="--tls --cacert /etc/redis-ca/ca.crt" + fi + + SENTINEL="rfs-{{ .Release.Prefix | default "redis-" }}{{ .Release.Name }}" + addr=$(redis-cli ${RTLS} -h "${SENTINEL}" -p 26379 sentinel get-master-addr-by-name mymaster) + MHOST=$(echo "${addr}" | sed -n '1p') + MPORT=$(echo "${addr}" | sed -n '2p') + if [ -z "${MHOST}" ] || [ -z "${MPORT}" ]; then + echo "could not resolve redis master from sentinel ${SENTINEL}" >&2 + exit 1 + fi + + tgt() { + if [ -n "${REDIS_PASSWORD:-}" ]; then + REDISCLI_AUTH="${REDIS_PASSWORD}" redis-cli ${RTLS} -h "${MHOST}" -p "${MPORT}" "$@" + else + redis-cli ${RTLS} -h "${MHOST}" -p "${MPORT}" "$@" + fi + } + + # Explicit modes only — an unrecognised MODE must NOT fall into the + # destructive restore branch. + if [ "${MODE}" = "backup" ]; then + echo "dumping RDB from master ${MHOST}:${MPORT}" + tgt --rdb /tmp/dump.rdb + echo "uploading s3://${S3_BUCKET}/${KEY}" + s3 -X PUT --upload-file /tmp/dump.rdb "${OBJ}" + elif [ "${MODE}" = "restore" ]; then + echo "downloading s3://${S3_BUCKET}/${KEY}" + s3 -o /tmp/dump.rdb "${OBJ}" + mkdir -p /tmp/restore + cp /tmp/dump.rdb /tmp/restore/dump.rdb + redis-server --dir /tmp/restore --dbfilename dump.rdb \ + --port 6399 --bind 127.0.0.1 --save "" --appendonly no & + for _ in $(seq 1 300); do + redis-cli -p 6399 PING >/dev/null 2>&1 && break + sleep 1 + done + if ! redis-cli -p 6399 PING >/dev/null 2>&1; then + echo "throwaway redis did not become ready: dataset likely exceeds the restore memory limit, or the RDB version is newer than the image" >&2 + exit 1 + fi + redis-cli -p 6399 CONFIG SET aof-use-rdb-preamble no >/dev/null + redis-cli -p 6399 CONFIG SET appendonly yes >/dev/null + # Wait for the rewrite to FINISH (no rewrite in flight, last one + # ok, base AOF on disk), not merely for in_progress:0 which can + # fall through before the scheduled rewrite starts. find (not + # `ls`) returns 0 on no-match under pipefail; REPLAY is set each + # iteration so it survives the loop. + REPLAY="" + for _ in $(seq 1 300); do + persistence=$(redis-cli -p 6399 INFO persistence | tr -d '\r') + inprog=$(echo "${persistence}" | awk -F: '/aof_rewrite_in_progress:/{print $2}') + status=$(echo "${persistence}" | awk -F: '/aof_last_bgrewrite_status:/{print $2}') + REPLAY=$(find /tmp/restore/appendonlydir -name '*.base.aof' 2>/dev/null | head -n1 || true) + if [ "${inprog}" = "0" ] && [ "${status}" = "ok" ] && [ -n "${REPLAY}" ]; then + break + fi + sleep 1 + done + if [ -z "${REPLAY}" ]; then + echo "throwaway redis produced no base AOF to replay (rewrite did not finish)" >&2 + exit 1 + fi + tgt FLUSHALL + tgt --pipe < "${REPLAY}" + redis-cli -p 6399 SHUTDOWN NOSAVE >/dev/null 2>&1 || true + echo "restore complete into ${MHOST}:${MPORT}" + else + echo "unknown mode: ${MODE}" >&2 + exit 1 + fi + resources: + requests: + cpu: 10m + memory: 64Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 4Gi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault diff --git a/examples/backups/redis/run-all.sh b/examples/backups/redis/run-all.sh new file mode 100755 index 0000000000..613803bdd3 --- /dev/null +++ b/examples/backups/redis/run-all.sh @@ -0,0 +1,153 @@ +#!/bin/bash +# End-to-end Redis backup/restore demo. It proves DATA INTEGRITY, not liveness: +# a sentinel key is seeded, backed up, and must reappear after a restore — first +# in place, then into a separate copy. +# +# The demo provisions its OWN Bucket + Redis strategy + BackupClass, filling the +# REPLACE_WITH_* markers in redis-strategy.yaml from the Bucket, so the +# round-trip is self-contained (its S3 objects tear down with the demo) and runs +# in CI, where the shared cozy-backups system bucket's advertised external +# endpoint is not routable. The platform-shipped cozy-default-redis strategy is +# left untouched; this demo only reads the client image off it. +# +# Env knobs: +# NAMESPACE (default tenant-root) +# SKIP_RESTORE=1 stop after a successful backup (steps 01-02) +# S3_ENDPOINT override the endpoint the backup Pod uses. BucketInfo +# advertises the EXTERNAL ingress endpoint, which in-cluster +# Pods cannot always reach or TLS-validate; the in-cluster +# alternative is https://seaweedfs-s3..svc:8333 — the .svc +# FQDN is the name the seaweedfs serving cert's SAN covers, so +# curl verifies TLS against the copied CA. CI sets this; a real +# cluster can leave it unset to use the advertised endpoint. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +# Fill the strategy manifest's placeholders from the provisioned Bucket and the +# platform client image. '|' as the sed delimiter keeps the image ref's slashes +# and the endpoint URL intact. +subst() { + sed \ + -e "s|REPLACE_WITH_COSI_BUCKET_NAME|${BUCKET}|g" \ + -e "s|REPLACE_WITH_S3_ENDPOINT|${S3_ENDPOINT}|g" \ + -e "s|REPLACE_WITH_BACKUP_CLIENT_IMAGE|${CLIENT_IMAGE}|g" \ + "$SCRIPT_DIR/$1" +} + +print_header "Redis backup/restore demo (namespace: $NAMESPACE)" + +# --- Bucket ------------------------------------------------------------------ +print_header "Step 00: Provision Bucket '${BUCKET_NAME}' in ${NAMESPACE}" +kubectl -n "$NAMESPACE" apply -f "$SCRIPT_DIR/bucket.yaml" +# 660s, above this generated release's 600s install timeout; see "Sizing an +# HR-Ready budget" in docs/agents/e2e-testing.md. +wait_hr_ready "bucket-${BUCKET_NAME}" 660 +wait_for_field bucketclaims.objectstorage.k8s.io "bucket-${BUCKET_NAME}" \ + '{.status.bucketReady}' true "$NAMESPACE" 300 +wait_for_field bucketaccesses.objectstorage.k8s.io "bucket-${BUCKET_NAME}-${BUCKET_USER}" \ + '{.status.accessGranted}' true "$NAMESPACE" 300 + +log_substep "Reading bucket coordinates from BucketInfo Secret..." +TMP=$(mktemp) +trap 'rm -f "$TMP"' EXIT +kubectl -n "$NAMESPACE" get secret "bucket-${BUCKET_NAME}-${BUCKET_USER}" \ + -o jsonpath='{.data.BucketInfo}' | base64 -d > "$TMP" +S3_ACCESS_KEY=$(jq -r '.spec.secretS3.accessKeyID' "$TMP") +S3_SECRET_KEY=$(jq -r '.spec.secretS3.accessSecretKey' "$TMP") +S3_ENDPOINT="${S3_ENDPOINT:-$(jq -r '.spec.secretS3.endpoint' "$TMP")}" +BUCKET=$(jq -r '.spec.bucketName' "$TMP") +for v in S3_ACCESS_KEY S3_SECRET_KEY S3_ENDPOINT BUCKET; do + [[ -n "${!v}" && "${!v}" != "null" ]] || { log_error "BucketInfo missing required field: ${v}"; exit 1; } +done +# The strategy script accepts a full URL (it prepends https:// only when the +# endpoint carries no scheme), so keep the scheme rather than stripping it. +case "$S3_ENDPOINT" in http://*|https://*) : ;; *) S3_ENDPOINT="https://${S3_ENDPOINT}" ;; esac +export S3_ENDPOINT +log_success "Bucket '${BUCKET}' at endpoint '${S3_ENDPOINT}'." + +# --- Secrets the strategy references ----------------------------------------- +print_header "Step 00b: Materialise the Secrets the Redis strategy references" +if [[ -n "$S3_CA_SECRET" ]] \ + && ! kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" >/dev/null 2>&1; then + log_warning "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} not found; discovering the seaweedfs CA Certificate..." + DISCOVERED_CA=$(kubectl -n "$S3_CA_NAMESPACE" get certificates.cert-manager.io \ + -l app.kubernetes.io/name=seaweedfs \ + -o jsonpath='{range .items[*]}{.spec.isCA}{" "}{.spec.secretName}{"\n"}{end}' \ + | awk '$1=="true"{print $2; exit}' || true) + if [[ -n "$DISCOVERED_CA" ]]; then + log_success "Discovered seaweedfs CA secret ${S3_CA_NAMESPACE}/${DISCOVERED_CA}" + S3_CA_SECRET="$DISCOVERED_CA" + else + log_error "No seaweedfs CA Certificate found in ${S3_CA_NAMESPACE}; set S3_CA_SECRET explicitly (or empty for a public-CA endpoint)." + exit 1 + fi +fi + +log_substep "Projecting bucket access keys into ${CREDS_SECRET}..." +kubectl -n "$NAMESPACE" create secret generic "$CREDS_SECRET" \ + --from-literal="AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY}" \ + --from-literal="AWS_SECRET_ACCESS_KEY=${S3_SECRET_KEY}" \ + --dry-run=client -o yaml | kubectl -n "$NAMESPACE" apply -f - + +if [[ -n "$S3_CA_SECRET" ]]; then + log_substep "Copying S3 CA ${S3_CA_NAMESPACE}/${S3_CA_SECRET}[${S3_CA_KEY}] -> ${CA_SECRET}..." + CA_PEM=$(kubectl -n "$S3_CA_NAMESPACE" get secret "$S3_CA_SECRET" \ + -o jsonpath="{.data.${S3_CA_KEY//./\\.}}" | base64 -d) + [[ -n "$CA_PEM" ]] || { log_error "S3 CA secret ${S3_CA_NAMESPACE}/${S3_CA_SECRET} has no ${S3_CA_KEY}"; exit 1; } + kubectl -n "$NAMESPACE" create secret generic "$CA_SECRET" \ + --from-literal="ca.crt=${CA_PEM}" \ + --dry-run=client -o yaml | kubectl -n "$NAMESPACE" apply -f - +else + # The strategy mounts CA_SECRET unconditionally; create an empty one so the + # Pod schedules. On a public-CA endpoint the script falls through to the + # system CA store (no /etc/s3-ca/ca.crt content to prefer). + log_warning "S3_CA_SECRET empty: creating an empty ${CA_SECRET} so the Pod schedules (public-CA endpoint)." + kubectl -n "$NAMESPACE" create secret generic "$CA_SECRET" \ + --from-literal="ca.crt=" \ + --dry-run=client -o yaml | kubectl -n "$NAMESPACE" apply -f - +fi + +# --- Strategy + BackupClass -------------------------------------------------- +print_header "Step 00c: Create the Redis strategy + BackupClass" +# Reuse the image the platform's cozy-default-redis strategy runs, so the demo +# Pod needs no package install at run time. Its presence also confirms the +# platform default backups stack (backupstrategy-controller + CRDs) is +# installed, without which the BackupJob below cannot reconcile at all. +# +# Wait for it rather than read it once. The strategy is rendered behind a +# lookup of the platform bucket, so it exists only once a Helm upgrade has run +# after that bucket was provisioned. The controller forces that upgrade, but +# helm-controller holds it while any release in its dependsOn is not Ready, so +# a platform change made just before this runs can keep it absent for minutes. +# kubectl wait keeps polling through NotFound and fails on any other error. +log_substep "Waiting for the platform's cozy-default-redis strategy..." +kubectl wait --for=create redis.strategy.backups.cozystack.io/cozy-default-redis \ + --timeout=15m >/dev/null \ + || { log_error "cozy-default-redis strategy did not appear: enable the platform default backups (backupstrategy-controller) before running this demo"; exit 1; } +CLIENT_IMAGE=$(kubectl get redis.strategy.backups.cozystack.io cozy-default-redis \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="redis-backup")].image}') +[[ -n "$CLIENT_IMAGE" ]] || { log_error "cozy-default-redis strategy has no redis-backup container image"; exit 1; } +log_substep "Reusing the platform strategy's client image: ${CLIENT_IMAGE}" +subst redis-strategy.yaml | kubectl apply -f - +kubectl apply -f "$SCRIPT_DIR/backupclass.yaml" + +# --- Round-trip (source + seed -> backup -> restore in place -> to a copy) --- +steps=(01 02 03 04) +[[ "${SKIP_RESTORE:-0}" == "1" ]] && steps=(01 02) + +for step in "${steps[@]}"; do + # find is glob-safe under set -euo pipefail and degrades to empty output + # when no file matches, unlike `ls glob | head`. + script="$(find "$SCRIPT_DIR" -maxdepth 1 -type f -name "${step}-*.sh" | sort | head -n1)" + if [[ -z "$script" ]]; then + echo "no script found for step $step" >&2 + exit 1 + fi + echo "=== Running $(basename "$script") ===" + bash "$script" +done + +[[ "${SKIP_RESTORE:-0}" == "1" ]] && log_warning "SKIP_RESTORE=1: stopped after a successful backup." +print_header "Demo complete — data round-tripped through object storage" diff --git a/examples/backups/vmdisk/00-helpers.sh b/examples/backups/vmdisk/00-helpers.sh new file mode 100755 index 0000000000..9bb2b046ef --- /dev/null +++ b/examples/backups/vmdisk/00-helpers.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Helper functions and variables for the VMDisk backup/restore demo. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +# ANSI color codes +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export WHITE='\033[1;37m' +export NC='\033[0m' +export BOLD='\033[1m' + +# Default settings +export NAMESPACE="${NAMESPACE:-tenant-root}" +# The platform ships the SeaweedFS-backed BackupStorageLocation as cozy-default +# (packages/system/backupstrategy-controller). Point the demo/e2e strategy at it +# by default so the flow works out of the box on a backups-enabled cluster; +# override for an external S3 whose BSL carries a different name. +export BACKUP_STORAGE_LOCATION="${BACKUP_STORAGE_LOCATION:-cozy-default}" + +# Logging functions (output to stderr to avoid polluting captured output) +log_info() { echo -e "${CYAN}ℹ${NC} $*" >&2; } +log_success() { echo -e "${GREEN}✔${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}⚠${NC} $*" >&2; } +log_error() { echo -e "${RED}✖${NC} $*" >&2; } +log_step() { echo -e "\n${MAGENTA}${BOLD}▶ $*${NC}" >&2; } +log_command() { echo -e "${WHITE} \$ $*${NC}" >&2; } +log_substep() { echo -e "${CYAN} → $*${NC}" >&2; } + +separator() { + echo -e "\n${CYAN}────────────────────────────────────────────────────────────${NC}\n" >&2 +} + +print_header() { + local title="$1" + echo -e "\n${MAGENTA}${BOLD}== ${title} ==${NC}\n" >&2 +} + +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" diff --git a/examples/backups/vmdisk/01-create-strategy.sh b/examples/backups/vmdisk/01-create-strategy.sh new file mode 100755 index 0000000000..d881b35d36 --- /dev/null +++ b/examples/backups/vmdisk/01-create-strategy.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Step 01: Create the Velero backup/restore strategy for a standalone VMDisk. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 1: Create VMDisk Velero strategy" + +log_step "Creating Velero strategy 'vmdisk-backup-strategy'..." +log_command "kubectl apply -f - (Velero strategy: vmdisk-backup-strategy)" + +# A standalone VMDisk is just an HelmRelease + a DataVolume-backed PVC (+ its +# configmaps), all labelled app.kubernetes.io/instance: vm-disk-. Unlike a +# VMInstance there is no VirtualMachine / VMI / launcher pod to capture. With +# snapshotMoveData the CSI data mover streams the volume through the +# BackupStorageLocation, so no VolumeSnapshotLocation is required (matching the +# platform's own cozy-default-velero-vmdisk strategy). +kubectl apply -f - <<'EOF' +apiVersion: strategy.backups.cozystack.io/v1alpha1 +kind: Velero +metadata: + name: vmdisk-backup-strategy +spec: + template: + restoreSpec: + existingResourcePolicy: update + includedNamespaces: + - '{{ .Application.metadata.namespace }}' + orLabelSelectors: + - matchLabels: + app.kubernetes.io/instance: 'vm-disk-{{ .Application.metadata.name }}' + - matchLabels: + apps.cozystack.io/application.kind: '{{ .Application.kind }}' + apps.cozystack.io/application.name: '{{ .Application.metadata.name }}' + includedResources: + - helmreleases.helm.toolkit.fluxcd.io + - persistentvolumeclaims + - configmaps + includeClusterResources: false + spec: + includedNamespaces: + - '{{ .Application.metadata.namespace }}' + orLabelSelectors: + - matchLabels: + app.kubernetes.io/instance: 'vm-disk-{{ .Application.metadata.name }}' + - matchLabels: + apps.cozystack.io/application.kind: '{{ .Application.kind }}' + apps.cozystack.io/application.name: '{{ .Application.metadata.name }}' + includedResources: + - helmreleases.helm.toolkit.fluxcd.io + - persistentvolumeclaims + - configmaps + includeClusterResources: false + storageLocation: '{{ .Parameters.backupStorageLocationName }}' + snapshotVolumes: true + snapshotMoveData: true + ttl: 720h0m0s + itemOperationTimeout: 24h0m0s +EOF + +log_success "Velero strategy created" +echo -e "\n${GREEN}${BOLD}Next step:${NC} ./02-create-backupclass.sh" >&2 diff --git a/examples/backups/vmdisk/02-create-backupclass.sh b/examples/backups/vmdisk/02-create-backupclass.sh new file mode 100755 index 0000000000..4c4bac79b8 --- /dev/null +++ b/examples/backups/vmdisk/02-create-backupclass.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Step 02: Create a BackupClass binding VMDisk to the Velero strategy. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 2: Create BackupClass" + +log_step "Creating BackupClass 'vmdisk-velero'..." +log_command "kubectl apply -f - (BackupClass: vmdisk-velero)" + +# BackupClass is cluster-scoped; a single instance covers every namespace. +kubectl apply -f - <&2 diff --git a/examples/backups/vmdisk/03-create-vmdisk.sh b/examples/backups/vmdisk/03-create-vmdisk.sh new file mode 100755 index 0000000000..aad26a6be9 --- /dev/null +++ b/examples/backups/vmdisk/03-create-vmdisk.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Step 03: Create a standalone VMDisk and wait for it to be ready to back up. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 3: Create VMDisk 'backup-src'" + +log_step "Creating VMDisk 'backup-src' in namespace $NAMESPACE..." +log_info "Imports the Ubuntu Noble cloud image into a 5Gi ${VMI_DISK_STORAGE_CLASS:-replicated} disk" +log_command "kubectl apply -f - (VMDisk: backup-src)" + +kubectl apply -f - <&2 diff --git a/examples/backups/vmdisk/04-create-backupjob.sh b/examples/backups/vmdisk/04-create-backupjob.sh new file mode 100755 index 0000000000..a0dc8ac6e9 --- /dev/null +++ b/examples/backups/vmdisk/04-create-backupjob.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Step 04: Back up the standalone VMDisk via a BackupJob. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 4: Create BackupJob" + +log_step "Creating BackupJob 'vmdisk-backup' in namespace $NAMESPACE..." +log_command "kubectl apply -f - (BackupJob: vmdisk-backup)" + +kubectl apply -f - <&2 || true +echo -e "\n${GREEN}${BOLD}Next step:${NC} ./05-restore-in-place.sh" >&2 diff --git a/examples/backups/vmdisk/05-restore-in-place.sh b/examples/backups/vmdisk/05-restore-in-place.sh new file mode 100755 index 0000000000..7547add227 --- /dev/null +++ b/examples/backups/vmdisk/05-restore-in-place.sh @@ -0,0 +1,159 @@ +#!/bin/bash +# Step 05: Restore the standalone VMDisk in-place from its backup. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 5: Restore VMDisk In-Place" + +PVC=vm-disk-backup-src +CONSUMER=vm-disk-backup-src-restore-consumer + +orig_uid=$(kubectl -n "$NAMESPACE" get pvc "$PVC" -o jsonpath='{.metadata.uid}') +[[ -n "$orig_uid" ]] || { log_error "PVC $PVC has no uid before the restore"; exit 1; } +log_info "PVC $PVC before restore: uid $orig_uid" + +log_step "Creating RestoreJob 'vmdisk-restore-in-place' in namespace $NAMESPACE..." +log_info "In-place restore: the original PVC is renamed and the disk is restored from backup" +log_command "kubectl apply -f - (RestoreJob: vmdisk-restore-in-place)" + +# The controller suspends the vm-disk- HelmRelease, deletes its DataVolume +# and (keepOriginalPVC) renames the live PVC to -orig-; Velero then +# creates a fresh PVC under the original name for the data mover to populate. +kubectl apply -f - <-orig- before restoring +EOF + +log_success "RestoreJob created" + +# On a WaitForFirstConsumer StorageClass the Velero data mover does not start +# until a scheduled Pod has picked a node for the restored PVC (Velero's +# WaitPVCConsumed). A VMInstance restore gets that from the virt-launcher Pod; a +# detached disk has no consumer, so stand in for the VM that would attach it. +# Only once the new PVC exists: a Pod mounting the original would pin it with +# pvc-protection and block the rename. +start_consumer() { + local mode volume mount + mode=$(kubectl -n "$NAMESPACE" get pvc "$PVC" -o jsonpath='{.spec.volumeMode}') + if [[ "$mode" == "Block" ]]; then + mount='volumeDevices: [{name: disk, devicePath: /dev/disk0}]' + else + mount='volumeMounts: [{name: disk, mountPath: /disk}]' + fi + volume="{name: disk, persistentVolumeClaim: {claimName: $PVC}}" + log_info "Starting consumer Pod $CONSUMER for the restored PVC ($mode)" + kubectl apply -f - <&2 +done +log_success "restorejob/vmdisk-restore-in-place reached 'Succeeded'" + +kubectl -n "$NAMESPACE" delete pod "$CONSUMER" --ignore-not-found --timeout=120s + +# Neither the restore-name label nor a new uid proves the data moved: Velero's +# existingResourcePolicy: update stamps the label on a PVC it only patched, and +# CDI recreating the disk would also give it a new uid. The DataDownload check +# below does; the uid and the -orig- PVC show the original was kept aside. +log_step "Verifying the PVC was recreated from the backup..." +wait_for_field pvc "$PVC" '{.status.phase}' Bound "$NAMESPACE" 120 +new_uid=$(kubectl -n "$NAMESPACE" get pvc "$PVC" -o jsonpath='{.metadata.uid}') +[[ "$new_uid" != "$orig_uid" ]] || { + log_error "PVC $PVC still has its pre-restore uid $orig_uid — the restore did not recreate it" + exit 1 +} +restore_label=$(kubectl -n "$NAMESPACE" get pvc "$PVC" \ + -o "jsonpath={.metadata.labels.velero\.io/restore-name}") +[[ -n "$restore_label" ]] || { + log_error "PVC $PVC carries no velero.io/restore-name label — Velero did not restore it" + exit 1 +} +all_pvcs=$(kubectl -n "$NAMESPACE" get pvc -o name) +[[ "$all_pvcs" == *"persistentvolumeclaim/${PVC}-orig-"* ]] || { + log_error "no ${PVC}-orig-* PVC — keepOriginalPVC did not rename the original disk" + exit 1 +} +# The DataDownload carries the same restore-name label as the PVC; one that +# Completed having written bytes is what shows the data mover filled the disk. +downloads=$(kubectl -n cozy-velero get datadownloads.velero.io -l "velero.io/restore-name=${restore_label}" \ + -o jsonpath='{range .items[*]}{.status.phase} {.status.progress.bytesDone}{"\n"}{end}') +grep -Eq '^Completed [1-9][0-9]*$' <<<"$downloads" || { + log_error "no Completed DataDownload with data for restore ${restore_label} — the data mover did not restore $PVC (got: ${downloads:-none})" + exit 1 +} + +log_step "Waiting for the restored VMDisk to become Ready again..." +# The HelmRelease was Ready before the controller suspended it, and that +# condition outlives the restore; only once Flux has reconciled the restored +# spec does Ready describe the restored release. +hr_generation=$(kubectl -n "$NAMESPACE" get hr vm-disk-backup-src -o jsonpath='{.metadata.generation}') +wait_for_field hr vm-disk-backup-src '{.status.observedGeneration}' "$hr_generation" "$NAMESPACE" 120 +wait_hr_ready vm-disk-backup-src 180 + +kubectl get restorejob vmdisk-restore-in-place -n "$NAMESPACE" -o wide >&2 || true +log_success "Restore completed; VMDisk 'backup-src' recreated from backup (${restore_label}, uid ${new_uid}), Bound and Ready" diff --git a/examples/backups/vmdisk/README.md b/examples/backups/vmdisk/README.md new file mode 100644 index 0000000000..3f67d81ac6 --- /dev/null +++ b/examples/backups/vmdisk/README.md @@ -0,0 +1,29 @@ +# VMDisk backup/restore example + +This directory shows how to back up and restore a standalone Cozystack `VMDisk` (a HelmRelease plus a DataVolume-backed PVC, with no VMInstance attached) using the cluster's `Velero` backup strategy driver. Each `BackupJob` drives a Velero CSI-snapshot backup whose data mover (kopia) streams the LINSTOR block volume to S3; each in-place `RestoreJob` recreates the disk from that artefact. + +The numbered scripts run in order (or via `run-all.sh`): + +1. `01-create-strategy.sh` — a `Velero` strategy for the `VMDisk` kind. +2. `02-create-backupclass.sh` — a `BackupClass` binding `VMDisk` to it. +3. `03-create-vmdisk.sh` — a standalone `VMDisk`, waited until imported. +4. `04-create-backupjob.sh` — an ad-hoc `BackupJob`, waited to `Succeeded`. +5. `05-restore-in-place.sh` — an in-place `RestoreJob`, then asserts that the PVC was recreated from the backup: it has a new uid, carries Velero's `velero.io/restore-name` label, and the original survives as `-orig-`. + +`cleanup.sh` tears everything down non-interactively. + +## In-place restore + +The driver prepares a `VMDisk` restore target the same way it does a VMInstance disk: it suspends the `vm-disk-` HelmRelease and deletes its DataVolume (so Flux/CDI do not recreate the PVC and race the data mover), and, with `keepOriginalPVC: true`, renames the live PVC to `-orig-` so that Velero creates a fresh PVC under the original name. `cleanup.sh` resumes the HelmRelease and reclaims the renamed PVC. + +A disk that a running VM mounts cannot be restored in place: the `RestoreJob` fails before it changes anything and names the Pod that holds the disk. Stop the VM, or restore the VMInstance instead. + +If the restore fails after this preparation, nothing is rolled back. The failure message and a Warning event name the HelmRelease that preparation suspended and the `-orig-` PVC that holds the original data. If Velero got as far as restoring the HelmRelease it is no longer suspended, and Flux may already have recreated the disk; with no restored PVC under the disk's name, CDI imports a fresh disk there. Either way, check what the disk holds, and restore again or bind the original PV back to a PVC under the disk's name if it is not what you expect. + +## Detached disks and WaitForFirstConsumer + +On a StorageClass with `volumeBindingMode: WaitForFirstConsumer`, such as the `local` class the CI lane uses, the Velero data mover does not start until a scheduled Pod has selected a node for the restored PVC. A VMInstance restore gets that from its virt-launcher Pod. A detached disk has no consumer, so its `RestoreJob` stays `Running` until something mounts the PVC, and fails once Velero's expose timeout passes. `05-restore-in-place.sh` therefore starts a minimal `pause` Pod on the restored PVC, standing in for the VM that would attach the disk, and deletes it once the restore has succeeded. On a real cluster, attaching the disk to a VMInstance does the same. + +## Overrides + +Environment variables (see `00-helpers.sh`) tune the flow: `NAMESPACE`, `BACKUP_STORAGE_LOCATION` (defaults to the platform `cozy-default` BSL), `VMI_DISK_STORAGE_CLASS` (defaults to `replicated`; set it to your cluster's class), and the `IMPORT_WAIT` / `BACKUP_WAIT` / `RESTORE_WAIT` budgets. `hack/e2e-chainsaw/vminstance/` drives this directory as the `vmdisk-2-backup-roundtrip` e2e test. diff --git a/examples/backups/vmdisk/cleanup.sh b/examples/backups/vmdisk/cleanup.sh new file mode 100755 index 0000000000..d66af53013 --- /dev/null +++ b/examples/backups/vmdisk/cleanup.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Clean up all resources created by the demo. Non-interactive: safe to call from +# the e2e harness's finally block, and idempotent (a no-op when a gated-out run +# created nothing). --ignore-not-found returns success for an absent resource but +# does NOT mask a stuck delete: the delete still blocks and, under set -e, a +# teardown that never settles fails this script instead of leaking state into the +# next run (docs/agents/e2e-testing.md convention 4). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Cleanup Demo Resources" + +# The controller suspends the VMDisk's HelmRelease before a restore and never +# resumes it. A restore that gets as far as Velero clears it, but one that fails +# earlier leaves it suspended, so resume it best-effort BEFORE deleting the +# RestoreJob (same reasoning as examples/backups/vmi/cleanup.sh). +if kubectl -n "$NAMESPACE" get hr vm-disk-backup-src >/dev/null 2>&1; then + log_substep "Resuming HelmRelease vm-disk-backup-src (no-op if not suspended)..." + kubectl -n "$NAMESPACE" patch hr vm-disk-backup-src --type=merge \ + -p '{"spec":{"suspend":false}}' >/dev/null || true +fi + +# Namespaced request objects first. Deleting the cozystack BackupJob/RestoreJob +# and Backup CRs cascades to the Velero Backup/Restore the controller created. +log_step "Deleting RestoreJob..." +kubectl delete restorejob vmdisk-restore-in-place -n "$NAMESPACE" --ignore-not-found + +log_step "Deleting BackupJob and Backup..." +kubectl delete backupjob vmdisk-backup -n "$NAMESPACE" --ignore-not-found +kubectl delete backup vmdisk-backup -n "$NAMESPACE" --ignore-not-found + +separator + +log_step "Deleting VMDisk and its PVC..." +kubectl delete pod vm-disk-backup-src-restore-consumer -n "$NAMESPACE" --ignore-not-found +kubectl delete vmdisk backup-src -n "$NAMESPACE" --ignore-not-found +kubectl delete pvc -n "$NAMESPACE" \ + -l app.kubernetes.io/instance=vm-disk-backup-src --ignore-not-found + +# keepOriginalPVC renamed the pre-restore PVC to -orig- with a bare +# ObjectMeta (no labels/owner) and flipped its PV to Retain, so the label delete +# above misses it. Reclaim it by name and flip the PV back to Delete BEFORE +# removing the PVC so the CSI provisioner reclaims the LINSTOR volume rather than +# stranding it. Let the lookup and patch fail the script (see the vmi cleanup). +if ! all_pvcs=$(kubectl -n "$NAMESPACE" get pvc -o name 2>&1); then + log_error "cannot list PVCs to reclaim orphaned disks: $all_pvcs" + exit 1 +fi +orig_pvcs=$(printf '%s\n' "$all_pvcs" | grep -E '/vm-disk-backup-src-orig-' || true) +for pvc in $orig_pvcs; do + pv=$(kubectl -n "$NAMESPACE" get "$pvc" -o jsonpath='{.spec.volumeName}') + log_substep "Reclaiming orphaned ${pvc#*/} and its LINSTOR volume..." + if [[ -n "$pv" ]]; then + kubectl patch pv "$pv" --type=merge \ + -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' >/dev/null + fi + kubectl -n "$NAMESPACE" delete "$pvc" --ignore-not-found +done + +separator + +log_step "Deleting BackupClass and Velero strategy..." +kubectl delete backupclass vmdisk-velero --ignore-not-found +kubectl delete velero.strategy.backups.cozystack.io vmdisk-backup-strategy --ignore-not-found + +separator + +log_success "Cleanup complete" diff --git a/examples/backups/vmdisk/run-all.sh b/examples/backups/vmdisk/run-all.sh new file mode 100755 index 0000000000..0956ad807e --- /dev/null +++ b/examples/backups/vmdisk/run-all.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Convenience runner + e2e harness for the standalone VMDisk backup/restore demo. +# +# Runs the numbered steps 01..05 a human reads, in order, non-interactively, +# stopping on the first failure (set -e). This is the SAME sequence the docs walk +# through, so the documented demo and the automated test can never drift. +# +# Flow: Velero strategy -> BackupClass -> a standalone VMDisk (waits imported) -> +# ad-hoc BackupJob (waits Succeeded) -> in-place RestoreJob (waits Succeeded, then +# waits for the restored disk to be Bound + Ready). +# +# Override NAMESPACE / BACKUP_STORAGE_LOCATION via the environment; see +# 00-helpers.sh. hack/e2e-chainsaw/vminstance/ drives this file as the +# vmdisk-2-backup-roundtrip e2e test, run un-gated in CI. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "VMDisk Backup & Restore Demo" +log_info "Namespace: ${NAMESPACE} BackupStorageLocation: ${BACKUP_STORAGE_LOCATION}" + +SCRIPTS=( + "01-create-strategy.sh" + "02-create-backupclass.sh" + "03-create-vmdisk.sh" + "04-create-backupjob.sh" + "05-restore-in-place.sh" +) + +for script in "${SCRIPTS[@]}"; do + if [[ -x "$SCRIPT_DIR/$script" ]]; then + "$SCRIPT_DIR/$script" + separator + else + log_error "Script not found or not executable: $script" + exit 1 + fi +done + +print_header "Demo Complete" +log_success "All steps completed successfully!" +log_info "To clean up resources: ./cleanup.sh" diff --git a/examples/backups/vmi/00-helpers.sh b/examples/backups/vmi/00-helpers.sh index 384054952d..30ca161f3b 100755 --- a/examples/backups/vmi/00-helpers.sh +++ b/examples/backups/vmi/00-helpers.sh @@ -15,7 +15,11 @@ export BOLD='\033[1m' # Default settings export NAMESPACE="${NAMESPACE:-tenant-root}" -export BACKUP_STORAGE_LOCATION="${BACKUP_STORAGE_LOCATION:-default}" +# The platform ships the SeaweedFS-backed BackupStorageLocation as cozy-default +# (packages/system/backupstrategy-controller). Point the demo/e2e strategies at +# it by default so the flow works out of the box on a backups-enabled cluster; +# override for an external S3 whose BSL carries a different name. +export BACKUP_STORAGE_LOCATION="${BACKUP_STORAGE_LOCATION:-cozy-default}" # Logging functions (output to stderr to avoid polluting captured output) log_info() { @@ -65,38 +69,10 @@ resource_exists() { fi } -# Wait for a resource field to reach a desired value -wait_for_field() { - local resource_type="$1" - local resource_name="$2" - local jsonpath="$3" - local desired="$4" - local namespace="${5:-}" - local timeout="${6:-300}" - - log_substep "Waiting for $resource_type/$resource_name $jsonpath to become '$desired'..." - - local elapsed=0 - local ns_flag="" - [[ -n "$namespace" ]] && ns_flag="-n $namespace" - - while true; do - local current - # shellcheck disable=SC2086 - current=$(kubectl get "$resource_type" "$resource_name" $ns_flag -o jsonpath="$jsonpath" 2>/dev/null || true) - if [[ "$current" == "$desired" ]]; then - log_success "$resource_type/$resource_name reached '$desired'" - return 0 - fi - if [[ $elapsed -ge $timeout ]]; then - log_error "Timeout waiting for $resource_type/$resource_name (current: '$current', expected: '$desired')" - return 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) - echo -n "." >&2 - done -} +# wait_for_field, wait_hr_ready and wait_deleted live in one file shared by +# every backup walkthrough, so a fix to one reaches all of them. +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" # Print a separator line separator() { diff --git a/examples/backups/vmi/01-create-strategies.sh b/examples/backups/vmi/01-create-strategies.sh index 78aa34d311..a12809c541 100755 --- a/examples/backups/vmi/01-create-strategies.sh +++ b/examples/backups/vmi/01-create-strategies.sh @@ -70,8 +70,11 @@ spec: - controllerrevisions.apps includeClusterResources: false storageLocation: '{{ .Parameters.backupStorageLocationName }}' - volumeSnapshotLocations: - - '{{ .Parameters.backupStorageLocationName }}' + # No volumeSnapshotLocations: with snapshotMoveData the CSI data mover + # uploads the volume data through the BackupStorageLocation, so a VSL of + # the same name is not required (and referencing a non-existent one fails + # the backup). The platform's own cozy-default-velero-vminstance strategy + # omits it for the same reason. snapshotVolumes: true snapshotMoveData: true ttl: 720h0m0s @@ -123,8 +126,8 @@ spec: - configmaps includeClusterResources: false storageLocation: '{{ .Parameters.backupStorageLocationName }}' - volumeSnapshotLocations: - - '{{ .Parameters.backupStorageLocationName }}' + # See the VMInstance strategy above: snapshotMoveData routes volume data + # through the BackupStorageLocation, so no VolumeSnapshotLocation is needed. snapshotVolumes: true snapshotMoveData: true ttl: 720h0m0s diff --git a/examples/backups/vmi/03-create-vmdisk.sh b/examples/backups/vmi/03-create-vmdisk.sh index 89d598867b..27234a09db 100755 --- a/examples/backups/vmi/03-create-vmdisk.sh +++ b/examples/backups/vmi/03-create-vmdisk.sh @@ -7,8 +7,13 @@ source "$SCRIPT_DIR/00-helpers.sh" print_header "Step 3: Create VMDisk" +# Image + size are overridable so the same flow can run a heavy ubuntu demo or a +# tiny cirros disk for a lightweight e2e (see hack/e2e-chainsaw/vminstance). +VMI_IMAGE_URL="${VMI_IMAGE_URL:-https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img}" +VMI_DISK_STORAGE="${VMI_DISK_STORAGE:-20Gi}" + log_step "Creating VMDisk 'ubuntu-source' in namespace $NAMESPACE..." -log_info "This will download the Ubuntu Noble cloud image (~700MB)" +log_info "Importing ${VMI_IMAGE_URL} into a ${VMI_DISK_STORAGE} disk" log_command "kubectl apply -f - (VMDisk: ubuntu-source)" kubectl apply -f - < @@ -34,11 +39,24 @@ log_success "VMInstance created" separator +# Wait for the VM to be fully booted before backing it up. Backing up a VM that +# is still importing its disk or has not started yields an incomplete, useless +# backup — so, like the etcd/postgres source steps, block on readiness here. +# The Ubuntu cloud image download (~700MB) plus nested KubeVirt boot is slow, +# hence the generous 900s bound. +log_step "Waiting for the VMInstance to boot (HelmRelease + VirtualMachine Ready)..." +wait_hr_ready vm-instance-test "${IMPORT_WAIT:-900}" +log_command "kubectl -n $NAMESPACE wait virtualmachine.kubevirt.io/vm-instance-test --for=condition=Ready" +kubectl -n "$NAMESPACE" wait virtualmachine.kubevirt.io/vm-instance-test \ + --for=condition=Ready --timeout="${VMI_BOOT_WAIT:-600}s" + +separator + log_step "Verifying VMInstance..." log_command "kubectl get vminstance test -n $NAMESPACE" kubectl get vminstance test -n "$NAMESPACE" separator -log_success "VMInstance is ready" +log_success "VMInstance is running and ready to back up" echo -e "\n${GREEN}${BOLD}Next step:${NC} ./05-create-backupjob.sh" diff --git a/examples/backups/vmi/05-create-backupjob.sh b/examples/backups/vmi/05-create-backupjob.sh index 581490a473..095e0b1692 100755 --- a/examples/backups/vmi/05-create-backupjob.sh +++ b/examples/backups/vmi/05-create-backupjob.sh @@ -30,7 +30,11 @@ log_success "BackupJob created" separator log_step "Waiting for BackupJob to complete..." -wait_for_field backupjob test-backup '{.status.phase}' Succeeded "$NAMESPACE" 600 +# Default 1800s: the Velero CSI data mover copies the whole VM disk (20Gi) to S3, +# far heavier than a database dump. Overridable (BACKUP_WAIT) so the CI harness +# can bound it well under the Chainsaw op timeout on its tiny guest. Fail fast if +# the job flips to Failed. +wait_for_field backupjob test-backup '{.status.phase}' Succeeded "$NAMESPACE" "${BACKUP_WAIT:-1800}" Failed separator diff --git a/examples/backups/vmi/06-restore-in-place.sh b/examples/backups/vmi/06-restore-in-place.sh index ea424ed56e..95cd5e1eea 100755 --- a/examples/backups/vmi/06-restore-in-place.sh +++ b/examples/backups/vmi/06-restore-in-place.sh @@ -7,6 +7,9 @@ source "$SCRIPT_DIR/00-helpers.sh" print_header "Step 6: Restore VMInstance In-Place" +orig_uid=$(kubectl -n "$NAMESPACE" get pvc vm-disk-ubuntu-source -o jsonpath='{.metadata.uid}') +[[ -n "$orig_uid" ]] || { log_error "PVC vm-disk-ubuntu-source has no uid before the restore"; exit 1; } + log_step "Creating RestoreJob 'restore-in-place-test' in namespace $NAMESPACE..." log_info "In-place restore: the VM will be halted, PVCs renamed, and data restored from backup" log_command "kubectl apply -f - (RestoreJob: restore-in-place-test)" @@ -25,8 +28,8 @@ spec: kind: VMInstance name: test options: - failIfTargetExists: true # if true, restore will fail when the target resource already exists - keepOriginalPVC: true # renames original VMI PVC before restore to `-orig-`, only for in-place restore + failIfTargetExists: true # only enforced for restore-to-copy; a no-op for in-place, whose source is expected to exist + keepOriginalPVC: true # renames original VMI PVC before restore to -orig-, only for in-place restore keepOriginalIpAndMac: true # restores original IP and MAC address of VMI via OVN annotations EOF @@ -35,7 +38,7 @@ log_success "RestoreJob created" separator log_step "Waiting for RestoreJob to complete..." -wait_for_field restorejob restore-in-place-test '{.status.phase}' Succeeded "$NAMESPACE" 600 +wait_for_field restorejob restore-in-place-test '{.status.phase}' Succeeded "$NAMESPACE" "${RESTORE_WAIT:-1800}" Failed separator @@ -45,5 +48,49 @@ kubectl get restorejob restore-in-place-test -n "$NAMESPACE" -o wide separator -log_success "In-place restore completed successfully" +# The RestoreJob reaching Succeeded means Velero re-created the objects; wait +# for the restored VM to come back before checking where its disk came from. +log_step "Waiting for the restored VMInstance to boot again..." +log_command "kubectl -n $NAMESPACE wait virtualmachine.kubevirt.io/vm-instance-test --for=condition=Ready" +kubectl -n "$NAMESPACE" wait virtualmachine.kubevirt.io/vm-instance-test \ + --for=condition=Ready --timeout="${VMI_BOOT_WAIT:-600}s" + +# VirtualMachine Ready is the virt-launcher Pod's readiness, reported before the +# guest runs, so it cannot tell a restored disk from a blank one, and this +# platform offers no guest-side signal (serial-console logging is disabled +# cluster-wide, the cirros guest has no qemu-guest-agent). Prove provenance on the +# PVC instead. Neither the restore-name label nor a new uid proves the data +# moved (existingResourcePolicy: update stamps the label on a PVC it only +# patched, and CDI recreating the disk gives it a new uid); the DataDownload +# check below does, and the uid and -orig- PVC show the original was kept aside. +log_step "Verifying the VM booted off a disk the restore recreated from the backup..." +wait_for_field pvc vm-disk-ubuntu-source '{.status.phase}' Bound "$NAMESPACE" 120 +new_uid=$(kubectl -n "$NAMESPACE" get pvc vm-disk-ubuntu-source -o jsonpath='{.metadata.uid}') +[[ "$new_uid" != "$orig_uid" ]] || { + log_error "PVC vm-disk-ubuntu-source still has its pre-restore uid $orig_uid — the restore did not recreate it" + exit 1 +} +restore_label=$(kubectl -n "$NAMESPACE" get pvc vm-disk-ubuntu-source \ + -o "jsonpath={.metadata.labels.velero\.io/restore-name}") +[[ -n "$restore_label" ]] || { + log_error "restored VM disk PVC carries no velero.io/restore-name label — Velero did not restore it" + exit 1 +} +all_pvcs=$(kubectl -n "$NAMESPACE" get pvc -o name) +[[ "$all_pvcs" == *"persistentvolumeclaim/vm-disk-ubuntu-source-orig-"* ]] || { + log_error "no vm-disk-ubuntu-source-orig-* PVC — keepOriginalPVC did not rename the original disk" + exit 1 +} +# The DataDownload carries the same restore-name label as the PVC; one that +# Completed having written bytes is what shows the data mover filled the disk. +downloads=$(kubectl -n cozy-velero get datadownloads.velero.io -l "velero.io/restore-name=${restore_label}" \ + -o jsonpath='{range .items[*]}{.status.phase} {.status.progress.bytesDone}{"\n"}{end}') +grep -Eq '^Completed [1-9][0-9]*$' <<<"$downloads" || { + log_error "no Completed DataDownload with data for restore ${restore_label} — the data mover did not restore vm-disk-ubuntu-source (got: ${downloads:-none})" + exit 1 +} + +separator + +log_success "In-place restore completed; VM disk recreated from backup (${restore_label}, uid ${new_uid}) and the VM is Running off it" echo -e "\n${GREEN}${BOLD}Next step:${NC} ./07-restore-to-copy.sh" diff --git a/examples/backups/vmi/07-restore-to-copy.sh b/examples/backups/vmi/07-restore-to-copy.sh index 9b84ce5587..d7f7b87fb7 100755 --- a/examples/backups/vmi/07-restore-to-copy.sh +++ b/examples/backups/vmi/07-restore-to-copy.sh @@ -40,7 +40,7 @@ spec: options: # runtime.RawExtension, typed based on targetApplicationRef and current controller implementation (for additional restore options) targetNamespace: ${TARGET_NAMESPACE} # when set to a different namespace, triggers cross-namespace restore via Velero namespaceMapping failIfTargetExists: true # if true, restore will fail when the target resource already exists - keepOriginalPVC: false # renames original VMI PVC before restore to `-orig-`, only for in-place restore + keepOriginalPVC: false # renames original VMI PVC before restore to -orig-, only for in-place restore keepOriginalIpAndMac: false # restores original IP and MAC address of VMI via OVN annotations EOF @@ -49,7 +49,7 @@ log_success "RestoreJob created" separator log_step "Waiting for RestoreJob to complete..." -wait_for_field restorejob restore-to-copy-test '{.status.phase}' Succeeded "$NAMESPACE" 600 +wait_for_field restorejob restore-to-copy-test '{.status.phase}' Succeeded "$NAMESPACE" 1800 Failed separator diff --git a/examples/backups/vmi/cleanup.sh b/examples/backups/vmi/cleanup.sh index 5965c3f036..3351ce9ee9 100755 --- a/examples/backups/vmi/cleanup.sh +++ b/examples/backups/vmi/cleanup.sh @@ -1,5 +1,10 @@ #!/bin/bash -# Clean up all resources created by the demo +# Clean up all resources created by the demo. Non-interactive: safe to call from +# the e2e harness's finally block, and idempotent (a no-op when a gated-out run +# created nothing). --ignore-not-found returns success for an absent resource +# but does NOT mask a stuck delete: the delete still blocks and, under set -e, +# a teardown that never settles fails this script instead of leaking state into +# the next run (docs/agents/e2e-testing.md convention 4). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -9,62 +14,74 @@ TARGET_NAMESPACE="${TARGET_NAMESPACE:-tenant-root-copy}" print_header "Cleanup Demo Resources" -log_warning "This script will delete all resources created during the demo" -echo -e "\n${YELLOW}The following will be deleted:${NC}" -echo " - RestoreJobs (restore-in-place-test, restore-to-copy-test)" -echo " - BackupJob (test-backup) and associated Backup" -echo " - VMInstance (test)" -echo " - VMDisk (ubuntu-source)" -echo " - BackupClass (velero)" -echo " - Velero strategies (vminstance-strategy, vmdisk-strategy)" -echo " - Target namespace ($TARGET_NAMESPACE)" -echo "" - -read -p "Continue? (y/N): " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - log_info "Cancelled" - exit 0 -fi - -separator - +# The driver suspends the VMInstance/VMDisk HelmReleases while a RestoreJob runs +# and never resumes them (suspendHelmRelease only ever sets suspend=true), so an +# aborted or completed restore leaves them suspended. Resume best-effort BEFORE +# deleting the RestoreJobs — deleting a RestoreJob without resuming strands the +# app suspended (same reasoning as examples/backups/postgres/cleanup.sh). +for hr in vm-instance-test vm-disk-ubuntu-source; do + if kubectl -n "$NAMESPACE" get hr "$hr" >/dev/null 2>&1; then + log_substep "Resuming HelmRelease $hr (no-op if not suspended)..." + kubectl -n "$NAMESPACE" patch hr "$hr" --type=merge \ + -p '{"spec":{"suspend":false}}' >/dev/null || true + fi +done + +# Namespaced request objects first. Deleting the cozystack BackupJob/RestoreJob +# and Backup CRs cascades to the Velero Backup/Restore the controller created. log_step "Deleting RestoreJobs..." -kubectl delete restorejob restore-to-copy-test -n "$NAMESPACE" 2>/dev/null || log_warning "RestoreJob restore-to-copy-test not found" -kubectl delete restorejob restore-in-place-test -n "$NAMESPACE" 2>/dev/null || log_warning "RestoreJob restore-in-place-test not found" - -separator +kubectl delete restorejob restore-to-copy-test restore-in-place-test \ + -n "$NAMESPACE" --ignore-not-found log_step "Deleting BackupJob and Backup..." -kubectl delete backupjob test-backup -n "$NAMESPACE" 2>/dev/null || log_warning "BackupJob test-backup not found" -kubectl delete backup test-backup -n "$NAMESPACE" 2>/dev/null || log_warning "Backup test-backup not found" - -separator - -log_step "Deleting VMInstance..." -kubectl delete vminstance test -n "$NAMESPACE" 2>/dev/null || log_warning "VMInstance test not found" - -log_step "Deleting VMDisk..." -kubectl delete vmdisk ubuntu-source -n "$NAMESPACE" 2>/dev/null || log_warning "VMDisk ubuntu-source not found" +kubectl delete backupjob test-backup -n "$NAMESPACE" --ignore-not-found +kubectl delete backup test-backup -n "$NAMESPACE" --ignore-not-found separator -log_step "Deleting BackupClass..." -kubectl delete backupclass velero 2>/dev/null || log_warning "BackupClass velero not found" +log_step "Deleting VMInstance and VMDisk..." +kubectl delete vminstance test -n "$NAMESPACE" --ignore-not-found +kubectl delete vmdisk ubuntu-source -n "$NAMESPACE" --ignore-not-found + +# keepOriginalPVC renames the VMInstance's disk PVC to -orig- with a +# bare ObjectMeta (no labels, no ownerRefs) and flips its PV to Retain. Nothing +# above reclaims it: the app delete does not own it and no label selector matches +# it, so under skipDelete a LINSTOR volume and a Released PV leak on every run. +# List loudly (a failed query must abort, not silently skip the reclaim it exists +# for); grep's no-match stays tolerant. +if ! all_pvcs=$(kubectl -n "$NAMESPACE" get pvc -o name 2>&1); then + log_error "cannot list PVCs to reclaim orphaned disks: $all_pvcs" + exit 1 +fi +orig_pvcs=$(printf '%s\n' "$all_pvcs" | grep -E '/vm-disk-ubuntu-source-orig-' || true) +for pvc in $orig_pvcs; do + # Flip the retained PV back to Delete BEFORE removing the PVC, so releasing it + # makes the CSI provisioner call DeleteVolume and reclaim the LINSTOR volume — + # deleting the PV object alone would drop the record and strand the volume. Let + # both the lookup and the patch fail the script: swallowing them would delete + # the PVC anyway and strand the volume with no PVC left to find it by. + pv=$(kubectl -n "$NAMESPACE" get "$pvc" -o jsonpath='{.spec.volumeName}') + log_substep "Reclaiming orphaned ${pvc#*/} and its LINSTOR volume..." + if [[ -n "$pv" ]]; then + kubectl patch pv "$pv" --type=merge \ + -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' >/dev/null + fi + kubectl -n "$NAMESPACE" delete "$pvc" --ignore-not-found +done separator -log_step "Deleting Velero strategies..." -kubectl delete velero.strategy.backups.cozystack.io vminstance-strategy 2>/dev/null || log_warning "Strategy vminstance-strategy not found" -kubectl delete velero.strategy.backups.cozystack.io vmdisk-strategy 2>/dev/null || log_warning "Strategy vmdisk-strategy not found" +# Cluster-scoped BackupClass + strategies. +log_step "Deleting BackupClass and Velero strategies..." +kubectl delete backupclass velero --ignore-not-found +kubectl delete velero.strategy.backups.cozystack.io \ + vminstance-strategy vmdisk-strategy --ignore-not-found separator log_step "Deleting target namespace..." -kubectl delete namespace "$TARGET_NAMESPACE" 2>/dev/null || log_warning "Namespace $TARGET_NAMESPACE not found" +kubectl delete namespace "$TARGET_NAMESPACE" --ignore-not-found separator log_success "Cleanup complete" -log_info "All demo resources have been deleted" -echo -e "\n${GREEN}${BOLD}To re-run the demo:${NC} ./01-create-strategies.sh" diff --git a/examples/backups/vmi/run-all.sh b/examples/backups/vmi/run-all.sh index b58f4b9d33..2585bdb303 100755 --- a/examples/backups/vmi/run-all.sh +++ b/examples/backups/vmi/run-all.sh @@ -1,15 +1,25 @@ #!/bin/bash -# Run the full VMInstance backup/restore demo sequentially +# Convenience runner + e2e harness for the VMInstance backup/restore demo. +# +# Runs the numbered steps 01..07 a human reads, in order, non-interactively, +# stopping on the first failure (set -e). This is the SAME sequence the docs +# walk through, so the documented flow and the automated test can never drift. +# +# Flow: Velero strategies -> BackupClass -> VMDisk + VMInstance (waits until the +# VM is booted) -> ad-hoc BackupJob (waits Succeeded) -> in-place RestoreJob +# (waits Succeeded, waits for the VM to boot again) -> cross-namespace to-copy +# RestoreJob (waits Succeeded). +# +# Override NAMESPACE / BACKUP_STORAGE_LOCATION / TARGET_NAMESPACE via the +# environment; see 00-helpers.sh. hack/e2e-chainsaw/vminstance/ drives this file +# as the vminstance-2-backup-roundtrip e2e test, run un-gated in CI. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/00-helpers.sh" print_header "VMInstance Backup & Restore Demo" - -log_info "This script will run all demo steps sequentially" -log_info "There will be a pause between steps for observation" -echo "" +log_info "Namespace: ${NAMESPACE} BackupStorageLocation: ${BACKUP_STORAGE_LOCATION}" SCRIPTS=( "01-create-strategies.sh" @@ -18,29 +28,18 @@ SCRIPTS=( "04-create-vminstance.sh" "05-create-backupjob.sh" "06-restore-in-place.sh" - "07-restore-to-copy.sh" ) - -echo -e "${CYAN}Demo steps:${NC}" -for i in "${!SCRIPTS[@]}"; do - echo " $((i+1)). ${SCRIPTS[$i]}" -done -echo "" - -read -p "Run demo? (y/N): " -n 1 -r -echo -if [[ ! $REPLY =~ ^[Yy]$ ]]; then - log_info "Cancelled" - exit 0 +# Cross-namespace to-copy restore is the heaviest step (a second full VM in a +# fresh namespace). SKIP_RESTORE_TO_COPY=1 drops it — used by the lightweight CI +# round-trip, which proves backup + in-place restore is enough. +if [[ "${SKIP_RESTORE_TO_COPY:-0}" != "1" ]]; then + SCRIPTS+=("07-restore-to-copy.sh") fi -separator - for script in "${SCRIPTS[@]}"; do if [[ -x "$SCRIPT_DIR/$script" ]]; then "$SCRIPT_DIR/$script" separator - wait_for_enter else log_error "Script not found or not executable: $script" exit 1 @@ -50,13 +49,13 @@ done print_header "Demo Complete" log_success "All steps completed successfully!" -echo "" log_info "What was demonstrated:" -echo " 1. Created Velero backup strategies for VMInstance and VMDisk" -echo " 2. Created BackupClass binding strategies to application types" -echo " 3. Provisioned a VMDisk and VMInstance" -echo " 4. Created a backup of the VMInstance via BackupJob" -echo " 5. Restored the VMInstance in-place" -echo " 6. Restored the VMInstance to a copy in a different namespace" -echo "" +echo " 1. Created Velero backup strategies for VMInstance and VMDisk" >&2 +echo " 2. Created BackupClass binding strategies to application types" >&2 +echo " 3. Provisioned a VMDisk and VMInstance" >&2 +echo " 4. Created a backup of the VMInstance via BackupJob" >&2 +echo " 5. Restored the VMInstance in-place" >&2 +if [[ "${SKIP_RESTORE_TO_COPY:-0}" != "1" ]]; then + echo " 6. Restored the VMInstance to a copy in a different namespace" >&2 +fi log_info "To clean up resources: ./cleanup.sh" diff --git a/examples/backups/vmi/scenario-user-restore.md b/examples/backups/vmi/scenario-user-restore.md index 1d35b22b07..b52453c450 100644 --- a/examples/backups/vmi/scenario-user-restore.md +++ b/examples/backups/vmi/scenario-user-restore.md @@ -39,6 +39,22 @@ The backup controller performs the following steps automatically: After the Velero Restore completes, the HelmReleases resume and the VM boots with the restored disk data while retaining its original IP and MAC addresses. +### Reclaiming the kept disks + +With `keepOriginalPVC: true`, the default, step 3 switches each disk's PV to the `Retain` reclaim policy so that it outlives the PVC it is renamed from. Nothing removes the `-orig` PVCs or sets the policy back afterwards: they stay until an operator deletes them. + +The policy the PV had before the restore is recorded in the `backups.cozystack.io/original-reclaim-policy` annotation, on the PV and on the `-orig` PVC. Set it back on the PV before deleting the `-orig` PVC, or a PV that was `Delete` stays `Retain` once released and its volume is never reclaimed. A PVC renamed by an older controller carries no annotation; the snippet then leaves the PV on `Retain` for you to decide. + +```bash +PVC=vm-disk-ubuntu-source-orig-1a2b +PV=$(kubectl -n "$NAMESPACE" get pvc "$PVC" -o jsonpath='{.spec.volumeName}') +POLICY=$(kubectl get pv "$PV" -o jsonpath='{.metadata.annotations.backups\.cozystack\.io/original-reclaim-policy}') +kubectl patch pv "$PV" --type=merge -p "{\"spec\":{\"persistentVolumeReclaimPolicy\":\"${POLICY:-Retain}\"}}" +kubectl -n "$NAMESPACE" delete pvc "$PVC" +``` + +The `` is derived from the RestoreJob name, so an `-orig` PVC left behind by a deleted RestoreJob takes the name a new RestoreJob of the same name would use. That RestoreJob fails before it halts the VM, naming the PVC, rather than restoring without keeping the current disk. Reclaim the leftover PVC as above, or give the new RestoreJob another name. + ## Method 2: Cross-Namespace Restore (Copy) ```bash diff --git a/go.mod b/go.mod index 7b251c6434..6109faf15a 100644 --- a/go.mod +++ b/go.mod @@ -2,44 +2,49 @@ module github.com/cozystack/cozystack -go 1.26.2 +go 1.27.1 require ( - github.com/cert-manager/cert-manager v1.17.4 + github.com/cert-manager/cert-manager v1.19.6 github.com/cozystack/cozystack-scheduler/pkg/apis v0.1.1 - github.com/emicklei/dot v1.10.0 - github.com/fluxcd/helm-controller/api v1.4.3 - github.com/fluxcd/source-controller/api v1.7.4 - github.com/fluxcd/source-watcher/api/v2 v2.0.3 - github.com/go-logr/logr v1.4.3 + github.com/emicklei/dot v1.11.0 + github.com/fluxcd/helm-controller/api v1.5.1 + github.com/fluxcd/pkg/apis/kustomize v1.15.0 + github.com/fluxcd/pkg/apis/meta v1.26.0 + github.com/fluxcd/source-controller/api v1.8.5 + github.com/fluxcd/source-watcher/api/v2 v2.1.1 + github.com/go-logr/logr v1.4.4 github.com/go-logr/zapr v1.3.0 github.com/go-task/slim-sprig/v3 v3.0.0 - github.com/google/gofuzz v1.2.0 - github.com/onsi/ginkgo/v2 v2.23.3 - github.com/onsi/gomega v1.37.0 - github.com/prometheus/client_golang v1.23.0 + github.com/onsi/ginkgo/v2 v2.33.0 + github.com/onsi/gomega v1.43.1 + github.com/prometheus/client_golang v1.24.1 github.com/robfig/cron/v3 v3.0.1 - github.com/spf13/cobra v1.9.1 - github.com/vmware-tanzu/velero v1.17.1 - go.uber.org/zap v1.27.0 - k8s.io/api v0.34.1 - k8s.io/apiextensions-apiserver v0.34.1 - k8s.io/apimachinery v0.34.2 - k8s.io/apiserver v0.34.1 - k8s.io/client-go v0.34.1 - k8s.io/component-base v0.34.1 - k8s.io/klog/v2 v2.130.1 - k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 - k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d + github.com/spf13/cobra v1.10.2 + github.com/spf13/pflag v1.0.10 + github.com/vmware-tanzu/velero v1.18.3 + go.uber.org/zap v1.28.0 + gomodules.xyz/jsonpatch/v2 v2.5.0 + k8s.io/api v0.37.1 + k8s.io/apiextensions-apiserver v0.37.1 + k8s.io/apimachinery v0.37.1 + k8s.io/apiserver v0.37.1 + k8s.io/client-go v0.37.1 + k8s.io/component-base v0.37.1 + k8s.io/klog/v2 v2.140.0 + k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad + k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 sigs.k8s.io/container-object-storage-interface-api v0.1.0 - sigs.k8s.io/controller-runtime v0.22.4 + sigs.k8s.io/controller-runtime v0.25.1 sigs.k8s.io/gateway-api v1.4.1 - sigs.k8s.io/structured-merge-diff/v6 v6.3.0 + sigs.k8s.io/randfill v1.0.0 + sigs.k8s.io/structured-merge-diff/v6 v6.4.2 sigs.k8s.io/yaml v1.6.0 ) require ( - cel.dev/expr v0.25.1 // indirect + cel.dev/expr v0.25.3 // indirect + github.com/Masterminds/semver/v3 v3.5.0 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -47,88 +52,96 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dustin/go-humanize v1.1.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch v5.9.11+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fluxcd/pkg/apis/acl v0.9.0 // indirect - github.com/fluxcd/pkg/apis/kustomize v1.13.0 // indirect - github.com/fluxcd/pkg/apis/meta v1.23.0 // indirect - github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/fluxcd/pkg/apis/acl v0.11.0 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/fxamacker/cbor/v2 v2.9.4 // indirect + github.com/gkampitakis/go-snaps v0.5.23 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.21.2 // indirect - github.com/go-openapi/jsonreference v0.21.0 // indirect - github.com/go-openapi/swag v0.23.1 // indirect - github.com/gogo/protobuf v1.3.2 // indirect + github.com/go-openapi/jsonpointer v1.0.1 // indirect + github.com/go-openapi/jsonreference v1.0.2 // indirect + github.com/go-openapi/swag v0.29.2 // indirect + github.com/go-openapi/swag/cmdutils v0.29.2 // indirect + github.com/go-openapi/swag/conv v0.29.2 // indirect + github.com/go-openapi/swag/fileutils v0.29.2 // indirect + github.com/go-openapi/swag/jsonutils v0.29.2 // indirect + github.com/go-openapi/swag/loading v0.29.2 // indirect + github.com/go-openapi/swag/mangling v0.29.2 // indirect + github.com/go-openapi/swag/netutils v0.29.2 // indirect + github.com/go-openapi/swag/pools v0.29.2 // indirect + github.com/go-openapi/swag/stringutils v0.29.2 // indirect + github.com/go-openapi/swag/typeutils v0.29.2 // indirect + github.com/go-openapi/swag/yamlutils v0.29.2 // indirect + github.com/go-openapi/testify/enable/yaml/v2 v2.8.0 // indirect + github.com/go-openapi/testify/v2 v2.8.0 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/btree v1.1.3 // indirect - github.com/google/cel-go v0.26.0 // indirect - github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/cel-go v0.29.2 // indirect + github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect + github.com/google/pprof v0.0.0-20260906184651-6331bc6350fe // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.4 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.20.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mailru/easyjson v0.9.0 // indirect github.com/moby/spdystream v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.65.0 // indirect - github.com/prometheus/procfs v0.17.0 // indirect - github.com/spf13/pflag v1.0.7 // indirect - github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/prometheus/client_model v0.6.3 // indirect + github.com/prometheus/common v0.71.0 // indirect + github.com/prometheus/procfs v0.22.0 // indirect + github.com/rogpeppe/go-internal v1.16.0 // indirect + github.com/sirupsen/logrus v1.10.2 // indirect + github.com/tidwall/match v1.2.0 // indirect github.com/x448/float16 v0.8.4 // indirect - go.etcd.io/etcd/api/v3 v3.6.4 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.6.4 // indirect - go.etcd.io/etcd/client/v3 v3.6.4 // indirect + go.etcd.io/etcd/api/v3 v3.7.0 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.7.0 // indirect + go.etcd.io/etcd/client/v3 v3.7.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect - go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect + go.opentelemetry.io/otel v1.46.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/sdk v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect - golang.org/x/net v0.52.0 // indirect - golang.org/x/oauth2 v0.35.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/term v0.42.0 // indirect - golang.org/x/text v0.36.0 // indirect - golang.org/x/time v0.12.0 // indirect - golang.org/x/tools v0.43.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect - google.golang.org/grpc v1.80.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba // indirect + golang.org/x/mod v0.41.0 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/oauth2 v0.37.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/term v0.46.0 // indirect + golang.org/x/text v0.42.0 // indirect + golang.org/x/time v0.16.0 // indirect + golang.org/x/tools v0.50.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/kms v0.34.1 // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect - sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect - sigs.k8s.io/randfill v1.0.0 // indirect + k8s.io/kms v0.37.1 // indirect + k8s.io/streaming v0.37.1 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 // indirect + sigs.k8s.io/json v0.0.0-20260909141634-11ed52e25bc5 // indirect ) - -// See: issues.k8s.io/135537 -replace k8s.io/apimachinery => github.com/cozystack/apimachinery v0.0.0-20251219010959-1f91eabae46c diff --git a/go.sum b/go.sum index c5f3e2cd40..dea6aaf08f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= @@ -12,16 +14,14 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/cert-manager/cert-manager v1.17.4 h1:pQrEur25zR23Mum1Au4jRH2p8eH3wY2v4/QahjPDzKo= -github.com/cert-manager/cert-manager v1.17.4/go.mod h1:zXVCSnEOu6vNDQOPpXrLO8a0iDnKd8uksgXe5s73p+w= +github.com/cert-manager/cert-manager v1.19.6 h1:HxoNU9l0/K3nuNT6lzWpXPj13+fl6nxjNUPJ+8JC+fI= +github.com/cert-manager/cert-manager v1.19.6/go.mod h1:uC9pPoxhw/+Jcv3eWBhYpN7jbqNlYwSZn7ioROZSPM4= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cozystack/apimachinery v0.0.0-20251219010959-1f91eabae46c h1:C2wIfH/OzhU9XOK/e6Ik9cg7nZ1z6fN4lf6a3yFdik8= -github.com/cozystack/apimachinery v0.0.0-20251219010959-1f91eabae46c/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= +github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cozystack/cozystack-scheduler/pkg/apis v0.1.1 h1:9uLa/8J4lRx3sNuaVH8UZqgZvnskHATPo5GxEKh0iSM= github.com/cozystack/cozystack-scheduler/pkg/apis v0.1.1/go.mod h1:kPeS9YPB4ENbvNINEkkp0SX8FB+gvwoOHGOHMT3Tg9Y= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -29,101 +29,128 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/emicklei/dot v1.10.0 h1:z17n0ce/FBMz3QbShSzVGhiW447Qhu7fljzvp3Gs6ig= -github.com/emicklei/dot v1.10.0/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= +github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= +github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= +github.com/emicklei/dot v1.11.0 h1:zsrhCuFHAJge/aZIC4N4LdHy5tqYu4tWEaUzIwdYj4Y= +github.com/emicklei/dot v1.11.0/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= -github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= +github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fluxcd/helm-controller/api v1.4.3 h1:CdZwjL1liXmYCWyk2jscmFEB59tICIlnWB9PfDDW5q4= -github.com/fluxcd/helm-controller/api v1.4.3/go.mod h1:0XrBhKEaqvxyDj/FziG1Q8Fmx2UATdaqLgYqmZh6wW4= -github.com/fluxcd/pkg/apis/acl v0.9.0 h1:wBpgsKT+jcyZEcM//OmZr9RiF8klL3ebrDp2u2ThsnA= -github.com/fluxcd/pkg/apis/acl v0.9.0/go.mod h1:TttNS+gocsGLwnvmgVi3/Yscwqrjc17+vhgYfqkfrV4= -github.com/fluxcd/pkg/apis/kustomize v1.13.0 h1:GGf0UBVRIku+gebY944icVeEIhyg1P/KE3IrhOyJJnE= -github.com/fluxcd/pkg/apis/kustomize v1.13.0/go.mod h1:TLKVqbtnzkhDuhWnAsN35977HvRfIjs+lgMuNro/LEc= -github.com/fluxcd/pkg/apis/meta v1.23.0 h1:fLis5YcHnOsyKYptzBtituBm5EWNx13I0bXQsy0FG4s= -github.com/fluxcd/pkg/apis/meta v1.23.0/go.mod h1:UWsIbBPCxYvoVklr2mV2uLFBf/n17dNAmKFjRfApdDo= -github.com/fluxcd/source-controller/api v1.7.4 h1:+EOVnRA9LmLxOx7J273l7IOEU39m+Slt/nQGBy69ygs= -github.com/fluxcd/source-controller/api v1.7.4/go.mod h1:ruf49LEgZRBfcP+eshl2n9SX1MfHayCcViAIGnZcaDY= -github.com/fluxcd/source-watcher/api/v2 v2.0.3 h1:SsVGAaMBxzvcgrOz/Kl6c2ybMHVqoiEFwtI+bDuSeSs= -github.com/fluxcd/source-watcher/api/v2 v2.0.3/go.mod h1:Nx3QZweVyuhaOtSNrw+oxifG+qrakPvjgNAN9qlUTb0= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/fluxcd/helm-controller/api v1.5.1 h1:yraWl0ImzO4yIy/N5d9i54N+OZxKuFZqjed8wrIjy8U= +github.com/fluxcd/helm-controller/api v1.5.1/go.mod h1:Yr0y7GKizbvQQGK5wBX6sGCZrTY86AN9n1PNEsji2XE= +github.com/fluxcd/pkg/apis/acl v0.11.0 h1:BxThatpA6qd5Qzwim6/hRu0gdAX0JxNm3PledDuJqXo= +github.com/fluxcd/pkg/apis/acl v0.11.0/go.mod h1:a87i2A7AlFO5N2J8CxtzaUCCDmuLLWOHwkKu3eJF5fY= +github.com/fluxcd/pkg/apis/kustomize v1.15.0 h1:p8wPIxdmn0vy0a664rsE9JKCfnliZz4HUsDcTy4ZOxA= +github.com/fluxcd/pkg/apis/kustomize v1.15.0/go.mod h1:XWdsx8P15OiMaQIvmUjYWdmD3zAwhl5q9osl5iCqcOk= +github.com/fluxcd/pkg/apis/meta v1.26.0 h1:dxP1FfBpTCYso6odzRcltVnnRuBb2VyhhgV0VX9YbUE= +github.com/fluxcd/pkg/apis/meta v1.26.0/go.mod h1:c7o6mJGLCMvNrfdinGZehkrdZuFT9vZdZNrn66DtVD0= +github.com/fluxcd/source-controller/api v1.8.5 h1:mLKc9YVMk46JCt1BQbkG6irkrpBZp95kiXh2+GYB6KQ= +github.com/fluxcd/source-controller/api v1.8.5/go.mod h1:sio4t49RDx+S1etHRFAEEw8qfVuw0KKlOg8bRVlEYPM= +github.com/fluxcd/source-watcher/api/v2 v2.1.1 h1:1LfT50ty+78MKKbschAZl28QbVqIyjaNq17KmW5wPJI= +github.com/fluxcd/source-watcher/api/v2 v2.1.1/go.mod h1:6M1BzBGQRoIuSenSQlfJHwMVVobFPiNPxXqfN0IILc4= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/fxamacker/cbor/v2 v2.9.4 h1:xwjVlxEMR3S605oUlgBjKLTTeGFciYPGYCtF/35LKGo= +github.com/fxamacker/cbor/v2 v2.9.4/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.4 h1:5eBSibVuSMbb/H6Elc0IIEFbkzCJi3lm94n0+U7Z0KY= +github.com/gkampitakis/ciinfo v0.3.4/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-snaps v0.5.23 h1:okh5QR48zpUjpWtu65AtqxdCY8huJq+dEDuUzd1PuKg= +github.com/gkampitakis/go-snaps v0.5.23/go.mod h1:uy3lVzCCRRsAwYqSocyw5fY8xRLCYEfqoOJNxr8HonM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.21.2 h1:AqQaNADVwq/VnkCmQg6ogE+M3FOsKTytwges0JdwVuA= -github.com/go-openapi/jsonpointer v0.21.2/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= -github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= -github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= -github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= -github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-openapi/jsonpointer v1.0.1 h1:2KxywRmNwJkT/FMBa3iRNHEaAxSJvjqoufQZy3au1Mg= +github.com/go-openapi/jsonpointer v1.0.1/go.mod h1:wI7ZYsFmbIi9nBXOZqgDaS/bqOchRGZjqxFli7FBYxY= +github.com/go-openapi/jsonreference v1.0.2 h1:oS4et8FOf3p3UQxEo4Xt0esijmBUM+F259Xl72OSZsc= +github.com/go-openapi/jsonreference v1.0.2/go.mod h1:TbUNSOo+fcorZjFaNoiSDSoaNnnZtqJtLGR1PuvE/Cs= +github.com/go-openapi/swag v0.29.2 h1:9n8frkcsuQRA0INU31VbxNxt1zBaHxrHYGEf5BUjFi4= +github.com/go-openapi/swag v0.29.2/go.mod h1:RkCiX1gCVXWgLOavDXE3ALJrQ/VO2ziiIzOUs3z10xo= +github.com/go-openapi/swag/cmdutils v0.29.2 h1:cXEzX/nWCODon251f+1HgL75/R4XXML5IZxgLCWMoro= +github.com/go-openapi/swag/cmdutils v0.29.2/go.mod h1:XziaSVzYqkDvpWXzASfWajw2YvJrLscq610O2XiyqJw= +github.com/go-openapi/swag/conv v0.29.2 h1:8c9shoB8l0QRSR6ymq1llHdBWDqpIgJfjTGHx3Vuhm0= +github.com/go-openapi/swag/conv v0.29.2/go.mod h1:AZS0YigTNf8qNtD6WqJz/N4RUNmpr76SyjFGkq4+p6Q= +github.com/go-openapi/swag/fileutils v0.29.2 h1:mdUL+Vw5ah1fO1AvFCoHeIyv7YZsCz2MN3KNqd/lLVI= +github.com/go-openapi/swag/fileutils v0.29.2/go.mod h1:7QKmmodjAebaq61lGVJa8ldLdGeMF4DSgYv8tXOGGo0= +github.com/go-openapi/swag/jsonutils v0.29.2 h1:uZNSD2/rJDYAfvsLYkykTzXuyxM3QpDKV9jhRHrMu6k= +github.com/go-openapi/swag/jsonutils v0.29.2/go.mod h1:ONTdNvj3Y+IXRzfa17E+Rgt2ns/qiSOYjIo2K7v2yDs= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.2 h1:w+Fd6EBOMGlC74GYGHqCLGyb3Bpams8TtNrgM/SG4jo= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.29.2/go.mod h1:HC2egPORFzbj/gf05Zrfgz8Mgplx3oW7hgGi7pjZVhM= +github.com/go-openapi/swag/loading v0.29.2 h1:QU1ry24e6r6Shoxe380Nx0X5iKkpRO+hJffsyJ2dVlY= +github.com/go-openapi/swag/loading v0.29.2/go.mod h1:DqilDjuiJKETecsS4TRopWG1acUDsfqN39ZmQhPu6uI= +github.com/go-openapi/swag/mangling v0.29.2 h1:ltdo3K0tTP4P6zWhVVWDU9D37/YYBuMJRdZxiS2Dtog= +github.com/go-openapi/swag/mangling v0.29.2/go.mod h1:RjDi4TnItAenS3cgCXSpIl1kKbjG7il6pnmQVl/8bp8= +github.com/go-openapi/swag/netutils v0.29.2 h1:dBli6jyUa93sDS/XYWShJzf4+nnVfFKfvY9SLUaUzB8= +github.com/go-openapi/swag/netutils v0.29.2/go.mod h1:1pM6Xg/Um8E1eps3umWl8Hme3ByWXOtHsvFC0CULHUE= +github.com/go-openapi/swag/pools v0.29.2 h1:PlGoDRF8WtSyXkedZZkpW3f9wDhFf3u8KtaBcmgmh8c= +github.com/go-openapi/swag/pools v0.29.2/go.mod h1:V3lhxVT4qDYRqc2MRSSbLsTYIYV/2HlHafhyEkmzLIc= +github.com/go-openapi/swag/stringutils v0.29.2 h1:lcnBxwAaysT3bMUVBfn9V/PkfVqkurpqufKU6rTUMGk= +github.com/go-openapi/swag/stringutils v0.29.2/go.mod h1:i9cdh7sGaa0nm3CqIvH5IM5h2QClk1wns2ktKeILvRI= +github.com/go-openapi/swag/typeutils v0.29.2 h1:O7aVvkTs3pXwikgtrPLenigEMdQFgONKRooQA9pgf2I= +github.com/go-openapi/swag/typeutils v0.29.2/go.mod h1:7+GDG+uz9Ke+eVt4rClZg7wklSDp8hT/mKNh/pC26TE= +github.com/go-openapi/swag/yamlutils v0.29.2 h1:IFKFFeDnuIwzfsuWRQU+rI8iL3g4XyAYjV/RlnlxPLM= +github.com/go-openapi/swag/yamlutils v0.29.2/go.mod h1:7MGqtcrK73sxQ4ceiyIf8qiXS/s09JLMdR5esK5euYQ= +github.com/go-openapi/testify/enable/yaml/v2 v2.8.0 h1:dvOQNZ2ovOdogn902NYq7i6jUI9Dma4qyjTXABY5YTY= +github.com/go-openapi/testify/enable/yaml/v2 v2.8.0/go.mod h1:3VwZZUT6nWcAjhIr2RypXxlsxd3mPPtXh3T8z65aRlQ= +github.com/go-openapi/testify/v2 v2.8.0 h1:19QDx5b57p8KjO2E4sgs/woL4Akp+FGLRlVm2xn0Rho= +github.com/go-openapi/testify/v2 v2.8.0/go.mod h1:4gUN8jC+RqOE2qjW/vDmfyTvl5BCht4qL7oca0odrak= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= -github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= -github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= -github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= -github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= -github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/cel-go v0.29.2 h1:ZtDxkeiMmz0mxbKDYiNkE5Lk7V5edMRcaaDf2jX002k= +github.com/google/cel-go v0.29.2/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= +github.com/google/gnostic-models v0.7.1 h1:SisTfuFKJSKM5CPZkffwi6coztzzeYUhc3v4yxLWH8c= +github.com/google/gnostic-models v0.7.1/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= -github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20260906184651-6331bc6350fe h1:QAinXoAFJdGQYztXn3VpFey7KCwpedbZ/EkzbplQ0cY= +github.com/google/pprof v0.0.0-20260906184651-6331bc6350fe/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= -github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= -github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 h1:FbSCl+KggFl+Ocym490i/EyXF4lPgLoUtcSWquBM0Rs= -github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0/go.mod h1:qOchhhIlmRcqk/O9uCo/puJlyo07YINaIqdZfZG3Jkc= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 h1:QGLs/O40yoNK9vmy4rhUGBVyMf1lISBGtXRpsu/Qu/o= +github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0/go.mod h1:hM2alZsMUni80N33RBe6J0e423LB+odMj7d3EMP9l20= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.4 h1:9ZJYjPEJpcleIKcqysXOo14NLQYwu23fzmKRFpIjPjc= +github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.4/go.mod h1:0xydIeg2omQ9DH6zYMP24Kk57793rnLUvP8fwdbFvz8= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= -github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= +github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -134,208 +161,184 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/onsi/ginkgo/v2 v2.23.3 h1:edHxnszytJ4lD9D5Jjc4tiDkPBZ3siDeJJkUZJJVkp0= -github.com/onsi/ginkgo/v2 v2.23.3/go.mod h1:zXTP6xIp3U8aVuXN8ENK9IXRaTjFnpVB9mGmaSRvxnM= -github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= -github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/ginkgo/v2 v2.33.0 h1:C8gBA6Uc2ZEubiV+SXiu5tZnMTwEmXHgkJwGozKtZf8= +github.com/onsi/ginkgo/v2 v2.33.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.43.1 h1:vGIPFuYrIO6/0Z09s0I0QQQgFchiX4+tb1re3MScJYo= +github.com/onsi/gomega v1.43.1/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= -github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= -github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= -github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= -github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= -github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= +github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= +github.com/prometheus/common v0.71.0 h1:9KDAKb7Mj3HEVKyFCK6Dc/HIwlBzZIN2l7/lrHl3KK8= +github.com/prometheus/common v0.71.0/go.mod h1:CLJ5H8TEsGX8bl31BdMkfhIZ+QmZ9tBPPotUxUbfcmk= +github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics= +github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.10.2 h1:G2SED73/qrAu6YwbdxOD6peLkCBI3z7L+ykJFTXJBBo= +github.com/sirupsen/logrus v1.10.2/go.mod h1:SLEg8TqYulVKKfIGHldVp2K2aYz2DKSVBq4g/H5bR7Q= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= -github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= -github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= +github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= -github.com/vmware-tanzu/velero v1.17.1 h1:ldKeiTuUwkThOw7zrUucNA1NwnLG66zl13YetWAoE0I= -github.com/vmware-tanzu/velero v1.17.1/go.mod h1:3KTxuUN6Un38JzmYAX+8U6j2k6EexGoNNxa8jrJML8U= +github.com/vmware-tanzu/velero v1.18.3 h1:9uwN5yxTWQl9PsF5ljX4xiG+7VIq3I9mxOQ0DyyZ31A= +github.com/vmware-tanzu/velero v1.18.3/go.mod h1:tiD1WgZxic89/1eWBW+C5kEYiZKZkztJB2n3dhNbdiw= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.4.2 h1:IrUHp260R8c+zYx/Tm8QZr04CX+qWS5PGfPdevhdm1I= -go.etcd.io/bbolt v1.4.2/go.mod h1:Is8rSHO/b4f3XigBC0lL0+4FwAQv3HXEEIgFMuKHceM= -go.etcd.io/etcd/api/v3 v3.6.4 h1:7F6N7toCKcV72QmoUKa23yYLiiljMrT4xCeBL9BmXdo= -go.etcd.io/etcd/api/v3 v3.6.4/go.mod h1:eFhhvfR8Px1P6SEuLT600v+vrhdDTdcfMzmnxVXXSbk= -go.etcd.io/etcd/client/pkg/v3 v3.6.4 h1:9HBYrjppeOfFjBjaMTRxT3R7xT0GLK8EJMVC4xg6ok0= -go.etcd.io/etcd/client/pkg/v3 v3.6.4/go.mod h1:sbdzr2cl3HzVmxNw//PH7aLGVtY4QySjQFuaCgcRFAI= -go.etcd.io/etcd/client/v3 v3.6.4 h1:YOMrCfMhRzY8NgtzUsHl8hC2EBSnuqbR3dh84Uryl7A= -go.etcd.io/etcd/client/v3 v3.6.4/go.mod h1:jaNNHCyg2FdALyKWnd7hxZXZxZANb0+KGY+YQaEMISo= -go.etcd.io/etcd/pkg/v3 v3.6.4 h1:fy8bmXIec1Q35/jRZ0KOes8vuFxbvdN0aAFqmEfJZWA= -go.etcd.io/etcd/pkg/v3 v3.6.4/go.mod h1:kKcYWP8gHuBRcteyv6MXWSN0+bVMnfgqiHueIZnKMtE= -go.etcd.io/etcd/server/v3 v3.6.4 h1:LsCA7CzjVt+8WGrdsnh6RhC0XqCsLkBly3ve5rTxMAU= -go.etcd.io/etcd/server/v3 v3.6.4/go.mod h1:aYCL/h43yiONOv0QIR82kH/2xZ7m+IWYjzRmyQfnCAg= -go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ= -go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo= +go.etcd.io/bbolt v1.5.0 h1:S7GAl7Fxv12yohbwFfIbQCGDWbQbtDGPET4P/bD4lxU= +go.etcd.io/bbolt v1.5.0/go.mod h1:mkltfYE5aUHQxUct9N9V+Kp7aSjFqjgrhcXIS70Lrdk= +go.etcd.io/etcd/api/v3 v3.7.0 h1:WZlGK7pRtYGDB8ti8wkrQ5D2oWGMbtNL9VA5e+vF2Fg= +go.etcd.io/etcd/api/v3 v3.7.0/go.mod h1:EcTihnwAQ0BQNh5dfAdaFVFdchuo7EP0HlX7TV3jz/A= +go.etcd.io/etcd/client/pkg/v3 v3.7.0 h1:sW9njJzS3vXKcAJjjLQ4nk+avNUJ12Bcijcx8ehUskE= +go.etcd.io/etcd/client/pkg/v3 v3.7.0/go.mod h1:cnzZGIUzSfjEwLC6UBVsSXlEK1eepS/JUD7wE6PLRT0= +go.etcd.io/etcd/client/v3 v3.7.0 h1:5MHO37VbPB87VRPKUXEcicjeQWiTSjpPv3Ume8xPx20= +go.etcd.io/etcd/client/v3 v3.7.0/go.mod h1:DJ382WuwjmbowjPDyaaQ0idWXy4dh91XRhe4FOrb9vM= +go.etcd.io/etcd/pkg/v3 v3.7.0 h1:aUBHGzXnYL2m3AlZfKp0ZhHQLVZIKfu9QsIlYdr+mxc= +go.etcd.io/etcd/pkg/v3 v3.7.0/go.mod h1:fDQYyc8rOC1Yl4EZLh0O1OjHzwJW7jAly80+BpD1M9o= +go.etcd.io/etcd/server/v3 v3.7.0 h1:ScdUdN8ljuimp0lZaNq0otLMrHcFSFT+dQyT1j7JSFo= +go.etcd.io/etcd/server/v3 v3.7.0/go.mod h1:v7N1dPdSW2vzyxGsbDwK4X0sCe3SYNlQ+9eNIMGzdxQ= +go.etcd.io/raft/v3 v3.7.0 h1:BGzlwx07bLv8PW6OU5HObuz1y4hlPZUXA07pM1mPUh4= +go.etcd.io/raft/v3 v3.7.0/go.mod h1:6gX6T2X907DjnjsFLODnTxba77stjs84W9gTTI0GUNA= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 h1:CqXxU8VOmDefoh0+ztfGaymYbhdB/tT3zs79QaZTNGY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0/go.mod h1:BuhAPThV8PBHBvg8ZzZ/Ok3idOdhWIodywz2xEcRbJo= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0 h1:RAE+JPfvEmvy+0LzyUA25/SGawPwIUbZ6u0Wug54sLc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.43.0/go.mod h1:AGmbycVGEsRx9mXMZ75CsOyhSP6MFIcj/6dnG+vhVjk= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= -go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= -go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 h1:B2h3uqicet1CT2N5TOFhS+Gq++9i0/CLmaxvhmhtP5s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0/go.mod h1:dylvB+ZiiwMvsDij9O84Uy7SijLgHMX4mbkncds+4Sw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0 h1:w53CDeOA/Kurp7yRsegSr6pbbr759dOvJ+yNmWM6Hxs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.46.0/go.mod h1:BOmGMCbAtvcJiSJ+hLuhgPLdDbimnraSl8irz3iY8sY= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= -golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= -golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= -golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= -gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU= +golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= +golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= +golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= +golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= +golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= +gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= +gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459 h1:GS9OIt/j7c8bvBjYNgnKQysVfmV7e4jM0H8ZK95G4t8= +google.golang.org/genproto/googleapis/api v0.0.0-20260921155816-b14227669459/go.mod h1:PX5/4vemwVoXtwEcRDWwcR1/r0qrosfx3qoVADMwnVE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 h1:b0xCahf3FK2m2Cv0p4vTozGPWncCvLfwV86UNg8xWU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= -k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= -k8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI= -k8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc= -k8s.io/apiserver v0.34.1 h1:U3JBGdgANK3dfFcyknWde1G6X1F4bg7PXuvlqt8lITA= -k8s.io/apiserver v0.34.1/go.mod h1:eOOc9nrVqlBI1AFCvVzsob0OxtPZUCPiUJL45JOTBG0= -k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= -k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= -k8s.io/component-base v0.34.1 h1:v7xFgG+ONhytZNFpIz5/kecwD+sUhVE6HU7qQUiRM4A= -k8s.io/component-base v0.34.1/go.mod h1:mknCpLlTSKHzAQJJnnHVKqjxR7gBeHRv0rPXA7gdtQ0= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kms v0.34.1 h1:iCFOvewDPzWM9fMTfyIPO+4MeuZ0tcZbugxLNSHFG4w= -k8s.io/kms v0.34.1/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= -k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 h1:liMHz39T5dJO1aOKHLvwaCjDbf07wVh6yaUlTpunnkE= -k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= -k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d h1:wAhiDyZ4Tdtt7e46e9M5ZSAJ/MnPGPs+Ki1gHw4w1R0= -k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +k8s.io/api v0.37.1 h1:l6N77U7tjwB5L056bgrBTJIEdevac/naBZ3iSvDNfpM= +k8s.io/api v0.37.1/go.mod h1:zSlbB1YpJ1YQlFVQy20UYll81UJSJJUMLhkhvg6Z78M= +k8s.io/apiextensions-apiserver v0.37.1 h1:7fIQG8eThDSTVYBWg/DOpI8v5wYfKCo0N8/TDMjj+zY= +k8s.io/apiextensions-apiserver v0.37.1/go.mod h1:1Q3ujWwHgnhBoGsdmGORflwK1sDFmuaqH1OsMDWk2Gs= +k8s.io/apimachinery v0.37.1 h1:hGCYyvKHCwtwMitj2vU4vYx0Z16N9GyZk9BBnz0wDAE= +k8s.io/apimachinery v0.37.1/go.mod h1:jF84AyUi/IRIXRot5f+lm6MpxoWI+F1XgjaMmwCdTFw= +k8s.io/apiserver v0.37.1 h1:dUsNwjsvItyVKNpC1N0AhQ0OBSiUJKfWz004vPifOe0= +k8s.io/apiserver v0.37.1/go.mod h1:EWXW6PntGfOC2uD9GGVobnOExgFoX1z0hZo7f7Rs7Ng= +k8s.io/client-go v0.37.1 h1:QTv/5ha4jAHtW9qxxVBkQVFBRDb4jHfFopQqqMdc+wM= +k8s.io/client-go v0.37.1/go.mod h1:dnAPtTnCNY38Ho04D2KdY1F4IKausa9UbqaAZKl60SY= +k8s.io/component-base v0.37.1 h1:93DMmlENnK7gNLkL4pMqLO5M/HVf3p3sM4q/GasLveY= +k8s.io/component-base v0.37.1/go.mod h1:bBrdziT4dreQG5lzTaPVxBwsZe1oxphG+ZVdD56dQWQ= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kms v0.37.1 h1:MD1kA8AdFF/N5sGP8gnvL6vTo/oeaQXbmLjWmmJHLc8= +k8s.io/kms v0.37.1/go.mod h1:0U6zfwLkfJr9O/jLgz2GARnKLh2uIhkq2EeaU66J5ow= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= +k8s.io/streaming v0.37.1 h1:TpzVfQeFuVndn2g9mFqxy1UcUYPwDzqjUmwR/IzJCWc= +k8s.io/streaming v0.37.1/go.mod h1:APlJR26ZWRcVy5bIEj0QRrKUXROtBHPcxl2NT7EAzPU= +k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0xvhQ5U686DPurkE= +k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 h1:/YpDJ4vReG7ZmzSpBGxduXgywWkJU9zHubgJG03MT+Y= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0/go.mod h1:tJo1aepTXyR+8Xs3sUsGBDk4Ub2AM5dPAPKJx0mpm5c= sigs.k8s.io/container-object-storage-interface-api v0.1.0 h1:8tB6JFQhbQIC1hwGQ+q4+tmSSNfjKemb7bFI6C0CK/4= sigs.k8s.io/container-object-storage-interface-api v0.1.0/go.mod h1:YiB+i/UGkzqgODDhRG3u7jkbWkQcoUeLEJ7hwOT/2Qk= -sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= -sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= +sigs.k8s.io/controller-runtime v0.25.1 h1:BKgU9OeE8xv8EbbM8cY0NVzTQs35rokkdq1jh12fMb4= +sigs.k8s.io/controller-runtime v0.25.1/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc= sigs.k8s.io/gateway-api v1.4.1 h1:NPxFutNkKNa8UfLd2CMlEuhIPMQgDQ6DXNKG9sHbJU8= sigs.k8s.io/gateway-api v1.4.1/go.mod h1:AR5RSqciWP98OPckEjOjh2XJhAe2Na4LHyXD2FUY7Qk= -sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= -sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/json v0.0.0-20260909141634-11ed52e25bc5 h1:k9jo7ED/fsfOcz3L1P20P5tjTNtsdQNwMOG2aKzHrl8= +sigs.k8s.io/json v0.0.0-20260909141634-11ed52e25bc5/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/admin-kubeconfig-invariant.bats b/hack/admin-kubeconfig-invariant.bats index a4b98b001f..fb764f557b 100644 --- a/hack/admin-kubeconfig-invariant.bats +++ b/hack/admin-kubeconfig-invariant.bats @@ -18,12 +18,13 @@ # project's CI runners and on the maintainer workstation. # ----------------------------------------------------------------------------- +load test_helper + @test "every Deployment mounting admin-kubeconfig has optional:true and wait-for-kubeconfig init" { values_file="packages/apps/kubernetes/tests/values-ci.yaml" [ -f "$values_file" ] tmp=$(mktemp -d) - trap 'rm -rf "$tmp"' EXIT helm template invariant packages/apps/kubernetes \ --namespace tenant-root \ @@ -69,6 +70,7 @@ fi echo "Invariant holds for $matched Deployment(s)" + rm -rf "$tmp" } @test "chart emits zero admin-kubeconfig Deployments when tenant has no etcd DataStore" { @@ -81,7 +83,6 @@ # with etcd empty and asserts no Deployment references the admin-kubeconfig # Secret. tmp=$(mktemp -d) - trap 'rm -rf "$tmp"' EXIT helm template invariant packages/apps/kubernetes \ --namespace tenant-root \ @@ -108,6 +109,7 @@ fi echo "No admin-kubeconfig Deployments rendered for empty etcd (as expected)" + rm -rf "$tmp" } @test "chart emits zero admin-kubeconfig HelmReleases when tenant has no etcd DataStore" { @@ -118,7 +120,6 @@ # the operator sees and contradicting the "awaiting-etcd beacon only" # contract of the soft-skip path. tmp=$(mktemp -d) - trap 'rm -rf "$tmp"' EXIT helm template invariant packages/apps/kubernetes \ --namespace tenant-root \ @@ -142,4 +143,5 @@ fi echo "No admin-kubeconfig HelmReleases rendered for empty etcd (as expected)" + rm -rf "$tmp" } diff --git a/hack/alert-severity-contract.bats b/hack/alert-severity-contract.bats new file mode 100644 index 0000000000..e1315eb966 --- /dev/null +++ b/hack/alert-severity-contract.bats @@ -0,0 +1,333 @@ +#!/usr/bin/env bats + +load test_helper + +# Alerta's default alarm model, the set the webhook accepts. `none` is left out +# on purpose: the Alertmanager config routes severity="none" to a blackhole +# receiver, so such an alert is never delivered, which is the outcome this +# contract exists to catch. Rule files under packages/**/charts/** are out of +# scope: they are re-vendored by `make update`, so a bad severity there is +# fixed upstream or in patches/, not by editing the label. +ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown " + +helm_actions_stripped() { + awk ' + function strip_comments(s, pre, rest) { + while (1) { + if (incomment) { + if (match(s, /\*\/[ \t]*-?[}][}]/)) { s = substr(s, RSTART + RLENGTH); incomment = 0 } else { return "" } + } + if (match(s, /[{][{]-?[ \t]*\/\*/)) { + pre = substr(s, 1, RSTART - 1); rest = substr(s, RSTART + RLENGTH) + if (match(rest, /\*\/[ \t]*-?[}][}]/)) { s = pre substr(rest, RSTART + RLENGTH) } else { incomment = 1; return pre } + } else { return s } + } + } + { + line = strip_comments($0) + if (incomment && line ~ /^[ \t]*$/) { next } + if (line ~ /^[ \t]*[{][{].*[}][}][ \t]*$/) { next } + gsub(/[{][{][^}]*[}][}]/, "TEMPLATED", line) + print line + } + ' "$1" +} + +alert_severity_rows() { + severity_root=${ALERT_SEVERITY_ROOT:-"$(dirname "${BATS_TEST_FILENAME:-$0}")/.."} + severity_files=$(mktemp) || return 1 + severity_rows=$(mktemp) || { + rm -f "$severity_files" + return 1 + } + severity_stripped=$(mktemp) || { + rm -f "$severity_files" "$severity_rows" + return 1 + } + + if ! ( + cd "$severity_root" && + git grep -Il -E "kind:[[:space:]]*[\"']?(PrometheusRule|VMRule)[\"']?([[:space:]#,}]|\$)" -- \ + 'packages/**/*.yaml' \ + 'packages/**/*.yml' \ + ':(exclude)packages/**/charts/**' + ) > "$severity_files"; then + rm -f "$severity_files" "$severity_rows" "$severity_stripped" + return 1 + fi + + if ! : > "$severity_rows"; then + rm -f "$severity_files" "$severity_rows" "$severity_stripped" + return 1 + fi + while IFS= read -r file; do + if ! (cd "$severity_root" && helm_actions_stripped "$file" > "$severity_stripped" && FILE="$file" yq 'select(.kind == "PrometheusRule" or .kind == "VMRule") | .spec.groups[]? | .rules[]? | select(has("alert")) | [strenv(FILE), .alert, ((.labels.severity // "") | tostring)] | @tsv' "$severity_stripped") >> "$severity_rows"; then + echo "$file: failed to parse alert rules" >&2 + rm -f "$severity_files" "$severity_rows" "$severity_stripped" + return 1 + fi + if ! severity_hidden=$(yq 'select(.kind == "PrometheusRule" or .kind == "VMRule") | ((select(((.spec.groups // []) | length) == 0) | ["a rule document without rule groups"] | @tsv), (.spec.groups[]? | select(((.rules // []) | map(select(tag == "!!map" and (has("alert") or has("record")))) | length) == 0) | ["rule group " + ((.name // "") | tostring)] | @tsv))' "$severity_stripped"); then + echo "$file: failed to parse alert rules" >&2 + rm -f "$severity_files" "$severity_rows" "$severity_stripped" + return 1 + fi + severity_hidden=$(printf '%s\n' "$severity_hidden" | sed '/^[[:space:]]*$/d') + if [ -n "$severity_hidden" ]; then + printf '%s\n' "$severity_hidden" | while IFS= read -r what; do + echo "$file: $what shows no alert or recording rule once template actions are stripped" >&2 + done + rm -f "$severity_files" "$severity_rows" "$severity_stripped" + return 1 + fi + done < "$severity_files" + + if ! sed '/^[[:space:]]*$/d' "$severity_rows"; then + rm -f "$severity_files" "$severity_rows" "$severity_stripped" + return 1 + fi + rm -f "$severity_files" "$severity_rows" "$severity_stripped" +} + +alert_severity_contract() { + rows=$(mktemp) || return 1 + if ! alert_severity_rows > "$rows"; then + rm -f "$rows" + return 1 + fi + bad=0 + tab=$(printf '\t') + while IFS="$tab" read -r file alert severity; do + accepted=0 + for allowed in $ALERT_SEVERITY_ALLOWED; do + if [ "$allowed" = "$severity" ]; then + accepted=1 + break + fi + done + if [ "$accepted" -eq 0 ]; then + echo "$file: $alert -> '$severity'" >&2 + bad=$((bad + 1)) + fi + done < "$rows" + rm -f "$rows" + [ "$bad" -eq 0 ] || return 1 +} + +@test "alert rules are discovered" { + rows=$(mktemp) || return 1 + if ! alert_severity_rows > "$rows"; then + rm -f "$rows" + return 1 + fi + count=$(wc -l < "$rows" | tr -d ' ') + rm -f "$rows" + [ "$count" -gt 0 ] +} + +@test "every alert rule carries a severity the alert sink accepts" { + alert_severity_contract +} + +@test "a parser failure cannot leave the contract green" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: valid' \ + ' rules:' \ + ' - alert: ValidAlert' \ + ' labels:' \ + ' severity: warning' \ + > "$fixture/packages/system/example/alerts/00-valid.yaml" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec: [' \ + > "$fixture/packages/system/example/alerts/99-malformed.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-valid.yaml packages/system/example/alerts/99-malformed.yaml + + parser_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_rows >/dev/null 2>&1 || parser_succeeded=$? + rm -rf "$fixture" + [ "$parser_succeeded" -ne 0 ] +} + +@test "template rule sources cannot bypass the severity contract" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/extra/example/templates" + printf '%s\n' \ + 'apiVersion: monitoring.coreos.com/v1' \ + 'kind: PrometheusRule' \ + 'spec:' \ + ' groups:' \ + ' - name: template' \ + ' rules:' \ + ' - alert: InvalidTemplateSeverity' \ + ' labels:' \ + ' severity: warn' \ + > "$fixture/packages/extra/example/templates/prometheus-rules.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/extra/example/templates/prometheus-rules.yaml + + contract_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$? + rm -rf "$fixture" + [ "$contract_succeeded" -ne 0 ] +} + +@test "helm template actions do not hide a rule from the contract" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/templates" + printf '%s\n' \ + '{{- if .Values.alerts.enabled }}' \ + '{{- /*' \ + 'Rules gated on a value, behind a comment that spans lines.' \ + '*/}}' \ + 'apiVersion: monitoring.coreos.com/v1' \ + 'kind: PrometheusRule' \ + 'metadata:' \ + ' name: example' \ + ' namespace: {{ .Release.Namespace }}' \ + 'spec:' \ + ' groups:' \ + ' - name: example' \ + ' rules:' \ + ' {{- /* one-line comment */}}' \ + ' - alert: TemplatedRuleWithBadSeverity' \ + ' expr: up == 0' \ + ' labels:' \ + ' severity: warn' \ + ' annotations:' \ + ' description: {{ "{{ $labels.instance }}" }} is down' \ + '{{- end }}' \ + > "$fixture/packages/system/example/templates/alerts.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/templates/alerts.yaml + + rows=$(mktemp) + parser_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_rows > "$rows" 2>/dev/null || parser_succeeded=$? + found=$(grep -c 'TemplatedRuleWithBadSeverity' "$rows" || true) + contract_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$? + rm -rf "$fixture" "$rows" + [ "$parser_succeeded" -eq 0 ] + [ "$found" -eq 1 ] + [ "$contract_succeeded" -ne 0 ] +} + +@test "two accepted words in one severity are rejected" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: joined' \ + ' rules:' \ + ' - alert: JoinedSeverity' \ + ' labels:' \ + ' severity: critical major' \ + > "$fixture/packages/system/example/alerts/00-joined.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-joined.yaml + + contract_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$? + rm -rf "$fixture" + [ "$contract_succeeded" -ne 0 ] +} + +@test "quoted kinds and trailing comments do not hide a rule source" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: "VMRule"' \ + 'spec:' \ + ' groups:' \ + ' - name: quoted' \ + ' rules:' \ + ' - alert: QuotedKindSeverity' \ + ' labels:' \ + ' severity: warn' \ + > "$fixture/packages/system/example/alerts/00-quoted.yaml" + printf '%s\n' \ + 'apiVersion: monitoring.coreos.com/v1' \ + 'kind: PrometheusRule # served by the operator' \ + 'spec:' \ + ' groups:' \ + ' - name: commented' \ + ' rules:' \ + ' - alert: CommentedKindSeverity' \ + ' labels:' \ + ' severity: warn' \ + > "$fixture/packages/system/example/alerts/01-commented.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-quoted.yaml packages/system/example/alerts/01-commented.yaml + + rows=$(mktemp) + ALERT_SEVERITY_ROOT="$fixture" alert_severity_rows > "$rows" 2>/dev/null || true + found=$(grep -cE 'QuotedKindSeverity|CommentedKindSeverity' "$rows" || true) + contract_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$? + rm -rf "$fixture" "$rows" + [ "$found" -eq 2 ] + [ "$contract_succeeded" -ne 0 ] +} + +@test "a rule group that only template actions fill fails the contract" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/templates" + printf '%s\n' \ + 'apiVersion: monitoring.coreos.com/v1' \ + 'kind: PrometheusRule' \ + 'metadata:' \ + ' name: example' \ + 'spec:' \ + ' groups:' \ + ' - name: generated' \ + ' rules:' \ + ' {{- include "example.rules" . | nindent 4 }}' \ + > "$fixture/packages/system/example/templates/alerts.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/templates/alerts.yaml + + errors=$(mktemp) + parser_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_rows >/dev/null 2>"$errors" || parser_succeeded=$? + named=$(grep -c '^packages/system/example/templates/alerts.yaml: rule group generated shows no alert or recording rule' "$errors" || true) + contract_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$? + rm -rf "$fixture" "$errors" + [ "$parser_succeeded" -ne 0 ] + [ "$named" -eq 1 ] + [ "$contract_succeeded" -ne 0 ] +} + +@test "a group of recording rules alone passes the contract" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: recording' \ + ' rules:' \ + ' - record: job:up:sum' \ + ' expr: sum by (job) (up)' \ + > "$fixture/packages/system/example/alerts/00-recording.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-recording.yaml + + contract_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$? + rm -rf "$fixture" + [ "$contract_succeeded" -eq 0 ] +} diff --git a/hack/app-cleanup-hooks_test.bats b/hack/app-cleanup-hooks_test.bats new file mode 100644 index 0000000000..672dcc541c --- /dev/null +++ b/hack/app-cleanup-hooks_test.bats @@ -0,0 +1,516 @@ +#!/usr/bin/env bats +# Behavioural tests for the two post-delete cleanup hooks that reclaim storage +# and ACME objects when an app release is uninstalled: +# +# packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml +# packages/apps/harbor/templates/hooks/cleanup.yaml +# +# The scripts are extracted from the rendered charts and RUN against a fake +# kubectl, because the property under test is the container's exit code and no +# amount of matching against the manifest text establishes it. The helm-unittest +# suites in packages/apps/*/tests/ match the source text instead. +# +# What is pinned here: +# * a delete that failed makes the hook exit non-zero. Without that, Helm +# records the hook as succeeded and the hook-succeeded delete policy removes +# the Job together with the pod that held the only account of the failure; +# * one failed delete does not skip the deletes that follow it; +# * an object that is already gone stays a success, so a re-run over an +# already-clean namespace exits 0; +# * a listing that FAILED is distinguishable from one that returned nothing. +# +# Run from the repo root through `make bats-unit-tests`; relative paths resolve +# against that cwd. The file also remains compatible with the legacy +# `hack/cozytest.sh` translator, so each @test builds and removes its fixture +# inline instead of relying on setup()/teardown(). No EXIT +# trap: docs/agents/e2e-testing.md bans them in hack/*.bats, and under the bats +# binary a test that installs one and then fails prints no TAP line at all. + +load test_helper + +QDRANT_CHART=packages/apps/qdrant +HARBOR_CHART=packages/apps/harbor + +# render_hook [extra helm args…] +# Extract the named Job's shell script from the rendered chart into . The +# non-empty check must be an early `return 1`: hack/cozytest.sh rewrites every +# line matching ^}$ into `return 0` plus `}`, helpers included, so a check left +# in last-command position has its status discarded. +render_hook() { + chart=$1 release=$2 namespace=$3 job=$4 out=$5 + shift 5 + helm template "$release" "$chart" --namespace "$namespace" "$@" \ + | yq eval "select(.kind == \"Job\" and .metadata.name == \"$job\") | .spec.template.spec.containers[0].command[2]" - > "$out" + [ -s "$out" ] || return 1 +} + +# write_fake_kubectl +# Emit a fake `kubectl` that logs every call to $KLOG and models the three +# kubectl behaviours these hooks depend on: +# * a listing prints the fixture held in $PVC_LIST / $CHALLENGE_LIST / +# $SOLVER_LIST and exits 0 — or, when its kind is named in $FAIL_GET, prints +# an error and exits 1 having printed nothing, which is what makes "the +# listing failed" and "the listing was empty" different events; +# * a delete whose kind is named in $FAIL_DELETE exits 1; +# * a delete of an object absent from $EXISTING is a NotFound: exit 0 under +# --ignore-not-found, exit 1 without it. Dropping that flag from a hook +# therefore turns the already-gone cases red rather than passing silently. +# A label-selector delete matches whatever is present and always exits 0, as +# kubectl's does. No line below is a bare column-0 `}`, so cozytest.sh's parser +# leaves the fixture intact. +write_fake_kubectl() { + cat > "$1/kubectl" <<'KEOF' +#!/bin/sh +echo "$*" >> "$KLOG" +verb=$1 +kind=${2:-} + +# The by-name deletes put the object name last. Flags and the value that +# follows -n / -l are skipped, so what survives is the trailing positional. +obj="" +skip=0 +for a in "$@"; do + if [ "$skip" = 1 ]; then skip=0; continue; fi + case "$a" in + -n|-l) skip=1 ;; + -*) ;; + *) obj=$a ;; + esac +done + +if [ "$verb" = get ]; then + case " ${FAIL_GET:-} " in + *" $kind "*) echo "error: the server could not list $kind" >&2; exit 1 ;; + esac + case "$kind" in + pvc) printf '%s' "${PVC_LIST:-}" ;; + challenge) printf '%s' "${CHALLENGE_LIST:-}" ;; + ingress) printf '%s' "${SOLVER_LIST:-}" ;; + *) echo "fake kubectl: unexpected get kind $kind" >&2; exit 64 ;; + esac + exit 0 +fi + +if [ "$verb" != delete ]; then + echo "fake kubectl: unexpected verb $verb" >&2 + exit 64 +fi + +# qdrant deletes through xargs as `delete -n NS --ignore-not-found …`, so +# there is no kind argument on that call; it is always a PVC delete. +case "$kind" in -*) kind=pvc ;; esac + +case " ${FAIL_DELETE:-} " in + *" $kind "*) echo "Error from server: failed to delete $kind $obj" >&2; exit 1 ;; +esac +case "$*" in *" -l "*) exit 0 ;; esac +case " ${EXISTING:-} " in + *" $obj "*) exit 0 ;; +esac +case "$*" in + *--ignore-not-found*) exit 0 ;; +esac +echo "Error from server (NotFound): $obj not found" >&2 +exit 1 +KEOF + chmod 0755 "$1/kubectl" +} + +@test "a render that produces no script fails instead of passing as a no-op" { + # Guards the guard: every test below runs whatever render_hook produced, so an + # empty render would satisfy the exit-0 assertions without exercising a line + # of the hook. This also fails if the emptiness check is ever moved into + # last-command position, where cozytest.sh's `return 0` rewrite swallows it. + tmp=$(mktemp -d) + if render_hook "$tmp/no-such-chart" r ns r-cleanup "$tmp/hook.sh" 2>/dev/null; then + echo "FAIL: render_hook reported success with no chart to render" + false + fi + if render_hook "$QDRANT_CHART" qdrant-test tenant-test no-such-job "$tmp/hook.sh"; then + echo "FAIL: render_hook reported success for a Job name the chart never emits" + false + fi + rm -rf "$tmp" +} + +@test "qdrant hook exits 0 and deletes exactly the release's data PVCs" { + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$QDRANT_CHART" qdrant-test tenant-test qdrant-test-qdrant-cleanup "$tmp/hook.sh" + + export KLOG="$tmp/calls" + # Two of this release's data PVCs, one belonging to the sibling release + # "qdrant-test-2", and one unrelated claim. + export PVC_LIST='persistentvolumeclaim/qdrant-storage-qdrant-test-0 +persistentvolumeclaim/qdrant-storage-qdrant-test-1 +persistentvolumeclaim/qdrant-storage-qdrant-test-2-0 +persistentvolumeclaim/db-qdrant-test-0' + export EXISTING='persistentvolumeclaim/qdrant-storage-qdrant-test-0 persistentvolumeclaim/qdrant-storage-qdrant-test-1' + + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc when every delete succeeded" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q 'PVC cleanup complete' "$tmp/out"; then + echo "FAIL: hook did not report completion" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q 'delete .*qdrant-storage-qdrant-test-0' "$tmp/calls"; then + echo "FAIL: the release's data PVCs were never deleted" + cat "$tmp/calls" + false + fi + if grep -q 'qdrant-storage-qdrant-test-2-0' "$tmp/calls"; then + echo "FAIL: the sibling release's PVC was included in the delete" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "qdrant hook exits 0 when the data PVCs are already gone" { + # The idempotent re-run: uninstalling twice, or uninstalling a release whose + # storage was reclaimed by hand, must not be reported as a failed cleanup. + # Covered in both shapes it takes — nothing matched the selector at all, and + # the delete itself came back NotFound. + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$QDRANT_CHART" qdrant-test tenant-test qdrant-test-qdrant-cleanup "$tmp/hook.sh" + + export KLOG="$tmp/calls" + export PVC_LIST='' + export EXISTING='' + + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc with no matching PVCs in the namespace" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q 'nothing to delete' "$tmp/out"; then + echo "FAIL: hook did not report an empty namespace" + cat "$tmp/out" + false + fi + + # Same release, but the PVCs are listed and disappear before the delete lands. + : > "$tmp/calls" + export PVC_LIST='persistentvolumeclaim/qdrant-storage-qdrant-test-0' + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out2" 2> "$tmp/err2" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc on a NotFound delete, which --ignore-not-found makes a success" + cat "$tmp/out2" "$tmp/err2" + false + fi + if ! grep -q 'delete .*qdrant-storage-qdrant-test-0' "$tmp/calls"; then + echo "FAIL: the delete was never attempted, so the NotFound path was not exercised" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "qdrant hook exits non-zero and names the PVCs when the delete fails" { + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$QDRANT_CHART" qdrant-test tenant-test qdrant-test-qdrant-cleanup "$tmp/hook.sh" + + export KLOG="$tmp/calls" + export PVC_LIST='persistentvolumeclaim/qdrant-storage-qdrant-test-0' + export EXISTING="$PVC_LIST" + export FAIL_DELETE='pvc' + + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -eq 0 ]; then + echo "FAIL: hook exited 0 although the PVC delete failed" + cat "$tmp/out" "$tmp/err" + false + fi + # The message must name what was left behind, not merely that something was. + # Anchored on the hook's own `ERROR: ` prefix: the fake kubectl names the + # object in its own diagnostic too, so an unanchored match is satisfied by + # the fake and stays green even when the hook stops naming anything. + if ! grep -q '^ERROR: .*qdrant-storage-qdrant-test-0' "$tmp/err"; then + echo "FAIL: the failure was not attributed to a named PVC" + cat "$tmp/err" + false + fi + if grep -q 'PVC cleanup complete' "$tmp/out"; then + echo "FAIL: hook reported completion after a failed delete" + cat "$tmp/out" + false + fi + rm -rf "$tmp" +} + +@test "qdrant hook exits non-zero when the PVC listing fails instead of reporting nothing to delete" { + # A failed listing used to reach the filter as an empty string, so the hook + # announced an empty namespace and exited 0 without having looked. + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$QDRANT_CHART" qdrant-test tenant-test qdrant-test-qdrant-cleanup "$tmp/hook.sh" + + export KLOG="$tmp/calls" + export PVC_LIST='' + export EXISTING='' + export FAIL_GET='pvc' + + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -eq 0 ]; then + echo "FAIL: hook exited 0 although it never managed to list the PVCs" + cat "$tmp/out" "$tmp/err" + false + fi + if grep -q 'nothing to delete' "$tmp/out"; then + echo "FAIL: a failed listing was reported as an empty namespace" + cat "$tmp/out" + false + fi + if ! grep -q 'failed to list PVCs' "$tmp/err"; then + echo "FAIL: the failed listing was not named" + cat "$tmp/err" + false + fi + rm -rf "$tmp" +} + +@test "harbor hook exits 0 and sweeps PVCs, Certificate and ACME solvers" { + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$HARBOR_CHART" harbor-test tenant-test harbor-test-cleanup "$tmp/hook.sh" \ + --set '_namespace.host=example.org' --set '_cluster.solver=http01' + + export KLOG="$tmp/calls" + export CHALLENGE_LIST='harbor-test-cert-1 ' + export SOLVER_LIST='cm-acme-http-solver-abcde ' + export EXISTING='harbor-test-ingress-tls harbor-test-cert-1 cm-acme-http-solver-abcde' + + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc when every delete succeeded" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q 'Cleanup complete' "$tmp/out"; then + echo "FAIL: hook did not report completion" + cat "$tmp/out" "$tmp/err" + false + fi + for expected in 'delete pvc' 'delete certificate' 'delete challenge' 'delete ingress'; do + if ! grep -q "$expected" "$tmp/calls"; then + echo "FAIL: the hook never issued: $expected" + cat "$tmp/calls" + false + fi + done + rm -rf "$tmp" +} + +@test "harbor hook exits 0 when every object is already gone" { + # No in-flight ACME challenge and no solver Ingress is the ordinary case, and + # the Certificate is usually collected with the release before the hook runs. + # All of it must stay a success. + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$HARBOR_CHART" harbor-test tenant-test harbor-test-cleanup "$tmp/hook.sh" \ + --set '_namespace.host=example.org' --set '_cluster.solver=http01' + + export KLOG="$tmp/calls" + export CHALLENGE_LIST='' + export SOLVER_LIST='' + export EXISTING='' + + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc on an already-clean namespace" + cat "$tmp/out" "$tmp/err" + false + fi + # The Certificate delete is the one that actually reaches a NotFound here, so + # it has to have been attempted for this test to mean anything. + if ! grep -q 'delete certificate .*harbor-test-ingress-tls' "$tmp/calls"; then + echo "FAIL: the Certificate delete was never attempted" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "harbor hook reports a failed delete, names the object and still runs the later sweeps" { + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$HARBOR_CHART" harbor-test tenant-test harbor-test-cleanup "$tmp/hook.sh" \ + --set '_namespace.host=example.org' --set '_cluster.solver=http01' + + export KLOG="$tmp/calls" + export CHALLENGE_LIST='harbor-test-cert-1 ' + export SOLVER_LIST='cm-acme-http-solver-abcde ' + export EXISTING='harbor-test-ingress-tls harbor-test-cert-1 cm-acme-http-solver-abcde' + export FAIL_DELETE='pvc' + + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -eq 0 ]; then + echo "FAIL: hook exited 0 although the PVC delete failed" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q '^ERROR: .*release=harbor-test-system' "$tmp/err"; then + echo "FAIL: the failure was not attributed to the objects it left behind" + cat "$tmp/err" + false + fi + # The first failure must not cost the release its Certificate and solver + # cleanup: those run to completion and the exit code is decided at the end. + for expected in 'delete certificate' 'delete challenge' 'delete ingress'; do + if ! grep -q "$expected" "$tmp/calls"; then + echo "FAIL: the sweep stopped at the first failure, skipping: $expected" + cat "$tmp/calls" + false + fi + done + if grep -q 'Cleanup complete' "$tmp/out"; then + echo "FAIL: hook reported completion after a failed delete" + cat "$tmp/out" + false + fi + rm -rf "$tmp" +} + +@test "harbor hook reports every ACME delete that failed, each by name" { + # The PVC sweep is only one of harbor's failure paths; the Certificate and the + # two per-object ACME deletes are three more, and #3566 names each of them + # separately. Covered together, with the PVC delete deliberately succeeding, + # so this test fails if ANY of the three stops reporting — reverting one of + # them to `|| echo "WARNING …"` is the exact defect being fixed. + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$HARBOR_CHART" harbor-test tenant-test harbor-test-cleanup "$tmp/hook.sh" \ + --set '_namespace.host=example.org' --set '_cluster.solver=http01' + + export KLOG="$tmp/calls" + export CHALLENGE_LIST='harbor-test-cert-1 ' + export SOLVER_LIST='cm-acme-http-solver-abcde ' + export EXISTING='harbor-test-ingress-tls harbor-test-cert-1 cm-acme-http-solver-abcde' + export FAIL_DELETE='certificate challenge ingress' + + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -eq 0 ]; then + echo "FAIL: hook exited 0 although three ACME deletes failed" + cat "$tmp/out" "$tmp/err" + false + fi + # Anchored on the hook's own ERROR: prefix — the fake kubectl names the object + # in its own diagnostic too, so an unanchored match would be satisfied by the + # fixture and stay green with the hook reporting nothing. + for object in 'Certificate tenant-test/harbor-test-ingress-tls' \ + 'Challenge tenant-test/harbor-test-cert-1' \ + 'solver Ingress tenant-test/cm-acme-http-solver-abcde'; do + if ! grep -q "^ERROR: .*$object" "$tmp/err"; then + echo "FAIL: the hook did not report the failed delete of: $object" + cat "$tmp/err" + false + fi + done + if grep -q 'Cleanup complete' "$tmp/out"; then + echo "FAIL: hook reported completion after three failed deletes" + cat "$tmp/out" + false + fi + rm -rf "$tmp" +} + +@test "harbor hook reports a failed solver Ingress listing, the second of its two loops" { + # The Challenge listing is covered below; this is the other loop. Both have + # the same shape and each needs its own case, because a revert of one is + # invisible to a test that only exercises the other. + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$HARBOR_CHART" harbor-test tenant-test harbor-test-cleanup "$tmp/hook.sh" \ + --set '_namespace.host=example.org' --set '_cluster.solver=http01' + + export KLOG="$tmp/calls" + export CHALLENGE_LIST='harbor-test-cert-1 ' + export SOLVER_LIST='' + export EXISTING='harbor-test-ingress-tls harbor-test-cert-1' + export FAIL_GET='ingress' + + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -eq 0 ]; then + echo "FAIL: hook exited 0 although it never managed to list the solver Ingresses" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q '^ERROR: .*failed to list solver Ingresses' "$tmp/err"; then + echo "FAIL: the failed solver Ingress listing was not reported by the hook" + cat "$tmp/err" + false + fi + # The Challenge sweep runs before it and must have completed normally: this + # failure is counted, not fatal. + if ! grep -q 'delete challenge .*harbor-test-cert-1' "$tmp/calls"; then + echo "FAIL: the Challenge sweep preceding the failed listing did not run" + cat "$tmp/calls" + false + fi + if grep -q 'Cleanup complete' "$tmp/out"; then + echo "FAIL: hook reported completion after a failed listing" + cat "$tmp/out" + false + fi + rm -rf "$tmp" +} + +@test "harbor hook exits non-zero when a listing fails instead of sweeping nothing" { + # Iterating $(kubectl get …) directly cannot tell a failed listing from an + # empty one: both leave the loop body unexecuted, so the sweep reported + # success without having looked. + tmp=$(mktemp -d) + write_fake_kubectl "$tmp" + render_hook "$HARBOR_CHART" harbor-test tenant-test harbor-test-cleanup "$tmp/hook.sh" \ + --set '_namespace.host=example.org' --set '_cluster.solver=http01' + + export KLOG="$tmp/calls" + export CHALLENGE_LIST='' + export SOLVER_LIST='cm-acme-http-solver-abcde ' + export EXISTING='harbor-test-ingress-tls cm-acme-http-solver-abcde' + export FAIL_GET='challenge' + + rc=0 + PATH="$tmp:$PATH" sh "$tmp/hook.sh" > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -eq 0 ]; then + echo "FAIL: hook exited 0 although it never managed to list the Challenges" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q 'failed to list Challenges' "$tmp/err"; then + echo "FAIL: the failed listing was not named" + cat "$tmp/err" + false + fi + # A failed listing is counted, not fatal: the solver Ingress sweep after it + # still runs and still deletes what it finds. + if ! grep -q 'delete ingress .*cm-acme-http-solver-abcde' "$tmp/calls"; then + echo "FAIL: the failed listing aborted the solver Ingress sweep that follows it" + cat "$tmp/calls" + false + fi + if grep -q 'Cleanup complete' "$tmp/out"; then + echo "FAIL: hook reported completion after a failed listing" + cat "$tmp/out" + false + fi + rm -rf "$tmp" +} diff --git a/hack/backup-access-preflight.bats b/hack/backup-access-preflight.bats new file mode 100644 index 0000000000..3892c8bcfa --- /dev/null +++ b/hack/backup-access-preflight.bats @@ -0,0 +1,530 @@ +#!/usr/bin/env bats +# Unit tests for the E2E-only S3 access gate that runs before expensive backup +# workloads. All cluster and S3 calls are shell stubs. + +load test_helper + +@test "preflight helper remains sourceable by the POSIX test runner" { + helper=hack/e2e-chainsaw/_lib/backup-access-preflight.sh + [ "$(sed -n '1p' "$helper")" = '#!/bin/sh' ] + if grep -nE '^[[:space:]]*(set[[:space:]].*pipefail|local([[:space:]]|$))' "$helper"; then + echo "backup preflight is sourced by /bin/sh and must not contain Bash-only shell state" >&2 + exit 1 + fi + sh -n "$helper" +} + +@test "preflight fails closed when the BucketInfo Secret cannot be read" { + . hack/e2e-chainsaw/_lib/backup-access-preflight.sh + out=$(mktemp) + kubectl() { + case "$*" in + *" get secret bucket-demo-backup "*) return 1 ;; + *) echo "preflight continued after the Secret read failed: $*" >&2; return 1 ;; + esac + } + + if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then + echo "failed BucketInfo Secret read passed the S3 preflight" >&2 + exit 1 + fi + grep -q 'failed to read BucketInfo Secret tenant-test/bucket-demo-backup' "$out" + if grep -q 'lacks access key' "$out"; then + echo "preflight continued into credential parsing after the Secret read failed" >&2 + exit 1 + fi +} + +@test "preflight retries IAM convergence then proves PUT GET compare and DELETE" { + . hack/e2e-chainsaw/_lib/backup-access-preflight.sh + attempts=$(mktemp) + uploaded_source=$(mktemp) + calls=$(mktemp) + printf '0\n' > "$attempts" + + kubectl() { + printf 'kubectl %s\n' "$*" >> "$calls" + case "$*" in + *" get secret bucket-demo-backup "*) + printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n' + ;; + *" port-forward "*) + while :; do command sleep 1; done + ;; + *) echo "unexpected kubectl call: $*" >&2; return 1 ;; + esac + } + timeout() { + shift + if [ "$1" = sh ]; then + return 0 + fi + "$@" + } + sleep() { :; } + mc() { + printf 'mc %s\n' "$*" >> "$calls" + case "$1" in + alias) return 0 ;; + cp) + case "$3" in + backup-preflight/*) + command cp "$(sed -n '1p' "$uploaded_source")" "$4" + ;; + *) + count=$(sed -n '1p' "$attempts") + count=$(( count + 1 )) + printf '%s\n' "$count" > "$attempts" + if [ "$count" -eq 1 ]; then + echo 'AccessDenied: IAM identity not loaded yet' >&2 + return 1 + fi + printf '%s\n' "$3" > "$uploaded_source" + ;; + esac + ;; + rm) return 0 ;; + *) echo "unexpected mc call: $*" >&2; return 1 ;; + esac + } + + cozy_backup_access_preflight tenant-test bucket-demo-backup 5 >/dev/null + [ "$(sed -n '1p' "$attempts")" -eq 2 ] || { + echo "preflight did not retry the not-yet-loaded IAM identity" >&2 + cat "$calls" >&2 + exit 1 + } + [ "$(grep -c '^mc rm ' "$calls")" -eq 1 ] || { + echo "preflight did not delete its successfully verified object exactly once" >&2 + cat "$calls" >&2 + exit 1 + } +} + +@test "preflight fails before backup work when granted credentials never reach S3" { + . hack/e2e-chainsaw/_lib/backup-access-preflight.sh + out=$(mktemp) + kubectl() { + case "$*" in + *" get secret bucket-demo-backup "*) + printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n' + ;; + *" port-forward "*) while :; do command sleep 1; done ;; + *) return 1 ;; + esac + } + timeout() { + shift + if [ "$1" = sh ]; then return 0; fi + "$@" + } + mc() { + case "$1" in + alias) return 0 ;; + cp) echo 'AccessDenied' >&2; return 1 ;; + *) return 0 ;; + esac + } + + if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then + echo "unusable credentials passed the S3 preflight" >&2 + exit 1 + fi + grep -q 'granted but did not pass S3 PUT/GET/DELETE within 0s' "$out" + grep -q 's3-preflight: AccessDenied' "$out" + grep -q 's3-preflight: attempt 1 failed at PUT' "$out" +} + +@test "preflight timeout reports the port-forward and how every attempt ended" { + . hack/e2e-chainsaw/_lib/backup-access-preflight.sh + out=$(mktemp) + now=$(mktemp) + pf_ready=$(mktemp) + attempts=$(mktemp) + uploaded_source=$(mktemp) + printf '100\n' > "$now" + printf '0\n' > "$attempts" + + kubectl() { + case "$*" in + *" get secret bucket-demo-backup "*) + printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n' + ;; + *" port-forward "*) + echo 'Forwarding from 127.0.0.1:18333 -> 8333' + printf 'ready\n' > "$pf_ready" + while :; do command sleep 1; done + ;; + *) return 1 ;; + esac + } + # A stubbed clock keeps the retry budget independent of how fast the + # stubs run: only a completed PUT advances it, so the loop ends after a + # known number of attempts. + date() { + if [ "${1:-}" = +%s ]; then sed -n '1p' "$now"; else command date "$@"; fi + } + timeout() { + shift + if [ "$1" = sh ]; then + waited=0 + while [ ! -s "$pf_ready" ] && [ "$waited" -lt 100 ]; do + waited=$(( waited + 1 )) + command sleep 0.1 + done + return 0 + fi + "$@" + } + sleep() { :; } + # The first three attempts break at a later stage than the one before, so + # the stages that follow a completed upload are told apart rather than + # merged. + mc() { + case "$1" in + alias) return 0 ;; + cp) + case "$3" in + backup-preflight/*) + case "$(sed -n '1p' "$attempts")" in + 1) + echo 'mc: Unable to download: connection reset by peer' >&2 + return 1 + ;; + 2) printf 'truncated\n' > "$4" ;; + *) command cp "$(sed -n '1p' "$uploaded_source")" "$4" ;; + esac + ;; + *) + printf '%s\n' "$3" > "$uploaded_source" + count=$(sed -n '1p' "$attempts") + printf '%s\n' "$(( count + 1 ))" > "$attempts" + stamp=$(sed -n '1p' "$now") + printf '%s\n' "$(( stamp + 10 ))" > "$now" + # Do not tidy this arm away into a clean ascending + # ladder. A ladder where every attempt fails later than + # the last can never catch a missing per-iteration + # stage reset, because the stale value is overwritten + # before anything reads it. + if [ "$count" -eq 3 ]; then + echo 'mc: Unable to upload: connection reset by peer' >&2 + return 1 + fi + echo 'source -> backup-preflight/bucket-real/object' + ;; + esac + ;; + rm) return 1 ;; + *) return 0 ;; + esac + } + + if cozy_backup_access_preflight tenant-test bucket-demo-backup 35 >"$out" 2>&1; then + echo "an S3 round trip that never completed passed the preflight" >&2 + exit 1 + fi + grep -q 'port-forward: Forwarding from 127.0.0.1:18333' "$out" || { + echo "timeout path dropped the port-forward log" >&2 + cat "$out" >&2 + exit 1 + } + for expected in \ + 'attempt 1 failed at GET' \ + 'attempt 2 failed at compare' \ + 'attempt 3 failed at DELETE' \ + 'attempt 4 failed at PUT'; do + grep -q "s3-preflight: $expected" "$out" || { + echo "timeout path lost '$expected'" >&2 + cat "$out" >&2 + exit 1 + } + done +} + +@test "all active database backup flows gate immediately after accessGranted" { + for file in \ + examples/backups/postgres/run-all.sh \ + examples/backups/mariadb/run-all.sh \ + examples/backups/mongodb/run-all.sh \ + examples/backups/clickhouse/03-create-bucket.sh \ + examples/backups/kafka/03-create-bucket.sh; do + granted=$(grep -n 'accessGranted' "$file" | head -n 1 | cut -d: -f1) + preflight=$(grep -n 'cozy_backup_access_preflight' "$file" | head -n 1 | cut -d: -f1) + coordinates=$(grep -n 'Reading bucket coordinates' "$file" | head -n 1 | cut -d: -f1) + if [ -z "$granted" ] || [ -z "$preflight" ] || [ -z "$coordinates" ] \ + || [ "$granted" -ge "$preflight" ] || [ "$preflight" -ge "$coordinates" ]; then + echo "$file must run the S3 preflight after accessGranted and before consuming bucket coordinates" >&2 + exit 1 + fi + done +} + +@test "the five preflight-wired Chainsaw backup roundtrips enable the E2E preflight" { + for file in \ + hack/e2e-chainsaw/postgres/chainsaw-test.yaml \ + hack/e2e-chainsaw/mariadb/chainsaw-test.yaml \ + hack/e2e-chainsaw/mongodb/chainsaw-test.yaml \ + hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml \ + hack/e2e-chainsaw/kafka/chainsaw-test.yaml; do + grep -q 'COZY_E2E_BACKUP_PREFLIGHT=1' "$file" || { + echo "$file does not enable the backup access preflight" >&2 + exit 1 + } + done +} + +@test "preflight reports the port-forward log when the S3 client cannot be configured" { + . hack/e2e-chainsaw/_lib/backup-access-preflight.sh + out=$(mktemp) + pf_ready=$(mktemp) + kubectl() { + case "$*" in + *" get secret bucket-demo-backup "*) + printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n' + ;; + *" port-forward "*) + echo 'Forwarding from 127.0.0.1:18333 -> 8333' + printf 'ready\n' > "$pf_ready" + while :; do command sleep 1; done + ;; + *) return 1 ;; + esac + } + timeout() { + shift + if [ "$1" = sh ]; then + waited=0 + while [ ! -s "$pf_ready" ] && [ "$waited" -lt 100 ]; do + waited=$(( waited + 1 )) + command sleep 0.1 + done + return 0 + fi + "$@" + } + mc() { + case "$1" in + alias) echo 'mc: Unable to initialize new alias: not an S3 endpoint' >&2; return 1 ;; + *) echo "preflight continued after the S3 client failed: mc $*" >&2; return 1 ;; + esac + } + + if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then + echo "a failed S3 client setup passed the preflight" >&2 + exit 1 + fi + grep -q 'failed to configure S3 client for BucketAccess tenant-test/bucket-demo-backup' "$out" + grep -q 'port-forward: Forwarding from 127.0.0.1:18333' "$out" || { + echo "the S3 client failure dropped the port-forward log" >&2 + cat "$out" >&2 + exit 1 + } +} + +@test "preflight cleanup deletes a failed round trip's object through a live tunnel" { + . hack/e2e-chainsaw/_lib/backup-access-preflight.sh + out=$(mktemp) + pf_state=$(mktemp) + calls=$(mktemp) + kubectl() { + case "$*" in + *" get secret bucket-demo-backup "*) + printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n' + ;; + *" port-forward "*) + # The PID of this background subshell, which is what the + # helper kills; the delete below probes it. + sh -c 'echo $PPID' > "$pf_state" + while :; do command sleep 1; done + ;; + *) return 1 ;; + esac + } + timeout() { + shift + if [ "$1" = sh ]; then + waited=0 + while [ ! -s "$pf_state" ] && [ "$waited" -lt 100 ]; do + waited=$(( waited + 1 )) + command sleep 0.1 + done + return 0 + fi + "$@" + } + mc() { + case "$1" in + alias) return 0 ;; + cp) + case "$3" in + backup-preflight/*) echo 'mc: Unable to download: connection reset by peer' >&2; return 1 ;; + *) return 0 ;; + esac + ;; + rm) + if kill -0 "$(cat "$pf_state")" 2>/dev/null; then + printf 'rm with tunnel alive\n' >> "$calls" + else + printf 'rm with tunnel gone\n' >> "$calls" + echo 'mc: dial tcp 127.0.0.1:18333: connect: connection refused' >&2 + return 1 + fi + ;; + *) return 1 ;; + esac + } + + if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then + echo "a failed download passed the preflight" >&2 + exit 1 + fi + [ "$(cat "$calls")" = "rm with tunnel alive" ] || { + echo "cleanup did not delete the uploaded object while the port-forward was still up:" >&2 + cat "$calls" "$out" >&2 + exit 1 + } + if grep -q 'could not remove' "$out"; then + echo "cleanup warned about a delete that succeeded" >&2 + cat "$out" >&2 + exit 1 + fi +} + +@test "preflight cleanup tries the delete when the upload timed out after landing" { + . hack/e2e-chainsaw/_lib/backup-access-preflight.sh + out=$(mktemp) + pf_state=$(mktemp) + calls=$(mktemp) + kubectl() { + case "$*" in + *" get secret bucket-demo-backup "*) + printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n' + ;; + *" port-forward "*) + printf alive > "$pf_state" + while :; do command sleep 1; done + ;; + *) return 1 ;; + esac + } + # The PUT reaches the server and then outlives its bound, so timeout + # reports 124 while the object is already in the bucket. + timeout() { + shift + if [ "$1" = sh ]; then + waited=0 + while [ ! -s "$pf_state" ] && [ "$waited" -lt 100 ]; do + waited=$(( waited + 1 )) + command sleep 0.1 + done + return 0 + fi + case "$1 $2 ${5:-}" in + "mc cp backup-preflight/"*) + printf 'put\n' >> "$calls" + return 124 + ;; + esac + "$@" + } + mc() { + case "$1" in + alias) return 0 ;; + rm) printf 'rm\n' >> "$calls" ;; + *) return 1 ;; + esac + } + + if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then + echo "a timed-out upload passed the preflight" >&2 + exit 1 + fi + grep -qx put "$calls" + grep -qx rm "$calls" || { + echo "cleanup skipped the delete of an upload that may have landed:" >&2 + cat "$calls" "$out" >&2 + exit 1 + } +} + +@test "preflight cleanup neither deletes nor warns when every PUT was refused" { + . hack/e2e-chainsaw/_lib/backup-access-preflight.sh + out=$(mktemp) + calls=$(mktemp) + kubectl() { + case "$*" in + *" get secret bucket-demo-backup "*) + printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n' + ;; + *" port-forward "*) while :; do command sleep 1; done ;; + *) return 1 ;; + esac + } + timeout() { + shift + if [ "$1" = sh ]; then return 0; fi + "$@" + } + mc() { + case "$1" in + alias) return 0 ;; + cp) echo 'AccessDenied' >&2; return 1 ;; + rm) printf 'rm\n' >> "$calls"; echo 'AccessDenied' >&2; return 1 ;; + *) return 1 ;; + esac + } + + if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then + echo "a refused PUT passed the preflight" >&2 + exit 1 + fi + if [ -s "$calls" ] || grep -q 'could not remove' "$out"; then + echo "cleanup tried to delete an object no PUT created:" >&2 + cat "$calls" "$out" >&2 + exit 1 + fi +} + +@test "preflight cleanup names the timeout when the delete itself times out" { + . hack/e2e-chainsaw/_lib/backup-access-preflight.sh + out=$(mktemp) + kubectl() { + case "$*" in + *" get secret bucket-demo-backup "*) + printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n' + ;; + *" port-forward "*) while :; do command sleep 1; done ;; + *) return 1 ;; + esac + } + # The delete hangs until its bound kills it: exit 124, nothing written. + timeout() { + shift + if [ "$1" = sh ]; then return 0; fi + if [ "$1 $2" = "mc rm" ]; then return 124; fi + "$@" + } + mc() { + case "$1" in + alias) return 0 ;; + cp) + case "$3" in + backup-preflight/*) echo 'mc: Unable to download: connection reset by peer' >&2; return 1 ;; + *) return 0 ;; + esac + ;; + *) return 1 ;; + esac + } + + if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then + echo "a failed download passed the preflight" >&2 + exit 1 + fi + grep -q 'could not remove failed S3 preflight object' "$out" + grep -q 's3-cleanup: mc rm timed out after 15s' "$out" || { + echo "a delete that timed out left the warning without a reason:" >&2 + cat "$out" >&2 + exit 1 + } +} diff --git a/hack/backup-examples-shared-helpers.bats b/hack/backup-examples-shared-helpers.bats new file mode 100644 index 0000000000..1821c01925 --- /dev/null +++ b/hack/backup-examples-shared-helpers.bats @@ -0,0 +1,95 @@ +#!/usr/bin/env bats +# Contract for the wait helpers the backup walkthroughs share: each is defined +# once, in examples/backups/_lib/wait-helpers.sh, so a fix made there reaches +# every walkthrough; each walkthrough's 00-helpers.sh provides them; and +# wait_for_field reports why kubectl could not read a field while staying quiet +# about a resource that does not exist yet. + +load test_helper + +SHARED=examples/backups/_lib/wait-helpers.sh +SHARED_HELPERS="wait_for_field wait_hr_ready wait_deleted" + +@test "each shared wait helper is defined once, in the shared file" { + fail= + for fn in $SHARED_HELPERS; do + definers=$(grep -rlE --include='*.sh' "^[[:space:]]*(function[[:space:]]+)?${fn}[[:space:]]*\(\)" examples packages | sort) + if [ "$definers" != "$SHARED" ]; then + fail="$fail +${fn} is defined in: +${definers:- (nowhere)}" + fi + done + [ -z "$fail" ] || { + echo "a walkthrough carries its own copy of a shared helper; use the one in $SHARED:$fail" >&2 + exit 1 + } +} + +@test "every walkthrough's helpers provide the shared wait helpers" { + # Some helpers read the cluster while being sourced (kafka resolves its + # image from the operator); a stub keeps this test off any real cluster. + stub=$(mktemp -d) + printf '#!/bin/sh\nexit 1\n' > "$stub/kubectl" + chmod +x "$stub/kubectl" + for helpers in examples/backups/*/00-helpers.sh; do + PATH="$stub:$PATH" bash -c ' + set -euo pipefail + . "$1" + for fn in $2; do + declare -F "$fn" >/dev/null || { echo "$1 does not provide $fn" >&2; exit 1; } + done + ' _ "$helpers" "$SHARED_HELPERS" + done +} + +@test "wait_for_field shows why kubectl could not read the field" { + # A Forbidden read has to reach the caller; swallowed, it looks like a + # field that is still empty. + stub=$(mktemp -d) + printf '#!/bin/sh\necho "Error from server (Forbidden): backupjobs is forbidden" >&2\nexit 1\n' > "$stub/kubectl" + chmod +x "$stub/kubectl" + err=$(mktemp) + if PATH="$stub:$PATH" bash -c ' + set -euo pipefail + . examples/backups/postgres/00-helpers.sh + wait_for_field backupjob demo "{.status.phase}" Succeeded tenant-test 0 + ' 2>"$err"; then + echo "wait_for_field succeeded against a kubectl that cannot read" >&2 + exit 1 + fi + grep -q 'backupjobs is forbidden' "$err" || { + echo "wait_for_field dropped kubectl's reason:" >&2 + cat "$err" >&2 + exit 1 + } +} + +@test "wait_for_field stays quiet while the resource does not exist yet" { + # The stub answers like kubectl: NotFound on stderr and exit 1, unless the + # read asks for --ignore-not-found, which makes an absent resource empty. + stub=$(mktemp -d) + cat > "$stub/kubectl" <<'EOF' +#!/bin/sh +case " $* " in +*" --ignore-not-found "*) exit 0 ;; +esac +echo 'Error from server (NotFound): backupjobs.backups.cozystack.io "demo" not found' >&2 +exit 1 +EOF + chmod +x "$stub/kubectl" + err=$(mktemp) + if PATH="$stub:$PATH" bash -c ' + set -euo pipefail + . examples/backups/postgres/00-helpers.sh + wait_for_field backupjob demo "{.status.phase}" Succeeded tenant-test 0 + ' 2>"$err"; then + echo "wait_for_field succeeded against a resource that does not exist" >&2 + exit 1 + fi + if grep -q 'NotFound' "$err"; then + echo "wait_for_field printed kubectl's NotFound for a resource it is waiting on:" >&2 + cat "$err" >&2 + exit 1 + fi +} diff --git a/hack/backupstrategy-crd-gate.bats b/hack/backupstrategy-crd-gate.bats new file mode 100644 index 0000000000..b4ad482a02 --- /dev/null +++ b/hack/backupstrategy-crd-gate.bats @@ -0,0 +1,135 @@ +#!/usr/bin/env bats +# backupstrategy-controller ships its strategy.backups.cozystack.io CRDs as +# ordinary templates in the same release as the default Strategy CRs. Helm maps +# every document of a release through the RESTMapper before it applies any, so +# one Strategy CR whose CRD is not served yet fails the whole release with "no +# matches for kind" and applies nothing, the CRDs included. That happens on the +# upgrade that first ships a kind, and on the first render of an external-S3 +# install, where the bucket gate is already open. Every default Strategy must +# therefore also gate on its CRD being established (the crdEstablished helper). +# +# helm-unittest can only pin the templates someone remembered to list. These +# cases discover the templates from the render itself, so a new +# strategy-*-default.yaml without the CRD gate fails here without anyone +# touching a test. +# +# Harness compatibility note: CI runs this file under Bats through +# `make bats-unit-tests`. It also remains compatible with the narrower legacy +# `hack/cozytest.sh` translator, so it avoids `run`, `$status`, `$output`, +# `skip`, and setup()/teardown(). Paths stay repo-root-relative for both +# runners. +# +# Run with: bats hack/backupstrategy-crd-gate.bats + +load test_helper + +CHART=packages/system/backupstrategy-controller +STRATEGY_API='strategy.backups.cozystack.io/' + +# Every Strategy CR in the render under $1, one "