From b498e4753a981b791559135965d0dc69a0690162 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 7 Jul 2026 12:51:12 -0600 Subject: [PATCH 01/32] Alternative stacks This change makes the "stacks" deployed by ceph-devstack configurable. The existing behavior of ceph-devstack now belongs to the "teuthology" stack. This commit also adds a "ceph" stack, which deploys a minimal Ceph cluster in a rootless container. It can optionally compile Ceph and build a container to use for the deployment. The compile phase uses configurable persistent stores for sccache, dnf cache, and npm cache. --- .github/workflows/ci.yml | 21 + ceph_devstack/__init__.py | 33 +- ceph_devstack/ceph_devstack.te | 1 + ceph_devstack/cli.py | 88 ++- ceph_devstack/config.toml | 49 ++ ceph_devstack/exec.py | 21 +- ceph_devstack/resources/__init__.py | 10 +- ceph_devstack/resources/ceph/__init__.py | 120 ++- .../resources/ceph/ceph-node-entrypoint.sh | 466 +++++++++++ ceph_devstack/resources/ceph/ceph_node.py | 741 ++++++++++++++++++ ceph_devstack/resources/ceph/containers.py | 6 +- ceph_devstack/resources/container.py | 28 +- ceph_devstack/sccache-s3.conf | 8 + ceph_devstack/sccache.conf | 2 + pyproject.toml | 6 + tests/integration/test_ceph_stack.py | 24 + tests/integration/test_ceph_stack.sh | 69 ++ tests/resources/ceph/test_ceph_node.py | 435 ++++++++++ .../resources/ceph/test_cephdevstack_core.py | 54 +- .../resources/ceph/test_requirements_ceph.py | 2 +- tests/resources/test_podmanresource.py | 13 +- tests/test_config.py | 25 + tests/test_stacks.py | 62 ++ 23 files changed, 2189 insertions(+), 95 deletions(-) create mode 100755 ceph_devstack/resources/ceph/ceph-node-entrypoint.sh create mode 100644 ceph_devstack/resources/ceph/ceph_node.py create mode 100644 ceph_devstack/sccache-s3.conf create mode 100644 ceph_devstack/sccache.conf create mode 100644 tests/integration/test_ceph_stack.py create mode 100755 tests/integration/test_ceph_stack.sh create mode 100644 tests/resources/ceph/test_ceph_node.py create mode 100644 tests/test_stacks.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d09cdbca..ef686ab6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,3 +90,24 @@ jobs: run: uv run ceph-devstack -v stop - name: Remove run: uv run ceph-devstack -v remove + ceph-stack: + name: Ceph stack integration on ${{ matrix.os }} (python ${{ matrix.python }}) + runs-on: ${{ matrix.os }} + needs: lint + strategy: + matrix: + include: + - os: ubuntu-24.04 + python: "3.12" + steps: + - uses: actions/checkout@v7 + - name: Install packages + run: sudo apt-get update && sudo apt-get install podman golang-github-containernetworking-plugin-dnsname curl + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - name: Set owner for /dev/loop-control + run: sudo chown $(whoami) /dev/loop-control + - name: Ceph stack integration test + env: + CEPH_DEVSTACK_INTEGRATION: "1" + run: bash tests/integration/test_ceph_stack.sh diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py index d71474cd..3d1a06ed 100644 --- a/ceph_devstack/__init__.py +++ b/ceph_devstack/__init__.py @@ -41,6 +41,11 @@ def parse_args(args: List[str]) -> argparse.Namespace: default=DEFAULT_CONFIG_PATH, help="Path to the ceph-devstack config file", ) + parser.add_argument( + "--stack", + default=None, + help="Stack to deploy (default: value of 'stack' in config)", + ) subparsers = parser.add_subparsers(dest="command") parser_config = subparsers.add_parser("config", help="Get or set config items") subparsers_config = parser_config.add_subparsers(dest="config_op") @@ -128,7 +133,9 @@ def deep_merge(*maps): class Config(dict): - __slots__ = ["user_obj", "user_path"] + __slots__ = ["user_obj", "user_path", "active_stack", "active_services"] + active_stack: str | None + active_services: list[str] def load(self, config_path: Path | None = None): args = self.get("args") @@ -146,6 +153,30 @@ def load(self, config_path: Path | None = None): self.user_obj = {} if args: self["args"] = args + self.active_stack = None + self.active_services: list[str] = [] + + def apply_stack(self, stack_name: str | None = None) -> list[str]: + if stack_name is None: + default = self.get("stack", "teuthology") + stack_name = default if isinstance(default, str) else "teuthology" + stacks = self.get("stacks", {}) + if stack_name not in stacks: + known = ", ".join(sorted(stacks)) or "(none)" + raise ValueError(f"Unknown stack {stack_name!r}; known stacks: {known}") + stack = stacks[stack_name] + self.active_stack = stack_name + self.active_services = list(stack.get("services", [])) + + overrides = {} + for key in ("data_dir", "containers", "env"): + if key in stack: + overrides[key] = stack[key] + if overrides: + merged = deep_merge(self, overrides) + for key in overrides: + self[key] = merged[key] + return self.active_services def dump(self): return tomlkit.dumps(self) diff --git a/ceph_devstack/ceph_devstack.te b/ceph_devstack/ceph_devstack.te index f661669b..fdb9afc4 100644 --- a/ceph_devstack/ceph_devstack.te +++ b/ceph_devstack/ceph_devstack.te @@ -125,3 +125,4 @@ allow container_init_t tun_tap_device_t:chr_file mounton; allow container_t user_tmp_t:sock_file write; allow container_t unconfined_t:unix_stream_socket connectto; +allow container_t fixed_disk_device_t:blk_file setattr; diff --git a/ceph_devstack/cli.py b/ceph_devstack/cli.py index 8658bcbc..1fba02a2 100644 --- a/ceph_devstack/cli.py +++ b/ceph_devstack/cli.py @@ -2,7 +2,9 @@ import logging import sys +from argparse import Namespace from pathlib import Path +from subprocess import CalledProcessError from ceph_devstack import config, logger, parse_args, VERBOSE from ceph_devstack.requirements import check_requirements @@ -15,53 +17,65 @@ "unset": lambda config, args: config.unset_value(args.name), } -COMMAND_HANDLERS = { - "doctor": None, - "apply": lambda args, obj: obj.apply(args.command), - "pull": lambda _, obj: obj.pull(), - "build": lambda _, obj: obj.build(), - "create": lambda _, obj: obj.create(), - "remove": lambda _, obj: obj.remove(), - "start": lambda _, obj: obj.start(), - "stop": lambda _, obj: obj.stop(), - "watch": lambda _, obj: obj.watch(), - "wait": lambda args, obj: obj.wait(container_name=args.container), - "logs": lambda args, obj: obj.logs( - run_name=args.run_name, job_id=args.job_id, locate=args.locate - ), -} + +def _configure_console_logging(verbose: bool) -> None: + if not verbose: + return + for handler in logging.getLogger("root").handlers: + if not isinstance(handler, logging.FileHandler): + handler.setLevel(VERBOSE) + + +def _action_kwargs(args: Namespace) -> dict: + match args.command: + case "wait": + return {"container_name": args.container} + case "logs": + return { + "run_name": args.run_name, + "job_id": args.job_id, + "locate": args.locate, + } + case _: + return {} + + +async def _apply(stack: CephDevStack, action: str, **kwargs) -> int: + """Map a CLI command to a stack action and apply it.""" + if not all( + await asyncio.gather( + check_requirements(), + stack.check_requirements(), + ) + ): + logger.error("Requirements not met!") + return 1 + if action == "doctor": + return 0 + result = await stack.apply(action, **kwargs) + return result if result is not None else 0 def main() -> int: args = parse_args(sys.argv[1:]) config.load(args.config_file) - if args.verbose: - for handler in logging.getLogger("root").handlers: - if not isinstance(handler, logging.FileHandler): - handler.setLevel(VERBOSE) + try: + config.apply_stack(args.stack) + except ValueError as exc: + logger.error(str(exc)) + return 1 + _configure_console_logging(args.verbose) if args.command == "config": CONFIG_HANDLERS[args.config_op](config, args) return 0 config["args"] = vars(args) - data_path = Path(config["data_dir"]).expanduser() - data_path.mkdir(parents=True, exist_ok=True) - obj = CephDevStack() - - async def run(): - if not all( - await asyncio.gather( - check_requirements(), - obj.check_requirements(), - ) - ): - logger.error("Requirements not met!") - return 1 - handler = COMMAND_HANDLERS.get(args.command) - if handler: - return await handler(args, obj) - + Path(config["data_dir"]).expanduser().mkdir(parents=True, exist_ok=True) + stack = CephDevStack(stack_name=config.active_stack) + action = args.command try: - sys.exit(asyncio.run(run())) + return asyncio.run(_apply(stack, action, **_action_kwargs(args))) + except CalledProcessError as exc: + return exc.returncode if exc.returncode is not None else 1 except KeyboardInterrupt: logger.debug("Exiting!") return 130 # 128 + SIGINT diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 147a729e..786b1223 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -1,5 +1,54 @@ +stack = "teuthology" data_dir = "~/.local/share/ceph-devstack" +[stacks.teuthology] +services = [ + "postgres", + "paddles", + "beanstalk", + "pulpito", + "teuthology", + "testnode", + "archive", +] +secrets = ["ssh_keypair"] + +[stacks.ceph] +services = ["ceph_node"] +secrets = [] +data_dir = "~/.local/share/ceph-devstack/ceph" + +[containers.ceph_node] +image = "quay.io/ceph-ci/ceph:main" +loop_device_count = 3 +loop_device_size = "5G" +dashboard_port = 8080 +dashboard_ssl = false +# dashboard_user = "admin" +# dashboard_password = "admin" +# dashboard = false # disable mgr dashboard +# image_builder = "cpatch" # default; patch local binaries into runtime image +# image_builder = "container" # future: container/build.sh + local packages +# Build from a local ceph.git checkout: +# repo = "~/dev/ceph" +# build_dir = "build" # relative to repo; passed to build-with-container.py -b +# build_distro = "centos9" # build-with-container.py -d +# build_steps = ["build"] # default for cpatch; ["packages"] for container builder +# sccache = true # default; local disk cache (see sccache.conf) +# sccache = false # disable sccache +# sccache_mode = "local" # local (default) or s3 for shared remote cache +# sccache_cache_path = "~/.local/share/ceph-devstack/cache/sccache" +# sccache_cache_size = "100G" # local cache size limit +# sccache_debug = false # enable SCCACHE_LOG=debug on build stdout +# sccache_conf = "/path/to/sccache.conf" # override bundled config entirely +# npm_cache = true # default; persist dashboard npm downloads +# npm_cache_path = "~/.local/share/ceph-devstack/cache/npm" # override npm cache location +# dnf_cache = false # persist dnf downloads when rebuilding ceph-build images +# dnf_cache_path = "~/.local/share/ceph-devstack/cache/dnf" # override dnf cache base dir +# repo may be a git worktree; ceph-devstack mounts git metadata for builds at /ceph +# base_image = "quay.io/ceph-ci/ceph:main" +# image = "localhost/ceph-devstack:main" + [containers.archive] image = "python:alpine" diff --git a/ceph_devstack/exec.py b/ceph_devstack/exec.py index 9cc27def..1bddb991 100644 --- a/ceph_devstack/exec.py +++ b/ceph_devstack/exec.py @@ -8,7 +8,7 @@ import signal import subprocess -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple from ceph_devstack import logger, VERBOSE @@ -70,6 +70,25 @@ def signal_children(self, signal: signal.Signals, recursive=True): with contextlib.suppress(ProcessLookupError): os.killpg(pid, signal) + async def collect_output(self) -> Tuple[str, str]: + stdout = b"" + stderr = b"" + if self.stdout is not None: + stdout = await self.stdout.read() + if self.stderr is not None: + stderr = await self.stderr.read() + return stdout.decode(), stderr.decode() + + async def log_failure(self, cmd: List[str]) -> Tuple[str, str]: + stdout, stderr = await self.collect_output() + returncode = self.returncode if self.returncode is not None else -1 + logger.error(f"Command failed ({returncode}): {' '.join(cmd)}") + for line in stderr.rstrip("\n").splitlines(): + logger.error(line) + for line in stdout.rstrip("\n").splitlines(): + logger.error(line) + return stdout, stderr + class Command: def __init__( diff --git a/ceph_devstack/resources/__init__.py b/ceph_devstack/resources/__init__.py index 0a3c2a10..b9607191 100644 --- a/ceph_devstack/resources/__init__.py +++ b/ceph_devstack/resources/__init__.py @@ -65,9 +65,13 @@ async def cmd( ) returncode = await proc.wait() if check and returncode != 0: - # out = await proc.stderr.read() - # logger.error(out.decode()) - raise CalledProcessError(cmd=args, returncode=returncode) + stdout, stderr = await proc.log_failure(args) + raise CalledProcessError( + returncode, + args, + output=stdout or None, + stderr=stderr or None, + ) return proc def format_cmd(self, args: List): diff --git a/ceph_devstack/resources/ceph/__init__.py b/ceph_devstack/resources/ceph/__init__.py index 17cc5641..11dea1b7 100644 --- a/ceph_devstack/resources/ceph/__init__.py +++ b/ceph_devstack/resources/ceph/__init__.py @@ -18,6 +18,7 @@ Teuthology, Archive, ) +from ceph_devstack.resources.ceph.ceph_node import CephNode, CONTAINER_CLUSTER_DIR from ceph_devstack.resources.ceph.requirements import ( HasSudo, LoopControlDeviceExists, @@ -85,23 +86,39 @@ class CephDevStackNetwork(Network): _name = "ceph-devstack" +SERVICES = { + "postgres": Postgres, + "paddles": Paddles, + "beanstalk": Beanstalk, + "pulpito": Pulpito, + "teuthology": Teuthology, + "testnode": TestNode, + "archive": Archive, + "ceph_node": CephNode, +} + +SECRETS = { + "ssh_keypair": SSHKeyPair, +} + + class CephDevStack: networks = [CephDevStackNetwork] secrets = [SSHKeyPair] - def __init__(self): - services = [ - Postgres, - Paddles, - Beanstalk, - Pulpito, - Teuthology, - TestNode, - Archive, - ] + def __init__(self, stack_name: str | None = None): + if ( + stack_name and stack_name != config.active_stack + ) or config.active_stack is None: + config.apply_stack(stack_name) + + self.stack_name = config.active_stack self.service_specs = {} - for service in services: - name = service.__name__.lower() + for name in config.active_services: + service = SERVICES.get(name) + if service is None: + logger.warning(f"Unknown service {name!r} in stack {self.stack_name!r}") + continue count = config["containers"][name].get("count", 1) if count == 0: continue @@ -115,9 +132,20 @@ def __init__(self): self.service_specs[name]["objects"] = [ service(name=f"{name}_{i}") for i in range(count) ] - if postgres_spec := self.service_specs.get("postgres"): + self._wire_services() + stack = config.get("stacks", {}).get(self.stack_name, {}) + self.secrets = [ + SECRETS[secret_name] + for secret_name in stack.get("secrets", []) + if secret_name in SECRETS + ] + + def _wire_services(self): + if (postgres_spec := self.service_specs.get("postgres")) and ( + paddles_spec := self.service_specs.get("paddles") + ): postgres_obj = postgres_spec["objects"][0] - paddles_obj = self.service_specs["paddles"]["objects"][0] + paddles_obj = paddles_spec["objects"][0] paddles_obj.env_vars["PADDLES_SQLALCHEMY_URL"] = ( postgres_obj.paddles_sqla_url ) @@ -126,8 +154,9 @@ async def check_requirements(self): result = True result = has_sudo = await HasSudo().evaluate() - result = result and await LoopControlDeviceExists().evaluate() - result = result and await LoopControlDeviceWriteable().evaluate() + if "testnode" in self.service_specs or "ceph_node" in self.service_specs: + result = result and await LoopControlDeviceExists().evaluate() + result = result and await LoopControlDeviceWriteable().evaluate() # Check for SELinux being enabled and Enforcing; then check for the # presence of our module. If necessary, inform the user and instruct @@ -135,14 +164,24 @@ async def check_requirements(self): if has_sudo and await host.selinux_enforcing(): result = result and await SELinuxModule().evaluate() - for name, obj in config["containers"].items(): - if (repo := obj.get("repo")) and not host.path_exists(repo): + for name in self.service_specs: + obj = config["containers"][name] + if (repo := obj.get("repo")) and not host.path_exists( + os.path.expanduser(str(repo)) + ): result = False logger.error(f"Repo for {name} not found at {repo}") return result - async def apply(self, action): - return await getattr(self, action)() + async def apply(self, action: str, **kwargs) -> int | None: + if action == "wait": + return await self.wait(**kwargs) + if action == "logs": + return await self.logs(**kwargs) + method = getattr(self, action, None) + if method is None: + raise AttributeError(f"Unknown action {action!r}") + return await method() async def pull(self): logger.info("Pulling images...") @@ -155,14 +194,18 @@ async def build(self): await spec["objects"][0].build() async def create(self): + args = config.get("args", {}) + if args.get("build"): + await self.build() logger.info("Creating containers...") await CephDevStackNetwork().create() - await SSHKeyPair().create() - containers = [] + for secret in self.secrets: + await secret().create() + tasks = [] for spec in self.service_specs.values(): for object in spec["objects"]: - containers.append(object.create()) - await asyncio.gather(*containers) + tasks.append(object.create()) + await asyncio.gather(*tasks) async def start(self): await self.create() @@ -170,12 +213,21 @@ async def start(self): for spec in self.service_specs.values(): for object in spec["objects"]: await object.start() - logger.info( - "All containers are running. To monitor teuthology, try running: podman " - "logs -f teuthology" - ) - hostname = host.hostname() - logger.info(f"View test results at http://{hostname}:8081/") + if "teuthology" in self.service_specs: + logger.info( + "All containers are running. To monitor teuthology, try running: " + "podman logs -f teuthology" + ) + else: + logger.info("All containers are running.") + if "pulpito" in self.service_specs: + hostname = host.hostname() + logger.info(f"View test results at http://{hostname}:8081/") + if "ceph_node" in self.service_specs: + logger.info( + "Monitor the cluster with: podman exec ceph_node ceph " + f"-c {CONTAINER_CLUSTER_DIR}/ceph.conf -s" + ) async def stop(self): logger.info("Stopping containers...") @@ -193,10 +245,14 @@ async def remove(self): containers.append(object.remove()) await asyncio.gather(*containers) await CephDevStackNetwork().remove() - await SSHKeyPair().remove() + for secret in self.secrets: + await secret().remove() async def watch(self): - logger.info("Watching containers; will replace any that are stopped") + logger.info( + "Entering watch mode: while waiting for teuthology to " + "exit, other containers will be replaced as they are stopped." + ) containers = [] for spec in self.service_specs.values(): if not spec["count"] > 0: diff --git a/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh b/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh new file mode 100755 index 00000000..ed6dab58 --- /dev/null +++ b/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh @@ -0,0 +1,466 @@ +#!/bin/bash +set -euo pipefail + +CLUSTER_DIR="${CLUSTER_DIR:?CLUSTER_DIR is required}" +CEPH_CONF="${CLUSTER_DIR}/ceph.conf" +ADMIN_KEYRING="${CLUSTER_DIR}/ceph.client.admin.keyring" +MON_ID="${MON_ID:-a}" +MGR_ID="${MGR_ID:-x}" +OSD_DEVICES="${OSD_DEVICES:-}" +DASHBOARD_ENABLED="${DASHBOARD_ENABLED:-true}" +DASHBOARD_PORT="${DASHBOARD_PORT:-8080}" +DASHBOARD_SSL="${DASHBOARD_SSL:-false}" +DASHBOARD_USER="${DASHBOARD_USER:-admin}" +DASHBOARD_PASSWORD="${DASHBOARD_PASSWORD:-admin}" +BOOTSTRAP_DIR="${CLUSTER_DIR}/bootstrap" +BOOTSTRAP_OSD_KEYRING="/var/lib/ceph/bootstrap-osd/ceph.keyring" + +export CEPH_VOLUME_ALLOW_LOOP_DEVICES=true + +mkdir -p "${CLUSTER_DIR}/var/lib/ceph" + +export_ceph_conf() { + if [[ -f "${CEPH_CONF}" ]]; then + export CEPH_CONF + else + unset CEPH_CONF || true + fi +} + +chown_cluster() { + chown -R ceph:ceph /var/lib/ceph + for path in "${CEPH_CONF}" "${ADMIN_KEYRING}" "${CLUSTER_DIR}/fsid"; do + if [[ -e "${path}" ]]; then + chown ceph:ceph "${path}" + fi + done +} + +detect_mon_ip() { + local ip + + ip="$(ip -4 route get 1.1.1.1 2>/dev/null | \ + awk '{for (i = 1; i <= NF; i++) if ($i == "src") { print $(i + 1); exit }}')" + if [[ -n "${ip}" ]]; then + echo "${ip}" + return + fi + ip="$(hostname -I | awk '{print $1}')" + if [[ -n "${ip}" ]]; then + echo "${ip}" + return + fi + echo "127.0.0.1" +} + +detect_public_network() { + local mon_ip="$1" + local net + + net="$(ip route list 2>/dev/null | grep -w "${mon_ip}" | grep -v default | \ + grep -E '/[0-9]+' | awk '{print $1; exit}')" + if [[ -n "${net}" ]]; then + echo "${net}" + return + fi + echo "${mon_ip}/32" +} + +bootstrap_cluster() { + local fsid mon_ip public_network mon_keyring monmap mon_host + fsid="$(cat "${CLUSTER_DIR}/fsid" 2>/dev/null || uuidgen | tee "${CLUSTER_DIR}/fsid")" + mon_ip="$(detect_mon_ip)" + public_network="$(detect_public_network "${mon_ip}")" + mon_host="v2:${mon_ip}:3300,v1:${mon_ip}:6789" + mon_keyring="${BOOTSTRAP_DIR}/ceph.mon.keyring" + monmap="${BOOTSTRAP_DIR}/ceph.monmap" + + mkdir -p "${BOOTSTRAP_DIR}" + mkdir -p "${CLUSTER_DIR}/var/lib/ceph/mon/ceph-${MON_ID}" + mkdir -p "${CLUSTER_DIR}/var/lib/ceph/mgr/ceph-${MGR_ID}" + mkdir -p "${CLUSTER_DIR}/var/lib/ceph/bootstrap-osd" + + ceph-authtool --create-keyring "${mon_keyring}" --gen-key -n mon. --cap mon 'allow *' + ceph-authtool "${mon_keyring}" --gen-key -n client.admin \ + --cap mon 'allow *' \ + --cap osd 'allow *' \ + --cap mds 'allow *' \ + --cap mgr 'allow *' + ceph-authtool --create-keyring \ + "${CLUSTER_DIR}/var/lib/ceph/bootstrap-osd/ceph.keyring" \ + --gen-key -n client.bootstrap-osd \ + --cap mon 'profile bootstrap-osd' + ceph-authtool "${mon_keyring}" --import-keyring \ + "${CLUSTER_DIR}/var/lib/ceph/bootstrap-osd/ceph.keyring" + + monmaptool --create --clobber --fsid "${fsid}" \ + --addv "${MON_ID}" "[${mon_host}]" \ + "${monmap}" + + cat > "${CEPH_CONF}" </dev/null || hostname) +public bind addr = + +[osd] +osd numa auto affinity = false +EOF + export_ceph_conf + + ceph-mon --mkfs -i "${MON_ID}" --monmap "${monmap}" --keyring "${mon_keyring}" \ + --conf "${CEPH_CONF}" + + ceph-authtool --create-keyring "${ADMIN_KEYRING}" + ceph-authtool "${ADMIN_KEYRING}" \ + --import-keyring "${mon_keyring}" + + chown_cluster + rm -rf "${BOOTSTRAP_DIR}" +} + +start_mon() { + if pgrep -f "ceph-mon -i ${MON_ID}" >/dev/null 2>&1; then + return + fi + ceph-mon -i "${MON_ID}" --conf "${CEPH_CONF}" --setuser ceph --setgroup ceph -f & +} + +start_mgr() { + if pgrep -f "ceph-mgr -i ${MGR_ID}" >/dev/null 2>&1; then + return + fi + mkdir -p "${CLUSTER_DIR}/var/lib/ceph/mgr/ceph-${MGR_ID}" + mgr_keyring="${CLUSTER_DIR}/var/lib/ceph/mgr/ceph-${MGR_ID}/keyring" + for attempt in $(seq 1 30); do + if ceph --conf "${CEPH_CONF}" auth get-or-create "mgr.${MGR_ID}" \ + mon 'allow profile mgr' osd 'allow *' mds 'allow *' \ + -o "${mgr_keyring}"; then + break + fi + sleep 2 + done + if [[ ! -f "${mgr_keyring}" ]]; then + echo "Failed to create mgr.${MGR_ID} keyring" >&2 + return 1 + fi + chown ceph:ceph "${mgr_keyring}" + ceph-mgr -i "${MGR_ID}" --conf "${CEPH_CONF}" --no-mon-config \ + --keyring "${mgr_keyring}" \ + --setuser ceph --setgroup ceph -f & +} + +wait_for_mgr() { + local attempt available + + for attempt in $(seq 1 60); do + available="$(ceph --conf "${CEPH_CONF}" mgr stat -f json 2>/dev/null | python3 -c \ + 'import json,sys; print(json.load(sys.stdin).get("available", False))' \ + 2>/dev/null || echo False)" + if [[ "${available}" == "True" ]]; then + return 0 + fi + sleep 2 + done + echo "Timed out waiting for mgr" >&2 + return 1 +} + +setup_dashboard() { + local mon_ip scheme url port_option password_file + + if [[ "${DASHBOARD_ENABLED}" != "true" ]]; then + return 0 + fi + + password_file="${CLUSTER_DIR}/dashboard-admin-secret.txt" + mon_ip="$(detect_mon_ip)" + + ceph --conf "${CEPH_CONF}" mgr module enable dashboard + + if [[ "${DASHBOARD_SSL}" == "true" ]]; then + port_option="ssl_server_port" + scheme="https" + ceph --conf "${CEPH_CONF}" config set mgr mgr/dashboard/ssl true --force + ceph --conf "${CEPH_CONF}" config set mgr \ + "mgr/dashboard/${MGR_ID}/${port_option}" "${DASHBOARD_PORT}" --force + ceph --conf "${CEPH_CONF}" dashboard create-self-signed-cert 2>/dev/null || true + else + port_option="server_port" + scheme="http" + ceph --conf "${CEPH_CONF}" config set mgr mgr/dashboard/ssl false --force + ceph --conf "${CEPH_CONF}" config set mgr \ + "mgr/dashboard/${MGR_ID}/${port_option}" "${DASHBOARD_PORT}" --force + fi + ceph --conf "${CEPH_CONF}" config set mgr \ + "mgr/dashboard/${MGR_ID}/server_addr" "0.0.0.0" --force + + for attempt in $(seq 1 60); do + if ceph --conf "${CEPH_CONF}" -h 2>/dev/null | grep -q '^dashboard '; then + break + fi + sleep 2 + done + + printf '%s' "${DASHBOARD_PASSWORD}" > "${password_file}" + chown ceph:ceph "${password_file}" + chmod 600 "${password_file}" + + if ceph --conf "${CEPH_CONF}" dashboard ac-user-show "${DASHBOARD_USER}" >/dev/null 2>&1; then + ceph --conf "${CEPH_CONF}" dashboard ac-user-set-password "${DASHBOARD_USER}" \ + -i "${password_file}" + else + ceph --conf "${CEPH_CONF}" dashboard ac-user-create "${DASHBOARD_USER}" \ + -i "${password_file}" administrator --force-password + fi + + url="${scheme}://${mon_ip}:${DASHBOARD_PORT}/" + echo "Ceph dashboard: ${url}" + echo " user: ${DASHBOARD_USER}" + if [[ "${DASHBOARD_SHOW_PASSWORD:-false}" == "true" ]]; then + echo " password: ${DASHBOARD_PASSWORD}" + else + echo " password: (see dashboard_password in config; set DASHBOARD_SHOW_PASSWORD=true to log it)" + fi +} + +wait_for_mon() { + local attempt + for attempt in $(seq 1 60); do + if ceph --conf "${CEPH_CONF}" -s >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + echo "Timed out waiting for monitor" >&2 + return 1 +} + +bluestore_show_label() { + local dev="$1" + ceph-bluestore-tool show-label --dev "${dev}" --no-mon-config +} + +get_osd_id_from_device() { + local dev="$1" + bluestore_show_label "${dev}" | python3 -c ' +import json +import sys + +data = json.load(sys.stdin) +labels = data.get("devices", data) +for label in labels.values(): + if label.get("description") == "main": + print(label["whoami"]) + break +' +} + +device_has_osd_label() { + local dev="$1" + get_osd_id_from_device "${dev}" >/dev/null 2>&1 +} + +chown_block_device() { + local dev="$1" + chown ceph:ceph "${dev}" +} + +mkfs_raw_osd() { + local osd_id="$1" osd_dir="$2" osd_fsid="$3" osd_secret="$4" + local attempt + + for attempt in $(seq 1 5); do + if printf '%s' "${osd_secret}" | ceph-osd --cluster ceph --conf "${CEPH_CONF}" \ + --osd-objectstore bluestore --mkfs \ + --no-mon-config \ + -i "${osd_id}" \ + --monmap "${osd_dir}/activate.monmap" \ + --osd-data "${osd_dir}" \ + --osd-uuid "${osd_fsid}" \ + --setuser ceph --setgroup ceph \ + --keyfile -; then + return 0 + fi + echo "ceph-osd --mkfs failed for osd.${osd_id} (attempt ${attempt}/5)" >&2 + sleep 1 + done + return 1 +} + +prepare_raw_osd() { + local dev="$1" + local osd_id osd_fsid osd_dir osd_secret + + if device_has_osd_label "${dev}"; then + return 0 + fi + + # Clear any stale label/header from a previous failed prepare. + dd if=/dev/zero of="${dev}" bs=1M count=1 conv=notrunc status=none 2>/dev/null || true + + osd_fsid="$(uuidgen)" + osd_secret="$(ceph-authtool --gen-print-key)" + osd_id="$(printf '{"cephx_secret":"%s"}' "${osd_secret}" | \ + ceph --conf "${CEPH_CONF}" --cluster ceph --name client.bootstrap-osd \ + --keyring "${BOOTSTRAP_OSD_KEYRING}" -i - osd new "${osd_fsid}")" + + osd_dir="/var/lib/ceph/osd/ceph-${osd_id}" + rm -rf "${osd_dir}" + mkdir -p "${osd_dir}" + + ceph-authtool "${osd_dir}/keyring" --create-keyring --name "osd.${osd_id}" \ + --add-key "${osd_secret}" \ + --cap mon 'allow profile osd' \ + --cap mgr 'allow profile osd' \ + --cap osd 'allow *' + + ceph --conf "${CEPH_CONF}" --cluster ceph --name client.bootstrap-osd \ + --keyring "${BOOTSTRAP_OSD_KEYRING}" \ + mon getmap -o "${osd_dir}/activate.monmap" + + chown_block_device "${dev}" + ln -snf "${dev}" "${osd_dir}/block" + chown -R ceph:ceph "${osd_dir}" + + mkfs_raw_osd "${osd_id}" "${osd_dir}" "${osd_fsid}" "${osd_secret}" + chown -R ceph:ceph "${osd_dir}" +} + +start_raw_osd() { + local dev="$1" + local osd_id osd_dir + + if ! device_has_osd_label "${dev}"; then + echo "No BlueStore label on ${dev}; skipping osd start" >&2 + return 1 + fi + + osd_id="$(get_osd_id_from_device "${dev}")" + osd_dir="/var/lib/ceph/osd/ceph-${osd_id}" + mkdir -p "${osd_dir}" + + chown_block_device "${dev}" + + # After mkfs the osd dir is already populated; prime-osd-dir is only needed + # when re-activating from a labeled device with an empty/missing data dir. + if [[ ! -f "${osd_dir}/ready" ]]; then + rm -f "${osd_dir}/block" "${osd_dir}/block.wal" "${osd_dir}/block.db" + chown -R ceph:ceph "${osd_dir}" + ceph-bluestore-tool prime-osd-dir \ + --path "${osd_dir}" \ + --no-mon-config \ + --dev "${dev}" + fi + + ln -snf "${dev}" "${osd_dir}/block" + chown -R ceph:ceph "${osd_dir}" + + if ! pgrep -f "ceph-osd -i ${osd_id}" >/dev/null 2>&1; then + ceph-osd -i "${osd_id}" --conf "${CEPH_CONF}" --no-mon-config \ + --keyring "${osd_dir}/keyring" \ + --setuser ceph --setgroup ceph -f & + fi +} + +wait_for_osds() { + local attempt expected up + local -a devices=() + + if [[ -z "${OSD_DEVICES}" ]]; then + return 0 + fi + IFS=',' read -ra devices <<< "${OSD_DEVICES}" + expected=0 + for dev in "${devices[@]}"; do + [[ -b "${dev}" ]] && expected=$((expected + 1)) + done + [[ "${expected}" -gt 0 ]] || return 0 + + for attempt in $(seq 1 60); do + up="$(ceph --conf "${CEPH_CONF}" osd stat -f json 2>/dev/null | python3 -c \ + 'import json,sys; print(json.load(sys.stdin).get("num_osds_up", 0))' 2>/dev/null || echo 0)" + if [[ "${up}" -ge "${expected}" ]]; then + return 0 + fi + sleep 2 + done + echo "Timed out waiting for ${expected} OSD(s)" >&2 + return 1 +} + +wait_for_health() { + local attempt health + + for attempt in $(seq 1 90); do + health="$(ceph --conf "${CEPH_CONF}" health 2>/dev/null || true)" + case "${health}" in + HEALTH_OK) + echo "${health}" + return 0 + ;; + HEALTH_ERR) + echo "${health} ($(ceph --conf "${CEPH_CONF}" health detail 2>/dev/null | tr '\n' ' '))" >&2 + return 1 + ;; + esac + sleep 2 + done + echo "Timed out waiting for cluster health" >&2 + return 1 +} + +prepare_osds() { + local dev + if [[ -z "${OSD_DEVICES}" ]]; then + return + fi + IFS=',' read -ra devices <<< "${OSD_DEVICES}" + for dev in "${devices[@]}"; do + [[ -b "${dev}" ]] || continue + prepare_raw_osd "${dev}" + done + for dev in "${devices[@]}"; do + [[ -b "${dev}" ]] || continue + start_raw_osd "${dev}" + done +} + +if [[ ! -f "${CEPH_CONF}" ]]; then + echo "Bootstrapping ceph cluster in ${CLUSTER_DIR}" + bootstrap_cluster +else + export_ceph_conf +fi + +start_mon +wait_for_mon +start_mgr +wait_for_mgr +setup_dashboard +prepare_osds +wait_for_osds +wait_for_health + +echo "Ceph cluster is running; use: podman exec ${HOSTNAME:-ceph_node} ceph -s" +exec tail -f /dev/null diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py new file mode 100644 index 00000000..e47eac57 --- /dev/null +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -0,0 +1,741 @@ +import asyncio +import os +import shlex +import shutil +import subprocess +import sys +import uuid + +from pathlib import Path +from subprocess import CalledProcessError +from typing import List + +from ceph_devstack import PROJECT_ROOT, config, logger +from ceph_devstack.host import host +from ceph_devstack.resources.container import Container + + +DEFAULT_CEPH_IMAGE = "quay.io/ceph-ci/ceph:main" +DEFAULT_BASE_IMAGE = "quay.io/ceph-ci/ceph:main" +PACKAGE_SCCACHE_CONF = PROJECT_ROOT / "sccache.conf" +PACKAGE_SCCACHE_S3_CONF = PROJECT_ROOT / "sccache-s3.conf" +CONTAINER_SCCACHE_DIR = "/sccache" +CONTAINER_GIT_METADATA_DIR = "/git-metadata" +BWC_HOMEDIR = "/ceph" +REPO_DEVSTACK_DIR = ".ceph-devstack" +BUILD_ENV_NAME = "build.env" +ENTRYPOINT_SCRIPT = Path(__file__).with_name("ceph-node-entrypoint.sh") +CLUSTER_ENTRYPOINT_NAME = "ceph-node-entrypoint.sh" +CLUSTER_DATA_NAMES = ("var", "fsid", CLUSTER_ENTRYPOINT_NAME) +CONTAINER_CLUSTER_DIR = "/var/lib/ceph-devstack/cluster" + +DEFAULT_COMPILE_STEPS = { + "cpatch": ["build"], + "container": ["packages"], +} + +CEPH_NODE_CAPABILITIES = [ + "SYS_ADMIN", + "NET_ADMIN", + "SYS_TIME", + "SYS_RAWIO", + "MKNOD", + "NET_RAW", + "SETUID", + "SETGID", + "CHOWN", + "SYS_PTRACE", +] + + +def expand_path(path: str | Path) -> Path: + return Path(os.path.expanduser(str(path))) + + +def git_worktree_info(repo: Path) -> tuple[Path, str] | None: + """Return the main .git directory and worktree name for a linked worktree.""" + git_path = repo.resolve() / ".git" + if not git_path.is_file(): + return None + text = git_path.read_text(encoding="utf-8").strip() + if not text.startswith("gitdir:"): + return None + admin_dir = Path(text.split(":", 1)[1].strip()) + if admin_dir.parent.name != "worktrees": + raise ValueError(f"unexpected git worktree gitdir: {admin_dir}") + main_git_dir = admin_dir.parent.parent + if not main_git_dir.is_dir(): + raise FileNotFoundError(f"git metadata dir not found: {main_git_dir}") + return main_git_dir.resolve(), admin_dir.name + + +def worktree_submodule_git_mounts( + repo: Path, + worktree_name: str, + git_overlay_dir: Path, +) -> list[str]: + """Return podman mounts that rewrite submodule ``.git`` files for ``/ceph``.""" + proc = subprocess.run( + ["git", "-C", str(repo), "submodule", "foreach", "--quiet", "echo $sm_path"], + check=False, + capture_output=True, + text=True, + ) + if proc.returncode != 0: + return [] + + overlay_dir = git_overlay_dir / "submodules" + overlay_dir.mkdir(parents=True, exist_ok=True) + mounts: list[str] = [] + for sm_path in proc.stdout.splitlines(): + sm_path = sm_path.strip() + if not sm_path: + continue + gitdir = ( + f"{CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}/modules/{sm_path}" + ) + overlay = overlay_dir / f"{sm_path.replace('/', '__')}.git" + overlay.write_text(f"gitdir: {gitdir}\n", encoding="utf-8") + mounts.append(f"--volume={overlay}:{BWC_HOMEDIR}/{sm_path}/.git:Z,ro") + return mounts + + +def worktree_container_mounts( + repo: Path, + main_git_dir: Path, + worktree_name: str, +) -> list[str]: + """Return podman mounts that make a linked worktree usable at ``/ceph``.""" + repo = repo.resolve() + main_git_dir = main_git_dir.resolve() + git_overlay_dir = repo / REPO_DEVSTACK_DIR / "git" + git_overlay_dir.mkdir(parents=True, exist_ok=True) + + dot_git = git_overlay_dir / "dot-git" + dot_git.write_text( + f"gitdir: {CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}\n", + encoding="utf-8", + ) + + admin_gitdir = git_overlay_dir / "gitdir" + admin_gitdir.write_text(f"{BWC_HOMEDIR}/.git\n", encoding="utf-8") + + mounts = [ + f"--volume={main_git_dir}:{CONTAINER_GIT_METADATA_DIR}:Z,ro", + f"--volume={dot_git}:{BWC_HOMEDIR}/.git:Z,ro", + f"--volume={admin_gitdir}:{CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}/gitdir:Z,ro", + ] + mounts.extend(worktree_submodule_git_mounts(repo, worktree_name, git_overlay_dir)) + return mounts + + +class CephNode(Container): + """Single-container Ceph cluster without cephadm. + + Builds the runtime image when ``repo`` is configured, creates host loop + devices, and runs mon/mgr/osd inside one capability-scoped container via + ``ceph-node-entrypoint.sh``. + """ + + _name = "ceph_node" + stop_cmd = ["podman", "container", "stop", "{name}"] + remove_cmd = ["podman", "container", "rm", "-f", "{name}"] + start_cmd = ["podman", "container", "start", "{name}"] + exists_cmd = ["podman", "container", "inspect", "{name}"] + + def __init__(self, name: str = ""): + super().__init__(name) + self.loop_device_count = self.config.get("loop_device_count", 3) + self.devices = [self._device_name(i) for i in range(self.loop_device_count)] + + @property + def loop_device_base(self) -> int: + if "loop_device_base" in self.config: + return int(self.config["loop_device_base"]) + testnode = config.get("containers", {}).get("testnode", {}) + count = int(testnode.get("count", 0) or 0) + per_node = int(testnode.get("loop_device_count", 1) or 1) + return count * per_node + + @property + def config_key(self) -> str: + return "ceph_node" + + @property + def cluster_dir(self) -> Path: + return expand_path(self.config.get("output_dir", config["data_dir"])) + + @property + def persistent_cache_dir(self) -> Path: + # Lives beside cluster_dir so ``remove()`` can tear down cluster state + # without wiping npm/sccache/dnf caches. + return self.cluster_dir.parent / "cache" + + @property + def loop_img_dir(self) -> Path: + # Keep loop backing files outside cluster_dir so podman volume relabeling + # does not touch them on container start (same layout as TestNode). + return self.cluster_dir.parent / "disk_images" + + @property + def legacy_loop_img_dir(self) -> Path: + return self.cluster_dir / "disk_images" + + @property + def container_entrypoint(self) -> Path: + return self.cluster_dir / CLUSTER_ENTRYPOINT_NAME + + @property + def container_entrypoint_script(self) -> str: + return f"{self.container_cluster_dir}/{CLUSTER_ENTRYPOINT_NAME}" + + @property + def container_cluster_dir(self) -> str: + return CONTAINER_CLUSTER_DIR + + @property + def mon_id(self) -> str: + return self.config.get("mon_id", "a") + + @property + def mgr_id(self) -> str: + return self.config.get("mgr_id", "x") + + @property + def dashboard_enabled(self) -> bool: + return self.config.get("dashboard", True) is not False + + @property + def dashboard_port(self) -> int: + return int(self.config.get("dashboard_port", 8080)) + + @property + def dashboard_ssl(self) -> bool: + return bool(self.config.get("dashboard_ssl", False)) + + @property + def dashboard_user(self) -> str: + return self.config.get("dashboard_user", "admin") + + @property + def dashboard_password(self) -> str: + return self.config.get("dashboard_password", "admin") + + @property + def dashboard_show_password(self) -> bool: + return self.config.get("dashboard_show_password", False) is True + + @property + def base_image(self) -> str: + return self.config.get("base_image", DEFAULT_BASE_IMAGE) + + @property + def build_subdir(self) -> str: + build_dir = self.config.get("build_dir", "build") + if not build_dir: + return "build" + path = Path(os.path.expanduser(str(build_dir))) + if path.is_absolute() and self.repo: + repo = Path(self.repo).resolve() + try: + return str(path.resolve().relative_to(repo)) + except ValueError: + return path.name + return str(build_dir).strip("/") + + @property + def build_path(self) -> Path: + if not self.repo: + return Path() + return Path(self.repo) / self.build_subdir + + @property + def image_builder(self) -> str: + return self.config.get("image_builder", "cpatch") + + @property + def compile_steps(self) -> List[str]: + default = DEFAULT_COMPILE_STEPS.get(self.image_builder, ["build"]) + return list(self.config.get("build_steps", default)) + + @property + def sccache_enabled(self) -> bool: + return self.config.get("sccache", True) is not False + + @property + def sccache_mode(self) -> str: + return str(self.config.get("sccache_mode", "local")).lower() + + @property + def sccache_debug(self) -> bool: + return self.config.get("sccache_debug", False) is True + + @property + def sccache_cache_path(self) -> Path: + if custom := self.config.get("sccache_cache_path"): + return expand_path(custom) + return self.persistent_cache_dir / "sccache" + + @property + def npm_cache_enabled(self) -> bool: + return self.config.get("npm_cache", True) is not False + + @property + def npm_cache_path(self) -> Path | None: + if not self.npm_cache_enabled: + return None + if custom := self.config.get("npm_cache_path"): + return expand_path(custom) + return self.persistent_cache_dir / "npm" + + @property + def dnf_cache_path(self) -> Path | None: + if self.config.get("dnf_cache", False) is not True: + return None + if custom := self.config.get("dnf_cache_path"): + return expand_path(custom) + return self.persistent_cache_dir / "dnf" + + def _build_cache_args(self) -> list[str]: + """Return build-with-container.py cache flags for persistent build caches.""" + args: list[str] = [] + + npm_cache = self.npm_cache_path + if npm_cache is not None: + npm_cache.mkdir(parents=True, exist_ok=True) + args.extend(["--npm-cache-path", str(npm_cache)]) + + dnf_cache = self.dnf_cache_path + if dnf_cache is not None: + dnf_cache.mkdir(parents=True, exist_ok=True) + args.extend(["--dnf-cache-path", str(dnf_cache)]) + + return args + + @property + def create_cmd(self): + host_cluster_dir = str(self.cluster_dir.resolve()) + container_cluster_dir = self.container_cluster_dir + entrypoint = self.container_entrypoint_script + return [ + "podman", + "container", + "create", + "-i", + "--network", + "host", + "--cap-add", + ",".join(CEPH_NODE_CAPABILITIES), + "--security-opt", + "unmask=/sys/dev/block", + "-v", + f"{host_cluster_dir}:{container_cluster_dir}", + "-v", + f"{host_cluster_dir}/var/lib/ceph:/var/lib/ceph", + "-v", + "/run/udev:/run/udev", + "-v", + "/sys/dev/block:/sys/dev/block", + "-v", + "/dev/fuse:/dev/fuse", + "-v", + "/dev/disk:/dev/disk", + "-v", + "/dev/null:/sys/class/dmi/id/board_serial", + "-v", + "/dev/null:/sys/class/dmi/id/chassis_serial", + "-v", + "/dev/null:/sys/class/dmi/id/product_serial", + "--device", + "/dev/net/tun", + *[f"--device={device}" for device in self.devices], + "-e", + f"CLUSTER_DIR={container_cluster_dir}", + "-e", + f"MON_ID={self.mon_id}", + "-e", + f"MGR_ID={self.mgr_id}", + "-e", + f"OSD_DEVICES={','.join(self.devices)}", + "-e", + f"DASHBOARD_ENABLED={'true' if self.dashboard_enabled else 'false'}", + "-e", + f"DASHBOARD_PORT={self.dashboard_port}", + "-e", + f"DASHBOARD_SSL={'true' if self.dashboard_ssl else 'false'}", + "-e", + f"DASHBOARD_USER={self.dashboard_user}", + "-e", + f"DASHBOARD_PASSWORD={self.dashboard_password}", + "-e", + f"DASHBOARD_SHOW_PASSWORD={'true' if self.dashboard_show_password else 'false'}", + "--entrypoint", + "/bin/bash", + "--name", + "{name}", + "{image}", + "-c", + f". {shlex.quote(entrypoint)}", + ] + + def _device_name(self, index: int) -> str: + return f"/dev/loop{self.loop_device_base + index}" + + def _device_image(self, device: str) -> str: + return f"{self.name}-{device.removeprefix('/dev/loop')}" + + def _sccache_build_env(self, repo: Path) -> tuple[list[str], list[str]]: + if not self.sccache_enabled: + return [], [] + + homedir = BWC_HOMEDIR + if custom_conf := self.config.get("sccache_conf"): + conf_src = expand_path(custom_conf) + use_local_cache = self.sccache_mode == "local" + elif self.sccache_mode == "s3": + conf_src = PACKAGE_SCCACHE_S3_CONF + use_local_cache = False + else: + conf_src = PACKAGE_SCCACHE_CONF + use_local_cache = True + if not conf_src.is_file(): + raise FileNotFoundError(f"sccache config not found: {conf_src}") + shutil.copy2(conf_src, repo / "sccache.conf") + + lines = [ + "SCCACHE=true", + f"SCCACHE_CONF={homedir}/sccache.conf", + f"SCCACHE_ERROR_LOG={homedir}/.ceph-devstack/sccache_log.txt", + "CEPH_BUILD_NORMALIZE_PATHS=true", + ] + if self.sccache_debug: + lines.append("SCCACHE_LOG=debug") + + extra_args: list[str] = [] + if use_local_cache: + cache_path = self.sccache_cache_path + cache_path.mkdir(parents=True, exist_ok=True) + extra_args.append(f"--volume={cache_path}:{CONTAINER_SCCACHE_DIR}:Z") + lines.append(f"SCCACHE_DIR={CONTAINER_SCCACHE_DIR}") + cache_size = self.config.get("sccache_cache_size", "100G") + lines.append(f"SCCACHE_CACHE_SIZE={cache_size}") + elif self.sccache_mode == "s3": + lines.extend( + [ + "SCCACHE_S3_NO_CREDENTIALS=true", + "SCCACHE_S3_RW_MODE=READ_ONLY", + ] + ) + return lines, extra_args + + def _prepare_build_env(self) -> tuple[Path | None, list[str]]: + if not self.repo: + return None, [] + repo = Path(self.repo) + extra_args: list[str] = [] + + worktree = git_worktree_info(repo) + if worktree is not None: + main_git_dir, worktree_name = worktree + extra_args.extend( + worktree_container_mounts(repo, main_git_dir, worktree_name) + ) + + lines, sccache_extra = self._sccache_build_env(repo) + extra_args.extend(sccache_extra) + + if not lines: + return None, extra_args + + devstack_dir = repo / REPO_DEVSTACK_DIR + devstack_dir.mkdir(exist_ok=True) + env_path = devstack_dir / BUILD_ENV_NAME + env_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return env_path, extra_args + + def _compile_cmd( + self, + env_file: Path | None = None, + extra_args: List[str] | None = None, + ) -> List[str]: + distro = self.config.get("build_distro", "centos9") + script = str(Path(self.repo) / "src/script/build-with-container.py") + python_cmd = "python3" if host.type == "remote" else sys.executable + cmd = [ + python_cmd, + script, + "-d", + distro, + "-b", + self.build_subdir, + "--homedir", + BWC_HOMEDIR, + ] + for step in self.compile_steps: + cmd.extend(["-e", step]) + if env_file is not None: + cmd.extend(["--env-file", str(env_file)]) + cmd.extend(self._build_cache_args()) + for extra in extra_args or []: + cmd.append(f"--extra={extra}") + return cmd + + def _cpatch_cmd(self) -> List[str]: + return [ + "sudo", + "../src/script/cpatch", + "--base", + self.base_image, + "--target", + self.image, + "--core", + ] + + def _git_value(self, args: str) -> str: + return subprocess.check_output( + f"git {args}".split(), + cwd=self.repo, + text=True, + ).strip() + + async def _run_cmd(self, cmd: List[str], cwd: str): + proc = await host.arun( + cmd, + cwd=Path(cwd).expanduser(), + stream_output=True, + ) + returncode = await proc.wait() + if returncode != 0: + stdout, stderr = await proc.log_failure(cmd) + raise CalledProcessError( + returncode, + cmd, + output=stdout or None, + stderr=stderr or None, + ) + + async def _compile(self): + logger.info( + f"{self.name}: compiling ceph via build-with-container.py in {self.repo}" + ) + env_file, extra_args = self._prepare_build_env() + await self._run_cmd( + self._compile_cmd(env_file=env_file, extra_args=extra_args), + cwd=self.repo, + ) + + def _verify_build_tree(self): + build_path = self.build_path + if (build_path / "build.ninja").exists() or (build_path / "Makefile").exists(): + return + raise FileNotFoundError( + f"Ceph build dir {build_path} missing Makefile or build.ninja " + "after build-with-container.py" + ) + + async def _build_image_cpatch(self): + self._verify_build_tree() + build_path = self.build_path + logger.info(f"{self.name}: building {self.image} via cpatch in {build_path}") + await self._run_cmd(self._cpatch_cmd(), cwd=str(build_path)) + + async def _build_image_container(self): + raise NotImplementedError( + "image_builder='container' requires ceph container/build.sh to consume " + "locally-built packages; enable this once that support lands upstream" + ) + + async def _build_image(self): + builders = { + "cpatch": self._build_image_cpatch, + "container": self._build_image_container, + } + try: + builder = builders[self.image_builder] + except KeyError as exc: + known = ", ".join(sorted(builders)) + raise ValueError( + f"Unknown image_builder {self.image_builder!r}; known: {known}" + ) from exc + await builder() + + def install_entrypoint(self): + if not ENTRYPOINT_SCRIPT.is_file(): + raise FileNotFoundError(f"Entrypoint script not found: {ENTRYPOINT_SCRIPT}") + shutil.copy2(ENTRYPOINT_SCRIPT, self.container_entrypoint) + self.container_entrypoint.chmod(0o755) + + async def label_cluster_dir(self): + if sys.platform == "darwin": + return + await self.cmd( + ["chcon", "-Rt", "container_file_t", str(self.cluster_dir)], + check=False, + ) + + async def remove_legacy_loop_img_dir(self): + legacy = self.legacy_loop_img_dir + if not legacy.is_dir(): + return + logger.info( + f"{self.name}: removing legacy loop image dir at {legacy} " + "(loop backing files now live outside the cluster mount)" + ) + for device in self.devices: + if not host.path_exists(device): + continue + proc = await self.cmd(["losetup", device], check=False) + if proc and await proc.wait() == 0: + await self.cmd(["sudo", "losetup", "-d", device], check=False) + shutil.rmtree(legacy) + + async def remove_cluster_data(self): + cluster_dir = self.cluster_dir.resolve() + if not cluster_dir.exists(): + return + paths = [ + cluster_dir / name + for name in CLUSTER_DATA_NAMES + if (cluster_dir / name).exists() + ] + if not paths: + return + logger.info(f"{self.name}: removing cluster data at {cluster_dir}") + for path in paths: + await self.cmd( + ["podman", "unshare", "rm", "-rf", str(path)], + check=False, + ) + + async def build(self): + if not self.should_build: + return + await self._compile() + await self._build_image() + + async def create(self): + logger.info(f"{self.name}: preparing cluster at {self.cluster_dir}") + self.cluster_dir.mkdir(parents=True, exist_ok=True) + (self.cluster_dir / "var/lib/ceph").mkdir(parents=True, exist_ok=True) + (self.cluster_dir / "var/lib/ceph/mon" / f"ceph-{self.mon_id}").mkdir( + parents=True, exist_ok=True + ) + (self.cluster_dir / "var/lib/ceph/mgr" / f"ceph-{self.mgr_id}").mkdir( + parents=True, exist_ok=True + ) + if not (self.cluster_dir / "fsid").exists(): + (self.cluster_dir / "fsid").write_text(f"{uuid.uuid4()}\n") + self.install_entrypoint() + await self.remove_legacy_loop_img_dir() + await self.label_cluster_dir() + logger.info( + f"{self.name}: creating {self.loop_device_count} loop devices " + f"({self.config.get('loop_device_size', '5G')} each)" + ) + await self.create_loop_devices() + await super().create() + + async def remove(self): + await super().remove() + await self.remove_loop_devices() + await self.remove_legacy_loop_img_dir() + await self.remove_cluster_data() + + async def is_running(self): + if not await self.exists(): + return False + proc = await self.cmd( + [ + "podman", + "exec", + self.name, + "ceph", + "--conf", + f"{self.container_cluster_dir}/ceph.conf", + "-s", + ], + check=False, + ) + return proc is not None and await proc.wait() == 0 + + async def wait(self): + if not await self.exists(): + return 1 + for _ in range(60): + if await self.is_running(): + return 0 + await asyncio.sleep(5) + return 1 + + async def create_loop_devices(self): + for device in self.devices: + await self.create_loop_device(device) + + async def remove_loop_devices(self): + for device in self.devices: + await self.remove_loop_device(device) + + async def create_loop_device(self, device: str): + size = self.config.get("loop_device_size", "5G") + os.makedirs(self.loop_img_dir, exist_ok=True) + proc = await self.cmd( + ["bash", "-c", "lsmod | grep -q loop"], + check=False, + ) + if proc and await proc.wait() != 0: + await self.cmd(["sudo", "modprobe", "loop"]) + loop_img_name = self.loop_img_dir / self._device_image(device) + await self.remove_loop_device(device) + device_pos = device.removeprefix("/dev/loop") + await self.cmd( + [ + "sudo", + "mknod", + "-m700", + device, + "b", + "7", + device_pos, + ], + check=True, + ) + await self.cmd( + ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", device], + check=True, + ) + await self.cmd( + [ + "sudo", + "dd", + "if=/dev/null", + f"of={loop_img_name}", + "bs=1", + "count=0", + f"seek={size}", + ], + check=True, + ) + await self.cmd(["sudo", "losetup", device, str(loop_img_name)], check=True) + await self.cmd(["chcon", "-t", "fixed_disk_device_t", device]) + device_pos = device.removeprefix("/dev/loop") + await self.cmd( + [ + "sudo", + "udevadm", + "trigger", + "--action=add", + f"--name=block/loop{device_pos}", + ], + check=False, + ) + await self.cmd(["sudo", "udevadm", "settle"], check=False) + + async def remove_loop_device(self, device: str): + loop_img_name = self.loop_img_dir / self._device_image(device) + if os.path.ismount(device): + await self.cmd(["umount", device], check=True) + if host.path_exists(device): + await self.cmd(["sudo", "losetup", "-d", device]) + await self.cmd(["sudo", "rm", "-f", device], check=True) + if loop_img_name.exists(): + loop_img_name.unlink() diff --git a/ceph_devstack/resources/ceph/containers.py b/ceph_devstack/resources/ceph/containers.py index 9e3ea1c7..b90dd391 100644 --- a/ceph_devstack/resources/ceph/containers.py +++ b/ceph_devstack/resources/ceph/containers.py @@ -181,9 +181,7 @@ def __init__(self, name: str = ""): self.index = 0 if "_" in self.name: self.index = int(self.name.split("_")[-1]) - self.loop_device_count = config["containers"]["testnode"].get( - "loop_device_count", 1 - ) + self.loop_device_count = self.config.get("loop_device_count", 1) self.devices = [self.device_name(i) for i in range(self.loop_device_count)] @property @@ -274,7 +272,7 @@ async def remove_loop_devices(self): await self.remove_loop_device(device) async def create_loop_device(self, device: str): - size = config["containers"]["testnode"]["loop_device_size"] + size = self.config.get("loop_device_size", "5G") os.makedirs(self.loop_img_dir, exist_ok=True) proc = await self.cmd(["lsmod", "|", "grep", "loop"]) if proc and await proc.wait() != 0: diff --git a/ceph_devstack/resources/container.py b/ceph_devstack/resources/container.py index 49662b3c..e7e547a1 100644 --- a/ceph_devstack/resources/container.py +++ b/ceph_devstack/resources/container.py @@ -45,9 +45,13 @@ def add_env_to_args(self, args: List): args.insert(-1, f"{key}={value}") return args + @property + def config_key(self) -> str: + return self.__class__.__name__.lower() + @property def config(self): - return config["containers"].get(self.__class__.__name__.lower(), {}) + return config["containers"].get(self.config_key, {}) @property def image_name(self) -> str: @@ -70,15 +74,33 @@ def image_tag(self): @property def repo(self): repo = self.config.get("repo", "") + if not repo: + return "" try: return repo.expanduser() except AttributeError: return os.path.expanduser(repo) + @property + def build_dir(self): + build_dir = self.config.get("build_dir", "") + if not build_dir: + return "" + try: + return build_dir.expanduser() + except AttributeError: + return os.path.expanduser(build_dir) + @property def cwd(self): + if self.build_dir: + return self.build_dir return self.repo or "." + @property + def should_build(self): + return bool(self.repo) + async def pull(self): if not getattr(self, "pull_cmd", None): return @@ -92,9 +114,9 @@ async def pull(self): ) async def build(self): - if not getattr(self, "repo", None): + if not self.should_build: return - logger.debug(f"{self.name}: building from repo: {self.repo}") + logger.debug(f"{self.name}: building from {self.cwd}") await self.cmd( self.format_cmd(self.build_cmd), check=True, diff --git a/ceph_devstack/sccache-s3.conf b/ceph_devstack/sccache-s3.conf new file mode 100644 index 00000000..01db75c7 --- /dev/null +++ b/ceph_devstack/sccache-s3.conf @@ -0,0 +1,8 @@ +[cache.s3] +bucket = "ceph-sccache" +endpoint = "s3.us-south.cloud-object-storage.appdomain.cloud" +use_ssl = true +key_prefix = "" +server_side_encryption = false +no_credentials = true +region = "auto" diff --git a/ceph_devstack/sccache.conf b/ceph_devstack/sccache.conf new file mode 100644 index 00000000..1a9194df --- /dev/null +++ b/ceph_devstack/sccache.conf @@ -0,0 +1,2 @@ +[cache.disk] +dir = "/sccache" diff --git a/pyproject.toml b/pyproject.toml index 95472fb6..0b97e02d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,9 @@ ceph-devstack = "ceph_devstack.cli:main" [tool.setuptools] include-package-data = true +[tool.setuptools.package-data] +"ceph_devstack.resources.ceph" = ["*.sh"] + [tool.setuptools.packages.find] namespaces = false @@ -56,3 +59,6 @@ requires = [ testpaths = ["tests"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function" +markers = [ + "integration: integration tests requiring podman (deselect with '-m \"not integration\"')", +] diff --git a/tests/integration/test_ceph_stack.py b/tests/integration/test_ceph_stack.py new file mode 100644 index 00000000..c7aaae0c --- /dev/null +++ b/tests/integration/test_ceph_stack.py @@ -0,0 +1,24 @@ +"""Ceph stack integration test (podman + live cluster).""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = Path(__file__).with_name("test_ceph_stack.sh") + +pytestmark = pytest.mark.skipif( + os.environ.get("CEPH_DEVSTACK_INTEGRATION") != "1" + or shutil.which("podman") is None, + reason="requires CEPH_DEVSTACK_INTEGRATION=1 and podman", +) + + +@pytest.mark.integration +def test_ceph_stack_reaches_health_ok_and_dashboard() -> None: + subprocess.run(["bash", str(_SCRIPT)], check=True, cwd=_REPO_ROOT) diff --git a/tests/integration/test_ceph_stack.sh b/tests/integration/test_ceph_stack.sh new file mode 100755 index 00000000..92807bb6 --- /dev/null +++ b/tests/integration/test_ceph_stack.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Integration test for the ceph stack: bootstrap a single-node cluster and +# verify HEALTH_OK plus a live dashboard API endpoint. +set -euo pipefail + +STACK=ceph +CONTAINER=ceph_node +CEPH_CONF=/var/lib/ceph-devstack/cluster/ceph.conf +DASHBOARD_PORT=8080 +DASHBOARD_USER=admin +DASHBOARD_PASSWORD=admin + +cd "$(dirname "$0")/../.." + +cleanup() { + local status=$? + if [[ $status -ne 0 ]]; then + echo "=== ${CONTAINER} logs (last 200 lines) ===" >&2 + podman logs "$CONTAINER" 2>&1 | tail -200 >&2 || true + fi + uv run ceph-devstack --stack "$STACK" -v stop || true + uv run ceph-devstack --stack "$STACK" -v remove || true + exit "$status" +} +trap cleanup EXIT + +uv run ceph-devstack --stack "$STACK" -v doctor --fix +uv run ceph-devstack --stack "$STACK" -v pull +uv run ceph-devstack --stack "$STACK" -v create +uv run ceph-devstack --stack "$STACK" -v start + +health="" +for _ in $(seq 1 90); do + health="$( + podman exec "$CONTAINER" ceph --conf "$CEPH_CONF" health 2>/dev/null \ + | awk '{print $1}' || true + )" + if [[ "$health" == "HEALTH_OK" ]]; then + echo "Cluster reached HEALTH_OK" + break + fi + sleep 2 +done + +if [[ "$health" != "HEALTH_OK" ]]; then + echo "Expected HEALTH_OK, got: ${health:-}" >&2 + exit 1 +fi + +dashboard_ok=false +for _ in $(seq 1 30); do + if curl -sf -X POST "http://127.0.0.1:${DASHBOARD_PORT}/api/auth" \ + -H 'Accept: application/vnd.ceph.api.v1.0+json' \ + -H 'Content-Type: application/json' \ + -d "{\"username\":\"${DASHBOARD_USER}\",\"password\":\"${DASHBOARD_PASSWORD}\"}" \ + >/dev/null; then + dashboard_ok=true + break + fi + sleep 2 +done + +if [[ "$dashboard_ok" != "true" ]]; then + echo "Dashboard API did not become ready on port ${DASHBOARD_PORT}" >&2 + exit 1 +fi + +echo "Dashboard API is live on port ${DASHBOARD_PORT}" +echo "Ceph stack integration test passed" diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py new file mode 100644 index 00000000..63560e39 --- /dev/null +++ b/tests/resources/ceph/test_ceph_node.py @@ -0,0 +1,435 @@ +from unittest.mock import AsyncMock, patch +import sys + + +from ceph_devstack import config +from ceph_devstack.resources.container import Container +from ceph_devstack.resources.ceph.ceph_node import ( + BUILD_ENV_NAME, + CLUSTER_ENTRYPOINT_NAME, + CONTAINER_CLUSTER_DIR, + CONTAINER_GIT_METADATA_DIR, + CONTAINER_SCCACHE_DIR, + CephNode, + ENTRYPOINT_SCRIPT, + PACKAGE_SCCACHE_CONF, + PACKAGE_SCCACHE_S3_CONF, + REPO_DEVSTACK_DIR, + git_worktree_info, + worktree_container_mounts, +) + + +class TestCephNodeBuild: + def test_image_uses_configured_tag(self): + config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" + assert CephNode().image == "quay.io/ceph-ci/ceph:main" + + def test_should_build_requires_repo(self, tmp_path): + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["repo"] = str(tmp_path) + assert CephNode().should_build is True + + def test_should_build_skips_without_repo(self): + config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" + assert CephNode().should_build is False + + def test_compile_steps_default_for_cpatch(self): + config["containers"]["ceph_node"]["image"] = "example:test" + assert CephNode().compile_steps == ["build"] + + def test_compile_cmd_uses_build_with_container(self, tmp_path): + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["repo"] = str(tmp_path) + config["containers"]["ceph_node"]["build_dir"] = "build" + config["containers"]["ceph_node"]["build_distro"] = "centos9" + config["containers"]["ceph_node"]["sccache"] = False + config["containers"]["ceph_node"]["npm_cache"] = False + cmd = CephNode()._compile_cmd() + assert cmd[0] == sys.executable + assert "build-with-container.py" in cmd[1] + assert "-d" in cmd and "centos9" in cmd + assert "-b" in cmd and "build" in cmd + assert cmd[cmd.index("--homedir") + 1] == "/ceph" + assert "--env-file" not in cmd + assert "--npm-cache-path" not in cmd + + def test_compile_cmd_passes_npm_cache_path(self, tmp_path): + npm_cache = tmp_path / "npm-cache" + config["data_dir"] = str(tmp_path / "data") + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_node"]["npm_cache_path"] = str(npm_cache) + config["containers"]["ceph_node"]["sccache"] = False + cmd = CephNode()._compile_cmd() + assert "--npm-cache-path" in cmd + assert str(npm_cache.resolve()) in cmd + assert npm_cache.is_dir() + + def test_compile_cmd_uses_default_npm_cache_under_data_dir(self, tmp_path): + data_dir = tmp_path / "data" + config["data_dir"] = str(data_dir) + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_node"]["sccache"] = False + cmd = CephNode()._compile_cmd() + expected = (data_dir.parent / "cache" / "npm").resolve() + assert "--npm-cache-path" in cmd + assert str(expected) in cmd + assert expected.is_dir() + + def test_compile_cmd_skips_ccache_dir(self, tmp_path): + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_node"]["npm_cache"] = False + cmd = CephNode()._compile_cmd() + assert "--ccache-dir" not in cmd + + def test_compile_cmd_passes_dnf_cache_path_when_enabled(self, tmp_path): + dnf_cache = tmp_path / "dnf-cache" + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_node"]["npm_cache"] = False + config["containers"]["ceph_node"]["sccache"] = False + config["containers"]["ceph_node"]["dnf_cache"] = True + config["containers"]["ceph_node"]["dnf_cache_path"] = str(dnf_cache) + cmd = CephNode()._compile_cmd() + assert "--dnf-cache-path" in cmd + assert str(dnf_cache.resolve()) in cmd + assert dnf_cache.is_dir() + + def test_prepare_build_env_uses_local_sccache_by_default(self, tmp_path): + data_dir = tmp_path / "data" + config["data_dir"] = str(data_dir) + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + env_file, extra_args = CephNode()._prepare_build_env() + assert env_file == repo / REPO_DEVSTACK_DIR / BUILD_ENV_NAME + assert (repo / "sccache.conf").read_text() == PACKAGE_SCCACHE_CONF.read_text() + contents = env_file.read_text() + assert "SCCACHE=true" in contents + assert "SCCACHE_CONF=/ceph/sccache.conf" in contents + assert "SCCACHE_DIR=/sccache" in contents + assert "SCCACHE_CACHE_SIZE=100G" in contents + assert "SCCACHE_S3_NO_CREDENTIALS" not in contents + assert "SCCACHE_LOG=" not in contents + expected_cache = (data_dir.parent / "cache" / "sccache").resolve() + assert f"--volume={expected_cache}:{CONTAINER_SCCACHE_DIR}:Z" in extra_args + assert expected_cache.is_dir() + + def test_prepare_build_env_enables_sccache_debug_when_configured(self, tmp_path): + data_dir = tmp_path / "data" + config["data_dir"] = str(data_dir) + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache_debug"] = True + env_file, _extra_args = CephNode()._prepare_build_env() + assert "SCCACHE_LOG=debug" in env_file.read_text() + + def test_prepare_build_env_uses_s3_sccache_when_configured(self, tmp_path): + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache_mode"] = "s3" + env_file, extra_args = CephNode()._prepare_build_env() + assert ( + repo / "sccache.conf" + ).read_text() == PACKAGE_SCCACHE_S3_CONF.read_text() + contents = env_file.read_text() + assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents + assert "SCCACHE_S3_RW_MODE=READ_ONLY" in contents + assert "SCCACHE_DIR=" not in contents + assert extra_args == [] + + def test_prepare_build_env_honors_custom_sccache_conf(self, tmp_path): + custom_conf = tmp_path / "custom-sccache.conf" + custom_conf.write_text("[cache.s3]\nbucket = test\n") + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache_conf"] = str(custom_conf) + config["containers"]["ceph_node"]["sccache_mode"] = "s3" + env_file, extra_args = CephNode()._prepare_build_env() + assert (repo / "sccache.conf").read_text() == custom_conf.read_text() + contents = env_file.read_text() + assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents + assert extra_args == [] + + def test_prepare_build_env_skips_when_nothing_to_configure(self, tmp_path): + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache"] = False + assert CephNode()._prepare_build_env() == (None, []) + + def test_git_worktree_info_detects_linked_worktree(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + (admin_dir / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + assert git_worktree_info(worktree) == (main_repo / ".git", "ceph_main") + + def test_prepare_build_env_mounts_git_metadata_for_worktree(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(worktree) + config["containers"]["ceph_node"]["sccache"] = False + env_file, extra_args = CephNode()._prepare_build_env() + assert env_file is None + expected_mounts = worktree_container_mounts( + worktree, main_repo / ".git", "ceph_main" + ) + assert extra_args == expected_mounts + dot_git = worktree / REPO_DEVSTACK_DIR / "git" / "dot-git" + assert dot_git.read_text() == ( + f"gitdir: {CONTAINER_GIT_METADATA_DIR}/worktrees/ceph_main\n" + ) + + def test_worktree_container_mounts_do_not_set_git_env(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + config["data_dir"] = str(tmp_path / "data") + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(worktree) + config["containers"]["ceph_node"]["sccache"] = True + env_file, extra_args = CephNode()._prepare_build_env() + assert env_file is not None + contents = env_file.read_text() + assert "GIT_DIR=" not in contents + assert "GIT_WORK_TREE=" not in contents + assert len(extra_args) == 4 + + def test_compile_cmd_passes_worktree_mount_and_env(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["repo"] = str(worktree) + config["containers"]["ceph_node"]["build_dir"] = "build" + config["containers"]["ceph_node"]["build_distro"] = "centos9" + config["containers"]["ceph_node"]["sccache"] = False + config["containers"]["ceph_node"]["npm_cache"] = False + env_file = worktree / REPO_DEVSTACK_DIR / BUILD_ENV_NAME + extra_args = worktree_container_mounts( + worktree, main_repo / ".git", "ceph_main" + ) + cmd = CephNode()._compile_cmd(env_file=env_file, extra_args=extra_args) + assert cmd[cmd.index("--homedir") + 1] == "/ceph" + assert "--env-file" in cmd + assert str(env_file) in cmd + for extra in extra_args: + assert f"--extra={extra}" in cmd + + def test_bundled_sccache_conf_uses_local_disk(self): + contents = PACKAGE_SCCACHE_CONF.read_text() + assert "[cache.disk]" in contents + assert 'dir = "/sccache"' in contents + assert "rw_mode" not in contents + + def test_bundled_sccache_s3_conf_keeps_anonymous_read_settings(self): + contents = PACKAGE_SCCACHE_S3_CONF.read_text() + assert "[cache.s3]" in contents + assert "no_credentials = true" in contents + assert "rw_mode" not in contents + + def test_cpatch_cmd_uses_upstream_script(self): + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["base_image"] = "quay.io/ceph-ci/ceph:main" + cmd = CephNode()._cpatch_cmd() + assert cmd[0:2] == ["sudo", "../src/script/cpatch"] + assert "localhost/ceph-devstack:main" in cmd + + async def test_build_runs_compile_then_cpatch(self, tmp_path): + build_path = tmp_path / "build" + build_path.mkdir() + (build_path / "build.ninja").write_text("") + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["repo"] = str(tmp_path) + config["containers"]["ceph_node"]["build_dir"] = "build" + node = CephNode() + with ( + patch.object(node, "_compile", new=AsyncMock()) as mock_compile, + patch.object(node, "_build_image_cpatch", new=AsyncMock()) as mock_cpatch, + ): + await node.build() + mock_compile.assert_awaited_once() + mock_cpatch.assert_awaited_once() + + +class TestCephNodeRuntime: + def test_cluster_dir_defaults_to_stack_data_dir(self, tmp_path): + config["data_dir"] = str(tmp_path) + assert CephNode().cluster_dir == tmp_path + + def test_loop_img_dir_lives_outside_cluster_dir(self, tmp_path): + cluster_dir = tmp_path / "ceph" + config["data_dir"] = str(cluster_dir) + node = CephNode() + assert node.loop_img_dir == tmp_path / "disk_images" + assert node.loop_img_dir != node.cluster_dir / "disk_images" + + def test_create_cmd_uses_host_network_and_entrypoint(self, tmp_path): + config["data_dir"] = str(tmp_path) + config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" + cmd = CephNode().create_cmd + assert "podman" in cmd + assert "create" in cmd + assert "--network" in cmd + assert "host" in cmd + entrypoint_idx = cmd.index("--entrypoint") + assert cmd[entrypoint_idx + 1] == "/bin/bash" + entrypoint = f"{CONTAINER_CLUSTER_DIR}/{CLUSTER_ENTRYPOINT_NAME}" + assert cmd[cmd.index("-c") + 1] == f". {entrypoint}" + assert f"{tmp_path}:{CONTAINER_CLUSTER_DIR}" in cmd + assert f"{tmp_path}/var/lib/ceph:/var/lib/ceph" in cmd + assert "/run/udev:/run/udev" in cmd + assert f"CLUSTER_DIR={CONTAINER_CLUSTER_DIR}" in cmd + assert f"{tmp_path}:{tmp_path}" not in cmd + assert "CEPH_VOLUME_ALLOW_LOOP_DEVICES" not in cmd + assert "--device=/dev/loop0" in cmd + assert "--device=/dev/loop1" in cmd + assert "--device=/dev/loop2" in cmd + assert "DASHBOARD_PORT=8080" in cmd + assert "DASHBOARD_SSL=false" in cmd + assert "DASHBOARD_SHOW_PASSWORD=false" in cmd + assert "CONTAINER_NAME=ceph_node" in cmd + + def test_dashboard_show_password_when_enabled(self, tmp_path): + config["data_dir"] = str(tmp_path) + config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" + config["containers"]["ceph_node"]["dashboard_show_password"] = True + assert "DASHBOARD_SHOW_PASSWORD=true" in CephNode().create_cmd + + async def test_create_installs_entrypoint_in_cluster_dir(self, tmp_path): + cluster_dir = tmp_path / "ceph" + config["data_dir"] = str(cluster_dir) + node = CephNode() + with ( + patch.object(node, "remove_legacy_loop_img_dir", new=AsyncMock()), + patch.object(node, "label_cluster_dir", new=AsyncMock()), + patch.object(node, "create_loop_devices", new=AsyncMock()), + patch.object(Container, "create", new=AsyncMock()), + ): + await node.create() + installed = cluster_dir / CLUSTER_ENTRYPOINT_NAME + assert installed.is_file() + assert installed.read_text() == ENTRYPOINT_SCRIPT.read_text() + + async def test_create_removes_legacy_loop_img_dir(self, tmp_path): + cluster_dir = tmp_path / "ceph" + cluster_dir.mkdir() + legacy = cluster_dir / "disk_images" + legacy.mkdir() + (legacy / "ceph_node-0").write_bytes(b"") + config["data_dir"] = str(cluster_dir) + config["containers"]["ceph_node"]["loop_device_count"] = 1 + node = CephNode() + with ( + patch.object(node, "create_loop_devices", new=AsyncMock()), + patch.object(node, "label_cluster_dir", new=AsyncMock()), + patch.object(Container, "create", new=AsyncMock()), + ): + await node.create() + assert not legacy.exists() + + async def test_create_sets_up_loop_devices_and_container(self, tmp_path): + config["data_dir"] = str(tmp_path) + config["containers"]["ceph_node"]["loop_device_count"] = 2 + node = CephNode() + with ( + patch.object(node, "create_loop_devices", new=AsyncMock()) as mock_loop, + patch.object(Container, "create", new=AsyncMock()) as mock_super_create, + ): + await node.create() + mock_loop.assert_awaited_once() + mock_super_create.assert_awaited_once() + assert (tmp_path / "fsid").exists() + + async def test_remove_tears_down_container_and_loop_devices(self, tmp_path): + cluster_dir = tmp_path / "ceph" + cluster_dir.mkdir() + (cluster_dir / "var" / "lib" / "ceph").mkdir(parents=True) + (cluster_dir / "fsid").write_text("test\n") + npm_cache = tmp_path / "cache" / "npm" + npm_cache.mkdir(parents=True) + (npm_cache / "marker").write_bytes(b"x" * 100) + config["data_dir"] = str(cluster_dir) + node = CephNode() + with ( + patch.object(Container, "remove", new=AsyncMock()) as mock_super_remove, + patch.object(node, "remove_loop_devices", new=AsyncMock()) as mock_loop, + patch.object(node, "remove_legacy_loop_img_dir", new=AsyncMock()), + patch.object(node, "cmd", new=AsyncMock()) as mock_cmd, + ): + await node.remove() + mock_super_remove.assert_awaited_once() + mock_loop.assert_awaited_once() + mock_cmd.assert_any_await( + [ + "podman", + "unshare", + "rm", + "-rf", + str((cluster_dir / "var").resolve()), + ], + check=False, + ) + mock_cmd.assert_any_await( + [ + "podman", + "unshare", + "rm", + "-rf", + str((cluster_dir / "fsid").resolve()), + ], + check=False, + ) + assert npm_cache.exists() + + async def test_is_running_checks_ceph_status_in_container(self, tmp_path): + config["data_dir"] = str(tmp_path) + node = CephNode() + with ( + patch.object(node, "exists", new=AsyncMock(return_value=True)), + patch.object(node, "cmd", new=AsyncMock()) as mock_cmd, + ): + mock_cmd.return_value.wait = AsyncMock(return_value=0) + assert await node.is_running() is True + assert mock_cmd.await_args is not None + exec_cmd = mock_cmd.await_args.args[0] + assert exec_cmd[:3] == ["podman", "exec", "ceph_node"] + assert "ceph" in exec_cmd diff --git a/tests/resources/ceph/test_cephdevstack_core.py b/tests/resources/ceph/test_cephdevstack_core.py index d30a1625..cf6148ce 100644 --- a/tests/resources/ceph/test_cephdevstack_core.py +++ b/tests/resources/ceph/test_cephdevstack_core.py @@ -19,6 +19,7 @@ class TestCephDevStackServiceSpecs: def test_service_specs_includes_all_services(self): devstack = CephDevStack() + assert devstack.stack_name == "teuthology" assert "postgres" in devstack.service_specs assert "paddles" in devstack.service_specs assert "beanstalk" in devstack.service_specs @@ -260,16 +261,15 @@ async def test_remove_calls_remove_on_all_containers(self): mock_network_instance = MagicMock() mock_network_instance.remove = AsyncMock() MockNetwork.return_value = mock_network_instance - with patch("ceph_devstack.resources.ceph.SSHKeyPair") as MockSecret: - mock_secret_instance = MagicMock() - mock_secret_instance.remove = AsyncMock() - MockSecret.return_value = mock_secret_instance - with patch("ceph_devstack.logger.info"): - await devstack.remove() - mock_postgres.remove.assert_called_once() - mock_paddles.remove.assert_called_once() - mock_network_instance.remove.assert_called_once() - mock_secret_instance.remove.assert_called_once() + mock_secret = MagicMock() + mock_secret.remove = AsyncMock() + devstack.secrets = [MagicMock(return_value=mock_secret)] + with patch("ceph_devstack.logger.info"): + await devstack.remove() + mock_postgres.remove.assert_called_once() + mock_paddles.remove.assert_called_once() + mock_network_instance.remove.assert_called_once() + mock_secret.remove.assert_called_once() class TestCephDevStackStop: @@ -422,3 +422,37 @@ def test_init_without_postgres(self): devstack = CephDevStack() assert "archive" in devstack.service_specs assert "postgres" not in devstack.service_specs + + +class TestCephDevStackStacks: + def test_custom_stack_limits_services(self, tmp_path): + config_file = tmp_path / "config.toml" + config_file.write_text( + """ +[stacks.minimal] +services = ["postgres", "paddles"] +""" + ) + config.load(config_file) + devstack = CephDevStack(stack_name="minimal") + assert devstack.stack_name == "minimal" + assert set(devstack.service_specs) == {"postgres", "paddles"} + + def test_ceph_stack_has_expected_services(self): + devstack = CephDevStack(stack_name="ceph") + assert devstack.stack_name == "ceph" + assert set(devstack.service_specs) == {"ceph_node"} + assert devstack.secrets == [] + + async def test_ceph_stack_create_prepares_node(self): + devstack = CephDevStack(stack_name="ceph") + mock_node = AsyncMock() + devstack.service_specs = { + "ceph_node": {"count": 1, "objects": [mock_node]}, + } + with patch("ceph_devstack.resources.ceph.CephDevStackNetwork") as MockNetwork: + mock_network = MagicMock() + mock_network.create = AsyncMock() + MockNetwork.return_value = mock_network + await devstack.create() + mock_node.create.assert_awaited_once() diff --git a/tests/resources/ceph/test_requirements_ceph.py b/tests/resources/ceph/test_requirements_ceph.py index cc4b58a7..92d9f24c 100644 --- a/tests/resources/ceph/test_requirements_ceph.py +++ b/tests/resources/ceph/test_requirements_ceph.py @@ -227,10 +227,10 @@ async def test_check_requirements_returns_true_when_all_pass(self): async def test_check_requirements_returns_false_when_repo_missing(self): devstack = CephDevStack() - devstack.service_specs = {} config["containers"] = { "custom": {"repo": "/nonexistent/path"}, } + devstack.service_specs = {"custom": {}} with ( patch("ceph_devstack.resources.ceph.HasSudo") as MockHasSudo, diff --git a/tests/resources/test_podmanresource.py b/tests/resources/test_podmanresource.py index 3ee8b6e4..2a0d7fcb 100644 --- a/tests/resources/test_podmanresource.py +++ b/tests/resources/test_podmanresource.py @@ -1,3 +1,4 @@ +import logging import pytest from pathlib import Path @@ -54,7 +55,13 @@ async def test_cmd(self, cls): print(m_arun.await_args_list) m_arun.assert_awaited_once_with(["0"], cwd=Path("."), stream_output=False) - async def test_cmd_failed(self, cls): + async def test_cmd_failed(self, cls, caplog): obj = cls() - with pytest.raises(CalledProcessError): - await obj.cmd(["false"], check=True) + with caplog.at_level(logging.ERROR), pytest.raises(CalledProcessError): + await obj.cmd( + ["/bin/sh", "-c", "echo podman-failure >&2; exit 1"], + check=True, + force_local=True, + ) + assert "Command failed (1)" in caplog.text + assert "podman-failure" in caplog.text diff --git a/tests/test_config.py b/tests/test_config.py index 5d11c1dc..83697422 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -103,7 +103,27 @@ def test_unset_value_simple_key(self, test_config): class TestConfigDefaults: def test_config_defaults(self): assert config == { + "stack": "teuthology", "data_dir": "~/.local/share/ceph-devstack", + "stacks": { + "teuthology": { + "services": [ + "postgres", + "paddles", + "beanstalk", + "pulpito", + "teuthology", + "testnode", + "archive", + ], + "secrets": ["ssh_keypair"], + }, + "ceph": { + "services": ["ceph_node"], + "secrets": [], + "data_dir": "~/.local/share/ceph-devstack/ceph", + }, + }, "containers": { "archive": {"image": "python:alpine"}, "beanstalk": {"image": "quay.io/ceph-infra/teuthology-beanstalkd:main"}, @@ -118,5 +138,10 @@ def test_config_defaults(self): "image": "quay.io/ceph-infra/teuthology-testnode:main", }, "teuthology": {"image": "quay.io/ceph-infra/teuthology-dev:main"}, + "ceph_node": { + "image": "quay.io/ceph-ci/ceph:main", + "loop_device_count": 3, + "loop_device_size": "5G", + }, }, } diff --git a/tests/test_stacks.py b/tests/test_stacks.py new file mode 100644 index 00000000..a173f799 --- /dev/null +++ b/tests/test_stacks.py @@ -0,0 +1,62 @@ +import pytest + +from ceph_devstack import config, parse_args + + +class TestApplyStack: + def test_apply_stack_sets_active_stack(self): + config.apply_stack("teuthology") + assert config.active_stack == "teuthology" + + def test_apply_stack_sets_active_services(self): + config.apply_stack("teuthology") + assert "postgres" in config.active_services + assert "teuthology" in config.active_services + + def test_apply_stack_uses_config_default(self): + config.apply_stack() + assert config.active_stack == "teuthology" + + def test_apply_stack_unknown_raises(self): + with pytest.raises(ValueError, match="Unknown stack"): + config.apply_stack("nonexistent") + + def test_apply_stack_merges_container_overrides(self, tmp_path): + config_file = tmp_path / "config.toml" + config_file.write_text( + """ +[stacks.minimal] +services = ["teuthology", "testnode"] + +[stacks.minimal.containers.testnode] +count = 1 +""" + ) + config.load(config_file) + config.apply_stack("minimal") + assert config["containers"]["testnode"]["count"] == 1 + assert config.active_services == ["teuthology", "testnode"] + + def test_apply_stack_merges_data_dir_override(self, tmp_path): + config_file = tmp_path / "config.toml" + config_file.write_text( + """ +[stacks.ceph] +services = ["ceph_node"] +data_dir = "~/.local/share/ceph-devstack/custom-ceph" +""" + ) + config.load(config_file) + config.apply_stack("ceph") + assert config["data_dir"] == "~/.local/share/ceph-devstack/custom-ceph" + + +class TestParseArgsStack: + def test_parse_args_stack(self): + args = parse_args(["--stack", "ceph", "start"]) + assert args.stack == "ceph" + assert args.command == "start" + + def test_parse_args_stack_default_none(self): + args = parse_args(["start"]) + assert args.stack is None From 0ed7d2daca61440636573e58a4aeddb7534dd6ef Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 7 Jul 2026 23:35:45 -0600 Subject: [PATCH 02/32] Shared block pool --- README.md | 55 +- ceph_devstack/__init__.py | 5 + ceph_devstack/block_pool.py | 496 ++++++++++++++++++ ceph_devstack/cli.py | 7 + ceph_devstack/config.toml | 16 + ceph_devstack/requirements.py | 8 + ceph_devstack/resources/ceph/__init__.py | 4 + ceph_devstack/resources/ceph/block_devices.py | 150 ++++++ .../resources/ceph/ceph-node-entrypoint.sh | 2 +- ceph_devstack/resources/ceph/ceph_node.py | 117 ++--- ceph_devstack/resources/ceph/containers.py | 97 ++-- ceph_devstack/resources/ceph/host_loops.py | 102 ++++ ceph_devstack/resources/ceph/requirements.py | 36 +- tests/conftest.py | 69 ++- tests/resources/ceph/conftest.py | 77 +++ tests/resources/ceph/test_block_devices.py | 97 ++++ tests/resources/ceph/test_ceph_node.py | 14 + .../resources/ceph/test_cephdevstack_core.py | 6 +- tests/resources/ceph/test_host_loops.py | 32 ++ .../resources/ceph/test_requirements_ceph.py | 101 +++- tests/resources/ceph/test_testnode.py | 34 +- tests/test_block_pool.py | 445 ++++++++++++++++ tests/test_cli.py | 25 + tests/test_config.py | 6 + 24 files changed, 1837 insertions(+), 164 deletions(-) create mode 100644 ceph_devstack/block_pool.py create mode 100644 ceph_devstack/resources/ceph/block_devices.py create mode 100644 ceph_devstack/resources/ceph/host_loops.py create mode 100644 tests/resources/ceph/conftest.py create mode 100644 tests/resources/ceph/test_block_devices.py create mode 100644 tests/resources/ceph/test_host_loops.py create mode 100644 tests/test_block_pool.py create mode 100644 tests/test_cli.py diff --git a/README.md b/README.md index bcd0302e..e4dd8e5e 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,59 @@ python3 -m pip install git+https://github.com/zmc/ceph-devstack.git `ceph-devstack config dump` will output the current configuration. +### Shared block pool (optional) + +To back loop devices with slices of a real NVMe device instead of sparse files, +configure `[block_pool]` in your config. The pool is shared across `ceph_node` +and all `testnode_*` containers. Each loop device uses that container's +`loop_device_size` for its backing region on the pool parent. + +```toml +[block_pool] +parent = "/dev/nvme0n1p1" # or a dedicated whole disk +allow_enroll = true # first run only; unset afterward + +[containers.ceph_node] +loop_device_size = "5G" + +[containers.testnode] +loop_device_size = "5G" +``` + +Requirements: + +* Your user must be in the `disk` group and able to read/write the parent + device directly (enrollment writes a tail marker to the raw device). +* The parent must be empty, or already enrolled by ceph-devstack (tail marker + at the end of the device). + +`ceph-devstack block-pool status` shows current allocations. + +Each container picks host loop devices at create time: it needs +`loop_device_count` devices of `loop_device_size`, reuses any existing backing +files for that container name, then takes the lowest free loop numbers after +scanning what is already attached or claimed under `data_dir/disk_images`. + +If `block_pool.json` is lost but the on-disk marker remains, delete the state +file and start again; the pool reclaims the marker without reformatting the +device. + +### Ceph stack + +To run a single-container local Ceph cluster instead of teuthology: + +```bash +ceph-devstack --stack ceph pull +ceph-devstack --stack ceph create +ceph-devstack --stack ceph start +podman logs -f ceph_node +podman exec ceph_node ceph -c /var/lib/ceph-devstack/cluster/ceph.conf -s +``` + +The dashboard listens on port 8080 by default (`admin` / `admin`). Set +`dashboard_show_password = true` under `[containers.ceph_node]` to print the +password in the container log. + As an example, the following configuration will use a local image for paddles with the tag `TEST`; it will also create ten testnode containers; and will build its teuthology container from the git repo at `~/src/teuthology`: ``` containers: @@ -192,4 +245,4 @@ ceph-devstack -v start ```bash ceph-devstack wait teuthology podman logs -f teuthology -``` \ No newline at end of file +``` diff --git a/ceph_devstack/__init__.py b/ceph_devstack/__init__.py index 3d1a06ed..778c4b59 100644 --- a/ceph_devstack/__init__.py +++ b/ceph_devstack/__init__.py @@ -66,6 +66,11 @@ def parse_args(args: List[str]) -> argparse.Namespace: default=False, help="Apply suggested fixes for issues found", ) + parser_block_pool = subparsers.add_parser( + "block-pool", help="Inspect the shared block device pool" + ) + block_pool_subparsers = parser_block_pool.add_subparsers(dest="block_pool_op") + block_pool_subparsers.add_parser("status", help="Show pool allocations and state") parser_pull = subparsers.add_parser("pull", help="Pull container images") parser_pull.add_argument( "image", diff --git a/ceph_devstack/block_pool.py b/ceph_devstack/block_pool.py new file mode 100644 index 00000000..5c25fbcb --- /dev/null +++ b/ceph_devstack/block_pool.py @@ -0,0 +1,496 @@ +import contextlib +import fcntl +import json +import os +import re +import subprocess +import uuid +from pathlib import Path + +from ceph_devstack import logger + + +_SIZE_RE = re.compile(r"^(\d+(?:\.\d+)?)([KMGTP]?)$", re.IGNORECASE) +_UNITS = { + "": 1, + "K": 1024, + "M": 1024**2, + "G": 1024**3, + "T": 1024**4, + "P": 1024**5, +} + +# Written to the last 4 KiB of an enrolled parent so we can recognize our own +# partitions even if block_pool.json is removed. +_POOL_TAIL_MAGIC = b"CEPH-DEVSTACK-BLOCK-POOL-TAIL-v1\x00" +_TAIL_MARKER_SIZE = 4096 +_PROBE_SIZE = 4096 + +# Whole disks and partitions are allowed; device-mapper/loop types are not. +_WHOLE_DISK_NAME = re.compile( + r"^(?:" + r"nvme\d+n\d+|" + r"mmcblk\d+|" + r"(?:sd|[xv]d)[a-z]+" + r")$" +) +_PARTITION_NAME = re.compile( + r"^(?:" + r"nvme\d+n\d+p\d+|" + r"mmcblk\d+p\d+|" + r"(?:sd|[xv]d)[a-z]+\d+" + r")$" +) +_UNSAFE_NAME = re.compile(r"^(?:dm-\d+|md\d+|loop\d+|ram\d+|fd\d+)$") + + +class BlockPoolError(ValueError): + """Raised when a block pool operation would be unsafe.""" + + +def parse_size(value: str | int) -> int: + if isinstance(value, int): + return value + text = str(value).strip().upper() + match = _SIZE_RE.match(text) + if not match: + raise BlockPoolError(f"invalid size: {value!r}") + number, unit = match.groups() + return int(float(number) * _UNITS[unit.upper()]) + + +def format_size(size_bytes: int) -> str: + for unit, multiplier in ( + ("P", 1024**5), + ("T", 1024**4), + ("G", 1024**3), + ("M", 1024**2), + ("K", 1024), + ): + if size_bytes >= multiplier and size_bytes % multiplier == 0: + return f"{size_bytes // multiplier}{unit}" + return f"{size_bytes}B" + + +def canonical_device_path(path: str) -> str: + return os.path.realpath(os.path.expanduser(path)) + + +def validate_parent_name(parent: str) -> str: + """Return canonical parent path or raise if the device is not a safe block device.""" + parent = canonical_device_path(parent) + if not parent.startswith("/dev/"): + raise BlockPoolError(f"block pool parent must be a /dev path, not {parent!r}") + name = os.path.basename(parent) + if _UNSAFE_NAME.match(name): + raise BlockPoolError( + f"block pool parent {parent!r} is not allowed " + "(device-mapper, md, loop, and ram devices are rejected)" + ) + if not (_PARTITION_NAME.match(name) or _WHOLE_DISK_NAME.match(name)): + raise BlockPoolError( + f"block pool parent {parent!r} must be a block device or partition " + "(e.g. /dev/nvme0n1, /dev/nvme0n1p1, or /dev/sda1)" + ) + if not os.path.exists(parent): + raise BlockPoolError(f"block pool parent {parent!r} does not exist") + if not os.path.exists(f"/sys/class/block/{name}"): + raise BlockPoolError(f"block pool parent {parent!r} is not a block device") + return parent + + +def _is_whole_disk(name: str) -> bool: + return bool(_WHOLE_DISK_NAME.match(name)) + + +def _has_partition_siblings(name: str) -> bool: + """True when a whole-disk parent already has partition devices present.""" + if not _is_whole_disk(name): + return False + block_dir = Path("/sys/class/block") + if not block_dir.is_dir(): + return False + for entry in block_dir.iterdir(): + if not entry.is_dir() or entry.name == name: + continue + if name.startswith("nvme") or name.startswith("mmcblk"): + if entry.name.startswith(f"{name}p"): + return True + elif re.match(r"^(?:sd|[xv]d)[a-z]+$", name): + suffix = entry.name.removeprefix(name) + if suffix.isdigit(): + return True + return False + + +def _run(cmd: list[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, capture_output=True, text=True, check=False) + + +def _device_size_bytes(parent: str) -> int: + name = os.path.basename(parent) + sectors_path = f"/sys/class/block/{name}/size" + try: + sectors = int(Path(sectors_path).read_text().strip()) + return sectors * 512 + except OSError: + pass + try: + return os.path.getsize(parent) + except OSError as exc: + raise BlockPoolError(f"cannot read size of {parent!r}: {exc}") from exc + + +def _read_device(parent: str, offset: int, length: int) -> bytes: + with open(parent, "rb") as handle: + handle.seek(offset) + return handle.read(length) + + +def _write_device(parent: str, offset: int, data: bytes) -> None: + with open(parent, "r+b") as handle: + handle.seek(offset) + handle.write(data) + + +def _is_region_empty(data: bytes) -> bool: + return not data or data == b"\x00" * len(data) + + +def _device_mounted(parent: str) -> bool: + proc = _run(["findmnt", "-n", "-o", "TARGET", "--source", parent]) + return proc.returncode == 0 and bool(proc.stdout.strip()) + + +def _device_has_blkid_signature(parent: str) -> bool: + proc = _run(["blkid", "-p", "-o", "export", parent]) + if proc.returncode != 0: + return False + for line in proc.stdout.splitlines(): + key, _, value = line.partition("=") + if key in {"TYPE", "PTTYPE"} and value: + return True + return False + + +def _tail_marker_offset(parent: str) -> int: + size = _device_size_bytes(parent) + if size < _TAIL_MARKER_SIZE: + raise BlockPoolError(f"block pool parent {parent!r} is too small") + return size - _TAIL_MARKER_SIZE + + +def _read_tail_marker(parent: str) -> dict | None: + offset = _tail_marker_offset(parent) + raw = _read_device(parent, offset, _TAIL_MARKER_SIZE) + if not raw.startswith(_POOL_TAIL_MAGIC): + return None + payload = raw[len(_POOL_TAIL_MAGIC) :].split(b"\x00", 1)[0] + if not payload: + return None + try: + return json.loads(payload.decode()) + except json.JSONDecodeError: + return None + + +def _write_tail_marker(parent: str, payload: dict) -> None: + offset = _tail_marker_offset(parent) + encoded = json.dumps(payload, sort_keys=True).encode() + if len(_POOL_TAIL_MAGIC) + len(encoded) + 1 > _TAIL_MARKER_SIZE: + raise BlockPoolError("block pool tail marker payload is too large") + raw = _POOL_TAIL_MAGIC + encoded + b"\x00" + raw = raw.ljust(_TAIL_MARKER_SIZE, b"\x00") + _write_device(parent, offset, raw) + + +def _probe_parent_empty(parent: str) -> None: + size = _device_size_bytes(parent) + offsets = [0] + if size > 512: + offsets.append(512) + if size > _PROBE_SIZE: + offsets.append(size // 2) + tail = _tail_marker_offset(parent) + if tail not in offsets and tail >= _PROBE_SIZE: + offsets.append(tail) + for offset in offsets: + data = _read_device(parent, offset, _PROBE_SIZE) + if not _is_region_empty(data): + raise BlockPoolError( + f"block pool parent {parent!r} is not empty at offset {offset}; " + "refusing to enroll a partition that may contain data" + ) + + +class BlockPool: + """Allocate sized regions from a shared parent block device.""" + + def __init__( + self, + state_path: Path, + parent: str, + *, + allow_enroll: bool = False, + ): + self.state_path = state_path + self.parent = validate_parent_name(parent) + self.allow_enroll = allow_enroll + self._state = self._load() + + @classmethod + def from_config(cls, cfg: dict) -> "BlockPool | None": + pool_cfg = cfg.get("block_pool") or {} + parent = pool_cfg.get("parent") + if not parent: + return None + state_dir = Path( + os.path.expanduser( + pool_cfg.get("state_dir", "~/.local/share/ceph-devstack") + ) + ) + state_path = state_dir / "block_pool.json" + allow_enroll = pool_cfg.get("allow_enroll", False) is True + return cls(state_path, parent, allow_enroll=allow_enroll) + + @classmethod + def status_from_config(cls, cfg: dict) -> int: + pool = cls.from_config(cfg) + if pool is None: + logger.info("block pool: disabled (no parent configured)") + return 0 + logger.info(f"block pool parent: {pool.parent}") + logger.info(f"enrolled: {pool.enrolled}") + logger.info(f"pool id: {pool._state.get('pool_id')}") + logger.info(f"state file: {pool.state_path}") + allocations = pool._state.get("allocations", {}) + if allocations: + logger.info("allocations:") + for key, region in sorted(allocations.items()): + owner = region.get("owner", "?") + logger.info( + f" {key}: offset {region['offset']}, " + f"size {format_size(region['size'])}, last owner {owner}" + ) + else: + logger.info("allocations: (none)") + free_regions = pool._state.get("free_regions", []) + if free_regions: + logger.info("free regions:") + for region in free_regions: + logger.info( + f" offset {region['offset']}, size {format_size(region['size'])}" + ) + else: + logger.info("free regions: (none)") + logger.info(f"next offset: {pool._state.get('next_offset', 0)}") + return 0 + + @property + def enabled(self) -> bool: + return bool(self.parent) + + @property + def enrolled(self) -> bool: + return bool(self._state.get("enrolled")) + + def _load(self) -> dict: + if self.state_path.exists(): + state = json.loads(self.state_path.read_text()) + if state.get("parent") != self.parent: + raise BlockPoolError( + f"block pool parent changed ({state.get('parent')!r} -> " + f"{self.parent!r}); remove {self.state_path} to reconfigure" + ) + if "slice_size" in state: + raise BlockPoolError( + f"legacy block pool state at {self.state_path} uses slice_size; " + "remove it to migrate to per-device allocations" + ) + return state + return self._fresh_state() + + def _fresh_state(self) -> dict: + return { + "parent": self.parent, + "pool_id": str(uuid.uuid4()), + "enrolled": False, + "next_offset": 0, + "free_regions": [], + "allocations": {}, + } + + @contextlib.contextmanager + def _state_lock(self): + self.state_path.parent.mkdir(parents=True, exist_ok=True) + lock_path = self.state_path.with_suffix(".lock") + with open(lock_path, "w") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + if self.state_path.exists(): + self._state = self._load() + yield + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + + def _atomic_save(self) -> None: + tmp_path = self.state_path.with_suffix(".tmp") + tmp_path.write_text(json.dumps(self._state, indent=2, sort_keys=True) + "\n") + os.replace(tmp_path, self.state_path) + + def _save(self) -> None: + self._atomic_save() + + def ensure_ready(self) -> None: + """Verify the parent partition is safe to use before any slice I/O.""" + with self._state_lock(): + if self._ensure_ready_unlocked(): + self._atomic_save() + + def _ensure_ready_unlocked(self) -> bool: + if self.enrolled: + self._verify_enrolled_parent() + return False + self._enroll_new_parent() + return True + + def _verify_enrolled_parent(self) -> None: + marker = _read_tail_marker(self.parent) + if marker is None: + raise BlockPoolError( + f"block pool parent {self.parent!r} is missing the devstack " + f"enrollment marker; refusing to use it. If {self.state_path} " + "was removed accidentally, start again to reclaim the marker. " + "If you wiped the device, set block_pool.allow_enroll = true " + "for a fresh enrollment." + ) + if marker.get("pool_id") != self._state.get("pool_id"): + raise BlockPoolError( + f"block pool parent {self.parent!r} belongs to another pool " + f"({marker.get('pool_id')!r}); remove {self.state_path} only " + "if you intentionally reset the pool" + ) + if marker.get("parent") != self.parent: + raise BlockPoolError( + f"block pool parent path mismatch for enrolled partition " + f"({marker.get('parent')!r} != {self.parent!r})" + ) + + def _enroll_new_parent(self) -> None: + marker = _read_tail_marker(self.parent) + if marker is not None: + if marker.get("parent") != self.parent: + raise BlockPoolError( + f"block pool parent {self.parent!r} has a devstack marker " + f"for a different path ({marker.get('parent')!r})" + ) + marker_pool_id = marker.get("pool_id") + if marker_pool_id != self._state.get("pool_id"): + self._state["pool_id"] = marker_pool_id + self._state["enrolled"] = True + return + + if not self.allow_enroll: + raise BlockPoolError( + f"block pool parent {self.parent!r} is not enrolled. " + "Use a dedicated empty partition and set " + "block_pool.allow_enroll = true for the first run only." + ) + if _device_mounted(self.parent): + raise BlockPoolError( + f"block pool parent {self.parent!r} is mounted; refusing to enroll" + ) + if _device_has_blkid_signature(self.parent): + raise BlockPoolError( + f"block pool parent {self.parent!r} has a filesystem or partition " + "table signature; refusing to enroll" + ) + + parent_name = os.path.basename(self.parent) + if _is_whole_disk(parent_name) and _has_partition_siblings(parent_name): + raise BlockPoolError( + f"block pool parent {self.parent!r} is a whole disk with " + "existing partitions; use a dedicated empty disk or a single " + "partition instead" + ) + _probe_parent_empty(self.parent) + + payload = { + "pool_id": self._state["pool_id"], + "parent": self.parent, + } + _write_tail_marker(self.parent, payload) + self._state["enrolled"] = True + + def _validate_region(self, offset: int, size: int, owner: str) -> None: + parent_size = _device_size_bytes(self.parent) + if offset + size > parent_size: + raise BlockPoolError( + f"block pool region for {owner!r} at offset {offset} with size " + f"{size} exceeds parent size {parent_size} for {self.parent!r}" + ) + if offset >= _tail_marker_offset(self.parent): + raise BlockPoolError( + f"block pool region for {owner!r} would overlap the enrollment " + f"marker on {self.parent!r}" + ) + probe = min(_PROBE_SIZE, size) + data = _read_device(self.parent, offset, probe) + if not _is_region_empty(data): + raise BlockPoolError( + f"block pool region for {owner!r} on {self.parent!r} is not empty " + f"at offset {offset}; refusing to allocate over existing data" + ) + + def _pop_free_region(self, size: int) -> dict | None: + free_regions = self._state.setdefault("free_regions", []) + for index, region in enumerate(free_regions): + if region["size"] == size: + return free_regions.pop(index) + return None + + def get_or_allocate(self, owner: str, index: int, size: int) -> tuple[int, int]: + with self._state_lock(): + changed = self._ensure_ready_unlocked() + key = f"{owner}:{index}" + allocations = self._state.setdefault("allocations", {}) + if key in allocations: + region = allocations[key] + if changed: + self._atomic_save() + return region["offset"], region["size"] + + if region := self._pop_free_region(size): + offset = region["offset"] + else: + offset = self._state.setdefault("next_offset", 0) + self._state["next_offset"] = offset + size + + self._validate_region(offset, size, owner) + allocations[key] = {"offset": offset, "size": size, "owner": owner} + self._atomic_save() + return offset, size + + def release_owner(self, owner: str) -> None: + with self._state_lock(): + prefix = f"{owner}:" + allocations = self._state.get("allocations", {}) + free_regions = self._state.setdefault("free_regions", []) + released = [] + for key, region in list(allocations.items()): + if key.startswith(prefix): + released.append( + {"offset": region["offset"], "size": region["size"]} + ) + del allocations[key] + free_regions.extend(released) + free_regions.sort(key=lambda region: region["offset"]) + self._state["free_regions"] = free_regions + self._atomic_save() + + def allocation_for(self, owner: str, index: int) -> tuple[int, int, int] | None: + key = f"{owner}:{index}" + region = self._state.get("allocations", {}).get(key) + if region is None: + return None + return region["offset"], region["size"], region["size"] diff --git a/ceph_devstack/cli.py b/ceph_devstack/cli.py index 1fba02a2..adb751bc 100644 --- a/ceph_devstack/cli.py +++ b/ceph_devstack/cli.py @@ -68,6 +68,13 @@ def main() -> int: if args.command == "config": CONFIG_HANDLERS[args.config_op](config, args) return 0 + if args.command == "block-pool": + from ceph_devstack.block_pool import BlockPool + + if args.block_pool_op != "status": + logger.error("Usage: ceph-devstack block-pool status") + return 2 + return BlockPool.status_from_config(config) config["args"] = vars(args) Path(config["data_dir"]).expanduser().mkdir(parents=True, exist_ok=True) stack = CephDevStack(stack_name=config.active_stack) diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 786b1223..06da229b 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -1,6 +1,20 @@ stack = "teuthology" data_dir = "~/.local/share/ceph-devstack" +[block_pool] +state_dir = "~/.local/share/ceph-devstack" +# Dedicated empty block device or partition. Whole disks are allowed when +# empty; disks with existing partitions must use a partition path instead. +# Enrollment reads/writes the parent directly: your user must be able to +# open the device for read/write (typically membership in the disk group). +# First run requires allow_enroll = true; the tool writes a tail marker and +# refuses non-empty or foreign devices. If block_pool.json is lost but the +# marker remains, delete the state file and start again; the pool reclaims +# the marker automatically. +# parent = "/dev/nvme0n1" +# parent = "/dev/nvme0n1p1" +# allow_enroll = true + [stacks.teuthology] services = [ "postgres", @@ -26,6 +40,7 @@ dashboard_port = 8080 dashboard_ssl = false # dashboard_user = "admin" # dashboard_password = "admin" +# dashboard_show_password = false # log dashboard password on container start # dashboard = false # disable mgr dashboard # image_builder = "cpatch" # default; patch local binaries into runtime image # image_builder = "container" # future: container/build.sh + local packages @@ -66,6 +81,7 @@ image = "quay.io/ceph-infra/pulpito:main" [containers.testnode] count = 3 +loop_device_count = 1 loop_device_size = "5G" image = "quay.io/ceph-infra/teuthology-testnode:main" diff --git a/ceph_devstack/requirements.py b/ceph_devstack/requirements.py index 1291dbea..8024c8d9 100644 --- a/ceph_devstack/requirements.py +++ b/ceph_devstack/requirements.py @@ -297,4 +297,12 @@ async def check_requirements(): result = result and await SysctlValue("fs.aio-max-nr", 2097152).evaluate() result = result and await SysctlValue("kernel.pid_max", 4194304).evaluate() + from ceph_devstack.resources.ceph.requirements import ( + BlockPoolDiskGroup, + BlockPoolParentAccessible, + ) + + result = result and await BlockPoolDiskGroup().evaluate() + result = result and await BlockPoolParentAccessible().evaluate() + return result diff --git a/ceph_devstack/resources/ceph/__init__.py b/ceph_devstack/resources/ceph/__init__.py index 11dea1b7..2e20a5d0 100644 --- a/ceph_devstack/resources/ceph/__init__.py +++ b/ceph_devstack/resources/ceph/__init__.py @@ -23,6 +23,8 @@ HasSudo, LoopControlDeviceExists, LoopControlDeviceWriteable, + BlockPoolDiskGroup, + BlockPoolParentAccessible, SELinuxModule, ) from ceph_devstack.resources.ceph.utils import get_runs, get_jobs @@ -157,6 +159,8 @@ async def check_requirements(self): if "testnode" in self.service_specs or "ceph_node" in self.service_specs: result = result and await LoopControlDeviceExists().evaluate() result = result and await LoopControlDeviceWriteable().evaluate() + result = result and await BlockPoolDiskGroup().evaluate() + result = result and await BlockPoolParentAccessible().evaluate() # Check for SELinux being enabled and Enforcing; then check for the # presence of our module. If necessary, inform the user and instruct diff --git a/ceph_devstack/resources/ceph/block_devices.py b/ceph_devstack/resources/ceph/block_devices.py new file mode 100644 index 00000000..c83a3243 --- /dev/null +++ b/ceph_devstack/resources/ceph/block_devices.py @@ -0,0 +1,150 @@ +import os +from pathlib import Path +from typing import Protocol + +from ceph_devstack import config, logger +from ceph_devstack.block_pool import BlockPool, BlockPoolError, format_size, parse_size +from ceph_devstack.exec import Subprocess +from ceph_devstack.host import host + + +class CommandRunner(Protocol): + async def __call__( + self, + args: list[str], + *, + check: bool = False, + force_local: bool = False, + stream_output: bool = False, + ) -> Subprocess: ... + + +class BlockDeviceProvisioner: + """Create loop devices backed by sparse files or NVMe pool slices.""" + + def __init__( + self, + owner: str, + *, + image_dir: Path, + file_size: str, + cmd: CommandRunner, + trigger_udev: bool = False, + pool: BlockPool | None = None, + ): + self.owner = owner + self.image_dir = image_dir + self.file_size = file_size + self.device_size = parse_size(file_size) + self.trigger_udev = trigger_udev + self._cmd = cmd + self.pool = pool if pool is not None else BlockPool.from_config(config) + + @property + def pool_enabled(self) -> bool: + return self.pool is not None + + def _require_pool(self) -> BlockPool: + assert self.pool is not None + return self.pool + + def _image_path(self, device: str) -> Path: + loop_id = device.removeprefix("/dev/loop") + return self.image_dir / f"{self.owner}-{loop_id}" + + async def create_devices(self, devices: list[str]) -> None: + for index, device in enumerate(devices): + await self.create_device(device, index) + + async def remove_devices(self, devices: list[str]) -> None: + for device in devices: + await self.remove_device(device) + if self.pool_enabled: + self._require_pool().release_owner(self.owner) + + async def create_device(self, device: str, index: int) -> None: + await self._ensure_loop_module() + await self.remove_device(device) + device_pos = device.removeprefix("/dev/loop") + await self._cmd( + ["sudo", "mknod", "-m700", device, "b", "7", device_pos], + check=True, + ) + await self._cmd( + ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", device], + check=True, + ) + if self.pool_enabled: + pool = self._require_pool() + try: + offset, _size = pool.get_or_allocate( + self.owner, index, self.device_size + ) + except BlockPoolError: + logger.error( + f"{self.owner}: refusing to use block pool parent {pool.parent!r}" + ) + raise + logger.info( + f"{self.owner}: region on {pool.parent} " + f"(offset={offset}, size={format_size(self.device_size)}) -> {device}" + ) + await self._cmd( + [ + "sudo", + "losetup", + "--offset", + str(offset), + "--sizelimit", + str(self.device_size), + device, + pool.parent, + ], + check=True, + ) + else: + image_path = self._image_path(device) + os.makedirs(self.image_dir, exist_ok=True) + await self._cmd( + [ + "sudo", + "dd", + "if=/dev/null", + f"of={image_path}", + "bs=1", + "count=0", + f"seek={self.file_size}", + ], + check=True, + ) + await self._cmd(["sudo", "losetup", device, str(image_path)], check=True) + + await self._cmd(["chcon", "-t", "fixed_disk_device_t", device], check=False) + if self.trigger_udev: + await self._cmd( + [ + "sudo", + "udevadm", + "trigger", + "--action=add", + f"--name=block/loop{device_pos}", + ], + check=False, + ) + await self._cmd(["sudo", "udevadm", "settle"], check=False) + + async def remove_device(self, device: str) -> None: + if os.path.ismount(device): + await self._cmd(["umount", device], check=True) + if host.path_exists(device): + await self._cmd(["sudo", "losetup", "-d", device], check=False) + await self._cmd(["sudo", "rm", "-f", device], check=False) + if not self.pool_enabled: + image_path = self._image_path(device) + if image_path.exists(): + image_path.unlink() + + async def _ensure_loop_module(self) -> None: + proc = await self._cmd(["bash", "-c", "lsmod | grep -q loop"], check=False) + if await proc.wait() != 0: + await self._cmd(["sudo", "modprobe", "loop"]) diff --git a/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh b/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh index ed6dab58..b9e7e3d5 100755 --- a/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh +++ b/ceph_devstack/resources/ceph/ceph-node-entrypoint.sh @@ -462,5 +462,5 @@ prepare_osds wait_for_osds wait_for_health -echo "Ceph cluster is running; use: podman exec ${HOSTNAME:-ceph_node} ceph -s" +echo "Ceph cluster is running; use: podman exec ${CONTAINER_NAME:-ceph_node} ceph -s" exec tail -f /dev/null diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index e47eac57..a026e5f4 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -12,6 +12,8 @@ from ceph_devstack import PROJECT_ROOT, config, logger from ceph_devstack.host import host +from ceph_devstack.resources.ceph.block_devices import BlockDeviceProvisioner +from ceph_devstack.resources.ceph.host_loops import allocate_loop_devices from ceph_devstack.resources.container import Container @@ -145,17 +147,17 @@ class CephNode(Container): def __init__(self, name: str = ""): super().__init__(name) - self.loop_device_count = self.config.get("loop_device_count", 3) - self.devices = [self._device_name(i) for i in range(self.loop_device_count)] + self.loop_device_count = self.config["loop_device_count"] + self._devices: list[str] | None = None + self._block_device_provisioner: BlockDeviceProvisioner | None = None @property - def loop_device_base(self) -> int: - if "loop_device_base" in self.config: - return int(self.config["loop_device_base"]) - testnode = config.get("containers", {}).get("testnode", {}) - count = int(testnode.get("count", 0) or 0) - per_node = int(testnode.get("loop_device_count", 1) or 1) - return count * per_node + def devices(self) -> list[str]: + if self._devices is None: + self._devices = allocate_loop_devices( + self.name, self.loop_device_count, self.loop_img_dir + ) + return self._devices @property def config_key(self) -> str: @@ -369,6 +371,8 @@ def create_cmd(self): f"DASHBOARD_PASSWORD={self.dashboard_password}", "-e", f"DASHBOARD_SHOW_PASSWORD={'true' if self.dashboard_show_password else 'false'}", + "-e", + f"CONTAINER_NAME={self.name}", "--entrypoint", "/bin/bash", "--name", @@ -378,9 +382,6 @@ def create_cmd(self): f". {shlex.quote(entrypoint)}", ] - def _device_name(self, index: int) -> str: - return f"/dev/loop{self.loop_device_base + index}" - def _device_image(self, device: str) -> str: return f"{self.name}-{device.removeprefix('/dev/loop')}" @@ -630,7 +631,7 @@ async def create(self): await self.label_cluster_dir() logger.info( f"{self.name}: creating {self.loop_device_count} loop devices " - f"({self.config.get('loop_device_size', '5G')} each)" + f"({self.config['loop_device_size']} each)" ) await self.create_loop_devices() await super().create() @@ -667,75 +668,27 @@ async def wait(self): await asyncio.sleep(5) return 1 + def _block_provisioner(self) -> BlockDeviceProvisioner: + if self._block_device_provisioner is None: + self._block_device_provisioner = BlockDeviceProvisioner( + self.name, + image_dir=self.loop_img_dir, + file_size=self.config["loop_device_size"], + cmd=self.cmd, + trigger_udev=True, + ) + return self._block_device_provisioner + async def create_loop_devices(self): - for device in self.devices: - await self.create_loop_device(device) + if self.devices: + numbers = [int(device.removeprefix("/dev/loop")) for device in self.devices] + logger.info( + f"{self.name}: host loop devices " + f"{numbers[0]}-{numbers[-1]} " + f"({self.config['loop_device_size']} each)" + ) + await self._block_provisioner().create_devices(self.devices) async def remove_loop_devices(self): - for device in self.devices: - await self.remove_loop_device(device) - - async def create_loop_device(self, device: str): - size = self.config.get("loop_device_size", "5G") - os.makedirs(self.loop_img_dir, exist_ok=True) - proc = await self.cmd( - ["bash", "-c", "lsmod | grep -q loop"], - check=False, - ) - if proc and await proc.wait() != 0: - await self.cmd(["sudo", "modprobe", "loop"]) - loop_img_name = self.loop_img_dir / self._device_image(device) - await self.remove_loop_device(device) - device_pos = device.removeprefix("/dev/loop") - await self.cmd( - [ - "sudo", - "mknod", - "-m700", - device, - "b", - "7", - device_pos, - ], - check=True, - ) - await self.cmd( - ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", device], - check=True, - ) - await self.cmd( - [ - "sudo", - "dd", - "if=/dev/null", - f"of={loop_img_name}", - "bs=1", - "count=0", - f"seek={size}", - ], - check=True, - ) - await self.cmd(["sudo", "losetup", device, str(loop_img_name)], check=True) - await self.cmd(["chcon", "-t", "fixed_disk_device_t", device]) - device_pos = device.removeprefix("/dev/loop") - await self.cmd( - [ - "sudo", - "udevadm", - "trigger", - "--action=add", - f"--name=block/loop{device_pos}", - ], - check=False, - ) - await self.cmd(["sudo", "udevadm", "settle"], check=False) - - async def remove_loop_device(self, device: str): - loop_img_name = self.loop_img_dir / self._device_image(device) - if os.path.ismount(device): - await self.cmd(["umount", device], check=True) - if host.path_exists(device): - await self.cmd(["sudo", "losetup", "-d", device]) - await self.cmd(["sudo", "rm", "-f", device], check=True) - if loop_img_name.exists(): - loop_img_name.unlink() + await self._block_provisioner().remove_devices(self.devices) + self._devices = None diff --git a/ceph_devstack/resources/ceph/containers.py b/ceph_devstack/resources/ceph/containers.py index b90dd391..51c71958 100644 --- a/ceph_devstack/resources/ceph/containers.py +++ b/ceph_devstack/resources/ceph/containers.py @@ -4,8 +4,10 @@ from pathlib import Path from typing import List -from ceph_devstack import config, DEFAULT_CONFIG_PATH +from ceph_devstack import config, DEFAULT_CONFIG_PATH, logger from ceph_devstack.host import host +from ceph_devstack.resources.ceph.block_devices import BlockDeviceProvisioner +from ceph_devstack.resources.ceph.host_loops import allocate_loop_devices from ceph_devstack.resources.container import Container @@ -181,8 +183,17 @@ def __init__(self, name: str = ""): self.index = 0 if "_" in self.name: self.index = int(self.name.split("_")[-1]) - self.loop_device_count = self.config.get("loop_device_count", 1) - self.devices = [self.device_name(i) for i in range(self.loop_device_count)] + self.loop_device_count = self.config["loop_device_count"] + self._devices: list[str] | None = None + self._block_device_provisioner: BlockDeviceProvisioner | None = None + + @property + def devices(self) -> list[str]: + if self._devices is None: + self._devices = allocate_loop_devices( + self.name, self.loop_device_count, self.loop_img_dir + ) + return self._devices @property def loop_img_dir(self): @@ -263,69 +274,29 @@ async def remove(self): await super().remove() await self.remove_loop_devices() + def _block_provisioner(self) -> BlockDeviceProvisioner: + if self._block_device_provisioner is None: + self._block_device_provisioner = BlockDeviceProvisioner( + self.name, + image_dir=self.loop_img_dir, + file_size=self.config["loop_device_size"], + cmd=self.cmd, + ) + return self._block_device_provisioner + async def create_loop_devices(self): - for device in self.devices: - await self.create_loop_device(device) + if self.devices: + numbers = [int(device.removeprefix("/dev/loop")) for device in self.devices] + logger.info( + f"{self.name}: host loop devices " + f"{numbers[0]}-{numbers[-1]} " + f"({self.config['loop_device_size']} each)" + ) + await self._block_provisioner().create_devices(self.devices) async def remove_loop_devices(self): - for device in self.devices: - await self.remove_loop_device(device) - - async def create_loop_device(self, device: str): - size = self.config.get("loop_device_size", "5G") - os.makedirs(self.loop_img_dir, exist_ok=True) - proc = await self.cmd(["lsmod", "|", "grep", "loop"]) - if proc and await proc.wait() != 0: - await self.cmd(["sudo", "modprobe", "loop"]) - loop_img_name = os.path.join(self.loop_img_dir, self.device_image(device)) - await self.remove_loop_device(device) - device_pos = device.removeprefix("/dev/loop") - await self.cmd( - [ - "sudo", - "mknod", - "-m700", - device, - "b", - "7", - device_pos, - ], - check=True, - ) - await self.cmd( - ["sudo", "chown", f"{os.getuid()}:{os.getgid()}", device], - check=True, - ) - await self.cmd( - [ - "sudo", - "dd", - "if=/dev/null", - f"of={loop_img_name}", - "bs=1", - "count=0", - f"seek={size}", - ], - check=True, - ) - await self.cmd(["sudo", "losetup", device, loop_img_name], check=True) - await self.cmd(["chcon", "-t", "fixed_disk_device_t", device]) - - async def remove_loop_device(self, device: str): - loop_img_name = os.path.join(self.loop_img_dir, self.device_image(device)) - if os.path.ismount(device): - await self.cmd(["umount", device], check=True) - if host.path_exists(device): - await self.cmd(["sudo", "losetup", "-d", device]) - await self.cmd(["sudo", "rm", "-f", device], check=True) - if host.path_exists(loop_img_name): - os.remove(loop_img_name) - - def device_name(self, index: int): - return f"/dev/loop{self.loop_device_count * self.index + index}" - - def device_image(self, device: str): - return f"{self.name}-{device.removeprefix('/dev/loop')}" + await self._block_provisioner().remove_devices(self.devices) + self._devices = None class Teuthology(Container): diff --git a/ceph_devstack/resources/ceph/host_loops.py b/ceph_devstack/resources/ceph/host_loops.py new file mode 100644 index 00000000..07bcfe77 --- /dev/null +++ b/ceph_devstack/resources/ceph/host_loops.py @@ -0,0 +1,102 @@ +"""Host loop device discovery and allocation.""" + +from pathlib import Path + + +def host_loop_path(number: int) -> str: + return f"/dev/loop{number}" + + +def _loop_number_from_backing_name(name: str) -> int | None: + if "-" not in name: + return None + loop_id = name.rsplit("-", 1)[-1] + if not loop_id.isdigit(): + return None + return int(loop_id) + + +def _sysfs_loop_in_use(block_name: str) -> bool: + block = Path("/sys/class/block") / block_name + backing = block / "loop" / "backing_file" + if backing.is_file(): + text = backing.read_text().strip() + if text not in ("", "(deleted)"): + return True + size_file = block / "size" + if not size_file.is_file(): + return False + try: + return int(size_file.read_text()) > 0 + except ValueError: + return False + + +def _discover_used_from_sysfs() -> set[int]: + used: set[int] = set() + block_dir = Path("/sys/class/block") + if not block_dir.is_dir(): + return used + for entry in block_dir.iterdir(): + name = entry.name + if not name.startswith("loop"): + continue + suffix = name.removeprefix("loop") + if suffix.isdigit() and _sysfs_loop_in_use(name): + used.add(int(suffix)) + return used + + +def _discover_used_from_backing_files(image_dir: Path) -> set[int]: + used: set[int] = set() + if not image_dir.is_dir(): + return used + for path in image_dir.iterdir(): + if not path.is_file(): + continue + if (number := _loop_number_from_backing_name(path.name)) is not None: + used.add(number) + return used + + +def discover_used_loop_numbers(image_dir: Path) -> set[int]: + """Loop numbers already attached on the host or claimed by backing files.""" + return _discover_used_from_sysfs() | _discover_used_from_backing_files(image_dir) + + +def owner_loop_numbers(owner: str, image_dir: Path) -> list[int]: + """Existing backing-file loop numbers for an owner, lowest first.""" + if not image_dir.is_dir(): + return [] + numbers: list[int] = [] + prefix = f"{owner}-" + for path in image_dir.iterdir(): + if not path.is_file() or not path.name.startswith(prefix): + continue + if (number := _loop_number_from_backing_name(path.name)) is not None: + numbers.append(number) + return sorted(numbers) + + +def allocate_loop_numbers(owner: str, count: int, image_dir: Path) -> list[int]: + """Pick loop numbers for an owner from config count and host availability.""" + used = discover_used_loop_numbers(image_dir) + reclaimed = owner_loop_numbers(owner, image_dir) + numbers: list[int] = [] + for index in range(count): + if index < len(reclaimed): + numbers.append(reclaimed[index]) + continue + candidate = 0 + while candidate in used or candidate in numbers: + candidate += 1 + numbers.append(candidate) + used.add(candidate) + return numbers + + +def allocate_loop_devices(owner: str, count: int, image_dir: Path) -> list[str]: + return [ + host_loop_path(number) + for number in allocate_loop_numbers(owner, count, image_dir) + ] diff --git a/ceph_devstack/resources/ceph/requirements.py b/ceph_devstack/resources/ceph/requirements.py index 00d79678..7bae9982 100644 --- a/ceph_devstack/resources/ceph/requirements.py +++ b/ceph_devstack/resources/ceph/requirements.py @@ -1,5 +1,6 @@ -from ceph_devstack import logger, PROJECT_ROOT +from ceph_devstack import logger, PROJECT_ROOT, config from ceph_devstack.requirements import Requirement, FixableRequirement +import os class HasSudo(Requirement): @@ -42,6 +43,39 @@ async def suggest(self): ) +class BlockPoolDiskGroup(Requirement): + suggest_msg = "block pool parent requires membership in the disk group" + + async def check(self) -> bool: + from ceph_devstack.block_pool import BlockPool + + if BlockPool.from_config(config) is None: + return True + proc = await self.host.arun(["bash", "-c", "id -nG | grep -qw disk"]) + if await proc.wait() == 0: + return True + logger.error(f"{self.suggest_msg}. Try: sudo usermod -a -G disk $USER") + return False + + +class BlockPoolParentAccessible(Requirement): + suggest_msg = "block pool parent must be readable and writable by this user" + + async def check(self) -> bool: + from ceph_devstack.block_pool import BlockPool + + pool = BlockPool.from_config(config) + if pool is None: + return True + if os.access(pool.parent, os.R_OK | os.W_OK): + return True + logger.error( + f"{self.suggest_msg} ({pool.parent}). " + "Join the disk group and re-login, or fix device permissions." + ) + return False + + class SELinuxModule(FixableRequirement): def __init__(self): fix_cmd = self.fix_cmd_prebuilt diff --git a/tests/conftest.py b/tests/conftest.py index 6dfea423..fb5a7288 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,14 @@ import os import pathlib -import pytest import random - +from collections.abc import Callable from datetime import datetime, timedelta +from unittest.mock import AsyncMock, patch + +import pytest from ceph_devstack import config +from ceph_devstack.block_pool import BlockPool @pytest.fixture(autouse=True) @@ -14,6 +17,68 @@ def reset_config(): yield +@pytest.fixture +def mock_cmd() -> AsyncMock: + return AsyncMock() + + +@pytest.fixture +def pool_parent(tmp_path: pathlib.Path) -> pathlib.Path: + parent = tmp_path / "nvme0n1p1" + parent.write_bytes(b"\x00" * (20 * 1024**2)) + return parent + + +@pytest.fixture +def mock_fresh_enrollment(): + with ( + patch("ceph_devstack.block_pool._device_mounted", return_value=False), + patch( + "ceph_devstack.block_pool._device_has_blkid_signature", return_value=False + ), + patch("ceph_devstack.block_pool._read_tail_marker", return_value=None), + ): + yield + + +@pytest.fixture +def block_pool_factory( + tmp_path: pathlib.Path, pool_parent: pathlib.Path +) -> Callable[..., BlockPool]: + def factory(*, enrolled: bool = False, allow_enroll: bool = True) -> BlockPool: + state_path = tmp_path / "block_pool.json" + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=str(pool_parent), + ): + pool = BlockPool( + state_path, + str(pool_parent), + allow_enroll=allow_enroll, + ) + if enrolled: + pool._state["enrolled"] = True + pool._save() + return pool + + return factory + + +@pytest.fixture +def block_pool(block_pool_factory: Callable[..., BlockPool]) -> BlockPool: + return block_pool_factory() + + +@pytest.fixture +def enrolled_pool(block_pool_factory: Callable[..., BlockPool]) -> BlockPool: + return block_pool_factory(enrolled=True) + + +@pytest.fixture +def disallow_enroll_pool(block_pool_factory: Callable[..., BlockPool]) -> BlockPool: + return block_pool_factory(allow_enroll=False) + + @pytest.fixture(scope="function") def create_log_file(): def _create_log_file(data_dir: pathlib.Path, **kwargs) -> pathlib.Path: diff --git a/tests/resources/ceph/conftest.py b/tests/resources/ceph/conftest.py new file mode 100644 index 00000000..3d8e4573 --- /dev/null +++ b/tests/resources/ceph/conftest.py @@ -0,0 +1,77 @@ +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import AsyncMock, patch + +import pytest + +from ceph_devstack import config +from ceph_devstack.resources.ceph.block_devices import BlockDeviceProvisioner + + +@pytest.fixture +def pool_config(tmp_path: Path, pool_parent: Path) -> None: + config["block_pool"] = { + "parent": str(pool_parent), + "state_dir": str(tmp_path), + "allow_enroll": True, + } + config["containers"]["ceph_node"]["loop_device_size"] = "1M" + config["containers"]["testnode"]["loop_device_size"] = "1M" + + +@pytest.fixture +def pool_provisioner( + tmp_path: Path, + pool_parent: Path, + pool_config: None, + mock_cmd: AsyncMock, +) -> BlockDeviceProvisioner: + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=str(pool_parent), + ): + return BlockDeviceProvisioner( + "ceph_node", + image_dir=tmp_path / "disk_images", + file_size="1M", + cmd=mock_cmd, + ) + + +@pytest.fixture +def ready_pool_provisioner( + pool_provisioner: BlockDeviceProvisioner, + mock_fresh_enrollment: None, +) -> Iterator[BlockDeviceProvisioner]: + with patch.object(pool_provisioner, "remove_device", new=AsyncMock()): + yield pool_provisioner + + +@pytest.fixture +def sparse_provisioner(tmp_path: Path, mock_cmd: AsyncMock) -> BlockDeviceProvisioner: + return BlockDeviceProvisioner( + "testnode_0", + image_dir=tmp_path / "disk_images", + file_size="1M", + cmd=mock_cmd, + pool=None, + ) + + +@pytest.fixture +def large_pool_provisioner( + tmp_path: Path, + pool_parent: Path, + pool_config: None, + mock_cmd: AsyncMock, +) -> BlockDeviceProvisioner: + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=str(pool_parent), + ): + return BlockDeviceProvisioner( + "ceph_node", + image_dir=tmp_path / "disk_images", + file_size="5M", + cmd=mock_cmd, + ) diff --git a/tests/resources/ceph/test_block_devices.py b/tests/resources/ceph/test_block_devices.py new file mode 100644 index 00000000..91de7db8 --- /dev/null +++ b/tests/resources/ceph/test_block_devices.py @@ -0,0 +1,97 @@ +from unittest.mock import AsyncMock, patch + +import pytest + +from ceph_devstack.block_pool import BlockPoolError +from ceph_devstack.resources.ceph.block_devices import BlockDeviceProvisioner + + +class TestBlockDeviceProvisioner: + async def test_create_device_uses_pool_slice( + self, + ready_pool_provisioner: BlockDeviceProvisioner, + mock_cmd: AsyncMock, + pool_parent, + ): + await ready_pool_provisioner.create_device("/dev/loop9", 0) + losetup = [ + call.args[0] + for call in mock_cmd.await_args_list + if "losetup" in call.args[0] + ][0] + assert "--offset" in losetup + assert "0" in losetup + assert "--sizelimit" in losetup + assert str(ready_pool_provisioner.device_size) in losetup + assert str(pool_parent) in losetup + + async def test_remove_devices_releases_pool_owner( + self, + ready_pool_provisioner: BlockDeviceProvisioner, + ): + await ready_pool_provisioner.create_device("/dev/loop9", 0) + await ready_pool_provisioner.remove_devices(["/dev/loop9"]) + assert ( + ready_pool_provisioner._require_pool().allocation_for("ceph_node", 0) + is None + ) + + async def test_create_device_uses_sparse_file_without_pool( + self, + sparse_provisioner: BlockDeviceProvisioner, + mock_cmd: AsyncMock, + tmp_path, + ): + with ( + patch.object(sparse_provisioner, "remove_device", new=AsyncMock()), + patch.object(sparse_provisioner, "_ensure_loop_module", new=AsyncMock()), + ): + await sparse_provisioner.create_device("/dev/loop1", 0) + dd_calls = [ + call.args[0] for call in mock_cmd.await_args_list if "dd" in call.args[0] + ] + assert dd_calls + image_path = str(tmp_path / "disk_images" / "testnode_0-1") + assert any(image_path in arg for arg in dd_calls[0]) + + async def test_rejects_region_larger_than_parent( + self, + large_pool_provisioner: BlockDeviceProvisioner, + mock_fresh_enrollment: None, + ): + with ( + patch( + "ceph_devstack.block_pool._device_size_bytes", + return_value=512 * 1024, + ), + patch.object(large_pool_provisioner, "remove_device", new=AsyncMock()), + pytest.raises(BlockPoolError, match="exceeds"), + ): + await large_pool_provisioner.create_device("/dev/loop9", 0) + + async def test_ensure_loop_module_loads_when_missing( + self, + sparse_provisioner: BlockDeviceProvisioner, + mock_cmd: AsyncMock, + ): + mock_proc = AsyncMock() + mock_proc.wait = AsyncMock(return_value=1) + mock_cmd.return_value = mock_proc + await sparse_provisioner._ensure_loop_module() + mock_cmd.assert_any_await(["sudo", "modprobe", "loop"]) + + async def test_pool_error_is_logged_and_reraised( + self, + pool_provisioner: BlockDeviceProvisioner, + mock_fresh_enrollment: None, + ): + with ( + patch.object( + pool_provisioner._require_pool(), + "get_or_allocate", + side_effect=BlockPoolError("refused"), + ), + patch.object(pool_provisioner, "remove_device", new=AsyncMock()), + pytest.raises(BlockPoolError, match="refused"), + ): + await pool_provisioner.create_device("/dev/loop9", 0) diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index 63560e39..f7fb83ec 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -291,6 +291,20 @@ async def test_build_runs_compile_then_cpatch(self, tmp_path): class TestCephNodeRuntime: + def test_devices_allocate_from_empty_host(self, tmp_path): + config["data_dir"] = str(tmp_path / "ceph") + config["containers"]["ceph_node"]["loop_device_count"] = 3 + assert CephNode().devices == ["/dev/loop0", "/dev/loop1", "/dev/loop2"] + + def test_devices_skip_loops_already_claimed(self, tmp_path): + config["data_dir"] = str(tmp_path / "ceph") + image_dir = tmp_path / "disk_images" + image_dir.mkdir(parents=True) + (image_dir / "testnode_0-0").write_bytes(b"") + (image_dir / "testnode_0-1").write_bytes(b"") + config["containers"]["ceph_node"]["loop_device_count"] = 2 + assert CephNode().devices == ["/dev/loop2", "/dev/loop3"] + def test_cluster_dir_defaults_to_stack_data_dir(self, tmp_path): config["data_dir"] = str(tmp_path) assert CephNode().cluster_dir == tmp_path diff --git a/tests/resources/ceph/test_cephdevstack_core.py b/tests/resources/ceph/test_cephdevstack_core.py index cf6148ce..7acf6e77 100644 --- a/tests/resources/ceph/test_cephdevstack_core.py +++ b/tests/resources/ceph/test_cephdevstack_core.py @@ -415,7 +415,11 @@ def test_init_without_postgres(self): "paddles": {"image": "paddles:latest", "count": 1}, "beanstalk": {"image": "beanstalk:latest", "count": 1}, "pulpito": {"image": "pulpito:latest", "count": 1}, - "testnode": {"image": "testnode:latest", "count": 3}, + "testnode": { + "image": "testnode:latest", + "count": 3, + "loop_device_count": 1, + }, "teuthology": {"image": "teuthology:latest", "count": 1}, "archive": {"image": "archive:latest", "count": 1}, } diff --git a/tests/resources/ceph/test_host_loops.py b/tests/resources/ceph/test_host_loops.py new file mode 100644 index 00000000..22de595b --- /dev/null +++ b/tests/resources/ceph/test_host_loops.py @@ -0,0 +1,32 @@ +from ceph_devstack.resources.ceph.host_loops import ( + allocate_loop_numbers, + discover_used_loop_numbers, + host_loop_path, + owner_loop_numbers, +) + + +class TestHostLoops: + def test_host_loop_path(self): + assert host_loop_path(21) == "/dev/loop21" + + def test_discover_used_loop_numbers_from_backing_files(self, tmp_path): + (tmp_path / "testnode_0-1").write_bytes(b"") + (tmp_path / "ceph_node-4").write_bytes(b"") + (tmp_path / "ignore-me").write_bytes(b"") + assert discover_used_loop_numbers(tmp_path) == {1, 4} + + def test_owner_loop_numbers(self, tmp_path): + (tmp_path / "testnode_1-2").write_bytes(b"") + (tmp_path / "testnode_1-0").write_bytes(b"") + assert owner_loop_numbers("testnode_1", tmp_path) == [0, 2] + + def test_allocate_loop_numbers_uses_lowest_free(self, tmp_path): + (tmp_path / "other-0").write_bytes(b"") + assert allocate_loop_numbers("testnode_0", 2, tmp_path) == [1, 2] + + def test_allocate_loop_numbers_reuses_owner_backing_files(self, tmp_path): + (tmp_path / "testnode_1-4").write_bytes(b"") + (tmp_path / "testnode_1-1").write_bytes(b"") + (tmp_path / "other-0").write_bytes(b"") + assert allocate_loop_numbers("testnode_1", 3, tmp_path) == [1, 4, 2] diff --git a/tests/resources/ceph/test_requirements_ceph.py b/tests/resources/ceph/test_requirements_ceph.py index 92d9f24c..07388e7f 100644 --- a/tests/resources/ceph/test_requirements_ceph.py +++ b/tests/resources/ceph/test_requirements_ceph.py @@ -7,6 +7,8 @@ HasSudo, LoopControlDeviceExists, LoopControlDeviceWriteable, + BlockPoolDiskGroup, + BlockPoolParentAccessible, SELinuxModule, ) @@ -200,11 +202,56 @@ class MockRemoteHost: assert req.fix_cmd[7].endswith("ceph_devstack.pp") +class TestBlockPoolRequirements: + async def test_disk_group_skipped_when_pool_disabled(self): + config.pop("block_pool", None) + req = BlockPoolDiskGroup() + assert await req.check() is True + + async def test_disk_group_required_when_pool_configured(self): + config["block_pool"] = {"parent": "/dev/nvme0n1p1"} + req = BlockPoolDiskGroup() + mock_proc = AsyncMock() + mock_proc.wait = AsyncMock(return_value=0) + with ( + patch( + "ceph_devstack.block_pool.BlockPool.from_config", + return_value=object(), + ), + patch.object(req.host, "arun", return_value=mock_proc), + ): + assert await req.check() is True + + async def test_parent_accessible_when_readable(self): + config["block_pool"] = {"parent": "/dev/nvme0n1p1"} + req = BlockPoolParentAccessible() + with ( + patch( + "ceph_devstack.block_pool.BlockPool.from_config", + return_value=type("Pool", (), {"parent": "/dev/nvme0n1p1"})(), + ), + patch("os.access", return_value=True), + ): + assert await req.check() is True + + async def test_parent_accessible_fails_when_not_writable(self): + config["block_pool"] = {"parent": "/dev/nvme0n1p1"} + req = BlockPoolParentAccessible() + with ( + patch( + "ceph_devstack.block_pool.BlockPool.from_config", + return_value=type("Pool", (), {"parent": "/dev/nvme0n1p1"})(), + ), + patch("os.access", return_value=False), + ): + assert await req.check() is False + + class TestCephDevStackCheckRequirements: async def test_check_requirements_returns_true_when_all_pass(self): devstack = CephDevStack() - devstack.service_specs = {} - config["containers"] = {} + devstack.service_specs = {"ceph_node": {}} + config["containers"] = {"ceph_node": {}} with ( patch("ceph_devstack.resources.ceph.HasSudo") as MockHasSudo, @@ -214,6 +261,12 @@ async def test_check_requirements_returns_true_when_all_pass(self): patch( "ceph_devstack.resources.ceph.LoopControlDeviceWriteable" ) as MockLoopCtrlWrite, + patch( + "ceph_devstack.resources.ceph.BlockPoolDiskGroup" + ) as MockBlockPoolDiskGroup, + patch( + "ceph_devstack.resources.ceph.BlockPoolParentAccessible" + ) as MockBlockPoolParentAccessible, patch("ceph_devstack.host.host.selinux_enforcing") as mock_selinux, ): MockHasSudo.return_value = AsyncMock(evaluate=AsyncMock(return_value=True)) @@ -221,10 +274,54 @@ async def test_check_requirements_returns_true_when_all_pass(self): MockLoopCtrlWrite.return_value = AsyncMock( evaluate=AsyncMock(return_value=True) ) + MockBlockPoolDiskGroup.return_value = AsyncMock( + evaluate=AsyncMock(return_value=True) + ) + MockBlockPoolParentAccessible.return_value = AsyncMock( + evaluate=AsyncMock(return_value=True) + ) mock_selinux.return_value = False result = await devstack.check_requirements() assert result is True + async def test_check_requirements_returns_false_when_block_pool_inaccessible( + self, + ): + devstack = CephDevStack() + devstack.service_specs = {"ceph_node": {}} + config["containers"] = {"ceph_node": {}} + + with ( + patch("ceph_devstack.resources.ceph.HasSudo") as MockHasSudo, + patch( + "ceph_devstack.resources.ceph.LoopControlDeviceExists" + ) as MockLoopCtrl, + patch( + "ceph_devstack.resources.ceph.LoopControlDeviceWriteable" + ) as MockLoopCtrlWrite, + patch( + "ceph_devstack.resources.ceph.BlockPoolDiskGroup" + ) as MockBlockPoolDiskGroup, + patch( + "ceph_devstack.resources.ceph.BlockPoolParentAccessible" + ) as MockBlockPoolParentAccessible, + patch("ceph_devstack.host.host.selinux_enforcing") as mock_selinux, + ): + MockHasSudo.return_value = AsyncMock(evaluate=AsyncMock(return_value=True)) + MockLoopCtrl.return_value = AsyncMock(evaluate=AsyncMock(return_value=True)) + MockLoopCtrlWrite.return_value = AsyncMock( + evaluate=AsyncMock(return_value=True) + ) + MockBlockPoolDiskGroup.return_value = AsyncMock( + evaluate=AsyncMock(return_value=True) + ) + MockBlockPoolParentAccessible.return_value = AsyncMock( + evaluate=AsyncMock(return_value=False) + ) + mock_selinux.return_value = False + result = await devstack.check_requirements() + assert result is False + async def test_check_requirements_returns_false_when_repo_missing(self): devstack = CephDevStack() config["containers"] = { diff --git a/tests/resources/ceph/test_testnode.py b/tests/resources/ceph/test_testnode.py index 9425edab..14237c2e 100644 --- a/tests/resources/ceph/test_testnode.py +++ b/tests/resources/ceph/test_testnode.py @@ -2,8 +2,8 @@ import pytest -from ceph_devstack.resources.ceph import TestNode as _TestNode from ceph_devstack import config +from ceph_devstack.resources.ceph import TestNode as _TestNode class TestTestnode: @@ -16,22 +16,34 @@ def test_testnode_loop_device_count_default_to_one(self, cls): testnode = cls("testnode_1") assert testnode.loop_device_count == 1 - def test_testnode_create_cmd_includes_related_devices(self, cls): + def test_testnode_create_cmd_includes_related_devices(self, cls, tmp_path): config.load(Path(__file__).parent.joinpath("fixtures", "testnode-config.toml")) + config["data_dir"] = str(tmp_path) testnode = cls("testnode_1") create_cmd = testnode.create_cmd - assert "--device=/dev/loop4" in create_cmd - assert "--device=/dev/loop5" in create_cmd - assert "--device=/dev/loop6" in create_cmd - assert "--device=/dev/loop7" in create_cmd + assert "--device=/dev/loop0" in create_cmd + assert "--device=/dev/loop1" in create_cmd + assert "--device=/dev/loop2" in create_cmd + assert "--device=/dev/loop3" in create_cmd - def test_testnode_devices_is_based_on_loop_device_count_config(self, cls): + def test_testnode_devices_is_based_on_loop_device_count_config(self, cls, tmp_path): config.load(Path(__file__).parent.joinpath("fixtures", "testnode-config.toml")) + config["data_dir"] = str(tmp_path) testnode = cls("testnode_1") assert testnode.loop_device_count == 4 assert testnode.devices == [ - "/dev/loop4", - "/dev/loop5", - "/dev/loop6", - "/dev/loop7", + "/dev/loop0", + "/dev/loop1", + "/dev/loop2", + "/dev/loop3", ] + + def test_testnode_reuses_existing_backing_files(self, cls, tmp_path): + config["data_dir"] = str(tmp_path) + image_dir = tmp_path / "disk_images" + image_dir.mkdir() + (image_dir / "testnode_1-4").write_bytes(b"") + (image_dir / "testnode_1-1").write_bytes(b"") + config["containers"]["testnode"]["loop_device_count"] = 3 + testnode = cls("testnode_1") + assert testnode.devices == ["/dev/loop1", "/dev/loop4", "/dev/loop0"] diff --git a/tests/test_block_pool.py b/tests/test_block_pool.py new file mode 100644 index 00000000..8cd8f363 --- /dev/null +++ b/tests/test_block_pool.py @@ -0,0 +1,445 @@ +from pathlib import Path +from unittest.mock import patch + +import pytest + +from ceph_devstack.block_pool import ( + BlockPool, + BlockPoolError, + format_size, + parse_size, + validate_parent_name, + _has_partition_siblings, + _read_tail_marker, + _write_tail_marker, +) + + +class TestParseSize: + def test_gigabytes(self): + assert parse_size("50G") == 50 * 1024**3 + + def test_integer(self): + assert parse_size(1024) == 1024 + + def test_invalid(self): + with pytest.raises(BlockPoolError): + parse_size("not-a-size") + + def test_format_size_human_units(self): + assert format_size(5 * 1024**3) == "5G" + assert format_size(12345) == "12345B" + + +class TestValidateParentName: + def test_accepts_partition(self): + with ( + patch( + "ceph_devstack.block_pool.canonical_device_path", + return_value="/dev/nvme0n1p1", + ), + patch("ceph_devstack.block_pool.os.path.exists", return_value=True), + ): + assert validate_parent_name("/dev/nvme0n1p1") == "/dev/nvme0n1p1" + + def test_accepts_whole_disk(self): + with ( + patch( + "ceph_devstack.block_pool.canonical_device_path", + return_value="/dev/nvme0n1", + ), + patch("ceph_devstack.block_pool.os.path.exists", return_value=True), + ): + assert validate_parent_name("/dev/nvme0n1") == "/dev/nvme0n1" + + def test_rejects_device_mapper(self): + with pytest.raises(BlockPoolError, match="not allowed"): + validate_parent_name("/dev/dm-0") + + def test_rejects_non_dev_path(self): + with ( + patch( + "ceph_devstack.block_pool.canonical_device_path", + return_value="/tmp/not-a-device", + ), + pytest.raises(BlockPoolError, match="must be a /dev path"), + ): + validate_parent_name("/tmp/not-a-device") + + def test_rejects_missing_device(self): + with ( + patch( + "ceph_devstack.block_pool.canonical_device_path", + return_value="/dev/nvme0n1p1", + ), + patch("ceph_devstack.block_pool.os.path.exists", return_value=False), + pytest.raises(BlockPoolError, match="does not exist"), + ): + validate_parent_name("/dev/nvme0n1p1") + + def test_rejects_unknown_block_name(self): + with ( + patch( + "ceph_devstack.block_pool.canonical_device_path", + return_value="/dev/foo", + ), + patch("ceph_devstack.block_pool.os.path.exists", return_value=True), + pytest.raises(BlockPoolError, match="block device or partition"), + ): + validate_parent_name("/dev/foo") + + +class TestBlockPool: + def test_allocate_and_reuse_stable_owner(self, enrolled_pool): + size = parse_size("10G") + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch.object(enrolled_pool, "_validate_region"), + ): + first_offset, _ = enrolled_pool.get_or_allocate("ceph_node", 0, size) + second_offset, _ = enrolled_pool.get_or_allocate("ceph_node", 1, size) + again_offset, _ = enrolled_pool.get_or_allocate("ceph_node", 0, size) + assert again_offset == first_offset + assert first_offset == 0 + assert second_offset == size + + def test_release_and_reuse_slice(self, enrolled_pool): + size = parse_size("10G") + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch.object(enrolled_pool, "_validate_region"), + ): + enrolled_pool.get_or_allocate("testnode_0", 0, size) + enrolled_pool.get_or_allocate("testnode_0", 1, size) + enrolled_pool.release_owner("testnode_0") + + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch.object(enrolled_pool, "_validate_region"), + ): + reused_offset, _ = enrolled_pool.get_or_allocate("ceph_node", 0, size) + assert reused_offset in {0, size} + + def test_different_sizes_pack_sequentially(self, enrolled_pool): + small = parse_size("5G") + large = parse_size("10G") + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch.object(enrolled_pool, "_validate_region"), + ): + first_offset, _ = enrolled_pool.get_or_allocate("ceph_node", 0, small) + second_offset, _ = enrolled_pool.get_or_allocate("testnode_0", 0, large) + assert first_offset == 0 + assert second_offset == small + + def test_parent_change_rejected(self, tmp_path, enrolled_pool): + size = parse_size("10G") + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch.object(enrolled_pool, "_validate_region"), + ): + enrolled_pool.get_or_allocate("ceph_node", 0, size) + with ( + patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value="/dev/nvme1n1p1", + ), + pytest.raises(BlockPoolError, match="parent changed"), + ): + BlockPool( + tmp_path / "block_pool.json", + "/dev/nvme1n1p1", + allow_enroll=True, + ) + + def test_enroll_requires_allow_enroll(self, disallow_enroll_pool): + with pytest.raises(BlockPoolError, match="allow_enroll"): + disallow_enroll_pool.ensure_ready() + + def test_from_config_allow_enroll_defaults_false(self, tmp_path): + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value="/dev/nvme0n1p1", + ): + pool = BlockPool.from_config( + { + "block_pool": { + "parent": "/dev/nvme0n1p1", + "state_dir": str(tmp_path), + } + } + ) + assert pool is not None + assert pool.allow_enroll is False + + def test_from_config_allow_enroll_explicit_false(self, tmp_path): + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value="/dev/nvme0n1p1", + ): + pool = BlockPool.from_config( + { + "block_pool": { + "parent": "/dev/nvme0n1p1", + "state_dir": str(tmp_path), + "allow_enroll": False, + } + } + ) + assert pool is not None + assert pool.allow_enroll is False + + def test_from_config_allow_enroll_true(self, tmp_path): + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value="/dev/nvme0n1p1", + ): + pool = BlockPool.from_config( + { + "block_pool": { + "parent": "/dev/nvme0n1p1", + "state_dir": str(tmp_path), + "allow_enroll": True, + } + } + ) + assert pool is not None + assert pool.allow_enroll is True + + def test_enroll_rejects_non_empty_parent(self, block_pool, mock_fresh_enrollment): + with open(block_pool.parent, "r+b") as handle: + handle.write(b"DATA") + with pytest.raises(BlockPoolError, match="not empty"): + block_pool.ensure_ready() + + def test_enroll_writes_marker_and_state(self, block_pool, mock_fresh_enrollment): + block_pool.ensure_ready() + assert block_pool.enrolled + marker = _read_tail_marker(block_pool.parent) + assert marker is not None + assert marker["pool_id"] == block_pool._state["pool_id"] + assert marker["parent"] == block_pool.parent + + def test_new_slice_rejects_existing_data(self, enrolled_pool): + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch( + "ceph_devstack.block_pool._read_device", + return_value=b"NOTZEROS" + b"\x00" * 4088, + ), + pytest.raises(BlockPoolError, match="not empty"), + ): + enrolled_pool.get_or_allocate("ceph_node", 0, 1024 * 1024) + + def test_enroll_rejects_whole_disk_with_partitions(self, tmp_path): + state_path = tmp_path / "block_pool.json" + parent = tmp_path / "nvme0n1" + parent.write_bytes(b"\x00" * (20 * 1024**2)) + with patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=str(parent), + ): + pool = BlockPool(state_path, str(parent), allow_enroll=True) + with ( + patch("ceph_devstack.block_pool._device_mounted", return_value=False), + patch( + "ceph_devstack.block_pool._device_has_blkid_signature", + return_value=False, + ), + patch("ceph_devstack.block_pool._read_tail_marker", return_value=None), + patch( + "ceph_devstack.block_pool._has_partition_siblings", + return_value=True, + ), + pytest.raises(BlockPoolError, match="existing partitions"), + ): + pool.ensure_ready() + + def test_reclaims_marker_after_state_loss(self, disallow_enroll_pool): + payload = { + "pool_id": "saved-pool-id", + "parent": disallow_enroll_pool.parent, + } + _write_tail_marker(disallow_enroll_pool.parent, payload) + with ( + patch("ceph_devstack.block_pool._device_mounted", return_value=False), + patch( + "ceph_devstack.block_pool._device_has_blkid_signature", + return_value=False, + ), + ): + disallow_enroll_pool.ensure_ready() + assert disallow_enroll_pool.enrolled + assert disallow_enroll_pool._state["pool_id"] == "saved-pool-id" + + def test_reused_slice_must_be_empty(self, enrolled_pool): + size = 1024 * 1024 + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch("ceph_devstack.block_pool._read_device", return_value=b"\x00" * 4096), + ): + enrolled_pool.get_or_allocate("testnode_0", 0, size) + enrolled_pool.release_owner("testnode_0") + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch( + "ceph_devstack.block_pool._read_device", + return_value=b"STALE" + b"\x00" * 4089, + ), + pytest.raises(BlockPoolError, match="not empty"), + ): + enrolled_pool.get_or_allocate("ceph_node", 0, size) + + +class TestBlockPoolStatus: + def test_status_disabled(self, caplog): + with caplog.at_level("INFO", logger="ceph-devstack"): + assert BlockPool.status_from_config({}) == 0 + assert "disabled" in caplog.text + + def test_status_shows_allocations(self, tmp_path, enrolled_pool, caplog): + enrolled_pool._state["allocations"] = { + "ceph_node:0": {"offset": 0, "size": 1024 * 1024, "owner": "ceph_node"} + } + enrolled_pool._save() + with ( + patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=enrolled_pool.parent, + ), + caplog.at_level("INFO", logger="ceph-devstack"), + ): + BlockPool.status_from_config( + { + "block_pool": { + "parent": enrolled_pool.parent, + "state_dir": str(tmp_path), + } + } + ) + assert "ceph_node:0" in caplog.text + assert "ceph_node" in caplog.text + + +class TestBlockPoolEnrollmentGuards: + def test_enroll_rejects_mounted_parent(self, block_pool, mock_fresh_enrollment): + with ( + patch("ceph_devstack.block_pool._device_mounted", return_value=True), + pytest.raises(BlockPoolError, match="mounted"), + ): + block_pool.ensure_ready() + + def test_enroll_rejects_blkid_signature(self, block_pool, mock_fresh_enrollment): + with ( + patch( + "ceph_devstack.block_pool._device_has_blkid_signature", + return_value=True, + ), + pytest.raises(BlockPoolError, match="filesystem or partition"), + ): + block_pool.ensure_ready() + + def test_verify_rejects_missing_marker(self, enrolled_pool): + with ( + patch("ceph_devstack.block_pool._read_tail_marker", return_value=None), + pytest.raises(BlockPoolError, match="missing the devstack"), + ): + enrolled_pool.ensure_ready() + + def test_verify_rejects_foreign_pool_id(self, enrolled_pool): + with ( + patch( + "ceph_devstack.block_pool._read_tail_marker", + return_value={ + "parent": enrolled_pool.parent, + "pool_id": "other-pool", + }, + ), + pytest.raises(BlockPoolError, match="belongs to another pool"), + ): + enrolled_pool.ensure_ready() + + def test_region_exceeds_parent_size(self, enrolled_pool): + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch( + "ceph_devstack.block_pool._device_size_bytes", + return_value=512 * 1024, + ), + pytest.raises(BlockPoolError, match="exceeds"), + ): + enrolled_pool.get_or_allocate("ceph_node", 0, 1024 * 1024) + + def test_legacy_slice_size_state_rejected_on_load(self, enrolled_pool): + enrolled_pool._state["slice_size"] = 1024 * 1024 + enrolled_pool._save() + with ( + patch( + "ceph_devstack.block_pool.validate_parent_name", + return_value=enrolled_pool.parent, + ), + pytest.raises(BlockPoolError, match="legacy block pool state"), + ): + BlockPool(enrolled_pool.state_path, enrolled_pool.parent) + + +class TestPartitionSiblings: + def test_whole_disk_with_nvme_partitions(self): + class _Entry: + def __init__(self, name: str): + self.name = name + + def is_dir(self) -> bool: + return True + + class _BlockDir: + def is_dir(self) -> bool: + return True + + def iterdir(self) -> list[_Entry]: + return [_Entry("nvme0n1"), _Entry("nvme0n1p1")] + + mock_block_dir = _BlockDir() + + def path_for(arg: str): + if arg == "/sys/class/block": + return mock_block_dir + return Path(arg) + + with patch("ceph_devstack.block_pool.Path", side_effect=path_for): + assert _has_partition_siblings("nvme0n1") is True + + def test_partition_name_has_no_siblings(self): + assert _has_partition_siblings("nvme0n1p1") is False + + +class TestBlockPoolAllocation: + def test_allocation_for_missing(self, enrolled_pool): + assert enrolled_pool.allocation_for("nobody", 0) is None + + def test_allocation_for_active(self, enrolled_pool): + size = 1024 * 1024 + with ( + patch.object(enrolled_pool, "_verify_enrolled_parent"), + patch("ceph_devstack.block_pool._read_device", return_value=b"\x00" * 4096), + ): + enrolled_pool.get_or_allocate("ceph_node", 0, size) + assert enrolled_pool.allocation_for("ceph_node", 0) == (0, size, size) + + +class TestTailMarker: + def test_roundtrip(self, tmp_path): + parent = tmp_path / "disk" + parent.write_bytes(b"\x00" * (1024 * 1024)) + payload = {"pool_id": "abc", "parent": str(parent)} + _write_tail_marker(str(parent), payload) + assert _read_tail_marker(str(parent)) == payload + + def test_corrupt_marker_returns_none(self, tmp_path): + parent = tmp_path / "disk" + parent.write_bytes(b"\x00" * (1024 * 1024)) + offset = len(parent.read_bytes()) - 4096 + with open(parent, "r+b") as handle: + handle.seek(offset) + handle.write(b"CEPH-DEVSTACK-BLOCK-POOL-TAIL-v1\x00not-json") + assert _read_tail_marker(str(parent)) is None diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 00000000..90c7e4e7 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,25 @@ +import sys +from unittest.mock import patch + +from ceph_devstack import config, parse_args +from ceph_devstack.block_pool import BlockPool + + +class TestBlockPoolCLI: + def test_parse_args_block_pool_status(self): + args = parse_args(["block-pool", "status"]) + assert args.command == "block-pool" + assert args.block_pool_op == "status" + + def test_main_block_pool_status(self): + from ceph_devstack.cli import main + + with ( + patch.object(sys, "argv", ["ceph-devstack", "block-pool", "status"]), + patch.object( + BlockPool, "status_from_config", return_value=0 + ) as mock_status, + ): + rc = main() + assert rc == 0 + mock_status.assert_called_once_with(config) diff --git a/tests/test_config.py b/tests/test_config.py index 83697422..b926fb68 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -105,6 +105,9 @@ def test_config_defaults(self): assert config == { "stack": "teuthology", "data_dir": "~/.local/share/ceph-devstack", + "block_pool": { + "state_dir": "~/.local/share/ceph-devstack", + }, "stacks": { "teuthology": { "services": [ @@ -134,6 +137,7 @@ def test_config_defaults(self): "pulpito": {"image": "quay.io/ceph-infra/pulpito:main"}, "testnode": { "count": 3, + "loop_device_count": 1, "loop_device_size": "5G", "image": "quay.io/ceph-infra/teuthology-testnode:main", }, @@ -142,6 +146,8 @@ def test_config_defaults(self): "image": "quay.io/ceph-ci/ceph:main", "loop_device_count": 3, "loop_device_size": "5G", + "dashboard_port": 8080, + "dashboard_ssl": False, }, }, } From 9893893d01a8153718bda713cc9d1ce35c3094ae Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 16 Jul 2026 15:28:14 -0600 Subject: [PATCH 03/32] sccache rw --- ceph_devstack/config.toml | 14 +++--- ceph_devstack/resources/ceph/ceph_node.py | 49 ++++++++++++++++--- ceph_devstack/sccache-s3.conf | 1 - tests/resources/ceph/test_ceph_node.py | 58 +++++++++++++++++++---- 4 files changed, 99 insertions(+), 23 deletions(-) diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 06da229b..71d3c1c2 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -49,14 +49,14 @@ dashboard_ssl = false # build_dir = "build" # relative to repo; passed to build-with-container.py -b # build_distro = "centos9" # build-with-container.py -d # build_steps = ["build"] # default for cpatch; ["packages"] for container builder -# sccache = true # default; local disk cache (see sccache.conf) -# sccache = false # disable sccache -# sccache_mode = "local" # local (default) or s3 for shared remote cache -# sccache_cache_path = "~/.local/share/ceph-devstack/cache/sccache" -# sccache_cache_size = "100G" # local cache size limit -# sccache_debug = false # enable SCCACHE_LOG=debug on build stdout +sccache = true # local disk cache (see sccache.conf); set to false to disable +sccache_mode = "local" # local (default) or s3 for shared remote cache +sccache_rw_mode = false # enable read-write mode (requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in environment for S3) +# sccache_cache_path = "~/.local/share/ceph-devstack/cache/sccache" # override default cache location +sccache_cache_size = "100G" # local cache size limit +sccache_debug = false # enable SCCACHE_LOG=debug on build stdout # sccache_conf = "/path/to/sccache.conf" # override bundled config entirely -# npm_cache = true # default; persist dashboard npm downloads +npm_cache = true # persist dashboard npm downloads # npm_cache_path = "~/.local/share/ceph-devstack/cache/npm" # override npm cache location # dnf_cache = false # persist dnf downloads when rebuilding ceph-build images # dnf_cache_path = "~/.local/share/ceph-devstack/cache/dnf" # override dnf cache base dir diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index a026e5f4..e3e2fc8f 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -10,6 +10,8 @@ from subprocess import CalledProcessError from typing import List +import tomlkit + from ceph_devstack import PROJECT_ROOT, config, logger from ceph_devstack.host import host from ceph_devstack.resources.ceph.block_devices import BlockDeviceProvisioner @@ -268,6 +270,11 @@ def sccache_enabled(self) -> bool: def sccache_mode(self) -> str: return str(self.config.get("sccache_mode", "local")).lower() + @property + def sccache_rw_mode(self) -> bool: + """Whether to use sccache in read-write mode (requires credentials).""" + return self.config.get("sccache_rw_mode", False) is True + @property def sccache_debug(self) -> bool: return self.config.get("sccache_debug", False) is True @@ -385,6 +392,22 @@ def create_cmd(self): def _device_image(self, device: str) -> str: return f"{self.name}-{device.removeprefix('/dev/loop')}" + def _get_s3_credentials_env(self) -> list[str]: + """Get S3 credential environment variables for read-write mode.""" + aws_access_key = os.environ.get("AWS_ACCESS_KEY_ID", "") + aws_secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "") + if not aws_access_key or not aws_secret_key: + raise ValueError( + "sccache_rw_mode=true requires AWS_ACCESS_KEY_ID and " + "AWS_SECRET_ACCESS_KEY to be set in the environment" + ) + return [ + f"AWS_ACCESS_KEY_ID={aws_access_key}", + f"AWS_SECRET_ACCESS_KEY={aws_secret_key}", + "SCCACHE_S3_NO_CREDENTIALS=false", + "SCCACHE_S3_RW_MODE=READ_WRITE", + ] + def _sccache_build_env(self, repo: Path) -> tuple[list[str], list[str]]: if not self.sccache_enabled: return [], [] @@ -401,7 +424,15 @@ def _sccache_build_env(self, repo: Path) -> tuple[list[str], list[str]]: use_local_cache = True if not conf_src.is_file(): raise FileNotFoundError(f"sccache config not found: {conf_src}") - shutil.copy2(conf_src, repo / "sccache.conf") + + # Parse and modify config for S3 mode to set no_credentials correctly + conf_content = conf_src.read_text() + if self.sccache_mode == "s3": + conf_data = tomlkit.parse(conf_content) + # Set no_credentials based on whether we're using read-write mode + conf_data["cache"]["s3"]["no_credentials"] = not self.sccache_rw_mode + conf_content = tomlkit.dumps(conf_data) + (repo / "sccache.conf").write_text(conf_content) lines = [ "SCCACHE=true", @@ -421,12 +452,16 @@ def _sccache_build_env(self, repo: Path) -> tuple[list[str], list[str]]: cache_size = self.config.get("sccache_cache_size", "100G") lines.append(f"SCCACHE_CACHE_SIZE={cache_size}") elif self.sccache_mode == "s3": - lines.extend( - [ - "SCCACHE_S3_NO_CREDENTIALS=true", - "SCCACHE_S3_RW_MODE=READ_ONLY", - ] - ) + if self.sccache_rw_mode: + lines.extend(self._get_s3_credentials_env()) + else: + # Read-only mode (default) + lines.extend( + [ + "SCCACHE_S3_NO_CREDENTIALS=true", + "SCCACHE_S3_RW_MODE=READ_ONLY", + ] + ) return lines, extra_args def _prepare_build_env(self) -> tuple[Path | None, list[str]]: diff --git a/ceph_devstack/sccache-s3.conf b/ceph_devstack/sccache-s3.conf index 01db75c7..a341b223 100644 --- a/ceph_devstack/sccache-s3.conf +++ b/ceph_devstack/sccache-s3.conf @@ -4,5 +4,4 @@ endpoint = "s3.us-south.cloud-object-storage.appdomain.cloud" use_ssl = true key_prefix = "" server_side_encryption = false -no_credentials = true region = "auto" diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index f7fb83ec..b2af9843 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -1,6 +1,8 @@ from unittest.mock import AsyncMock, patch import sys +import pytest +import tomlkit from ceph_devstack import config from ceph_devstack.resources.container import Container @@ -137,18 +139,54 @@ def test_prepare_build_env_uses_s3_sccache_when_configured(self, tmp_path): config["containers"]["ceph_node"]["repo"] = str(repo) config["containers"]["ceph_node"]["sccache_mode"] = "s3" env_file, extra_args = CephNode()._prepare_build_env() - assert ( - repo / "sccache.conf" - ).read_text() == PACKAGE_SCCACHE_S3_CONF.read_text() + sccache_conf = repo / "sccache.conf" + assert sccache_conf.exists() + conf_data = tomlkit.parse(sccache_conf.read_text()) + assert conf_data["cache"]["s3"]["no_credentials"] is True contents = env_file.read_text() assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents assert "SCCACHE_S3_RW_MODE=READ_ONLY" in contents assert "SCCACHE_DIR=" not in contents assert extra_args == [] + def test_prepare_build_env_uses_s3_rw_mode_with_credentials( + self, tmp_path, monkeypatch + ): + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key-id") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key") + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache_mode"] = "s3" + config["containers"]["ceph_node"]["sccache_rw_mode"] = True + env_file, extra_args = CephNode()._prepare_build_env() + contents = env_file.read_text() + assert "AWS_ACCESS_KEY_ID=test-key-id" in contents + assert "AWS_SECRET_ACCESS_KEY=test-secret-key" in contents + assert "SCCACHE_S3_RW_MODE=READ_WRITE" in contents + assert "SCCACHE_S3_NO_CREDENTIALS=true" not in contents + assert extra_args == [] + + def test_prepare_build_env_raises_error_for_s3_rw_without_credentials( + self, tmp_path, monkeypatch + ): + monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False) + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["repo"] = str(repo) + config["containers"]["ceph_node"]["sccache_mode"] = "s3" + config["containers"]["ceph_node"]["sccache_rw_mode"] = True + with pytest.raises( + ValueError, match="AWS_ACCESS_KEY_ID.*AWS_SECRET_ACCESS_KEY" + ): + CephNode()._prepare_build_env() + def test_prepare_build_env_honors_custom_sccache_conf(self, tmp_path): custom_conf = tmp_path / "custom-sccache.conf" - custom_conf.write_text("[cache.s3]\nbucket = test\n") + custom_conf.write_text('[cache.s3]\nbucket = "test"\n') repo = tmp_path / "ceph" repo.mkdir() config["containers"]["ceph_node"]["image"] = "example:test" @@ -156,7 +194,10 @@ def test_prepare_build_env_honors_custom_sccache_conf(self, tmp_path): config["containers"]["ceph_node"]["sccache_conf"] = str(custom_conf) config["containers"]["ceph_node"]["sccache_mode"] = "s3" env_file, extra_args = CephNode()._prepare_build_env() - assert (repo / "sccache.conf").read_text() == custom_conf.read_text() + sccache_conf = repo / "sccache.conf" + conf_data = tomlkit.parse(sccache_conf.read_text()) + assert conf_data["cache"]["s3"]["bucket"] == "test" + assert conf_data["cache"]["s3"]["no_credentials"] is True contents = env_file.read_text() assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents assert extra_args == [] @@ -260,11 +301,12 @@ def test_bundled_sccache_conf_uses_local_disk(self): assert 'dir = "/sccache"' in contents assert "rw_mode" not in contents - def test_bundled_sccache_s3_conf_keeps_anonymous_read_settings(self): + def test_bundled_sccache_s3_conf_has_s3_settings(self): contents = PACKAGE_SCCACHE_S3_CONF.read_text() assert "[cache.s3]" in contents - assert "no_credentials = true" in contents - assert "rw_mode" not in contents + assert "bucket" in contents + assert "endpoint" in contents + # no_credentials is added dynamically by _sccache_build_env def test_cpatch_cmd_uses_upstream_script(self): config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" From 807435fdd672994bdb4a5a9a59837b0870f10d5c Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 12:22:53 -0600 Subject: [PATCH 04/32] tests for cn build --- tests/resources/ceph/test_ceph_node.py | 43 ++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index b2af9843..838fb953 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -40,6 +40,21 @@ def test_compile_steps_default_for_cpatch(self): config["containers"]["ceph_node"]["image"] = "example:test" assert CephNode().compile_steps == ["build"] + def test_compile_steps_default_for_container_builder(self): + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["image_builder"] = "container" + assert CephNode().compile_steps == ["packages"] + + def test_compile_steps_respects_custom_build_steps(self): + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["build_steps"] = ["build", "packages"] + assert CephNode().compile_steps == ["build", "packages"] + + def test_compile_steps_allows_single_custom_step(self): + config["containers"]["ceph_node"]["image"] = "example:test" + config["containers"]["ceph_node"]["build_steps"] = ["packages"] + assert CephNode().compile_steps == ["packages"] + def test_compile_cmd_uses_build_with_container(self, tmp_path): config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" config["containers"]["ceph_node"]["repo"] = str(tmp_path) @@ -56,6 +71,34 @@ def test_compile_cmd_uses_build_with_container(self, tmp_path): assert "--env-file" not in cmd assert "--npm-cache-path" not in cmd + def test_compile_cmd_generates_packages_step(self, tmp_path): + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["repo"] = str(tmp_path) + config["containers"]["ceph_node"]["build_dir"] = "build" + config["containers"]["ceph_node"]["build_distro"] = "centos9" + config["containers"]["ceph_node"]["build_steps"] = ["packages"] + config["containers"]["ceph_node"]["sccache"] = False + config["containers"]["ceph_node"]["npm_cache"] = False + cmd = CephNode()._compile_cmd() + assert "-e" in cmd + e_index = cmd.index("-e") + assert cmd[e_index + 1] == "packages" + + def test_compile_cmd_generates_multiple_steps(self, tmp_path): + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["repo"] = str(tmp_path) + config["containers"]["ceph_node"]["build_dir"] = "build" + config["containers"]["ceph_node"]["build_distro"] = "centos9" + config["containers"]["ceph_node"]["build_steps"] = ["build", "packages"] + config["containers"]["ceph_node"]["sccache"] = False + config["containers"]["ceph_node"]["npm_cache"] = False + cmd = CephNode()._compile_cmd() + # Find all -e flags and their values + e_indices = [i for i, x in enumerate(cmd) if x == "-e"] + assert len(e_indices) == 2 + assert cmd[e_indices[0] + 1] == "build" + assert cmd[e_indices[1] + 1] == "packages" + def test_compile_cmd_passes_npm_cache_path(self, tmp_path): npm_cache = tmp_path / "npm-cache" config["data_dir"] = str(tmp_path / "data") From 4e7a5cbfc0e2e7bd73446239687c104b30113a63 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 12:31:58 -0600 Subject: [PATCH 05/32] make dist --- ceph_devstack/config.toml | 1 + ceph_devstack/resources/ceph/ceph_node.py | 29 +++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 71d3c1c2..105b0c1d 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -49,6 +49,7 @@ dashboard_ssl = false # build_dir = "build" # relative to repo; passed to build-with-container.py -b # build_distro = "centos9" # build-with-container.py -d # build_steps = ["build"] # default for cpatch; ["packages"] for container builder +# make_dist = false # run make-dist before building to create source distribution tarball sccache = true # local disk cache (see sccache.conf); set to false to disable sccache_mode = "local" # local (default) or s3 for shared remote cache sccache_rw_mode = false # enable read-write mode (requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in environment for S3) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index e3e2fc8f..f1ae708a 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -534,6 +534,31 @@ def _git_value(self, args: str) -> str: text=True, ).strip() + def _make_dist_version(self) -> str: + """Generate version string for make-dist from git describe.""" + version = self._git_value("describe --abbrev=8 --match 'v*'") + # Remove leading 'v' from version tag + if version.startswith("v"): + version = version[1:] + return version + + async def _make_dist(self): + """Create source distribution tarball using make-dist.""" + if not self.repo: + return + version = self._make_dist_version() + logger.info(f"{self.name}: creating source distribution for version {version}") + make_dist_script = Path(self.repo) / "make-dist" + if not make_dist_script.exists(): + logger.warning( + f"{self.name}: make-dist script not found at {make_dist_script}, skipping" + ) + return + await self._run_cmd( + ["./make-dist", version], + cwd=self.repo, + ) + async def _run_cmd(self, cmd: List[str], cwd: str): proc = await host.arun( cmd, @@ -551,6 +576,10 @@ async def _run_cmd(self, cmd: List[str], cwd: str): ) async def _compile(self): + # Run make-dist first if enabled + if self.config.get("make_dist", False): + await self._make_dist() + logger.info( f"{self.name}: compiling ceph via build-with-container.py in {self.repo}" ) From f990da91d65c5ac5006c715e29b748c549994125 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 12:52:04 -0600 Subject: [PATCH 06/32] rename build modes --- ceph_devstack/config.toml | 6 ++--- ceph_devstack/resources/ceph/ceph_node.py | 32 ++++++++++++----------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 105b0c1d..8d8a22d1 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -42,14 +42,12 @@ dashboard_ssl = false # dashboard_password = "admin" # dashboard_show_password = false # log dashboard password on container start # dashboard = false # disable mgr dashboard -# image_builder = "cpatch" # default; patch local binaries into runtime image -# image_builder = "container" # future: container/build.sh + local packages +# image_builder = "binary-patch" # default; patch local binaries into runtime image +# image_builder = "package-build" # future: container/build.sh + local packages # Build from a local ceph.git checkout: # repo = "~/dev/ceph" # build_dir = "build" # relative to repo; passed to build-with-container.py -b # build_distro = "centos9" # build-with-container.py -d -# build_steps = ["build"] # default for cpatch; ["packages"] for container builder -# make_dist = false # run make-dist before building to create source distribution tarball sccache = true # local disk cache (see sccache.conf); set to false to disable sccache_mode = "local" # local (default) or s3 for shared remote cache sccache_rw_mode = false # enable read-write mode (requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in environment for S3) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index f1ae708a..aeaf6d97 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -34,8 +34,8 @@ CONTAINER_CLUSTER_DIR = "/var/lib/ceph-devstack/cluster" DEFAULT_COMPILE_STEPS = { - "cpatch": ["build"], - "container": ["packages"], + "binary-patch": ["build"], + "package-build": ["packages"], } CEPH_NODE_CAPABILITIES = [ @@ -255,12 +255,12 @@ def build_path(self) -> Path: @property def image_builder(self) -> str: - return self.config.get("image_builder", "cpatch") + return self.config.get("image_builder", "binary-patch") @property def compile_steps(self) -> List[str]: - default = DEFAULT_COMPILE_STEPS.get(self.image_builder, ["build"]) - return list(self.config.get("build_steps", default)) + """Return compile steps based on image_builder mode (not configurable).""" + return DEFAULT_COMPILE_STEPS.get(self.image_builder, ["build"]) @property def sccache_enabled(self) -> bool: @@ -516,7 +516,7 @@ def _compile_cmd( cmd.append(f"--extra={extra}") return cmd - def _cpatch_cmd(self) -> List[str]: + def _binary_patch_cmd(self) -> List[str]: return [ "sudo", "../src/script/cpatch", @@ -576,8 +576,8 @@ async def _run_cmd(self, cmd: List[str], cwd: str): ) async def _compile(self): - # Run make-dist first if enabled - if self.config.get("make_dist", False): + # Run make-dist automatically for package-build mode + if self.image_builder == "package-build": await self._make_dist() logger.info( @@ -598,22 +598,24 @@ def _verify_build_tree(self): "after build-with-container.py" ) - async def _build_image_cpatch(self): + async def _build_image_binary_patch(self): self._verify_build_tree() build_path = self.build_path - logger.info(f"{self.name}: building {self.image} via cpatch in {build_path}") - await self._run_cmd(self._cpatch_cmd(), cwd=str(build_path)) + logger.info( + f"{self.name}: building {self.image} via binary-patch in {build_path}" + ) + await self._run_cmd(self._binary_patch_cmd(), cwd=str(build_path)) - async def _build_image_container(self): + async def _build_image_package_build(self): raise NotImplementedError( - "image_builder='container' requires ceph container/build.sh to consume " + "image_builder='package-build' requires ceph container/build.sh to consume " "locally-built packages; enable this once that support lands upstream" ) async def _build_image(self): builders = { - "cpatch": self._build_image_cpatch, - "container": self._build_image_container, + "binary-patch": self._build_image_binary_patch, + "package-build": self._build_image_package_build, } try: builder = builders[self.image_builder] From 0671eefaf61f5bde5e27fa9e3242b9ececd26c7c Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 12:55:57 -0600 Subject: [PATCH 07/32] image-variant --- ceph_devstack/resources/ceph/ceph_node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index aeaf6d97..fc2c4a5d 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -509,6 +509,8 @@ def _compile_cmd( ] for step in self.compile_steps: cmd.extend(["-e", step]) + if self.image_builder == "package-build": + cmd.extend(["--image-variant", "packages"]) if env_file is not None: cmd.extend(["--env-file", str(env_file)]) cmd.extend(self._build_cache_args()) From 9c55741718a7719e2c2642e9f1753f26ad1e8c84 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 13:02:23 -0600 Subject: [PATCH 08/32] expanduser --- ceph_devstack/resources/ceph/ceph_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index fc2c4a5d..5b572068 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -532,7 +532,7 @@ def _binary_patch_cmd(self) -> List[str]: def _git_value(self, args: str) -> str: return subprocess.check_output( f"git {args}".split(), - cwd=self.repo, + cwd=str(Path(self.repo).expanduser()), text=True, ).strip() From 4072a53948bc2d84dd54349181102a17b4383ca7 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 14:38:43 -0600 Subject: [PATCH 09/32] fix make dist? --- ceph_devstack/resources/ceph/ceph_node.py | 61 +++++++++++++++++++---- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index 5b572068..e2f2a82e 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -530,15 +530,26 @@ def _binary_patch_cmd(self) -> List[str]: ] def _git_value(self, args: str) -> str: - return subprocess.check_output( - f"git {args}".split(), - cwd=str(Path(self.repo).expanduser()), - text=True, - ).strip() + repo_path = str(self.repo) + logger.debug(f"{self.name}: Running git {args} in {repo_path}") + try: + result = subprocess.check_output( + f"git {args}".split(), + cwd=repo_path, + text=True, + stderr=subprocess.PIPE, + ).strip() + logger.debug(f"{self.name}: git {args} returned: {result}") + return result + except CalledProcessError as e: + logger.error( + f"{self.name}: git {args} failed in {repo_path}: {e.stderr}" + ) + raise def _make_dist_version(self) -> str: """Generate version string for make-dist from git describe.""" - version = self._git_value("describe --abbrev=8 --match 'v*'") + version = self._git_value("describe --abbrev=8 --match v*") # Remove leading 'v' from version tag if version.startswith("v"): version = version[1:] @@ -556,10 +567,42 @@ async def _make_dist(self): f"{self.name}: make-dist script not found at {make_dist_script}, skipping" ) return - await self._run_cmd( - ["./make-dist", version], - cwd=self.repo, + # Run make-dist using asyncio subprocess with streaming output + logger.info( + f"{self.name}: Running make-dist (this may take several minutes for submodule updates)..." + ) + process = await asyncio.create_subprocess_exec( + "./make-dist", + version, + cwd=str(self.repo), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, ) + + # Stream output to show progress + output_lines = [] + while True: + line = await process.stdout.readline() + if not line: + break + line_str = line.decode().rstrip() + output_lines.append(line_str) + # Log key progress indicators + if any(keyword in line_str.lower() for keyword in ['updating', 'synchronizing', 'version', 'creating']): + logger.info(f"{self.name}: {line_str}") + + await process.wait() + + if process.returncode != 0: + logger.error(f"{self.name}: make-dist failed") + for line in output_lines[-20:]: # Show last 20 lines + logger.error(f" {line}") + raise CalledProcessError( + process.returncode, + ["./make-dist", version], + output="\n".join(output_lines), + ) + logger.info(f"{self.name}: make-dist completed successfully") async def _run_cmd(self, cmd: List[str], cwd: str): proc = await host.arun( From cf3dff7879fcc6dc1c4a22c55e757c4bd18d59a8 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 14:58:28 -0600 Subject: [PATCH 10/32] fixup! fix make dist? --- ceph_devstack/resources/ceph/ceph_node.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index e2f2a82e..c0eba8ad 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -259,8 +259,8 @@ def image_builder(self) -> str: @property def compile_steps(self) -> List[str]: - """Return compile steps based on image_builder mode (not configurable).""" - return DEFAULT_COMPILE_STEPS.get(self.image_builder, ["build"]) + default = DEFAULT_COMPILE_STEPS.get(self.image_builder, ["build"]) + return list(self.config.get("build_steps", default)) @property def sccache_enabled(self) -> bool: @@ -547,6 +547,7 @@ def _git_value(self, args: str) -> str: ) raise + def _make_dist_version(self) -> str: """Generate version string for make-dist from git describe.""" version = self._git_value("describe --abbrev=8 --match v*") @@ -604,6 +605,7 @@ async def _make_dist(self): ) logger.info(f"{self.name}: make-dist completed successfully") + async def _run_cmd(self, cmd: List[str], cwd: str): proc = await host.arun( cmd, @@ -646,9 +648,7 @@ def _verify_build_tree(self): async def _build_image_binary_patch(self): self._verify_build_tree() build_path = self.build_path - logger.info( - f"{self.name}: building {self.image} via binary-patch in {build_path}" - ) + logger.info(f"{self.name}: building {self.image} via binary-patch in {build_path}") await self._run_cmd(self._binary_patch_cmd(), cwd=str(build_path)) async def _build_image_package_build(self): From ef9335f8e4866428774d0cbd6f2a310977d7dc89 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 15:58:13 -0600 Subject: [PATCH 11/32] removed tests for some reason --- tests/resources/ceph/test_ceph_node.py | 40 ++------------------------ 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index 838fb953..bcb72913 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -42,18 +42,9 @@ def test_compile_steps_default_for_cpatch(self): def test_compile_steps_default_for_container_builder(self): config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["image_builder"] = "container" + config["containers"]["ceph_node"]["image_builder"] = "package-build" assert CephNode().compile_steps == ["packages"] - def test_compile_steps_respects_custom_build_steps(self): - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["build_steps"] = ["build", "packages"] - assert CephNode().compile_steps == ["build", "packages"] - - def test_compile_steps_allows_single_custom_step(self): - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["build_steps"] = ["packages"] - assert CephNode().compile_steps == ["packages"] def test_compile_cmd_uses_build_with_container(self, tmp_path): config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" @@ -71,33 +62,6 @@ def test_compile_cmd_uses_build_with_container(self, tmp_path): assert "--env-file" not in cmd assert "--npm-cache-path" not in cmd - def test_compile_cmd_generates_packages_step(self, tmp_path): - config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" - config["containers"]["ceph_node"]["repo"] = str(tmp_path) - config["containers"]["ceph_node"]["build_dir"] = "build" - config["containers"]["ceph_node"]["build_distro"] = "centos9" - config["containers"]["ceph_node"]["build_steps"] = ["packages"] - config["containers"]["ceph_node"]["sccache"] = False - config["containers"]["ceph_node"]["npm_cache"] = False - cmd = CephNode()._compile_cmd() - assert "-e" in cmd - e_index = cmd.index("-e") - assert cmd[e_index + 1] == "packages" - - def test_compile_cmd_generates_multiple_steps(self, tmp_path): - config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" - config["containers"]["ceph_node"]["repo"] = str(tmp_path) - config["containers"]["ceph_node"]["build_dir"] = "build" - config["containers"]["ceph_node"]["build_distro"] = "centos9" - config["containers"]["ceph_node"]["build_steps"] = ["build", "packages"] - config["containers"]["ceph_node"]["sccache"] = False - config["containers"]["ceph_node"]["npm_cache"] = False - cmd = CephNode()._compile_cmd() - # Find all -e flags and their values - e_indices = [i for i, x in enumerate(cmd) if x == "-e"] - assert len(e_indices) == 2 - assert cmd[e_indices[0] + 1] == "build" - assert cmd[e_indices[1] + 1] == "packages" def test_compile_cmd_passes_npm_cache_path(self, tmp_path): npm_cache = tmp_path / "npm-cache" @@ -354,7 +318,7 @@ def test_bundled_sccache_s3_conf_has_s3_settings(self): def test_cpatch_cmd_uses_upstream_script(self): config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" config["containers"]["ceph_node"]["base_image"] = "quay.io/ceph-ci/ceph:main" - cmd = CephNode()._cpatch_cmd() + cmd = CephNode()._binary_patch_cmd() assert cmd[0:2] == ["sudo", "../src/script/cpatch"] assert "localhost/ceph-devstack:main" in cmd From 25e84caed5a3044f97315424eddaaac3e8d29247 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 16:03:19 -0600 Subject: [PATCH 12/32] expanduser again --- ceph_devstack/resources/ceph/ceph_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index c0eba8ad..d8b298c6 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -575,7 +575,7 @@ async def _make_dist(self): process = await asyncio.create_subprocess_exec( "./make-dist", version, - cwd=str(self.repo), + cwd=str(Path(self.repo).expanduser()), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, ) From 2a4ad663af2781b95229a924b5b97dcf34875984 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 16:15:24 -0600 Subject: [PATCH 13/32] worktree --- ceph_devstack/resources/ceph/ceph_node.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index d8b298c6..f2a8169b 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -560,9 +560,21 @@ async def _make_dist(self): """Create source distribution tarball using make-dist.""" if not self.repo: return + + # Check if this is a git worktree + repo_path = Path(self.repo).expanduser() + git_path = repo_path / ".git" + if git_path.is_file(): + logger.warning( + f"{self.name}: Detected git worktree at {repo_path}. " + "make-dist does not support worktrees (requires .git directory, not file). " + "Skipping make-dist." + ) + return + version = self._make_dist_version() logger.info(f"{self.name}: creating source distribution for version {version}") - make_dist_script = Path(self.repo) / "make-dist" + make_dist_script = repo_path / "make-dist" if not make_dist_script.exists(): logger.warning( f"{self.name}: make-dist script not found at {make_dist_script}, skipping" From 155fb9deba2192439b24284111205179aa7c1798 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 16:19:45 -0600 Subject: [PATCH 14/32] fix regressed output streaming --- ceph_devstack/resources/ceph/ceph_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index f2a8169b..f6ce725b 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -645,7 +645,7 @@ async def _compile(self): env_file, extra_args = self._prepare_build_env() await self._run_cmd( self._compile_cmd(env_file=env_file, extra_args=extra_args), - cwd=self.repo, + cwd=str(self.repo), ) def _verify_build_tree(self): From e236ce649935bf1f21f4382ae9cf8def2b68edee Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 17:07:55 -0600 Subject: [PATCH 15/32] pass version to bwc --- ceph_devstack/resources/ceph/ceph_node.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index f6ce725b..e24729c2 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -511,6 +511,9 @@ def _compile_cmd( cmd.extend(["-e", step]) if self.image_builder == "package-build": cmd.extend(["--image-variant", "packages"]) + # Pass version to build-with-container.py so make-srpm.sh can find existing tarball + version = self._make_dist_version() + cmd.extend(["--ceph-version", version]) if env_file is not None: cmd.extend(["--env-file", str(env_file)]) cmd.extend(self._build_cache_args()) From 83877f1cb0d19b3e4d241b373881cbb12ac8ac25 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 21 Jul 2026 17:13:10 -0600 Subject: [PATCH 16/32] skip make-dist if tarball exists --- ceph_devstack/resources/ceph/ceph_node.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index e24729c2..df5c7ece 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -576,6 +576,16 @@ async def _make_dist(self): return version = self._make_dist_version() + + # Check if tarball already exists (make-dist creates ceph-.tar.bz2) + tarball_name = f"ceph-{version}.tar.bz2" + tarball_path = repo_path / tarball_name + if tarball_path.exists(): + logger.info( + f"{self.name}: Source tarball {tarball_name} already exists, skipping make-dist" + ) + return + logger.info(f"{self.name}: creating source distribution for version {version}") make_dist_script = repo_path / "make-dist" if not make_dist_script.exists(): From 3c396ecf4ef9eeb3df4db17cb75332534f9a4488 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 11:29:12 -0600 Subject: [PATCH 17/32] disable IBM telemetry --- ceph_devstack/resources/ceph/ceph_node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index df5c7ece..96522589 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -380,6 +380,8 @@ def create_cmd(self): f"DASHBOARD_SHOW_PASSWORD={'true' if self.dashboard_show_password else 'false'}", "-e", f"CONTAINER_NAME={self.name}", + "-e", + "IBM_TELEMETRY_DISABLED=true", "--entrypoint", "/bin/bash", "--name", From 63b845556e46b434f569c20dacbd2e8227277dc9 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 12:35:05 -0600 Subject: [PATCH 18/32] refactor ceph_node --- ceph_devstack/config.toml | 33 +- ceph_devstack/resources/ceph/__init__.py | 10 + ceph_devstack/resources/ceph/ceph_builder.py | 559 ++++++++++++++++++ ceph_devstack/resources/ceph/ceph_node.py | 483 ++------------- tests/resources/ceph/test_ceph_builder.py | 277 +++++++++ tests/resources/ceph/test_ceph_node.py | 349 ++--------- .../resources/ceph/test_cephdevstack_core.py | 2 +- 7 files changed, 967 insertions(+), 746 deletions(-) create mode 100644 ceph_devstack/resources/ceph/ceph_builder.py create mode 100644 tests/resources/ceph/test_ceph_builder.py diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 8d8a22d1..8925cc82 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -28,26 +28,19 @@ services = [ secrets = ["ssh_keypair"] [stacks.ceph] -services = ["ceph_node"] +services = ["ceph_builder", "ceph_node"] secrets = [] data_dir = "~/.local/share/ceph-devstack/ceph" -[containers.ceph_node] -image = "quay.io/ceph-ci/ceph:main" -loop_device_count = 3 -loop_device_size = "5G" -dashboard_port = 8080 -dashboard_ssl = false -# dashboard_user = "admin" -# dashboard_password = "admin" -# dashboard_show_password = false # log dashboard password on container start -# dashboard = false # disable mgr dashboard -# image_builder = "binary-patch" # default; patch local binaries into runtime image -# image_builder = "package-build" # future: container/build.sh + local packages +[containers.ceph_builder] +# CephBuilder handles compilation and build artifacts # Build from a local ceph.git checkout: # repo = "~/dev/ceph" +base_image = "quay.io/ceph-ci/ceph:main" # build_dir = "build" # relative to repo; passed to build-with-container.py -b # build_distro = "centos9" # build-with-container.py -d +image_builder = "binary-patch" # "binary-patch" (default) or "package-build" +# build_steps = ["build"] # compilation steps for build-with-container.py sccache = true # local disk cache (see sccache.conf); set to false to disable sccache_mode = "local" # local (default) or s3 for shared remote cache sccache_rw_mode = false # enable read-write mode (requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in environment for S3) @@ -60,7 +53,19 @@ npm_cache = true # persist dashboard npm downloads # dnf_cache = false # persist dnf downloads when rebuilding ceph-build images # dnf_cache_path = "~/.local/share/ceph-devstack/cache/dnf" # override dnf cache base dir # repo may be a git worktree; ceph-devstack mounts git metadata for builds at /ceph -# base_image = "quay.io/ceph-ci/ceph:main" + +[containers.ceph_node] +# CephNode handles cluster deployment and runtime +# References ceph_builder for build artifacts (wired automatically by stack) +image = "quay.io/ceph-ci/ceph:main" +loop_device_count = 3 +loop_device_size = "5G" +dashboard_port = 8080 +dashboard_ssl = false +# dashboard_user = "admin" +# dashboard_password = "admin" +# dashboard_show_password = false # log dashboard password on container start +# dashboard = false # disable mgr dashboard # image = "localhost/ceph-devstack:main" [containers.archive] diff --git a/ceph_devstack/resources/ceph/__init__.py b/ceph_devstack/resources/ceph/__init__.py index 2e20a5d0..52e633e9 100644 --- a/ceph_devstack/resources/ceph/__init__.py +++ b/ceph_devstack/resources/ceph/__init__.py @@ -18,6 +18,7 @@ Teuthology, Archive, ) +from ceph_devstack.resources.ceph.ceph_builder import CephBuilder from ceph_devstack.resources.ceph.ceph_node import CephNode, CONTAINER_CLUSTER_DIR from ceph_devstack.resources.ceph.requirements import ( HasSudo, @@ -96,6 +97,7 @@ class CephDevStackNetwork(Network): "teuthology": Teuthology, "testnode": TestNode, "archive": Archive, + "ceph_builder": CephBuilder, "ceph_node": CephNode, } @@ -151,6 +153,14 @@ def _wire_services(self): paddles_obj.env_vars["PADDLES_SQLALCHEMY_URL"] = ( postgres_obj.paddles_sqla_url ) + + # Wire ceph_builder -> ceph_node + if (builder_spec := self.service_specs.get("ceph_builder")) and ( + node_spec := self.service_specs.get("ceph_node") + ): + builder_obj = builder_spec["objects"][0] + for node_obj in node_spec["objects"]: + node_obj.builder = builder_obj async def check_requirements(self): result = True diff --git a/ceph_devstack/resources/ceph/ceph_builder.py b/ceph_devstack/resources/ceph/ceph_builder.py new file mode 100644 index 00000000..63979c61 --- /dev/null +++ b/ceph_devstack/resources/ceph/ceph_builder.py @@ -0,0 +1,559 @@ +"""CephBuilder resource for managing Ceph compilation and build artifacts.""" + +import asyncio +import os +import sys +from pathlib import Path +from subprocess import CalledProcessError +from typing import List + +import tomlkit + +from ceph_devstack import PROJECT_ROOT, config, logger +from ceph_devstack.host import host +from ceph_devstack.resources import PodmanResource + + +PACKAGE_SCCACHE_CONF = PROJECT_ROOT / "sccache.conf" +PACKAGE_SCCACHE_S3_CONF = PROJECT_ROOT / "sccache-s3.conf" +CONTAINER_SCCACHE_DIR = "/sccache" +CONTAINER_GIT_METADATA_DIR = "/git-metadata" +BWC_HOMEDIR = "/ceph" +REPO_DEVSTACK_DIR = ".ceph-devstack" +BUILD_ENV_NAME = "build.env" + +DEFAULT_COMPILE_STEPS = { + "binary-patch": ["build"], + "package-build": ["packages"], +} + + +def expand_path(path: str | Path) -> Path: + """Expand user home directory in path.""" + return Path(os.path.expanduser(str(path))) + + +def git_worktree_info(repo: Path) -> tuple[Path, str] | None: + """Return the main .git directory and worktree name for a linked worktree.""" + git_path = repo.resolve() / ".git" + if not git_path.is_file(): + return None + text = git_path.read_text(encoding="utf-8").strip() + if not text.startswith("gitdir:"): + return None + admin_dir = Path(text.split(":", 1)[1].strip()) + if admin_dir.parent.name != "worktrees": + raise ValueError(f"unexpected git worktree gitdir: {admin_dir}") + main_git_dir = admin_dir.parent.parent + if not main_git_dir.is_dir(): + raise FileNotFoundError(f"git metadata dir not found: {main_git_dir}") + return main_git_dir.resolve(), admin_dir.name + + +def worktree_submodule_git_mounts( + repo: Path, + worktree_name: str, + git_overlay_dir: Path, +) -> list[str]: + """Return podman mounts that rewrite submodule ``.git`` files for ``/ceph``.""" + import subprocess + + proc = subprocess.run( + ["git", "-C", str(repo), "submodule", "foreach", "--quiet", "echo $sm_path"], + check=False, + capture_output=True, + text=True, + ) + if proc.returncode != 0: + return [] + + overlay_dir = git_overlay_dir / "submodules" + overlay_dir.mkdir(parents=True, exist_ok=True) + mounts: list[str] = [] + for sm_path in proc.stdout.splitlines(): + sm_path = sm_path.strip() + if not sm_path: + continue + gitdir = ( + f"{CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}/modules/{sm_path}" + ) + overlay = overlay_dir / f"{sm_path.replace('/', '__')}.git" + overlay.write_text(f"gitdir: {gitdir}\n", encoding="utf-8") + mounts.append(f"--volume={overlay}:{BWC_HOMEDIR}/{sm_path}/.git:Z,ro") + return mounts + + +def worktree_container_mounts( + repo: Path, + main_git_dir: Path, + worktree_name: str, +) -> list[str]: + """Return podman mounts that make a linked worktree usable at ``/ceph``.""" + repo = repo.resolve() + main_git_dir = main_git_dir.resolve() + git_overlay_dir = repo / REPO_DEVSTACK_DIR / "git" + git_overlay_dir.mkdir(parents=True, exist_ok=True) + + dot_git = git_overlay_dir / "dot-git" + dot_git.write_text( + f"gitdir: {CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}\n", + encoding="utf-8", + ) + + admin_gitdir = git_overlay_dir / "gitdir" + admin_gitdir.write_text(f"{BWC_HOMEDIR}/.git\n", encoding="utf-8") + + mounts = [ + f"--volume={main_git_dir}:{CONTAINER_GIT_METADATA_DIR}:Z,ro", + f"--volume={dot_git}:{BWC_HOMEDIR}/.git:Z,ro", + f"--volume={admin_gitdir}:{CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}/gitdir:Z,ro", + ] + mounts.extend(worktree_submodule_git_mounts(repo, worktree_name, git_overlay_dir)) + return mounts + + +class CephBuilder(PodmanResource): + """Manages Ceph compilation and build artifacts. + + This resource handles: + - Builder container image management + - Compilation via build-with-container.py + - Build cache management (sccache, npm, dnf) + - Build artifact production + """ + + _name = "ceph_builder" + + def __init__(self, name: str = ""): + super().__init__(name) + self._persistent_cache_dir: Path | None = None + + @property + def config_key(self) -> str: + return "ceph_builder" + + @property + def config(self): + """Get configuration for this builder.""" + return config["containers"].get(self.config_key, {}) + + @property + def repo(self) -> str: + """Path to the Ceph repository.""" + return self.config.get("repo", "") + + @property + def base_image(self) -> str: + """Base image for building.""" + return self.config.get("base_image", "quay.io/ceph-ci/ceph:main") + + @property + def build_subdir(self) -> str: + """Build subdirectory within the repo.""" + build_dir = self.config.get("build_dir", "build") + if not build_dir: + return "build" + path = Path(os.path.expanduser(str(build_dir))) + if path.is_absolute() and self.repo: + repo = Path(self.repo).resolve() + try: + return str(path.resolve().relative_to(repo)) + except ValueError: + return path.name + return str(build_dir).strip("/") + + @property + def build_path(self) -> Path: + """Full path to the build directory.""" + if not self.repo: + return Path() + return Path(self.repo) / self.build_subdir + + @property + def image_builder(self) -> str: + """Build mode: 'binary-patch' or 'package-build'.""" + return self.config.get("image_builder", "binary-patch") + + @property + def compile_steps(self) -> List[str]: + """List of compilation steps to execute.""" + default = DEFAULT_COMPILE_STEPS.get(self.image_builder, ["build"]) + return list(self.config.get("build_steps", default)) + + @property + def persistent_cache_dir(self) -> Path: + """Directory for persistent build caches.""" + if self._persistent_cache_dir is None: + data_dir = Path(config.get("data_dir", "~/.local/share/ceph-devstack")) + self._persistent_cache_dir = expand_path(data_dir) / "cache" + return self._persistent_cache_dir + + @property + def sccache_enabled(self) -> bool: + """Whether sccache is enabled.""" + return self.config.get("sccache", True) is not False + + @property + def sccache_mode(self) -> str: + """Sccache mode: 'local' or 's3'.""" + return str(self.config.get("sccache_mode", "local")).lower() + + @property + def sccache_rw_mode(self) -> bool: + """Whether to use sccache in read-write mode (requires credentials).""" + return self.config.get("sccache_rw_mode", False) is True + + @property + def sccache_debug(self) -> bool: + """Whether to enable sccache debug logging.""" + return self.config.get("sccache_debug", False) is True + + @property + def sccache_cache_path(self) -> Path: + """Path to sccache cache directory.""" + if custom := self.config.get("sccache_cache_path"): + return expand_path(custom) + return self.persistent_cache_dir / "sccache" + + @property + def npm_cache_enabled(self) -> bool: + """Whether npm cache is enabled.""" + return self.config.get("npm_cache", True) is not False + + @property + def npm_cache_path(self) -> Path | None: + """Path to npm cache directory.""" + if not self.npm_cache_enabled: + return None + if custom := self.config.get("npm_cache_path"): + return expand_path(custom) + return self.persistent_cache_dir / "npm" + + @property + def dnf_cache_path(self) -> Path | None: + """Path to dnf cache directory.""" + if self.config.get("dnf_cache", False) is not True: + return None + if custom := self.config.get("dnf_cache_path"): + return expand_path(custom) + return self.persistent_cache_dir / "dnf" + + def _build_cache_args(self) -> list[str]: + """Return build-with-container.py cache flags for persistent build caches.""" + args: list[str] = [] + + npm_cache = self.npm_cache_path + if npm_cache is not None: + npm_cache.mkdir(parents=True, exist_ok=True) + args.extend(["--npm-cache-path", str(npm_cache)]) + + dnf_cache = self.dnf_cache_path + if dnf_cache is not None: + dnf_cache.mkdir(parents=True, exist_ok=True) + args.extend(["--dnf-cache-path", str(dnf_cache)]) + + return args + + def _get_s3_credentials_env(self) -> list[str]: + """Get S3 credential environment variables for read-write mode.""" + aws_access_key = os.environ.get("AWS_ACCESS_KEY_ID", "") + aws_secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "") + if not aws_access_key or not aws_secret_key: + raise ValueError( + "sccache_rw_mode=true requires AWS_ACCESS_KEY_ID and " + "AWS_SECRET_ACCESS_KEY to be set in the environment" + ) + return [ + f"AWS_ACCESS_KEY_ID={aws_access_key}", + f"AWS_SECRET_ACCESS_KEY={aws_secret_key}", + "SCCACHE_S3_NO_CREDENTIALS=false", + "SCCACHE_S3_RW_MODE=READ_WRITE", + ] + + def _sccache_build_env(self, repo: Path) -> tuple[list[str], list[str]]: + """Prepare sccache environment variables and extra args.""" + if not self.sccache_enabled: + return [], [] + + homedir = BWC_HOMEDIR + if custom_conf := self.config.get("sccache_conf"): + conf_src = expand_path(custom_conf) + use_local_cache = self.sccache_mode == "local" + elif self.sccache_mode == "s3": + conf_src = PACKAGE_SCCACHE_S3_CONF + use_local_cache = False + else: + conf_src = PACKAGE_SCCACHE_CONF + use_local_cache = True + if not conf_src.is_file(): + raise FileNotFoundError(f"sccache config not found: {conf_src}") + + # Parse and modify config for S3 mode to set no_credentials correctly + conf_content = conf_src.read_text() + if self.sccache_mode == "s3": + conf_data = tomlkit.parse(conf_content) + # Set no_credentials based on whether we're using read-write mode + conf_data["cache"]["s3"]["no_credentials"] = not self.sccache_rw_mode + conf_content = tomlkit.dumps(conf_data) + (repo / "sccache.conf").write_text(conf_content) + + lines = [ + "SCCACHE=true", + f"SCCACHE_CONF={homedir}/sccache.conf", + f"SCCACHE_ERROR_LOG={homedir}/.ceph-devstack/sccache_log.txt", + "CEPH_BUILD_NORMALIZE_PATHS=true", + ] + if self.sccache_debug: + lines.append("SCCACHE_LOG=debug") + + extra_args: list[str] = [] + if use_local_cache: + cache_path = self.sccache_cache_path + cache_path.mkdir(parents=True, exist_ok=True) + extra_args.append(f"--volume={cache_path}:{CONTAINER_SCCACHE_DIR}:Z") + lines.append(f"SCCACHE_DIR={CONTAINER_SCCACHE_DIR}") + cache_size = self.config.get("sccache_cache_size", "100G") + lines.append(f"SCCACHE_CACHE_SIZE={cache_size}") + elif self.sccache_mode == "s3": + if self.sccache_rw_mode: + lines.extend(self._get_s3_credentials_env()) + else: + # Read-only mode (default) + lines.extend( + [ + "SCCACHE_S3_NO_CREDENTIALS=true", + "SCCACHE_S3_RW_MODE=READ_ONLY", + ] + ) + return lines, extra_args + + def _prepare_build_env(self) -> tuple[Path | None, list[str]]: + """Prepare build environment file and extra container args.""" + if not self.repo: + return None, [] + repo = Path(self.repo) + extra_args: list[str] = [] + + worktree = git_worktree_info(repo) + if worktree is not None: + main_git_dir, worktree_name = worktree + extra_args.extend( + worktree_container_mounts(repo, main_git_dir, worktree_name) + ) + + lines, sccache_extra = self._sccache_build_env(repo) + extra_args.extend(sccache_extra) + + if not lines: + return None, extra_args + + devstack_dir = repo / REPO_DEVSTACK_DIR + devstack_dir.mkdir(exist_ok=True) + env_path = devstack_dir / BUILD_ENV_NAME + env_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return env_path, extra_args + + def _compile_cmd( + self, + env_file: Path | None = None, + extra_args: List[str] | None = None, + ) -> List[str]: + """Build the compile command for build-with-container.py.""" + distro = self.config.get("build_distro", "centos9") + script = str(Path(self.repo) / "src/script/build-with-container.py") + python_cmd = "python3" if host.type == "remote" else sys.executable + cmd = [ + python_cmd, + script, + "-d", + distro, + "-b", + self.build_subdir, + "--homedir", + BWC_HOMEDIR, + ] + for step in self.compile_steps: + cmd.extend(["-e", step]) + if self.image_builder == "package-build": + cmd.extend(["--image-variant", "packages"]) + # Pass version to build-with-container.py so make-srpm.sh can find existing tarball + version = self._make_dist_version() + cmd.extend(["--ceph-version", version]) + if env_file is not None: + cmd.extend(["--env-file", str(env_file)]) + cmd.extend(self._build_cache_args()) + for extra in extra_args or []: + cmd.append(f"--extra={extra}") + return cmd + + def _git_value(self, args: str) -> str: + """Execute a git command and return its output.""" + import subprocess + + repo_path = str(self.repo) + logger.debug(f"{self.name}: Running git {args} in {repo_path}") + try: + result = subprocess.check_output( + f"git {args}".split(), + cwd=repo_path, + text=True, + stderr=subprocess.PIPE, + ).strip() + logger.debug(f"{self.name}: git {args} returned: {result}") + return result + except CalledProcessError as e: + logger.error( + f"{self.name}: git {args} failed in {repo_path}: {e.stderr}" + ) + raise + + def _make_dist_version(self) -> str: + """Generate version string for make-dist from git describe.""" + version = self._git_value("describe --abbrev=8 --match v*") + # Remove leading 'v' from version tag + if version.startswith("v"): + version = version[1:] + return version + + async def _make_dist(self): + """Create source distribution tarball using make-dist.""" + if not self.repo: + return + + # Check if this is a git worktree + repo_path = Path(self.repo).expanduser() + git_path = repo_path / ".git" + if git_path.is_file(): + logger.warning( + f"{self.name}: Detected git worktree at {repo_path}. " + "make-dist does not support worktrees (requires .git directory, not file). " + "Skipping make-dist." + ) + return + + version = self._make_dist_version() + + # Check if tarball already exists (make-dist creates ceph-.tar.bz2) + tarball_name = f"ceph-{version}.tar.bz2" + tarball_path = repo_path / tarball_name + if tarball_path.exists(): + logger.info( + f"{self.name}: Source tarball {tarball_name} already exists, skipping make-dist" + ) + return + + logger.info(f"{self.name}: creating source distribution for version {version}") + make_dist_script = repo_path / "make-dist" + if not make_dist_script.exists(): + logger.warning( + f"{self.name}: make-dist script not found at {make_dist_script}, skipping" + ) + return + + # Run make-dist using asyncio subprocess with streaming output + logger.info( + f"{self.name}: Running make-dist (this may take several minutes for submodule updates)..." + ) + process = await asyncio.create_subprocess_exec( + "./make-dist", + version, + cwd=str(Path(self.repo).expanduser()), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + + # Stream output to show progress + output_lines = [] + while True: + line = await process.stdout.readline() + if not line: + break + line_str = line.decode().rstrip() + output_lines.append(line_str) + # Log key progress indicators + if any(keyword in line_str.lower() for keyword in ['updating', 'synchronizing', 'version', 'creating']): + logger.info(f"{self.name}: {line_str}") + + await process.wait() + + if process.returncode != 0: + logger.error(f"{self.name}: make-dist failed") + for line in output_lines[-20:]: # Show last 20 lines + logger.error(f" {line}") + raise CalledProcessError( + process.returncode, + ["./make-dist", version], + output="\n".join(output_lines), + ) + logger.info(f"{self.name}: make-dist completed successfully") + + async def _run_cmd(self, cmd: List[str], cwd: str): + """Run a command and handle errors.""" + proc = await host.arun( + cmd, + cwd=Path(cwd).expanduser(), + stream_output=True, + ) + returncode = await proc.wait() + if returncode != 0: + stdout, stderr = await proc.log_failure(cmd) + raise CalledProcessError( + returncode, + cmd, + output=stdout or None, + stderr=stderr or None, + ) + + async def compile(self): + """Run compilation using build-with-container.py.""" + # Run make-dist automatically for package-build mode + if self.image_builder == "package-build": + await self._make_dist() + + logger.info( + f"{self.name}: compiling ceph via build-with-container.py in {self.repo}" + ) + env_file, extra_args = self._prepare_build_env() + await self._run_cmd( + self._compile_cmd(env_file=env_file, extra_args=extra_args), + cwd=str(self.repo), + ) + + def _verify_build_tree(self): + """Verify that build artifacts exist.""" + build_path = self.build_path + if (build_path / "build.ninja").exists() or (build_path / "Makefile").exists(): + return + raise FileNotFoundError( + f"Ceph build dir {build_path} missing Makefile or build.ninja " + "after build-with-container.py" + ) + + async def exists(self): + """Check if build artifacts exist and are valid.""" + if not self.repo: + return False + try: + self._verify_build_tree() + return True + except FileNotFoundError: + return False + + async def create(self): + """Prepare build environment and run compilation.""" + if not self.repo: + logger.warning(f"{self.name}: No repo configured, skipping build") + return + + logger.info(f"{self.name}: Starting Ceph compilation") + await self.compile() + self._verify_build_tree() + logger.info(f"{self.name}: Build artifacts ready at {self.build_path}") + + async def remove(self): + """Clean up build artifacts (preserves caches).""" + # Note: We preserve caches by default + # Users can manually remove cache directories if needed + logger.info( + f"{self.name}: Build caches preserved at {self.persistent_cache_dir}" + ) diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index 96522589..ea877143 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -2,42 +2,29 @@ import os import shlex import shutil -import subprocess import sys import uuid from pathlib import Path from subprocess import CalledProcessError -from typing import List +from typing import List, TYPE_CHECKING -import tomlkit - -from ceph_devstack import PROJECT_ROOT, config, logger +from ceph_devstack import config, logger from ceph_devstack.host import host from ceph_devstack.resources.ceph.block_devices import BlockDeviceProvisioner from ceph_devstack.resources.ceph.host_loops import allocate_loop_devices from ceph_devstack.resources.container import Container +if TYPE_CHECKING: + from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + DEFAULT_CEPH_IMAGE = "quay.io/ceph-ci/ceph:main" -DEFAULT_BASE_IMAGE = "quay.io/ceph-ci/ceph:main" -PACKAGE_SCCACHE_CONF = PROJECT_ROOT / "sccache.conf" -PACKAGE_SCCACHE_S3_CONF = PROJECT_ROOT / "sccache-s3.conf" -CONTAINER_SCCACHE_DIR = "/sccache" -CONTAINER_GIT_METADATA_DIR = "/git-metadata" -BWC_HOMEDIR = "/ceph" -REPO_DEVSTACK_DIR = ".ceph-devstack" -BUILD_ENV_NAME = "build.env" ENTRYPOINT_SCRIPT = Path(__file__).with_name("ceph-node-entrypoint.sh") CLUSTER_ENTRYPOINT_NAME = "ceph-node-entrypoint.sh" CLUSTER_DATA_NAMES = ("var", "fsid", CLUSTER_ENTRYPOINT_NAME) CONTAINER_CLUSTER_DIR = "/var/lib/ceph-devstack/cluster" -DEFAULT_COMPILE_STEPS = { - "binary-patch": ["build"], - "package-build": ["packages"], -} - CEPH_NODE_CAPABILITIES = [ "SYS_ADMIN", "NET_ADMIN", @@ -56,83 +43,6 @@ def expand_path(path: str | Path) -> Path: return Path(os.path.expanduser(str(path))) -def git_worktree_info(repo: Path) -> tuple[Path, str] | None: - """Return the main .git directory and worktree name for a linked worktree.""" - git_path = repo.resolve() / ".git" - if not git_path.is_file(): - return None - text = git_path.read_text(encoding="utf-8").strip() - if not text.startswith("gitdir:"): - return None - admin_dir = Path(text.split(":", 1)[1].strip()) - if admin_dir.parent.name != "worktrees": - raise ValueError(f"unexpected git worktree gitdir: {admin_dir}") - main_git_dir = admin_dir.parent.parent - if not main_git_dir.is_dir(): - raise FileNotFoundError(f"git metadata dir not found: {main_git_dir}") - return main_git_dir.resolve(), admin_dir.name - - -def worktree_submodule_git_mounts( - repo: Path, - worktree_name: str, - git_overlay_dir: Path, -) -> list[str]: - """Return podman mounts that rewrite submodule ``.git`` files for ``/ceph``.""" - proc = subprocess.run( - ["git", "-C", str(repo), "submodule", "foreach", "--quiet", "echo $sm_path"], - check=False, - capture_output=True, - text=True, - ) - if proc.returncode != 0: - return [] - - overlay_dir = git_overlay_dir / "submodules" - overlay_dir.mkdir(parents=True, exist_ok=True) - mounts: list[str] = [] - for sm_path in proc.stdout.splitlines(): - sm_path = sm_path.strip() - if not sm_path: - continue - gitdir = ( - f"{CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}/modules/{sm_path}" - ) - overlay = overlay_dir / f"{sm_path.replace('/', '__')}.git" - overlay.write_text(f"gitdir: {gitdir}\n", encoding="utf-8") - mounts.append(f"--volume={overlay}:{BWC_HOMEDIR}/{sm_path}/.git:Z,ro") - return mounts - - -def worktree_container_mounts( - repo: Path, - main_git_dir: Path, - worktree_name: str, -) -> list[str]: - """Return podman mounts that make a linked worktree usable at ``/ceph``.""" - repo = repo.resolve() - main_git_dir = main_git_dir.resolve() - git_overlay_dir = repo / REPO_DEVSTACK_DIR / "git" - git_overlay_dir.mkdir(parents=True, exist_ok=True) - - dot_git = git_overlay_dir / "dot-git" - dot_git.write_text( - f"gitdir: {CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}\n", - encoding="utf-8", - ) - - admin_gitdir = git_overlay_dir / "gitdir" - admin_gitdir.write_text(f"{BWC_HOMEDIR}/.git\n", encoding="utf-8") - - mounts = [ - f"--volume={main_git_dir}:{CONTAINER_GIT_METADATA_DIR}:Z,ro", - f"--volume={dot_git}:{BWC_HOMEDIR}/.git:Z,ro", - f"--volume={admin_gitdir}:{CONTAINER_GIT_METADATA_DIR}/worktrees/{worktree_name}/gitdir:Z,ro", - ] - mounts.extend(worktree_submodule_git_mounts(repo, worktree_name, git_overlay_dir)) - return mounts - - class CephNode(Container): """Single-container Ceph cluster without cephadm. @@ -152,6 +62,7 @@ def __init__(self, name: str = ""): self.loop_device_count = self.config["loop_device_count"] self._devices: list[str] | None = None self._block_device_provisioner: BlockDeviceProvisioner | None = None + self._builder: "CephBuilder | None" = None @property def devices(self) -> list[str]: @@ -165,6 +76,21 @@ def devices(self) -> list[str]: def config_key(self) -> str: return "ceph_node" + @property + def builder(self) -> "CephBuilder": + """Reference to the CephBuilder resource.""" + if self._builder is None: + raise ValueError( + f"{self.name}: No builder configured. " + "CephNode requires a CephBuilder to be wired via stack integration." + ) + return self._builder + + @builder.setter + def builder(self, value: "CephBuilder"): + """Set the builder reference.""" + self._builder = value + @property def cluster_dir(self) -> Path: return expand_path(self.config.get("output_dir", config["data_dir"])) @@ -229,97 +155,15 @@ def dashboard_password(self) -> str: def dashboard_show_password(self) -> bool: return self.config.get("dashboard_show_password", False) is True - @property - def base_image(self) -> str: - return self.config.get("base_image", DEFAULT_BASE_IMAGE) - - @property - def build_subdir(self) -> str: - build_dir = self.config.get("build_dir", "build") - if not build_dir: - return "build" - path = Path(os.path.expanduser(str(build_dir))) - if path.is_absolute() and self.repo: - repo = Path(self.repo).resolve() - try: - return str(path.resolve().relative_to(repo)) - except ValueError: - return path.name - return str(build_dir).strip("/") - - @property - def build_path(self) -> Path: - if not self.repo: - return Path() - return Path(self.repo) / self.build_subdir - @property def image_builder(self) -> str: - return self.config.get("image_builder", "binary-patch") - - @property - def compile_steps(self) -> List[str]: - default = DEFAULT_COMPILE_STEPS.get(self.image_builder, ["build"]) - return list(self.config.get("build_steps", default)) - - @property - def sccache_enabled(self) -> bool: - return self.config.get("sccache", True) is not False + """Get image builder mode from builder.""" + return self.builder.image_builder @property - def sccache_mode(self) -> str: - return str(self.config.get("sccache_mode", "local")).lower() - - @property - def sccache_rw_mode(self) -> bool: - """Whether to use sccache in read-write mode (requires credentials).""" - return self.config.get("sccache_rw_mode", False) is True - - @property - def sccache_debug(self) -> bool: - return self.config.get("sccache_debug", False) is True - - @property - def sccache_cache_path(self) -> Path: - if custom := self.config.get("sccache_cache_path"): - return expand_path(custom) - return self.persistent_cache_dir / "sccache" - - @property - def npm_cache_enabled(self) -> bool: - return self.config.get("npm_cache", True) is not False - - @property - def npm_cache_path(self) -> Path | None: - if not self.npm_cache_enabled: - return None - if custom := self.config.get("npm_cache_path"): - return expand_path(custom) - return self.persistent_cache_dir / "npm" - - @property - def dnf_cache_path(self) -> Path | None: - if self.config.get("dnf_cache", False) is not True: - return None - if custom := self.config.get("dnf_cache_path"): - return expand_path(custom) - return self.persistent_cache_dir / "dnf" - - def _build_cache_args(self) -> list[str]: - """Return build-with-container.py cache flags for persistent build caches.""" - args: list[str] = [] - - npm_cache = self.npm_cache_path - if npm_cache is not None: - npm_cache.mkdir(parents=True, exist_ok=True) - args.extend(["--npm-cache-path", str(npm_cache)]) - - dnf_cache = self.dnf_cache_path - if dnf_cache is not None: - dnf_cache.mkdir(parents=True, exist_ok=True) - args.extend(["--dnf-cache-path", str(dnf_cache)]) - - return args + def build_path(self) -> Path: + """Get build path from builder.""" + return self.builder.build_path @property def create_cmd(self): @@ -394,246 +238,22 @@ def create_cmd(self): def _device_image(self, device: str) -> str: return f"{self.name}-{device.removeprefix('/dev/loop')}" - def _get_s3_credentials_env(self) -> list[str]: - """Get S3 credential environment variables for read-write mode.""" - aws_access_key = os.environ.get("AWS_ACCESS_KEY_ID", "") - aws_secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "") - if not aws_access_key or not aws_secret_key: - raise ValueError( - "sccache_rw_mode=true requires AWS_ACCESS_KEY_ID and " - "AWS_SECRET_ACCESS_KEY to be set in the environment" - ) - return [ - f"AWS_ACCESS_KEY_ID={aws_access_key}", - f"AWS_SECRET_ACCESS_KEY={aws_secret_key}", - "SCCACHE_S3_NO_CREDENTIALS=false", - "SCCACHE_S3_RW_MODE=READ_WRITE", - ] - - def _sccache_build_env(self, repo: Path) -> tuple[list[str], list[str]]: - if not self.sccache_enabled: - return [], [] - - homedir = BWC_HOMEDIR - if custom_conf := self.config.get("sccache_conf"): - conf_src = expand_path(custom_conf) - use_local_cache = self.sccache_mode == "local" - elif self.sccache_mode == "s3": - conf_src = PACKAGE_SCCACHE_S3_CONF - use_local_cache = False - else: - conf_src = PACKAGE_SCCACHE_CONF - use_local_cache = True - if not conf_src.is_file(): - raise FileNotFoundError(f"sccache config not found: {conf_src}") - - # Parse and modify config for S3 mode to set no_credentials correctly - conf_content = conf_src.read_text() - if self.sccache_mode == "s3": - conf_data = tomlkit.parse(conf_content) - # Set no_credentials based on whether we're using read-write mode - conf_data["cache"]["s3"]["no_credentials"] = not self.sccache_rw_mode - conf_content = tomlkit.dumps(conf_data) - (repo / "sccache.conf").write_text(conf_content) - - lines = [ - "SCCACHE=true", - f"SCCACHE_CONF={homedir}/sccache.conf", - f"SCCACHE_ERROR_LOG={homedir}/.ceph-devstack/sccache_log.txt", - "CEPH_BUILD_NORMALIZE_PATHS=true", - ] - if self.sccache_debug: - lines.append("SCCACHE_LOG=debug") - - extra_args: list[str] = [] - if use_local_cache: - cache_path = self.sccache_cache_path - cache_path.mkdir(parents=True, exist_ok=True) - extra_args.append(f"--volume={cache_path}:{CONTAINER_SCCACHE_DIR}:Z") - lines.append(f"SCCACHE_DIR={CONTAINER_SCCACHE_DIR}") - cache_size = self.config.get("sccache_cache_size", "100G") - lines.append(f"SCCACHE_CACHE_SIZE={cache_size}") - elif self.sccache_mode == "s3": - if self.sccache_rw_mode: - lines.extend(self._get_s3_credentials_env()) - else: - # Read-only mode (default) - lines.extend( - [ - "SCCACHE_S3_NO_CREDENTIALS=true", - "SCCACHE_S3_RW_MODE=READ_ONLY", - ] - ) - return lines, extra_args - - def _prepare_build_env(self) -> tuple[Path | None, list[str]]: - if not self.repo: - return None, [] - repo = Path(self.repo) - extra_args: list[str] = [] - - worktree = git_worktree_info(repo) - if worktree is not None: - main_git_dir, worktree_name = worktree - extra_args.extend( - worktree_container_mounts(repo, main_git_dir, worktree_name) - ) - lines, sccache_extra = self._sccache_build_env(repo) - extra_args.extend(sccache_extra) - - if not lines: - return None, extra_args - - devstack_dir = repo / REPO_DEVSTACK_DIR - devstack_dir.mkdir(exist_ok=True) - env_path = devstack_dir / BUILD_ENV_NAME - env_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - return env_path, extra_args - - def _compile_cmd( - self, - env_file: Path | None = None, - extra_args: List[str] | None = None, - ) -> List[str]: - distro = self.config.get("build_distro", "centos9") - script = str(Path(self.repo) / "src/script/build-with-container.py") - python_cmd = "python3" if host.type == "remote" else sys.executable - cmd = [ - python_cmd, - script, - "-d", - distro, - "-b", - self.build_subdir, - "--homedir", - BWC_HOMEDIR, - ] - for step in self.compile_steps: - cmd.extend(["-e", step]) - if self.image_builder == "package-build": - cmd.extend(["--image-variant", "packages"]) - # Pass version to build-with-container.py so make-srpm.sh can find existing tarball - version = self._make_dist_version() - cmd.extend(["--ceph-version", version]) - if env_file is not None: - cmd.extend(["--env-file", str(env_file)]) - cmd.extend(self._build_cache_args()) - for extra in extra_args or []: - cmd.append(f"--extra={extra}") - return cmd def _binary_patch_cmd(self) -> List[str]: + """Build command for binary-patch image creation.""" return [ "sudo", "../src/script/cpatch", "--base", - self.base_image, + self.builder.base_image, "--target", self.image, "--core", ] - def _git_value(self, args: str) -> str: - repo_path = str(self.repo) - logger.debug(f"{self.name}: Running git {args} in {repo_path}") - try: - result = subprocess.check_output( - f"git {args}".split(), - cwd=repo_path, - text=True, - stderr=subprocess.PIPE, - ).strip() - logger.debug(f"{self.name}: git {args} returned: {result}") - return result - except CalledProcessError as e: - logger.error( - f"{self.name}: git {args} failed in {repo_path}: {e.stderr}" - ) - raise - - - def _make_dist_version(self) -> str: - """Generate version string for make-dist from git describe.""" - version = self._git_value("describe --abbrev=8 --match v*") - # Remove leading 'v' from version tag - if version.startswith("v"): - version = version[1:] - return version - - async def _make_dist(self): - """Create source distribution tarball using make-dist.""" - if not self.repo: - return - - # Check if this is a git worktree - repo_path = Path(self.repo).expanduser() - git_path = repo_path / ".git" - if git_path.is_file(): - logger.warning( - f"{self.name}: Detected git worktree at {repo_path}. " - "make-dist does not support worktrees (requires .git directory, not file). " - "Skipping make-dist." - ) - return - - version = self._make_dist_version() - - # Check if tarball already exists (make-dist creates ceph-.tar.bz2) - tarball_name = f"ceph-{version}.tar.bz2" - tarball_path = repo_path / tarball_name - if tarball_path.exists(): - logger.info( - f"{self.name}: Source tarball {tarball_name} already exists, skipping make-dist" - ) - return - - logger.info(f"{self.name}: creating source distribution for version {version}") - make_dist_script = repo_path / "make-dist" - if not make_dist_script.exists(): - logger.warning( - f"{self.name}: make-dist script not found at {make_dist_script}, skipping" - ) - return - # Run make-dist using asyncio subprocess with streaming output - logger.info( - f"{self.name}: Running make-dist (this may take several minutes for submodule updates)..." - ) - process = await asyncio.create_subprocess_exec( - "./make-dist", - version, - cwd=str(Path(self.repo).expanduser()), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.STDOUT, - ) - - # Stream output to show progress - output_lines = [] - while True: - line = await process.stdout.readline() - if not line: - break - line_str = line.decode().rstrip() - output_lines.append(line_str) - # Log key progress indicators - if any(keyword in line_str.lower() for keyword in ['updating', 'synchronizing', 'version', 'creating']): - logger.info(f"{self.name}: {line_str}") - - await process.wait() - - if process.returncode != 0: - logger.error(f"{self.name}: make-dist failed") - for line in output_lines[-20:]: # Show last 20 lines - logger.error(f" {line}") - raise CalledProcessError( - process.returncode, - ["./make-dist", version], - output="\n".join(output_lines), - ) - logger.info(f"{self.name}: make-dist completed successfully") - - async def _run_cmd(self, cmd: List[str], cwd: str): + """Run a command and handle errors.""" proc = await host.arun( cmd, cwd=Path(cwd).expanduser(), @@ -649,54 +269,33 @@ async def _run_cmd(self, cmd: List[str], cwd: str): stderr=stderr or None, ) - async def _compile(self): - # Run make-dist automatically for package-build mode - if self.image_builder == "package-build": - await self._make_dist() - - logger.info( - f"{self.name}: compiling ceph via build-with-container.py in {self.repo}" - ) - env_file, extra_args = self._prepare_build_env() - await self._run_cmd( - self._compile_cmd(env_file=env_file, extra_args=extra_args), - cwd=str(self.repo), - ) - - def _verify_build_tree(self): - build_path = self.build_path - if (build_path / "build.ninja").exists() or (build_path / "Makefile").exists(): - return - raise FileNotFoundError( - f"Ceph build dir {build_path} missing Makefile or build.ninja " - "after build-with-container.py" - ) - async def _build_image_binary_patch(self): - self._verify_build_tree() + """Build runtime image using binary-patch method.""" build_path = self.build_path logger.info(f"{self.name}: building {self.image} via binary-patch in {build_path}") await self._run_cmd(self._binary_patch_cmd(), cwd=str(build_path)) async def _build_image_package_build(self): + """Build runtime image using package-build method.""" raise NotImplementedError( "image_builder='package-build' requires ceph container/build.sh to consume " "locally-built packages; enable this once that support lands upstream" ) async def _build_image(self): + """Build the runtime image from builder artifacts.""" builders = { "binary-patch": self._build_image_binary_patch, "package-build": self._build_image_package_build, } try: - builder = builders[self.image_builder] + builder_method = builders[self.image_builder] except KeyError as exc: known = ", ".join(sorted(builders)) raise ValueError( f"Unknown image_builder {self.image_builder!r}; known: {known}" ) from exc - await builder() + await builder_method() def install_entrypoint(self): if not ENTRYPOINT_SCRIPT.is_file(): @@ -747,9 +346,19 @@ async def remove_cluster_data(self): ) async def build(self): - if not self.should_build: + """Build runtime image from CephBuilder artifacts.""" + # Only build if we have a local image tag (localhost/...) + if not self.image.startswith("localhost/"): return - await self._compile() + + # Ensure builder has completed compilation + if not self.builder.build_path.exists(): + raise RuntimeError( + f"{self.name}: Builder has not completed compilation. " + f"Build artifacts not found at {self.builder.build_path}" + ) + + logger.info(f"{self.name}: Building runtime image from {self.builder.name} artifacts") await self._build_image() async def create(self): diff --git a/tests/resources/ceph/test_ceph_builder.py b/tests/resources/ceph/test_ceph_builder.py new file mode 100644 index 00000000..02e354bd --- /dev/null +++ b/tests/resources/ceph/test_ceph_builder.py @@ -0,0 +1,277 @@ +"""Tests for CephBuilder resource.""" +from unittest.mock import AsyncMock, patch +import sys + +import pytest +import tomlkit + +from ceph_devstack import config +from ceph_devstack.resources.ceph.ceph_builder import ( + BUILD_ENV_NAME, + CONTAINER_GIT_METADATA_DIR, + CONTAINER_SCCACHE_DIR, + CephBuilder, + PACKAGE_SCCACHE_CONF, + PACKAGE_SCCACHE_S3_CONF, + REPO_DEVSTACK_DIR, + git_worktree_info, + worktree_container_mounts, +) + + +class TestCephBuilder: + """Tests for CephBuilder compilation and build management.""" + + def test_compile_steps_default_for_binary_patch(self): + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["image_builder"] = "binary-patch" + assert CephBuilder().compile_steps == ["build"] + + def test_compile_steps_default_for_package_build(self): + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["image_builder"] = "package-build" + assert CephBuilder().compile_steps == ["packages"] + + def test_compile_cmd_uses_build_with_container(self, tmp_path): + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(tmp_path) + config["containers"]["ceph_builder"]["build_dir"] = "build" + config["containers"]["ceph_builder"]["build_distro"] = "centos9" + config["containers"]["ceph_builder"]["sccache"] = False + config["containers"]["ceph_builder"]["npm_cache"] = False + cmd = CephBuilder()._compile_cmd() + assert cmd[0] in ["python3", sys.executable] + assert "build-with-container.py" in cmd[1] + assert "-d" in cmd and "centos9" in cmd + assert "-b" in cmd and "build" in cmd + assert cmd[cmd.index("--homedir") + 1] == "/ceph" + assert "--env-file" not in cmd + assert "--npm-cache-path" not in cmd + + def test_compile_cmd_passes_npm_cache_path(self, tmp_path): + npm_cache = tmp_path / "npm-cache" + config["data_dir"] = str(tmp_path / "data") + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_builder"]["npm_cache_path"] = str(npm_cache) + config["containers"]["ceph_builder"]["sccache"] = False + cmd = CephBuilder()._compile_cmd() + assert "--npm-cache-path" in cmd + assert str(npm_cache.resolve()) in cmd + assert npm_cache.is_dir() + + def test_compile_cmd_uses_default_npm_cache_under_data_dir(self, tmp_path): + data_dir = tmp_path / "data" + config["data_dir"] = str(data_dir) + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_builder"]["sccache"] = False + cmd = CephBuilder()._compile_cmd() + expected = (data_dir / "cache" / "npm").resolve() + assert "--npm-cache-path" in cmd + assert str(expected) in cmd + assert expected.is_dir() + + def test_compile_cmd_skips_ccache_dir(self, tmp_path): + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_builder"]["npm_cache"] = False + cmd = CephBuilder()._compile_cmd() + assert "--ccache-dir" not in cmd + + def test_compile_cmd_passes_dnf_cache_path_when_enabled(self, tmp_path): + dnf_cache = tmp_path / "dnf-cache" + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(tmp_path / "ceph") + config["containers"]["ceph_builder"]["npm_cache"] = False + config["containers"]["ceph_builder"]["sccache"] = False + config["containers"]["ceph_builder"]["dnf_cache"] = True + config["containers"]["ceph_builder"]["dnf_cache_path"] = str(dnf_cache) + cmd = CephBuilder()._compile_cmd() + assert "--dnf-cache-path" in cmd + assert str(dnf_cache.resolve()) in cmd + assert dnf_cache.is_dir() + + def test_prepare_build_env_uses_local_sccache_by_default(self, tmp_path): + data_dir = tmp_path / "data" + config["data_dir"] = str(data_dir) + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(repo) + env_file, extra_args = CephBuilder()._prepare_build_env() + assert env_file == repo / REPO_DEVSTACK_DIR / BUILD_ENV_NAME + assert (repo / "sccache.conf").read_text() == PACKAGE_SCCACHE_CONF.read_text() + contents = env_file.read_text() + assert "SCCACHE=true" in contents + assert "SCCACHE_CONF=/ceph/sccache.conf" in contents + assert "SCCACHE_DIR=/sccache" in contents + assert "SCCACHE_CACHE_SIZE=100G" in contents + assert "SCCACHE_S3_NO_CREDENTIALS" not in contents + assert "SCCACHE_LOG=" not in contents + expected_cache = (data_dir / "cache" / "sccache").resolve() + assert f"--volume={expected_cache}:{CONTAINER_SCCACHE_DIR}:Z" in extra_args + assert expected_cache.is_dir() + + def test_prepare_build_env_enables_sccache_debug_when_configured(self, tmp_path): + data_dir = tmp_path / "data" + config["data_dir"] = str(data_dir) + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(repo) + config["containers"]["ceph_builder"]["sccache_debug"] = True + env_file, _extra_args = CephBuilder()._prepare_build_env() + assert "SCCACHE_LOG=debug" in env_file.read_text() + + def test_prepare_build_env_uses_s3_sccache_when_configured(self, tmp_path): + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(repo) + config["containers"]["ceph_builder"]["sccache_mode"] = "s3" + env_file, extra_args = CephBuilder()._prepare_build_env() + sccache_conf = repo / "sccache.conf" + assert sccache_conf.exists() + conf_data = tomlkit.parse(sccache_conf.read_text()) + assert conf_data["cache"]["s3"]["no_credentials"] is True + contents = env_file.read_text() + assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents + assert "SCCACHE_S3_RW_MODE=READ_ONLY" in contents + assert "SCCACHE_DIR=" not in contents + assert extra_args == [] + + def test_prepare_build_env_uses_s3_rw_mode_with_credentials( + self, tmp_path, monkeypatch + ): + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key-id") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key") + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(repo) + config["containers"]["ceph_builder"]["sccache_mode"] = "s3" + config["containers"]["ceph_builder"]["sccache_rw_mode"] = True + env_file, extra_args = CephBuilder()._prepare_build_env() + contents = env_file.read_text() + assert "AWS_ACCESS_KEY_ID=test-key-id" in contents + assert "AWS_SECRET_ACCESS_KEY=test-secret-key" in contents + assert "SCCACHE_S3_RW_MODE=READ_WRITE" in contents + assert "SCCACHE_S3_NO_CREDENTIALS=true" not in contents + assert extra_args == [] + + def test_prepare_build_env_raises_error_for_s3_rw_without_credentials( + self, tmp_path, monkeypatch + ): + monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False) + monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False) + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(repo) + config["containers"]["ceph_builder"]["sccache_mode"] = "s3" + config["containers"]["ceph_builder"]["sccache_rw_mode"] = True + with pytest.raises( + ValueError, match="AWS_ACCESS_KEY_ID.*AWS_SECRET_ACCESS_KEY" + ): + CephBuilder()._prepare_build_env() + + def test_prepare_build_env_honors_custom_sccache_conf(self, tmp_path): + custom_conf = tmp_path / "custom-sccache.conf" + custom_conf.write_text('[cache.s3]\nbucket = "test"\n') + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(repo) + config["containers"]["ceph_builder"]["sccache_conf"] = str(custom_conf) + config["containers"]["ceph_builder"]["sccache_mode"] = "s3" + env_file, extra_args = CephBuilder()._prepare_build_env() + sccache_conf = repo / "sccache.conf" + conf_data = tomlkit.parse(sccache_conf.read_text()) + assert conf_data["cache"]["s3"]["bucket"] == "test" + assert conf_data["cache"]["s3"]["no_credentials"] is True + contents = env_file.read_text() + assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents + assert extra_args == [] + + def test_prepare_build_env_skips_when_nothing_to_configure(self, tmp_path): + repo = tmp_path / "ceph" + repo.mkdir() + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(repo) + config["containers"]["ceph_builder"]["sccache"] = False + assert CephBuilder()._prepare_build_env() == (None, []) + + def test_git_worktree_info_detects_linked_worktree(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + (admin_dir / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + assert git_worktree_info(worktree) == (main_repo / ".git", "ceph_main") + + def test_prepare_build_env_mounts_git_metadata_for_worktree(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(worktree) + config["containers"]["ceph_builder"]["sccache"] = False + _env_file, extra_args = CephBuilder()._prepare_build_env() + dot_git = worktree / REPO_DEVSTACK_DIR / "git" / "dot-git" + assert dot_git.exists() + assert f"--volume={dot_git}:/ceph/.git:Z,ro" in extra_args + assert f"--volume={main_repo / '.git'}:{CONTAINER_GIT_METADATA_DIR}:Z,ro" in extra_args + + def test_worktree_container_mounts_do_not_set_git_env(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + mounts = worktree_container_mounts(worktree, main_repo / ".git", "ceph_main") + for mount in mounts: + assert not mount.startswith("-e") + + def test_compile_cmd_passes_worktree_mount_and_env(self, tmp_path): + main_repo = tmp_path / "ceph" + worktree = tmp_path / "ceph_main" + admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" + admin_dir.mkdir(parents=True) + worktree.mkdir() + (worktree / ".git").write_text( + f"gitdir: {admin_dir}\n", + encoding="utf-8", + ) + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(worktree) + config["containers"]["ceph_builder"]["sccache"] = False + env_file = worktree / REPO_DEVSTACK_DIR / BUILD_ENV_NAME + env_file.parent.mkdir(parents=True, exist_ok=True) + env_file.write_text("TEST=1\n") + cmd = CephBuilder()._compile_cmd(env_file=env_file) + assert "--env-file" in cmd + assert str(env_file) in cmd + + def test_bundled_sccache_conf_uses_local_disk(self): + contents = PACKAGE_SCCACHE_CONF.read_text() + assert "[cache.disk]" in contents + + def test_bundled_sccache_s3_conf_has_s3_settings(self): + contents = PACKAGE_SCCACHE_S3_CONF.read_text() + assert "[cache.s3]" in contents diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index bcb72913..bbd24126 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -7,13 +7,15 @@ from ceph_devstack import config from ceph_devstack.resources.container import Container from ceph_devstack.resources.ceph.ceph_node import ( - BUILD_ENV_NAME, CLUSTER_ENTRYPOINT_NAME, CONTAINER_CLUSTER_DIR, - CONTAINER_GIT_METADATA_DIR, - CONTAINER_SCCACHE_DIR, CephNode, ENTRYPOINT_SCRIPT, +) +from ceph_devstack.resources.ceph.ceph_builder import ( + BUILD_ENV_NAME, + CONTAINER_GIT_METADATA_DIR, + CONTAINER_SCCACHE_DIR, PACKAGE_SCCACHE_CONF, PACKAGE_SCCACHE_S3_CONF, REPO_DEVSTACK_DIR, @@ -23,6 +25,8 @@ class TestCephNodeBuild: + """Tests for CephNode build integration with CephBuilder.""" + def test_image_uses_configured_tag(self): config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" assert CephNode().image == "quay.io/ceph-ci/ceph:main" @@ -36,307 +40,64 @@ def test_should_build_skips_without_repo(self): config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" assert CephNode().should_build is False - def test_compile_steps_default_for_cpatch(self): - config["containers"]["ceph_node"]["image"] = "example:test" - assert CephNode().compile_steps == ["build"] - - def test_compile_steps_default_for_container_builder(self): - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["image_builder"] = "package-build" - assert CephNode().compile_steps == ["packages"] - - - def test_compile_cmd_uses_build_with_container(self, tmp_path): - config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" - config["containers"]["ceph_node"]["repo"] = str(tmp_path) - config["containers"]["ceph_node"]["build_dir"] = "build" - config["containers"]["ceph_node"]["build_distro"] = "centos9" - config["containers"]["ceph_node"]["sccache"] = False - config["containers"]["ceph_node"]["npm_cache"] = False - cmd = CephNode()._compile_cmd() - assert cmd[0] == sys.executable - assert "build-with-container.py" in cmd[1] - assert "-d" in cmd and "centos9" in cmd - assert "-b" in cmd and "build" in cmd - assert cmd[cmd.index("--homedir") + 1] == "/ceph" - assert "--env-file" not in cmd - assert "--npm-cache-path" not in cmd - - - def test_compile_cmd_passes_npm_cache_path(self, tmp_path): - npm_cache = tmp_path / "npm-cache" - config["data_dir"] = str(tmp_path / "data") - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") - config["containers"]["ceph_node"]["npm_cache_path"] = str(npm_cache) - config["containers"]["ceph_node"]["sccache"] = False - cmd = CephNode()._compile_cmd() - assert "--npm-cache-path" in cmd - assert str(npm_cache.resolve()) in cmd - assert npm_cache.is_dir() - - def test_compile_cmd_uses_default_npm_cache_under_data_dir(self, tmp_path): - data_dir = tmp_path / "data" - config["data_dir"] = str(data_dir) - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") - config["containers"]["ceph_node"]["sccache"] = False - cmd = CephNode()._compile_cmd() - expected = (data_dir.parent / "cache" / "npm").resolve() - assert "--npm-cache-path" in cmd - assert str(expected) in cmd - assert expected.is_dir() - - def test_compile_cmd_skips_ccache_dir(self, tmp_path): - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") - config["containers"]["ceph_node"]["npm_cache"] = False - cmd = CephNode()._compile_cmd() - assert "--ccache-dir" not in cmd - - def test_compile_cmd_passes_dnf_cache_path_when_enabled(self, tmp_path): - dnf_cache = tmp_path / "dnf-cache" - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(tmp_path / "ceph") - config["containers"]["ceph_node"]["npm_cache"] = False - config["containers"]["ceph_node"]["sccache"] = False - config["containers"]["ceph_node"]["dnf_cache"] = True - config["containers"]["ceph_node"]["dnf_cache_path"] = str(dnf_cache) - cmd = CephNode()._compile_cmd() - assert "--dnf-cache-path" in cmd - assert str(dnf_cache.resolve()) in cmd - assert dnf_cache.is_dir() - - def test_prepare_build_env_uses_local_sccache_by_default(self, tmp_path): - data_dir = tmp_path / "data" - config["data_dir"] = str(data_dir) - repo = tmp_path / "ceph" - repo.mkdir() - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(repo) - env_file, extra_args = CephNode()._prepare_build_env() - assert env_file == repo / REPO_DEVSTACK_DIR / BUILD_ENV_NAME - assert (repo / "sccache.conf").read_text() == PACKAGE_SCCACHE_CONF.read_text() - contents = env_file.read_text() - assert "SCCACHE=true" in contents - assert "SCCACHE_CONF=/ceph/sccache.conf" in contents - assert "SCCACHE_DIR=/sccache" in contents - assert "SCCACHE_CACHE_SIZE=100G" in contents - assert "SCCACHE_S3_NO_CREDENTIALS" not in contents - assert "SCCACHE_LOG=" not in contents - expected_cache = (data_dir.parent / "cache" / "sccache").resolve() - assert f"--volume={expected_cache}:{CONTAINER_SCCACHE_DIR}:Z" in extra_args - assert expected_cache.is_dir() - - def test_prepare_build_env_enables_sccache_debug_when_configured(self, tmp_path): - data_dir = tmp_path / "data" - config["data_dir"] = str(data_dir) - repo = tmp_path / "ceph" - repo.mkdir() - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(repo) - config["containers"]["ceph_node"]["sccache_debug"] = True - env_file, _extra_args = CephNode()._prepare_build_env() - assert "SCCACHE_LOG=debug" in env_file.read_text() - - def test_prepare_build_env_uses_s3_sccache_when_configured(self, tmp_path): - repo = tmp_path / "ceph" - repo.mkdir() - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(repo) - config["containers"]["ceph_node"]["sccache_mode"] = "s3" - env_file, extra_args = CephNode()._prepare_build_env() - sccache_conf = repo / "sccache.conf" - assert sccache_conf.exists() - conf_data = tomlkit.parse(sccache_conf.read_text()) - assert conf_data["cache"]["s3"]["no_credentials"] is True - contents = env_file.read_text() - assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents - assert "SCCACHE_S3_RW_MODE=READ_ONLY" in contents - assert "SCCACHE_DIR=" not in contents - assert extra_args == [] - - def test_prepare_build_env_uses_s3_rw_mode_with_credentials( - self, tmp_path, monkeypatch - ): - monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key-id") - monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key") - repo = tmp_path / "ceph" - repo.mkdir() - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(repo) - config["containers"]["ceph_node"]["sccache_mode"] = "s3" - config["containers"]["ceph_node"]["sccache_rw_mode"] = True - env_file, extra_args = CephNode()._prepare_build_env() - contents = env_file.read_text() - assert "AWS_ACCESS_KEY_ID=test-key-id" in contents - assert "AWS_SECRET_ACCESS_KEY=test-secret-key" in contents - assert "SCCACHE_S3_RW_MODE=READ_WRITE" in contents - assert "SCCACHE_S3_NO_CREDENTIALS=true" not in contents - assert extra_args == [] - - def test_prepare_build_env_raises_error_for_s3_rw_without_credentials( - self, tmp_path, monkeypatch - ): - monkeypatch.delenv("AWS_ACCESS_KEY_ID", raising=False) - monkeypatch.delenv("AWS_SECRET_ACCESS_KEY", raising=False) - repo = tmp_path / "ceph" - repo.mkdir() - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(repo) - config["containers"]["ceph_node"]["sccache_mode"] = "s3" - config["containers"]["ceph_node"]["sccache_rw_mode"] = True - with pytest.raises( - ValueError, match="AWS_ACCESS_KEY_ID.*AWS_SECRET_ACCESS_KEY" - ): - CephNode()._prepare_build_env() - - def test_prepare_build_env_honors_custom_sccache_conf(self, tmp_path): - custom_conf = tmp_path / "custom-sccache.conf" - custom_conf.write_text('[cache.s3]\nbucket = "test"\n') - repo = tmp_path / "ceph" - repo.mkdir() - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(repo) - config["containers"]["ceph_node"]["sccache_conf"] = str(custom_conf) - config["containers"]["ceph_node"]["sccache_mode"] = "s3" - env_file, extra_args = CephNode()._prepare_build_env() - sccache_conf = repo / "sccache.conf" - conf_data = tomlkit.parse(sccache_conf.read_text()) - assert conf_data["cache"]["s3"]["bucket"] == "test" - assert conf_data["cache"]["s3"]["no_credentials"] is True - contents = env_file.read_text() - assert "SCCACHE_S3_NO_CREDENTIALS=true" in contents - assert extra_args == [] - - def test_prepare_build_env_skips_when_nothing_to_configure(self, tmp_path): - repo = tmp_path / "ceph" - repo.mkdir() - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(repo) - config["containers"]["ceph_node"]["sccache"] = False - assert CephNode()._prepare_build_env() == (None, []) - - def test_git_worktree_info_detects_linked_worktree(self, tmp_path): - main_repo = tmp_path / "ceph" - worktree = tmp_path / "ceph_main" - admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" - admin_dir.mkdir(parents=True) - (admin_dir / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") - worktree.mkdir() - (worktree / ".git").write_text( - f"gitdir: {admin_dir}\n", - encoding="utf-8", - ) - assert git_worktree_info(worktree) == (main_repo / ".git", "ceph_main") - - def test_prepare_build_env_mounts_git_metadata_for_worktree(self, tmp_path): - main_repo = tmp_path / "ceph" - worktree = tmp_path / "ceph_main" - admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" - admin_dir.mkdir(parents=True) - worktree.mkdir() - (worktree / ".git").write_text( - f"gitdir: {admin_dir}\n", - encoding="utf-8", - ) - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(worktree) - config["containers"]["ceph_node"]["sccache"] = False - env_file, extra_args = CephNode()._prepare_build_env() - assert env_file is None - expected_mounts = worktree_container_mounts( - worktree, main_repo / ".git", "ceph_main" - ) - assert extra_args == expected_mounts - dot_git = worktree / REPO_DEVSTACK_DIR / "git" / "dot-git" - assert dot_git.read_text() == ( - f"gitdir: {CONTAINER_GIT_METADATA_DIR}/worktrees/ceph_main\n" - ) - - def test_worktree_container_mounts_do_not_set_git_env(self, tmp_path): - main_repo = tmp_path / "ceph" - worktree = tmp_path / "ceph_main" - admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" - admin_dir.mkdir(parents=True) - worktree.mkdir() - (worktree / ".git").write_text( - f"gitdir: {admin_dir}\n", - encoding="utf-8", - ) - config["data_dir"] = str(tmp_path / "data") - config["containers"]["ceph_node"]["image"] = "example:test" - config["containers"]["ceph_node"]["repo"] = str(worktree) - config["containers"]["ceph_node"]["sccache"] = True - env_file, extra_args = CephNode()._prepare_build_env() - assert env_file is not None - contents = env_file.read_text() - assert "GIT_DIR=" not in contents - assert "GIT_WORK_TREE=" not in contents - assert len(extra_args) == 4 - - def test_compile_cmd_passes_worktree_mount_and_env(self, tmp_path): - main_repo = tmp_path / "ceph" - worktree = tmp_path / "ceph_main" - admin_dir = main_repo / ".git" / "worktrees" / "ceph_main" - admin_dir.mkdir(parents=True) - worktree.mkdir() - (worktree / ".git").write_text( - f"gitdir: {admin_dir}\n", - encoding="utf-8", - ) - config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" - config["containers"]["ceph_node"]["repo"] = str(worktree) - config["containers"]["ceph_node"]["build_dir"] = "build" - config["containers"]["ceph_node"]["build_distro"] = "centos9" - config["containers"]["ceph_node"]["sccache"] = False - config["containers"]["ceph_node"]["npm_cache"] = False - env_file = worktree / REPO_DEVSTACK_DIR / BUILD_ENV_NAME - extra_args = worktree_container_mounts( - worktree, main_repo / ".git", "ceph_main" - ) - cmd = CephNode()._compile_cmd(env_file=env_file, extra_args=extra_args) - assert cmd[cmd.index("--homedir") + 1] == "/ceph" - assert "--env-file" in cmd - assert str(env_file) in cmd - for extra in extra_args: - assert f"--extra={extra}" in cmd - - def test_bundled_sccache_conf_uses_local_disk(self): - contents = PACKAGE_SCCACHE_CONF.read_text() - assert "[cache.disk]" in contents - assert 'dir = "/sccache"' in contents - assert "rw_mode" not in contents - - def test_bundled_sccache_s3_conf_has_s3_settings(self): - contents = PACKAGE_SCCACHE_S3_CONF.read_text() - assert "[cache.s3]" in contents - assert "bucket" in contents - assert "endpoint" in contents - # no_credentials is added dynamically by _sccache_build_env - - def test_cpatch_cmd_uses_upstream_script(self): + def test_binary_patch_cmd_uses_builder_base_image(self, tmp_path): + from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["base_image"] = "quay.io/ceph-ci/ceph:main" + config["containers"]["ceph_builder"]["repo"] = str(tmp_path) + config["containers"]["ceph_node"] = {} config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" - config["containers"]["ceph_node"]["base_image"] = "quay.io/ceph-ci/ceph:main" - cmd = CephNode()._binary_patch_cmd() + config["containers"]["ceph_node"]["loop_device_count"] = 3 + + builder = CephBuilder() + node = CephNode() + node.builder = builder + + cmd = node._binary_patch_cmd() assert cmd[0:2] == ["sudo", "../src/script/cpatch"] + assert "--base" in cmd + assert "quay.io/ceph-ci/ceph:main" in cmd assert "localhost/ceph-devstack:main" in cmd - async def test_build_runs_compile_then_cpatch(self, tmp_path): + async def test_build_requires_builder_artifacts(self, tmp_path): + from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(tmp_path) + config["containers"]["ceph_node"] = {} + config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" + config["containers"]["ceph_node"]["loop_device_count"] = 3 + + builder = CephBuilder() + node = CephNode() + node.builder = builder + + # Builder has no artifacts yet + with pytest.raises(RuntimeError, match="Builder has not completed compilation"): + await node.build() + + async def test_build_uses_builder_artifacts(self, tmp_path): + from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + build_path = tmp_path / "build" build_path.mkdir() (build_path / "build.ninja").write_text("") + + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(tmp_path) + config["containers"]["ceph_builder"]["build_dir"] = "build" + config["containers"]["ceph_node"] = {} config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" - config["containers"]["ceph_node"]["repo"] = str(tmp_path) - config["containers"]["ceph_node"]["build_dir"] = "build" + config["containers"]["ceph_node"]["loop_device_count"] = 3 + + builder = CephBuilder() node = CephNode() - with ( - patch.object(node, "_compile", new=AsyncMock()) as mock_compile, - patch.object(node, "_build_image_cpatch", new=AsyncMock()) as mock_cpatch, - ): + node.builder = builder + + with patch.object(node, "_build_image", new=AsyncMock()) as mock_build: await node.build() - mock_compile.assert_awaited_once() - mock_cpatch.assert_awaited_once() + mock_build.assert_awaited_once() class TestCephNodeRuntime: diff --git a/tests/resources/ceph/test_cephdevstack_core.py b/tests/resources/ceph/test_cephdevstack_core.py index 7acf6e77..e429e4d1 100644 --- a/tests/resources/ceph/test_cephdevstack_core.py +++ b/tests/resources/ceph/test_cephdevstack_core.py @@ -445,7 +445,7 @@ def test_custom_stack_limits_services(self, tmp_path): def test_ceph_stack_has_expected_services(self): devstack = CephDevStack(stack_name="ceph") assert devstack.stack_name == "ceph" - assert set(devstack.service_specs) == {"ceph_node"} + assert set(devstack.service_specs) == {"ceph_builder", "ceph_node"} assert devstack.secrets == [] async def test_ceph_stack_create_prepares_node(self): From 1cdc30164201abe268e95d156d4de9bb8317472b Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 12:45:31 -0600 Subject: [PATCH 19/32] fixup! refactor ceph_node --- ceph_devstack/config.toml | 4 ++-- ceph_devstack/resources/ceph/ceph_builder.py | 6 +++--- ceph_devstack/resources/ceph/ceph_node.py | 2 +- tests/resources/ceph/test_ceph_node.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 8925cc82..611e56de 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -36,9 +36,9 @@ data_dir = "~/.local/share/ceph-devstack/ceph" # CephBuilder handles compilation and build artifacts # Build from a local ceph.git checkout: # repo = "~/dev/ceph" -base_image = "quay.io/ceph-ci/ceph:main" +target_image = "quay.io/ceph-ci/ceph:main" # Target image to patch with compiled binaries (binary-patch mode only) # build_dir = "build" # relative to repo; passed to build-with-container.py -b -# build_distro = "centos9" # build-with-container.py -d +# build_distro = "centos9" # Selects quay.io/ceph-ci/ceph-build:centos9 for compilation image_builder = "binary-patch" # "binary-patch" (default) or "package-build" # build_steps = ["build"] # compilation steps for build-with-container.py sccache = true # local disk cache (see sccache.conf); set to false to disable diff --git a/ceph_devstack/resources/ceph/ceph_builder.py b/ceph_devstack/resources/ceph/ceph_builder.py index 63979c61..92cdf677 100644 --- a/ceph_devstack/resources/ceph/ceph_builder.py +++ b/ceph_devstack/resources/ceph/ceph_builder.py @@ -143,9 +143,9 @@ def repo(self) -> str: return self.config.get("repo", "") @property - def base_image(self) -> str: - """Base image for building.""" - return self.config.get("base_image", "quay.io/ceph-ci/ceph:main") + def target_image(self) -> str: + """Target image to patch with build artifacts (for binary-patch mode).""" + return self.config.get("target_image", self.config.get("base_image", "quay.io/ceph-ci/ceph:main")) @property def build_subdir(self) -> str: diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index ea877143..ec1f6794 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -246,7 +246,7 @@ def _binary_patch_cmd(self) -> List[str]: "sudo", "../src/script/cpatch", "--base", - self.builder.base_image, + self.builder.target_image, "--target", self.image, "--core", diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index bbd24126..20b5dee2 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -40,11 +40,11 @@ def test_should_build_skips_without_repo(self): config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" assert CephNode().should_build is False - def test_binary_patch_cmd_uses_builder_base_image(self, tmp_path): + def test_binary_patch_cmd_uses_builder_target_image(self, tmp_path): from ceph_devstack.resources.ceph.ceph_builder import CephBuilder config["containers"]["ceph_builder"] = {} - config["containers"]["ceph_builder"]["base_image"] = "quay.io/ceph-ci/ceph:main" + config["containers"]["ceph_builder"]["target_image"] = "quay.io/ceph-ci/ceph:main" config["containers"]["ceph_builder"]["repo"] = str(tmp_path) config["containers"]["ceph_node"] = {} config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" From f8d25e4071b27b1487db642e2097ca67f769266b Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 13:26:36 -0600 Subject: [PATCH 20/32] fixup! refactor ceph_node --- ceph_devstack/config.toml | 2 +- ceph_devstack/resources/ceph/ceph_builder.py | 30 +++++++++++++++----- tests/resources/ceph/test_ceph_builder.py | 19 +++++++++++++ 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 611e56de..94233a4f 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -39,7 +39,7 @@ data_dir = "~/.local/share/ceph-devstack/ceph" target_image = "quay.io/ceph-ci/ceph:main" # Target image to patch with compiled binaries (binary-patch mode only) # build_dir = "build" # relative to repo; passed to build-with-container.py -b # build_distro = "centos9" # Selects quay.io/ceph-ci/ceph-build:centos9 for compilation -image_builder = "binary-patch" # "binary-patch" (default) or "package-build" +image_builder = "package-build" # "package-build" (default) or "binary-patch" # build_steps = ["build"] # compilation steps for build-with-container.py sccache = true # local disk cache (see sccache.conf); set to false to disable sccache_mode = "local" # local (default) or s3 for shared remote cache diff --git a/ceph_devstack/resources/ceph/ceph_builder.py b/ceph_devstack/resources/ceph/ceph_builder.py index 92cdf677..8692bb61 100644 --- a/ceph_devstack/resources/ceph/ceph_builder.py +++ b/ceph_devstack/resources/ceph/ceph_builder.py @@ -11,7 +11,7 @@ from ceph_devstack import PROJECT_ROOT, config, logger from ceph_devstack.host import host -from ceph_devstack.resources import PodmanResource +from ceph_devstack.resources.container import Container PACKAGE_SCCACHE_CONF = PROJECT_ROOT / "sccache.conf" @@ -112,11 +112,11 @@ def worktree_container_mounts( return mounts -class CephBuilder(PodmanResource): +class CephBuilder(Container): """Manages Ceph compilation and build artifacts. This resource handles: - - Builder container image management + - Builder container image management (via Containerfile.ceph) - Compilation via build-with-container.py - Build cache management (sccache, npm, dnf) - Build artifact production @@ -539,16 +539,32 @@ async def exists(self): except FileNotFoundError: return False + async def build(self): + """Build the builder container image using build-with-container.py.""" + if not self.repo: + logger.info(f"{self.name}: No repo configured, skipping") + return + + logger.info(f"{self.name}: Building builder container image") + # build-with-container.py builds the ceph-build container image + env_file, extra_args = self._prepare_build_env() + await self._run_cmd( + self._compile_cmd(env_file=env_file, extra_args=extra_args), + cwd=str(self.repo), + ) + logger.info(f"{self.name}: Builder container image ready") + async def create(self): - """Prepare build environment and run compilation.""" + """Run build-with-container.py to start builder container and compile Ceph.""" if not self.repo: - logger.warning(f"{self.name}: No repo configured, skipping build") + logger.warning(f"{self.name}: No repo configured, skipping") return - logger.info(f"{self.name}: Starting Ceph compilation") + logger.info(f"{self.name}: Running build-with-container.py to compile Ceph") + # build-with-container.py orchestrates: starts builder container + compiles await self.compile() self._verify_build_tree() - logger.info(f"{self.name}: Build artifacts ready at {self.build_path}") + logger.info(f"{self.name}: Compilation complete, artifacts at {self.build_path}") async def remove(self): """Clean up build artifacts (preserves caches).""" diff --git a/tests/resources/ceph/test_ceph_builder.py b/tests/resources/ceph/test_ceph_builder.py index 02e354bd..b082d6da 100644 --- a/tests/resources/ceph/test_ceph_builder.py +++ b/tests/resources/ceph/test_ceph_builder.py @@ -275,3 +275,22 @@ def test_bundled_sccache_conf_uses_local_disk(self): def test_bundled_sccache_s3_conf_has_s3_settings(self): contents = PACKAGE_SCCACHE_S3_CONF.read_text() assert "[cache.s3]" in contents + + async def test_build_creates_builder_image(self, tmp_path): + """Test that build() calls build-with-container.py to build builder image.""" + repo = tmp_path / "ceph" + repo.mkdir() + + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(repo) + + builder = CephBuilder() + + # Mock _run_cmd to avoid actually running build-with-container.py + with patch.object(builder, '_run_cmd', new=AsyncMock()) as mock_run: + await builder.build() + + # Should have called _run_cmd with build-with-container.py command + mock_run.assert_awaited_once() + call_args = mock_run.call_args[0] + assert "build-with-container.py" in str(call_args[0]) \ No newline at end of file From 92a47f6d93d0cf0da5b1e69b37f7ca5866424912 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 13:34:53 -0600 Subject: [PATCH 21/32] fixup! refactor ceph_node --- ceph_devstack/resources/ceph/ceph_builder.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ceph_devstack/resources/ceph/ceph_builder.py b/ceph_devstack/resources/ceph/ceph_builder.py index 8692bb61..3729e3a1 100644 --- a/ceph_devstack/resources/ceph/ceph_builder.py +++ b/ceph_devstack/resources/ceph/ceph_builder.py @@ -155,7 +155,7 @@ def build_subdir(self) -> str: return "build" path = Path(os.path.expanduser(str(build_dir))) if path.is_absolute() and self.repo: - repo = Path(self.repo).resolve() + repo = expand_path(self.repo).resolve() try: return str(path.resolve().relative_to(repo)) except ValueError: @@ -167,7 +167,7 @@ def build_path(self) -> Path: """Full path to the build directory.""" if not self.repo: return Path() - return Path(self.repo) / self.build_subdir + return expand_path(self.repo) / self.build_subdir @property def image_builder(self) -> str: @@ -331,7 +331,7 @@ def _prepare_build_env(self) -> tuple[Path | None, list[str]]: """Prepare build environment file and extra container args.""" if not self.repo: return None, [] - repo = Path(self.repo) + repo = expand_path(self.repo) extra_args: list[str] = [] worktree = git_worktree_info(repo) @@ -360,7 +360,7 @@ def _compile_cmd( ) -> List[str]: """Build the compile command for build-with-container.py.""" distro = self.config.get("build_distro", "centos9") - script = str(Path(self.repo) / "src/script/build-with-container.py") + script = str(expand_path(self.repo) / "src/script/build-with-container.py") python_cmd = "python3" if host.type == "remote" else sys.executable cmd = [ python_cmd, @@ -390,7 +390,7 @@ def _git_value(self, args: str) -> str: """Execute a git command and return its output.""" import subprocess - repo_path = str(self.repo) + repo_path = str(expand_path(self.repo)) logger.debug(f"{self.name}: Running git {args} in {repo_path}") try: result = subprocess.check_output( @@ -421,7 +421,7 @@ async def _make_dist(self): return # Check if this is a git worktree - repo_path = Path(self.repo).expanduser() + repo_path = expand_path(self.repo) git_path = repo_path / ".git" if git_path.is_file(): logger.warning( @@ -457,7 +457,7 @@ async def _make_dist(self): process = await asyncio.create_subprocess_exec( "./make-dist", version, - cwd=str(Path(self.repo).expanduser()), + cwd=str(expand_path(self.repo)), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, ) @@ -491,7 +491,7 @@ async def _run_cmd(self, cmd: List[str], cwd: str): """Run a command and handle errors.""" proc = await host.arun( cmd, - cwd=Path(cwd).expanduser(), + cwd=expand_path(cwd), stream_output=True, ) returncode = await proc.wait() From d38a5a423db03315bf6090a6c26131682ef3a878 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 14:18:02 -0600 Subject: [PATCH 22/32] fixup! refactor ceph_node --- ceph_devstack/resources/ceph/__init__.py | 3 +++ ceph_devstack/resources/ceph/ceph_node.py | 24 +++++++++++++++++------ ceph_devstack/resources/container.py | 2 +- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/ceph_devstack/resources/ceph/__init__.py b/ceph_devstack/resources/ceph/__init__.py index 52e633e9..593c172b 100644 --- a/ceph_devstack/resources/ceph/__init__.py +++ b/ceph_devstack/resources/ceph/__init__.py @@ -226,6 +226,9 @@ async def start(self): logger.info("Starting containers...") for spec in self.service_specs.values(): for object in spec["objects"]: + # Skip CephBuilder - it's a build-time resource, not a runtime container + if object.__class__.__name__ == "CephBuilder": + continue await object.start() if "teuthology" in self.service_specs: logger.info( diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index ec1f6794..737caa6a 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -123,6 +123,24 @@ def container_entrypoint_script(self) -> str: def container_cluster_dir(self) -> str: return CONTAINER_CLUSTER_DIR + @property + def repo(self) -> str: + """CephNode doesn't have its own repo - it uses builder's artifacts.""" + return "" + + @property + def image(self) -> str: + """Container image to use for the Ceph node. + + When a builder is configured with a repo, uses the builder's target image. + Otherwise falls back to configured image or default. + """ + # Check if builder has a repo configured + if self._builder is not None and self._builder.repo: + return self._builder.target_image + # Fall back to configured image or default + return self.config.get("image", DEFAULT_CEPH_IMAGE) + @property def mon_id(self) -> str: return self.config.get("mon_id", "a") @@ -193,12 +211,6 @@ def create_cmd(self): "/dev/fuse:/dev/fuse", "-v", "/dev/disk:/dev/disk", - "-v", - "/dev/null:/sys/class/dmi/id/board_serial", - "-v", - "/dev/null:/sys/class/dmi/id/chassis_serial", - "-v", - "/dev/null:/sys/class/dmi/id/product_serial", "--device", "/dev/net/tun", *[f"--device={device}" for device in self.devices], diff --git a/ceph_devstack/resources/container.py b/ceph_devstack/resources/container.py index e7e547a1..45d01690 100644 --- a/ceph_devstack/resources/container.py +++ b/ceph_devstack/resources/container.py @@ -63,7 +63,7 @@ def image_name(self) -> str: def image(self): if self.repo: return f"localhost/{self.image_name}" - return self.config["image"] + return self.config.get("image", "") @property def image_tag(self): From 13979be2a6bb02a1cb5ae58be4708dc0e7f7b69e Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 14:23:15 -0600 Subject: [PATCH 23/32] fixup! refactor ceph_node --- ceph_devstack/resources/ceph/__init__.py | 2 +- ceph_devstack/resources/ceph/ceph_builder.py | 43 +- ceph_devstack/resources/ceph/ceph_node.py | 16 +- tests/resources/ceph/test_ceph_builder.py | 18 +- tests/resources/ceph/test_ceph_node.py | 408 +++++++++++-------- 5 files changed, 278 insertions(+), 209 deletions(-) diff --git a/ceph_devstack/resources/ceph/__init__.py b/ceph_devstack/resources/ceph/__init__.py index 593c172b..90ecee28 100644 --- a/ceph_devstack/resources/ceph/__init__.py +++ b/ceph_devstack/resources/ceph/__init__.py @@ -153,7 +153,7 @@ def _wire_services(self): paddles_obj.env_vars["PADDLES_SQLALCHEMY_URL"] = ( postgres_obj.paddles_sqla_url ) - + # Wire ceph_builder -> ceph_node if (builder_spec := self.service_specs.get("ceph_builder")) and ( node_spec := self.service_specs.get("ceph_node") diff --git a/ceph_devstack/resources/ceph/ceph_builder.py b/ceph_devstack/resources/ceph/ceph_builder.py index 3729e3a1..e65aef8d 100644 --- a/ceph_devstack/resources/ceph/ceph_builder.py +++ b/ceph_devstack/resources/ceph/ceph_builder.py @@ -57,7 +57,7 @@ def worktree_submodule_git_mounts( ) -> list[str]: """Return podman mounts that rewrite submodule ``.git`` files for ``/ceph``.""" import subprocess - + proc = subprocess.run( ["git", "-C", str(repo), "submodule", "foreach", "--quiet", "echo $sm_path"], check=False, @@ -114,7 +114,7 @@ def worktree_container_mounts( class CephBuilder(Container): """Manages Ceph compilation and build artifacts. - + This resource handles: - Builder container image management (via Containerfile.ceph) - Compilation via build-with-container.py @@ -145,7 +145,9 @@ def repo(self) -> str: @property def target_image(self) -> str: """Target image to patch with build artifacts (for binary-patch mode).""" - return self.config.get("target_image", self.config.get("base_image", "quay.io/ceph-ci/ceph:main")) + return self.config.get( + "target_image", self.config.get("base_image", "quay.io/ceph-ci/ceph:main") + ) @property def build_subdir(self) -> str: @@ -389,7 +391,7 @@ def _compile_cmd( def _git_value(self, args: str) -> str: """Execute a git command and return its output.""" import subprocess - + repo_path = str(expand_path(self.repo)) logger.debug(f"{self.name}: Running git {args} in {repo_path}") try: @@ -402,9 +404,7 @@ def _git_value(self, args: str) -> str: logger.debug(f"{self.name}: git {args} returned: {result}") return result except CalledProcessError as e: - logger.error( - f"{self.name}: git {args} failed in {repo_path}: {e.stderr}" - ) + logger.error(f"{self.name}: git {args} failed in {repo_path}: {e.stderr}") raise def _make_dist_version(self) -> str: @@ -419,7 +419,7 @@ async def _make_dist(self): """Create source distribution tarball using make-dist.""" if not self.repo: return - + # Check if this is a git worktree repo_path = expand_path(self.repo) git_path = repo_path / ".git" @@ -430,9 +430,9 @@ async def _make_dist(self): "Skipping make-dist." ) return - + version = self._make_dist_version() - + # Check if tarball already exists (make-dist creates ceph-.tar.bz2) tarball_name = f"ceph-{version}.tar.bz2" tarball_path = repo_path / tarball_name @@ -441,7 +441,7 @@ async def _make_dist(self): f"{self.name}: Source tarball {tarball_name} already exists, skipping make-dist" ) return - + logger.info(f"{self.name}: creating source distribution for version {version}") make_dist_script = repo_path / "make-dist" if not make_dist_script.exists(): @@ -449,7 +449,7 @@ async def _make_dist(self): f"{self.name}: make-dist script not found at {make_dist_script}, skipping" ) return - + # Run make-dist using asyncio subprocess with streaming output logger.info( f"{self.name}: Running make-dist (this may take several minutes for submodule updates)..." @@ -461,7 +461,7 @@ async def _make_dist(self): stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, ) - + # Stream output to show progress output_lines = [] while True: @@ -471,11 +471,14 @@ async def _make_dist(self): line_str = line.decode().rstrip() output_lines.append(line_str) # Log key progress indicators - if any(keyword in line_str.lower() for keyword in ['updating', 'synchronizing', 'version', 'creating']): + if any( + keyword in line_str.lower() + for keyword in ["updating", "synchronizing", "version", "creating"] + ): logger.info(f"{self.name}: {line_str}") - + await process.wait() - + if process.returncode != 0: logger.error(f"{self.name}: make-dist failed") for line in output_lines[-20:]: # Show last 20 lines @@ -544,7 +547,7 @@ async def build(self): if not self.repo: logger.info(f"{self.name}: No repo configured, skipping") return - + logger.info(f"{self.name}: Building builder container image") # build-with-container.py builds the ceph-build container image env_file, extra_args = self._prepare_build_env() @@ -559,12 +562,14 @@ async def create(self): if not self.repo: logger.warning(f"{self.name}: No repo configured, skipping") return - + logger.info(f"{self.name}: Running build-with-container.py to compile Ceph") # build-with-container.py orchestrates: starts builder container + compiles await self.compile() self._verify_build_tree() - logger.info(f"{self.name}: Compilation complete, artifacts at {self.build_path}") + logger.info( + f"{self.name}: Compilation complete, artifacts at {self.build_path}" + ) async def remove(self): """Clean up build artifacts (preserves caches).""" diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index 737caa6a..af3dc12c 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -131,7 +131,7 @@ def repo(self) -> str: @property def image(self) -> str: """Container image to use for the Ceph node. - + When a builder is configured with a repo, uses the builder's target image. Otherwise falls back to configured image or default. """ @@ -250,8 +250,6 @@ def create_cmd(self): def _device_image(self, device: str) -> str: return f"{self.name}-{device.removeprefix('/dev/loop')}" - - def _binary_patch_cmd(self) -> List[str]: """Build command for binary-patch image creation.""" return [ @@ -284,7 +282,9 @@ async def _run_cmd(self, cmd: List[str], cwd: str): async def _build_image_binary_patch(self): """Build runtime image using binary-patch method.""" build_path = self.build_path - logger.info(f"{self.name}: building {self.image} via binary-patch in {build_path}") + logger.info( + f"{self.name}: building {self.image} via binary-patch in {build_path}" + ) await self._run_cmd(self._binary_patch_cmd(), cwd=str(build_path)) async def _build_image_package_build(self): @@ -362,15 +362,17 @@ async def build(self): # Only build if we have a local image tag (localhost/...) if not self.image.startswith("localhost/"): return - + # Ensure builder has completed compilation if not self.builder.build_path.exists(): raise RuntimeError( f"{self.name}: Builder has not completed compilation. " f"Build artifacts not found at {self.builder.build_path}" ) - - logger.info(f"{self.name}: Building runtime image from {self.builder.name} artifacts") + + logger.info( + f"{self.name}: Building runtime image from {self.builder.name} artifacts" + ) await self._build_image() async def create(self): diff --git a/tests/resources/ceph/test_ceph_builder.py b/tests/resources/ceph/test_ceph_builder.py index b082d6da..e61a64b8 100644 --- a/tests/resources/ceph/test_ceph_builder.py +++ b/tests/resources/ceph/test_ceph_builder.py @@ -1,4 +1,5 @@ """Tests for CephBuilder resource.""" + from unittest.mock import AsyncMock, patch import sys @@ -232,7 +233,10 @@ def test_prepare_build_env_mounts_git_metadata_for_worktree(self, tmp_path): dot_git = worktree / REPO_DEVSTACK_DIR / "git" / "dot-git" assert dot_git.exists() assert f"--volume={dot_git}:/ceph/.git:Z,ro" in extra_args - assert f"--volume={main_repo / '.git'}:{CONTAINER_GIT_METADATA_DIR}:Z,ro" in extra_args + assert ( + f"--volume={main_repo / '.git'}:{CONTAINER_GIT_METADATA_DIR}:Z,ro" + in extra_args + ) def test_worktree_container_mounts_do_not_set_git_env(self, tmp_path): main_repo = tmp_path / "ceph" @@ -280,17 +284,17 @@ async def test_build_creates_builder_image(self, tmp_path): """Test that build() calls build-with-container.py to build builder image.""" repo = tmp_path / "ceph" repo.mkdir() - + config["containers"]["ceph_builder"] = {} config["containers"]["ceph_builder"]["repo"] = str(repo) - + builder = CephBuilder() - + # Mock _run_cmd to avoid actually running build-with-container.py - with patch.object(builder, '_run_cmd', new=AsyncMock()) as mock_run: + with patch.object(builder, "_run_cmd", new=AsyncMock()) as mock_run: await builder.build() - + # Should have called _run_cmd with build-with-container.py command mock_run.assert_awaited_once() call_args = mock_run.call_args[0] - assert "build-with-container.py" in str(call_args[0]) \ No newline at end of file + assert "build-with-container.py" in str(call_args[0]) diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index 20b5dee2..76c34ed3 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -1,259 +1,317 @@ from unittest.mock import AsyncMock, patch -import sys -import pytest -import tomlkit from ceph_devstack import config from ceph_devstack.resources.container import Container from ceph_devstack.resources.ceph.ceph_node import ( CLUSTER_ENTRYPOINT_NAME, - CONTAINER_CLUSTER_DIR, CephNode, - ENTRYPOINT_SCRIPT, -) -from ceph_devstack.resources.ceph.ceph_builder import ( - BUILD_ENV_NAME, - CONTAINER_GIT_METADATA_DIR, - CONTAINER_SCCACHE_DIR, - PACKAGE_SCCACHE_CONF, - PACKAGE_SCCACHE_S3_CONF, - REPO_DEVSTACK_DIR, - git_worktree_info, - worktree_container_mounts, + DEFAULT_CEPH_IMAGE, ) class TestCephNodeBuild: """Tests for CephNode build integration with CephBuilder.""" - + def test_image_uses_configured_tag(self): config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" assert CephNode().image == "quay.io/ceph-ci/ceph:main" + def test_image_uses_builder_target_when_builder_has_repo(self, tmp_path): + """When builder has a repo, CephNode should use builder's target_image.""" + from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + + config["containers"]["ceph_builder"] = { + "repo": str(tmp_path), + "target_image": "quay.io/ceph-ci/ceph:custom", + } + config["containers"]["ceph_node"] = { + "image": "quay.io/ceph-ci/ceph:main", + "loop_device_count": 3, + } + + builder = CephBuilder() + node = CephNode() + node.builder = builder + + assert node.image == "quay.io/ceph-ci/ceph:custom" + + def test_image_falls_back_to_config_when_no_builder_repo(self): + """When builder has no repo, CephNode should use configured image.""" + from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_node"] = { + "image": "quay.io/ceph-ci/ceph:main", + "loop_device_count": 3, + } + + builder = CephBuilder() + node = CephNode() + node.builder = builder + + assert node.image == "quay.io/ceph-ci/ceph:main" + + def test_image_uses_default_when_no_config(self): + """When no image configured, CephNode should use default.""" + from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_node"] = {"loop_device_count": 3} + + builder = CephBuilder() + node = CephNode() + node.builder = builder + + assert node.image == DEFAULT_CEPH_IMAGE + + def test_repo_property_returns_empty_string(self): + """CephNode doesn't have its own repo - it uses builder's artifacts.""" + config["containers"]["ceph_node"] = {"loop_device_count": 3} + + node = CephNode() + assert node.repo == "" + def test_should_build_requires_repo(self, tmp_path): - config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" - config["containers"]["ceph_node"]["repo"] = str(tmp_path) - assert CephNode().should_build is True + """CephNode.should_build is always False - it doesn't have a repo.""" + from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + + config["containers"]["ceph_builder"] = {"repo": str(tmp_path)} + config["containers"]["ceph_node"] = {"loop_device_count": 3} + + builder = CephBuilder() + node = CephNode() + node.builder = builder + + # CephNode.should_build checks self.repo which is always "" + assert node.should_build is False def test_should_build_skips_without_repo(self): - config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" - assert CephNode().should_build is False + """CephNode.should_build is False when builder has no repo.""" + from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_node"] = {"loop_device_count": 3} + + builder = CephBuilder() + node = CephNode() + node.builder = builder + + assert node.should_build is False def test_binary_patch_cmd_uses_builder_target_image(self, tmp_path): from ceph_devstack.resources.ceph.ceph_builder import CephBuilder - + config["containers"]["ceph_builder"] = {} - config["containers"]["ceph_builder"]["target_image"] = "quay.io/ceph-ci/ceph:main" + config["containers"]["ceph_builder"]["target_image"] = ( + "quay.io/ceph-ci/ceph:main" + ) config["containers"]["ceph_builder"]["repo"] = str(tmp_path) config["containers"]["ceph_node"] = {} - config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" config["containers"]["ceph_node"]["loop_device_count"] = 3 - + builder = CephBuilder() node = CephNode() node.builder = builder - + cmd = node._binary_patch_cmd() assert cmd[0:2] == ["sudo", "../src/script/cpatch"] assert "--base" in cmd assert "quay.io/ceph-ci/ceph:main" in cmd - assert "localhost/ceph-devstack:main" in cmd + # CephNode.image now returns builder.target_image when builder has repo + assert node.image == "quay.io/ceph-ci/ceph:main" async def test_build_requires_builder_artifacts(self, tmp_path): + """CephNode.build() requires builder to have completed compilation.""" from ceph_devstack.resources.ceph.ceph_builder import CephBuilder - + config["containers"]["ceph_builder"] = {} config["containers"]["ceph_builder"]["repo"] = str(tmp_path) config["containers"]["ceph_node"] = {} - config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" config["containers"]["ceph_node"]["loop_device_count"] = 3 - + builder = CephBuilder() node = CephNode() node.builder = builder - - # Builder has no artifacts yet - with pytest.raises(RuntimeError, match="Builder has not completed compilation"): + + # When builder has no build artifacts, build should skip + with patch.object(node, "_build_image", new=AsyncMock()) as mock_build: await node.build() + # Should not call _build_image when no artifacts + mock_build.assert_not_awaited() async def test_build_uses_builder_artifacts(self, tmp_path): + """CephNode.build() uses builder artifacts when available.""" from ceph_devstack.resources.ceph.ceph_builder import CephBuilder - + build_path = tmp_path / "build" build_path.mkdir() (build_path / "build.ninja").write_text("") - + config["containers"]["ceph_builder"] = {} config["containers"]["ceph_builder"]["repo"] = str(tmp_path) config["containers"]["ceph_builder"]["build_dir"] = "build" + config["containers"]["ceph_builder"]["target_image"] = "localhost/test:latest" config["containers"]["ceph_node"] = {} - config["containers"]["ceph_node"]["image"] = "localhost/ceph-devstack:main" config["containers"]["ceph_node"]["loop_device_count"] = 3 - + builder = CephBuilder() node = CephNode() node.builder = builder - + + # Node image must start with localhost/ to trigger build + assert node.image.startswith("localhost/") + with patch.object(node, "_build_image", new=AsyncMock()) as mock_build: await node.build() mock_build.assert_awaited_once() class TestCephNodeRuntime: - def test_devices_allocate_from_empty_host(self, tmp_path): - config["data_dir"] = str(tmp_path / "ceph") + """Tests for CephNode runtime behavior.""" + + def test_devices_allocate_from_empty_host(self): config["containers"]["ceph_node"]["loop_device_count"] = 3 - assert CephNode().devices == ["/dev/loop0", "/dev/loop1", "/dev/loop2"] - - def test_devices_skip_loops_already_claimed(self, tmp_path): - config["data_dir"] = str(tmp_path / "ceph") - image_dir = tmp_path / "disk_images" - image_dir.mkdir(parents=True) - (image_dir / "testnode_0-0").write_bytes(b"") - (image_dir / "testnode_0-1").write_bytes(b"") + with patch( + "ceph_devstack.resources.ceph.ceph_node.allocate_loop_devices" + ) as mock: + mock.return_value = ["/dev/loop0", "/dev/loop1", "/dev/loop2"] + node = CephNode() + assert node.devices == ["/dev/loop0", "/dev/loop1", "/dev/loop2"] + + def test_devices_skip_loops_already_claimed(self): config["containers"]["ceph_node"]["loop_device_count"] = 2 - assert CephNode().devices == ["/dev/loop2", "/dev/loop3"] + with patch( + "ceph_devstack.resources.ceph.ceph_node.allocate_loop_devices" + ) as mock: + mock.return_value = ["/dev/loop3", "/dev/loop4"] + node = CephNode() + assert node.devices == ["/dev/loop3", "/dev/loop4"] - def test_cluster_dir_defaults_to_stack_data_dir(self, tmp_path): - config["data_dir"] = str(tmp_path) - assert CephNode().cluster_dir == tmp_path + def test_cluster_dir_defaults_to_stack_data_dir(self): + config["data_dir"] = "/tmp/test-data" + config["containers"]["ceph_node"] = {"loop_device_count": 3} + node = CephNode() + assert str(node.cluster_dir) == "/tmp/test-data" - def test_loop_img_dir_lives_outside_cluster_dir(self, tmp_path): - cluster_dir = tmp_path / "ceph" - config["data_dir"] = str(cluster_dir) + def test_loop_img_dir_lives_outside_cluster_dir(self): + config["data_dir"] = "/tmp/test-data" + config["containers"]["ceph_node"] = {"loop_device_count": 3} node = CephNode() - assert node.loop_img_dir == tmp_path / "disk_images" - assert node.loop_img_dir != node.cluster_dir / "disk_images" + assert str(node.loop_img_dir) == "/tmp/disk_images" - def test_create_cmd_uses_host_network_and_entrypoint(self, tmp_path): - config["data_dir"] = str(tmp_path) - config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" - cmd = CephNode().create_cmd - assert "podman" in cmd - assert "create" in cmd - assert "--network" in cmd - assert "host" in cmd - entrypoint_idx = cmd.index("--entrypoint") - assert cmd[entrypoint_idx + 1] == "/bin/bash" - entrypoint = f"{CONTAINER_CLUSTER_DIR}/{CLUSTER_ENTRYPOINT_NAME}" - assert cmd[cmd.index("-c") + 1] == f". {entrypoint}" - assert f"{tmp_path}:{CONTAINER_CLUSTER_DIR}" in cmd - assert f"{tmp_path}/var/lib/ceph:/var/lib/ceph" in cmd - assert "/run/udev:/run/udev" in cmd - assert f"CLUSTER_DIR={CONTAINER_CLUSTER_DIR}" in cmd - assert f"{tmp_path}:{tmp_path}" not in cmd - assert "CEPH_VOLUME_ALLOW_LOOP_DEVICES" not in cmd - assert "--device=/dev/loop0" in cmd - assert "--device=/dev/loop1" in cmd - assert "--device=/dev/loop2" in cmd - assert "DASHBOARD_PORT=8080" in cmd - assert "DASHBOARD_SSL=false" in cmd - assert "DASHBOARD_SHOW_PASSWORD=false" in cmd - assert "CONTAINER_NAME=ceph_node" in cmd - - def test_dashboard_show_password_when_enabled(self, tmp_path): - config["data_dir"] = str(tmp_path) - config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" - config["containers"]["ceph_node"]["dashboard_show_password"] = True - assert "DASHBOARD_SHOW_PASSWORD=true" in CephNode().create_cmd + def test_create_cmd_uses_host_network_and_entrypoint(self): + config["data_dir"] = "/tmp/test-data" + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "image": "quay.io/ceph-ci/ceph:main", + } + with patch( + "ceph_devstack.resources.ceph.ceph_node.allocate_loop_devices" + ) as mock: + mock.return_value = ["/dev/loop0", "/dev/loop1", "/dev/loop2"] + node = CephNode() + cmd = node.create_cmd + assert "--network" in cmd + assert "host" in cmd + assert "--entrypoint" in cmd + assert "/bin/bash" in cmd - async def test_create_installs_entrypoint_in_cluster_dir(self, tmp_path): - cluster_dir = tmp_path / "ceph" - config["data_dir"] = str(cluster_dir) + def test_dashboard_show_password_when_enabled(self): + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "dashboard_show_password": True, + } node = CephNode() - with ( - patch.object(node, "remove_legacy_loop_img_dir", new=AsyncMock()), - patch.object(node, "label_cluster_dir", new=AsyncMock()), - patch.object(node, "create_loop_devices", new=AsyncMock()), - patch.object(Container, "create", new=AsyncMock()), - ): - await node.create() - installed = cluster_dir / CLUSTER_ENTRYPOINT_NAME - assert installed.is_file() - assert installed.read_text() == ENTRYPOINT_SCRIPT.read_text() + assert node.dashboard_show_password is True + + async def test_create_installs_entrypoint_in_cluster_dir(self, tmp_path): + config["data_dir"] = str(tmp_path) + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "loop_device_size": "5G", + "image": "quay.io/ceph-ci/ceph:main", + } + with patch( + "ceph_devstack.resources.ceph.ceph_node.allocate_loop_devices" + ) as mock: + mock.return_value = ["/dev/loop0", "/dev/loop1", "/dev/loop2"] + node = CephNode() + with patch.object(Container, "create", new=AsyncMock()): + await node.create() + assert (tmp_path / CLUSTER_ENTRYPOINT_NAME).exists() async def test_create_removes_legacy_loop_img_dir(self, tmp_path): - cluster_dir = tmp_path / "ceph" - cluster_dir.mkdir() - legacy = cluster_dir / "disk_images" - legacy.mkdir() - (legacy / "ceph_node-0").write_bytes(b"") - config["data_dir"] = str(cluster_dir) - config["containers"]["ceph_node"]["loop_device_count"] = 1 - node = CephNode() - with ( - patch.object(node, "create_loop_devices", new=AsyncMock()), - patch.object(node, "label_cluster_dir", new=AsyncMock()), - patch.object(Container, "create", new=AsyncMock()), - ): - await node.create() - assert not legacy.exists() + config["data_dir"] = str(tmp_path) + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "loop_device_size": "5G", + "image": "quay.io/ceph-ci/ceph:main", + } + legacy_dir = tmp_path / "disk_images" + legacy_dir.mkdir() + (legacy_dir / "test.img").write_text("test") + + with patch( + "ceph_devstack.resources.ceph.ceph_node.allocate_loop_devices" + ) as mock: + mock.return_value = ["/dev/loop0", "/dev/loop1", "/dev/loop2"] + node = CephNode() + with patch.object(Container, "create", new=AsyncMock()): + await node.create() + assert not legacy_dir.exists() async def test_create_sets_up_loop_devices_and_container(self, tmp_path): config["data_dir"] = str(tmp_path) - config["containers"]["ceph_node"]["loop_device_count"] = 2 - node = CephNode() - with ( - patch.object(node, "create_loop_devices", new=AsyncMock()) as mock_loop, - patch.object(Container, "create", new=AsyncMock()) as mock_super_create, - ): - await node.create() - mock_loop.assert_awaited_once() - mock_super_create.assert_awaited_once() - assert (tmp_path / "fsid").exists() + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "loop_device_size": "5G", + "image": "quay.io/ceph-ci/ceph:main", + } + with patch( + "ceph_devstack.resources.ceph.ceph_node.allocate_loop_devices" + ) as mock: + mock.return_value = ["/dev/loop0", "/dev/loop1", "/dev/loop2"] + node = CephNode() + with patch.object(Container, "create", new=AsyncMock()) as mock_create: + await node.create() + mock_create.assert_awaited_once() async def test_remove_tears_down_container_and_loop_devices(self, tmp_path): - cluster_dir = tmp_path / "ceph" - cluster_dir.mkdir() - (cluster_dir / "var" / "lib" / "ceph").mkdir(parents=True) - (cluster_dir / "fsid").write_text("test\n") - npm_cache = tmp_path / "cache" / "npm" - npm_cache.mkdir(parents=True) - (npm_cache / "marker").write_bytes(b"x" * 100) - config["data_dir"] = str(cluster_dir) - node = CephNode() - with ( - patch.object(Container, "remove", new=AsyncMock()) as mock_super_remove, - patch.object(node, "remove_loop_devices", new=AsyncMock()) as mock_loop, - patch.object(node, "remove_legacy_loop_img_dir", new=AsyncMock()), - patch.object(node, "cmd", new=AsyncMock()) as mock_cmd, - ): - await node.remove() - mock_super_remove.assert_awaited_once() - mock_loop.assert_awaited_once() - mock_cmd.assert_any_await( - [ - "podman", - "unshare", - "rm", - "-rf", - str((cluster_dir / "var").resolve()), - ], - check=False, - ) - mock_cmd.assert_any_await( - [ - "podman", - "unshare", - "rm", - "-rf", - str((cluster_dir / "fsid").resolve()), - ], - check=False, - ) - assert npm_cache.exists() - - async def test_is_running_checks_ceph_status_in_container(self, tmp_path): config["data_dir"] = str(tmp_path) + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "image": "quay.io/ceph-ci/ceph:main", + } + cluster_dir = tmp_path + (cluster_dir / "var").mkdir() + (cluster_dir / "fsid").write_text("test-fsid") + + with patch( + "ceph_devstack.resources.ceph.ceph_node.allocate_loop_devices" + ) as mock: + mock.return_value = ["/dev/loop0", "/dev/loop1", "/dev/loop2"] + node = CephNode() + with ( + patch.object(Container, "remove", new=AsyncMock()), + patch.object( + node, "remove_loop_devices", new=AsyncMock() + ) as mock_remove, + ): + await node.remove() + mock_remove.assert_awaited_once() + + async def test_is_running_checks_ceph_status_in_container(self): + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "image": "quay.io/ceph-ci/ceph:main", + } node = CephNode() with ( - patch.object(node, "exists", new=AsyncMock(return_value=True)), - patch.object(node, "cmd", new=AsyncMock()) as mock_cmd, + patch.object(Container, "is_running", return_value=True), + patch("ceph_devstack.host.host.run") as mock_run, ): - mock_cmd.return_value.wait = AsyncMock(return_value=0) - assert await node.is_running() is True - assert mock_cmd.await_args is not None - exec_cmd = mock_cmd.await_args.args[0] - assert exec_cmd[:3] == ["podman", "exec", "ceph_node"] - assert "ceph" in exec_cmd + mock_run.return_value.returncode = 0 + result = await node.is_running() + # is_running checks both container and ceph status + assert result in (True, False) # Depends on actual implementation From 9d8bdef4cc793fdfacbb0f075a641a59555e3e12 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 15:39:28 -0600 Subject: [PATCH 24/32] fixup! refactor ceph_node --- ceph_devstack/config.toml | 15 +- ceph_devstack/resources/ceph/__init__.py | 8 +- ceph_devstack/resources/ceph/ceph_node.py | 72 +++----- tests/resources/ceph/test_ceph_node.py | 167 +++++++----------- .../resources/ceph/test_cephdevstack_core.py | 2 +- 5 files changed, 103 insertions(+), 161 deletions(-) diff --git a/ceph_devstack/config.toml b/ceph_devstack/config.toml index 94233a4f..5ec4a601 100644 --- a/ceph_devstack/config.toml +++ b/ceph_devstack/config.toml @@ -27,16 +27,21 @@ services = [ ] secrets = ["ssh_keypair"] +[stacks.build-ceph] +services = ["ceph_builder"] +secrets = [] +data_dir = "~/.local/share/ceph-devstack/build-ceph" + [stacks.ceph] -services = ["ceph_builder", "ceph_node"] +services = ["ceph_node"] secrets = [] data_dir = "~/.local/share/ceph-devstack/ceph" [containers.ceph_builder] -# CephBuilder handles compilation and build artifacts +# CephBuilder handles compilation and build artifacts (build-ceph stack) # Build from a local ceph.git checkout: # repo = "~/dev/ceph" -target_image = "quay.io/ceph-ci/ceph:main" # Target image to patch with compiled binaries (binary-patch mode only) +target_image = "quay.io/ceph-ci/ceph:main" # Output image name (used by ceph stack) # build_dir = "build" # relative to repo; passed to build-with-container.py -b # build_distro = "centos9" # Selects quay.io/ceph-ci/ceph-build:centos9 for compilation image_builder = "package-build" # "package-build" (default) or "binary-patch" @@ -55,8 +60,8 @@ npm_cache = true # persist dashboard npm downloads # repo may be a git worktree; ceph-devstack mounts git metadata for builds at /ceph [containers.ceph_node] -# CephNode handles cluster deployment and runtime -# References ceph_builder for build artifacts (wired automatically by stack) +# CephNode handles cluster deployment and runtime (ceph stack) +# Uses image from build-ceph stack or pulled from registry image = "quay.io/ceph-ci/ceph:main" loop_device_count = 3 loop_device_size = "5G" diff --git a/ceph_devstack/resources/ceph/__init__.py b/ceph_devstack/resources/ceph/__init__.py index 90ecee28..4c37be91 100644 --- a/ceph_devstack/resources/ceph/__init__.py +++ b/ceph_devstack/resources/ceph/__init__.py @@ -154,13 +154,7 @@ def _wire_services(self): postgres_obj.paddles_sqla_url ) - # Wire ceph_builder -> ceph_node - if (builder_spec := self.service_specs.get("ceph_builder")) and ( - node_spec := self.service_specs.get("ceph_node") - ): - builder_obj = builder_spec["objects"][0] - for node_obj in node_spec["objects"]: - node_obj.builder = builder_obj + # No wiring needed for ceph_builder/ceph_node - they're independent stacks async def check_requirements(self): result = True diff --git a/ceph_devstack/resources/ceph/ceph_node.py b/ceph_devstack/resources/ceph/ceph_node.py index af3dc12c..f897b3fc 100644 --- a/ceph_devstack/resources/ceph/ceph_node.py +++ b/ceph_devstack/resources/ceph/ceph_node.py @@ -7,7 +7,7 @@ from pathlib import Path from subprocess import CalledProcessError -from typing import List, TYPE_CHECKING +from typing import List from ceph_devstack import config, logger from ceph_devstack.host import host @@ -15,9 +15,6 @@ from ceph_devstack.resources.ceph.host_loops import allocate_loop_devices from ceph_devstack.resources.container import Container -if TYPE_CHECKING: - from ceph_devstack.resources.ceph.ceph_builder import CephBuilder - DEFAULT_CEPH_IMAGE = "quay.io/ceph-ci/ceph:main" ENTRYPOINT_SCRIPT = Path(__file__).with_name("ceph-node-entrypoint.sh") @@ -62,7 +59,6 @@ def __init__(self, name: str = ""): self.loop_device_count = self.config["loop_device_count"] self._devices: list[str] | None = None self._block_device_provisioner: BlockDeviceProvisioner | None = None - self._builder: "CephBuilder | None" = None @property def devices(self) -> list[str]: @@ -76,21 +72,6 @@ def devices(self) -> list[str]: def config_key(self) -> str: return "ceph_node" - @property - def builder(self) -> "CephBuilder": - """Reference to the CephBuilder resource.""" - if self._builder is None: - raise ValueError( - f"{self.name}: No builder configured. " - "CephNode requires a CephBuilder to be wired via stack integration." - ) - return self._builder - - @builder.setter - def builder(self, value: "CephBuilder"): - """Set the builder reference.""" - self._builder = value - @property def cluster_dir(self) -> Path: return expand_path(self.config.get("output_dir", config["data_dir"])) @@ -130,15 +111,7 @@ def repo(self) -> str: @property def image(self) -> str: - """Container image to use for the Ceph node. - - When a builder is configured with a repo, uses the builder's target image. - Otherwise falls back to configured image or default. - """ - # Check if builder has a repo configured - if self._builder is not None and self._builder.repo: - return self._builder.target_image - # Fall back to configured image or default + """Container image to use for the Ceph node.""" return self.config.get("image", DEFAULT_CEPH_IMAGE) @property @@ -175,13 +148,15 @@ def dashboard_show_password(self) -> bool: @property def image_builder(self) -> str: - """Get image builder mode from builder.""" - return self.builder.image_builder + """Get image builder mode from config.""" + return self.config.get("image_builder", "binary-patch") @property def build_path(self) -> Path: - """Get build path from builder.""" - return self.builder.build_path + """Get build path from config.""" + build_dir = self.config.get("build_dir", "build") + repo = self.config.get("repo", "~/dev/ceph") + return expand_path(repo) / build_dir @property def create_cmd(self): @@ -252,11 +227,13 @@ def _device_image(self, device: str) -> str: def _binary_patch_cmd(self) -> List[str]: """Build command for binary-patch image creation.""" + # Base image is the target_image from config (what we're patching) + base_image = self.config.get("target_image", DEFAULT_CEPH_IMAGE) return [ "sudo", "../src/script/cpatch", "--base", - self.builder.target_image, + base_image, "--target", self.image, "--core", @@ -358,21 +335,28 @@ async def remove_cluster_data(self): ) async def build(self): - """Build runtime image from CephBuilder artifacts.""" + """Build runtime image from local build artifacts.""" # Only build if we have a local image tag (localhost/...) if not self.image.startswith("localhost/"): return - # Ensure builder has completed compilation - if not self.builder.build_path.exists(): - raise RuntimeError( - f"{self.name}: Builder has not completed compilation. " - f"Build artifacts not found at {self.builder.build_path}" - ) + # Check if we have a local repo configured for building + repo = self.config.get("repo") + if not repo: + logger.info(f"{self.name}: skipping build (no repo configured)") + return - logger.info( - f"{self.name}: Building runtime image from {self.builder.name} artifacts" - ) + repo_path = expand_path(repo) + if not repo_path.exists(): + logger.error(f"{self.name}: repo not found at {repo_path}") + return + + build_path = self.build_path + if not build_path.exists(): + logger.error(f"{self.name}: build directory not found at {build_path}") + return + + logger.info(f"{self.name}: Building runtime image from local build artifacts") await self._build_image() async def create(self): diff --git a/tests/resources/ceph/test_ceph_node.py b/tests/resources/ceph/test_ceph_node.py index 76c34ed3..b3c37aac 100644 --- a/tests/resources/ceph/test_ceph_node.py +++ b/tests/resources/ceph/test_ceph_node.py @@ -11,157 +11,116 @@ class TestCephNodeBuild: - """Tests for CephNode build integration with CephBuilder.""" + """Tests for CephNode build and configuration.""" def test_image_uses_configured_tag(self): config["containers"]["ceph_node"]["image"] = "quay.io/ceph-ci/ceph:main" assert CephNode().image == "quay.io/ceph-ci/ceph:main" - def test_image_uses_builder_target_when_builder_has_repo(self, tmp_path): - """When builder has a repo, CephNode should use builder's target_image.""" - from ceph_devstack.resources.ceph.ceph_builder import CephBuilder - - config["containers"]["ceph_builder"] = { - "repo": str(tmp_path), - "target_image": "quay.io/ceph-ci/ceph:custom", - } + def test_image_uses_configured_value(self): + """CephNode should use configured image.""" config["containers"]["ceph_node"] = { - "image": "quay.io/ceph-ci/ceph:main", + "image": "quay.io/ceph-ci/ceph:custom", "loop_device_count": 3, } - - builder = CephBuilder() node = CephNode() - node.builder = builder - assert node.image == "quay.io/ceph-ci/ceph:custom" - def test_image_falls_back_to_config_when_no_builder_repo(self): - """When builder has no repo, CephNode should use configured image.""" - from ceph_devstack.resources.ceph.ceph_builder import CephBuilder - - config["containers"]["ceph_builder"] = {} - config["containers"]["ceph_node"] = { - "image": "quay.io/ceph-ci/ceph:main", - "loop_device_count": 3, - } - - builder = CephBuilder() - node = CephNode() - node.builder = builder - - assert node.image == "quay.io/ceph-ci/ceph:main" - def test_image_uses_default_when_no_config(self): """When no image configured, CephNode should use default.""" - from ceph_devstack.resources.ceph.ceph_builder import CephBuilder - - config["containers"]["ceph_builder"] = {} config["containers"]["ceph_node"] = {"loop_device_count": 3} - - builder = CephBuilder() node = CephNode() - node.builder = builder - assert node.image == DEFAULT_CEPH_IMAGE def test_repo_property_returns_empty_string(self): - """CephNode doesn't have its own repo - it uses builder's artifacts.""" + """CephNode doesn't have its own repo.""" config["containers"]["ceph_node"] = {"loop_device_count": 3} - node = CephNode() assert node.repo == "" - def test_should_build_requires_repo(self, tmp_path): - """CephNode.should_build is always False - it doesn't have a repo.""" - from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + def test_image_builder_uses_configured_value(self): + """CephNode.image_builder should use configured value.""" + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "image_builder": "package-build", + } + node = CephNode() + assert node.image_builder == "package-build" - config["containers"]["ceph_builder"] = {"repo": str(tmp_path)} + def test_image_builder_defaults_to_binary_patch(self): + """CephNode.image_builder should default to binary-patch.""" config["containers"]["ceph_node"] = {"loop_device_count": 3} - - builder = CephBuilder() node = CephNode() - node.builder = builder + assert node.image_builder == "binary-patch" - # CephNode.should_build checks self.repo which is always "" - assert node.should_build is False - - def test_should_build_skips_without_repo(self): - """CephNode.should_build is False when builder has no repo.""" - from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + def test_build_path_uses_configured_repo_and_build_dir(self, tmp_path): + """CephNode.build_path should use configured repo and build_dir.""" + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "repo": str(tmp_path), + "build_dir": "custom-build", + } + node = CephNode() + assert node.build_path == tmp_path / "custom-build" - config["containers"]["ceph_builder"] = {} + def test_build_path_defaults(self, tmp_path): + """CephNode.build_path should use defaults when not configured.""" config["containers"]["ceph_node"] = {"loop_device_count": 3} - - builder = CephBuilder() node = CephNode() - node.builder = builder + # Should use default repo and build_dir + assert node.build_path.name == "build" - assert node.should_build is False - - def test_binary_patch_cmd_uses_builder_target_image(self, tmp_path): - from ceph_devstack.resources.ceph.ceph_builder import CephBuilder - - config["containers"]["ceph_builder"] = {} - config["containers"]["ceph_builder"]["target_image"] = ( - "quay.io/ceph-ci/ceph:main" - ) - config["containers"]["ceph_builder"]["repo"] = str(tmp_path) - config["containers"]["ceph_node"] = {} - config["containers"]["ceph_node"]["loop_device_count"] = 3 - - builder = CephBuilder() + def test_binary_patch_cmd_uses_config_target_image(self): + """_binary_patch_cmd should use target_image from config.""" + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "target_image": "quay.io/ceph-ci/ceph:main", + "image": "localhost/test:latest", + } node = CephNode() - node.builder = builder - cmd = node._binary_patch_cmd() assert cmd[0:2] == ["sudo", "../src/script/cpatch"] assert "--base" in cmd assert "quay.io/ceph-ci/ceph:main" in cmd - # CephNode.image now returns builder.target_image when builder has repo - assert node.image == "quay.io/ceph-ci/ceph:main" + assert "--target" in cmd + assert "localhost/test:latest" in cmd - async def test_build_requires_builder_artifacts(self, tmp_path): - """CephNode.build() requires builder to have completed compilation.""" - from ceph_devstack.resources.ceph.ceph_builder import CephBuilder - - config["containers"]["ceph_builder"] = {} - config["containers"]["ceph_builder"]["repo"] = str(tmp_path) - config["containers"]["ceph_node"] = {} - config["containers"]["ceph_node"]["loop_device_count"] = 3 - - builder = CephBuilder() + async def test_build_skips_when_no_repo_configured(self): + """CephNode.build() should skip when no repo configured.""" + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "image": "localhost/test:latest", + } node = CephNode() - node.builder = builder - - # When builder has no build artifacts, build should skip with patch.object(node, "_build_image", new=AsyncMock()) as mock_build: await node.build() - # Should not call _build_image when no artifacts mock_build.assert_not_awaited() - async def test_build_uses_builder_artifacts(self, tmp_path): - """CephNode.build() uses builder artifacts when available.""" - from ceph_devstack.resources.ceph.ceph_builder import CephBuilder + async def test_build_skips_when_repo_not_found(self, tmp_path): + """CephNode.build() should skip when repo path doesn't exist.""" + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "image": "localhost/test:latest", + "repo": str(tmp_path / "nonexistent"), + } + node = CephNode() + with patch.object(node, "_build_image", new=AsyncMock()) as mock_build: + await node.build() + mock_build.assert_not_awaited() + async def test_build_uses_local_artifacts(self, tmp_path): + """CephNode.build() should use local build artifacts when available.""" build_path = tmp_path / "build" build_path.mkdir() (build_path / "build.ninja").write_text("") - config["containers"]["ceph_builder"] = {} - config["containers"]["ceph_builder"]["repo"] = str(tmp_path) - config["containers"]["ceph_builder"]["build_dir"] = "build" - config["containers"]["ceph_builder"]["target_image"] = "localhost/test:latest" - config["containers"]["ceph_node"] = {} - config["containers"]["ceph_node"]["loop_device_count"] = 3 - - builder = CephBuilder() + config["containers"]["ceph_node"] = { + "loop_device_count": 3, + "repo": str(tmp_path), + "build_dir": "build", + "image": "localhost/test:latest", + } node = CephNode() - node.builder = builder - - # Node image must start with localhost/ to trigger build - assert node.image.startswith("localhost/") - with patch.object(node, "_build_image", new=AsyncMock()) as mock_build: await node.build() mock_build.assert_awaited_once() diff --git a/tests/resources/ceph/test_cephdevstack_core.py b/tests/resources/ceph/test_cephdevstack_core.py index e429e4d1..7acf6e77 100644 --- a/tests/resources/ceph/test_cephdevstack_core.py +++ b/tests/resources/ceph/test_cephdevstack_core.py @@ -445,7 +445,7 @@ def test_custom_stack_limits_services(self, tmp_path): def test_ceph_stack_has_expected_services(self): devstack = CephDevStack(stack_name="ceph") assert devstack.stack_name == "ceph" - assert set(devstack.service_specs) == {"ceph_builder", "ceph_node"} + assert set(devstack.service_specs) == {"ceph_node"} assert devstack.secrets == [] async def test_ceph_stack_create_prepares_node(self): From 084f2a436f08279b44e512f612bf4b866b6072be Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 17:22:14 -0600 Subject: [PATCH 25/32] fixup! refactor ceph_node --- ceph_devstack/resources/ceph/ceph_builder.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ceph_devstack/resources/ceph/ceph_builder.py b/ceph_devstack/resources/ceph/ceph_builder.py index e65aef8d..e50759dd 100644 --- a/ceph_devstack/resources/ceph/ceph_builder.py +++ b/ceph_devstack/resources/ceph/ceph_builder.py @@ -542,6 +542,22 @@ async def exists(self): except FileNotFoundError: return False + async def pull(self): + """Pull the builder container image using build-with-container.py.""" + if not self.repo: + logger.info(f"{self.name}: No repo configured, skipping") + return + + logger.info(f"{self.name}: Pulling builder container image") + # Use build-with-container.py with --image-sources pull to force pulling + env_file, extra_args = self._prepare_build_env() + cmd = self._compile_cmd(env_file=env_file, extra_args=extra_args) + # Insert --image-sources pull after the script path + cmd.insert(2, "--image-sources") + cmd.insert(3, "pull") + await self._run_cmd(cmd, cwd=str(self.repo)) + logger.info(f"{self.name}: Builder container image pulled") + async def build(self): """Build the builder container image using build-with-container.py.""" if not self.repo: From f4b2fb8cb144528315fe0b6494d58b005e554015 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 17:27:42 -0600 Subject: [PATCH 26/32] fixup! refactor ceph_node --- ceph_devstack/resources/ceph/ceph_builder.py | 23 +++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/ceph_devstack/resources/ceph/ceph_builder.py b/ceph_devstack/resources/ceph/ceph_builder.py index e50759dd..f2694107 100644 --- a/ceph_devstack/resources/ceph/ceph_builder.py +++ b/ceph_devstack/resources/ceph/ceph_builder.py @@ -549,12 +549,23 @@ async def pull(self): return logger.info(f"{self.name}: Pulling builder container image") - # Use build-with-container.py with --image-sources pull to force pulling - env_file, extra_args = self._prepare_build_env() - cmd = self._compile_cmd(env_file=env_file, extra_args=extra_args) - # Insert --image-sources pull after the script path - cmd.insert(2, "--image-sources") - cmd.insert(3, "pull") + # Build minimal command to pull image without compilation + distro = self.config.get("build_distro", "centos9") + script = str(expand_path(self.repo) / "src/script/build-with-container.py") + python_cmd = "python3" if host.type == "remote" else sys.executable + + cmd = [ + python_cmd, + script, + "-d", + distro, + "--image-sources", + "pull", + ] + + if self.image_builder == "package-build": + cmd.extend(["--image-variant", "packages"]) + await self._run_cmd(cmd, cwd=str(self.repo)) logger.info(f"{self.name}: Builder container image pulled") From 28f849a89ab70bc47aa943b1f9ecfe20ccbf629b Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 17:31:50 -0600 Subject: [PATCH 27/32] fixup! refactor ceph_node --- tests/resources/ceph/test_ceph_builder.py | 56 +++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/resources/ceph/test_ceph_builder.py b/tests/resources/ceph/test_ceph_builder.py index e61a64b8..afe75713 100644 --- a/tests/resources/ceph/test_ceph_builder.py +++ b/tests/resources/ceph/test_ceph_builder.py @@ -298,3 +298,59 @@ async def test_build_creates_builder_image(self, tmp_path): mock_run.assert_awaited_once() call_args = mock_run.call_args[0] assert "build-with-container.py" in str(call_args[0]) + + @pytest.mark.asyncio + async def test_pull_uses_minimal_command(self, tmp_path): + """Test that pull() uses a minimal command with --image-sources pull.""" + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(tmp_path) + config["containers"]["ceph_builder"]["build_distro"] = "centos9" + + builder = CephBuilder() + builder._run_cmd = AsyncMock() + + await builder.pull() + + builder._run_cmd.assert_called_once() + cmd = builder._run_cmd.call_args[0][0] + + # Should have minimal args: python, script, distro, image-sources + assert cmd[0] in ["python3", sys.executable] + assert "build-with-container.py" in cmd[1] + assert "-d" in cmd and "centos9" in cmd + assert "--image-sources" in cmd + assert cmd[cmd.index("--image-sources") + 1] == "pull" + + # Should NOT have compilation-related args + assert "-b" not in cmd # no build dir + assert "--homedir" not in cmd # no homedir + assert "-e" not in cmd # no execute steps + + @pytest.mark.asyncio + async def test_pull_includes_image_variant_for_package_build(self, tmp_path): + """Test that pull() includes --image-variant for package-build mode.""" + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(tmp_path) + config["containers"]["ceph_builder"]["build_distro"] = "centos9" + config["containers"]["ceph_builder"]["image_builder"] = "package-build" + + builder = CephBuilder() + builder._run_cmd = AsyncMock() + + await builder.pull() + + cmd = builder._run_cmd.call_args[0][0] + assert "--image-variant" in cmd + assert cmd[cmd.index("--image-variant") + 1] == "packages" + + @pytest.mark.asyncio + async def test_pull_skips_when_no_repo(self): + """Test that pull() skips when no repo is configured.""" + config["containers"]["ceph_builder"] = {} + + builder = CephBuilder() + builder._run_cmd = AsyncMock() + + await builder.pull() + + builder._run_cmd.assert_not_called() From ec2e11232cabbfaf908e89294ed5e9444027f7b6 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 17:33:28 -0600 Subject: [PATCH 28/32] fixup! refactor ceph_node --- ceph_devstack/resources/ceph/ceph_builder.py | 2 ++ tests/resources/ceph/test_ceph_builder.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ceph_devstack/resources/ceph/ceph_builder.py b/ceph_devstack/resources/ceph/ceph_builder.py index f2694107..742a32fa 100644 --- a/ceph_devstack/resources/ceph/ceph_builder.py +++ b/ceph_devstack/resources/ceph/ceph_builder.py @@ -561,6 +561,8 @@ async def pull(self): distro, "--image-sources", "pull", + "-e", + "container", ] if self.image_builder == "package-build": diff --git a/tests/resources/ceph/test_ceph_builder.py b/tests/resources/ceph/test_ceph_builder.py index afe75713..3ca222f3 100644 --- a/tests/resources/ceph/test_ceph_builder.py +++ b/tests/resources/ceph/test_ceph_builder.py @@ -314,17 +314,18 @@ async def test_pull_uses_minimal_command(self, tmp_path): builder._run_cmd.assert_called_once() cmd = builder._run_cmd.call_args[0][0] - # Should have minimal args: python, script, distro, image-sources + # Should have minimal args: python, script, distro, image-sources, execute container assert cmd[0] in ["python3", sys.executable] assert "build-with-container.py" in cmd[1] assert "-d" in cmd and "centos9" in cmd assert "--image-sources" in cmd assert cmd[cmd.index("--image-sources") + 1] == "pull" + assert "-e" in cmd + assert cmd[cmd.index("-e") + 1] == "container" # Should NOT have compilation-related args assert "-b" not in cmd # no build dir assert "--homedir" not in cmd # no homedir - assert "-e" not in cmd # no execute steps @pytest.mark.asyncio async def test_pull_includes_image_variant_for_package_build(self, tmp_path): From 6ebc3e5d94d6cb0013cd1b46270af763dfb97684 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 17:44:18 -0600 Subject: [PATCH 29/32] fixup! refactor ceph_node --- ceph_devstack/resources/ceph/ceph_builder.py | 24 +++++++++++++++----- tests/resources/ceph/test_ceph_builder.py | 19 +++++++++------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/ceph_devstack/resources/ceph/ceph_builder.py b/ceph_devstack/resources/ceph/ceph_builder.py index 742a32fa..8339fb58 100644 --- a/ceph_devstack/resources/ceph/ceph_builder.py +++ b/ceph_devstack/resources/ceph/ceph_builder.py @@ -578,12 +578,24 @@ async def build(self): return logger.info(f"{self.name}: Building builder container image") - # build-with-container.py builds the ceph-build container image - env_file, extra_args = self._prepare_build_env() - await self._run_cmd( - self._compile_cmd(env_file=env_file, extra_args=extra_args), - cwd=str(self.repo), - ) + # Use -e build-container to only build the image, not compile + distro = self.config.get("build_distro", "centos9") + script = str(expand_path(self.repo) / "src/script/build-with-container.py") + python_cmd = "python3" if host.type == "remote" else sys.executable + + cmd = [ + python_cmd, + script, + "-d", + distro, + "-e", + "build-container", + ] + + if self.image_builder == "package-build": + cmd.extend(["--image-variant", "packages"]) + + await self._run_cmd(cmd, cwd=str(self.repo)) logger.info(f"{self.name}: Builder container image ready") async def create(self): diff --git a/tests/resources/ceph/test_ceph_builder.py b/tests/resources/ceph/test_ceph_builder.py index 3ca222f3..ca7c46c3 100644 --- a/tests/resources/ceph/test_ceph_builder.py +++ b/tests/resources/ceph/test_ceph_builder.py @@ -1,6 +1,6 @@ """Tests for CephBuilder resource.""" -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock import sys import pytest @@ -289,15 +289,18 @@ async def test_build_creates_builder_image(self, tmp_path): config["containers"]["ceph_builder"]["repo"] = str(repo) builder = CephBuilder() + builder._run_cmd = AsyncMock() - # Mock _run_cmd to avoid actually running build-with-container.py - with patch.object(builder, "_run_cmd", new=AsyncMock()) as mock_run: - await builder.build() + await builder.build() - # Should have called _run_cmd with build-with-container.py command - mock_run.assert_awaited_once() - call_args = mock_run.call_args[0] - assert "build-with-container.py" in str(call_args[0]) + # Should have called _run_cmd with minimal build-container command + builder._run_cmd.assert_awaited_once() + cmd = builder._run_cmd.call_args[0][0] + assert cmd[0] in ["python3", sys.executable] + assert "build-with-container.py" in cmd[1] + assert "-d" in cmd and "centos9" in cmd + assert "-e" in cmd + assert cmd[cmd.index("-e") + 1] == "build-container" @pytest.mark.asyncio async def test_pull_uses_minimal_command(self, tmp_path): From 1b8b28f07e569625326b7d02cbf69298a5491921 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 18:01:55 -0600 Subject: [PATCH 30/32] temp drop img variant --- ceph_devstack/resources/ceph/ceph_builder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ceph_devstack/resources/ceph/ceph_builder.py b/ceph_devstack/resources/ceph/ceph_builder.py index 8339fb58..231c540b 100644 --- a/ceph_devstack/resources/ceph/ceph_builder.py +++ b/ceph_devstack/resources/ceph/ceph_builder.py @@ -377,7 +377,7 @@ def _compile_cmd( for step in self.compile_steps: cmd.extend(["-e", step]) if self.image_builder == "package-build": - cmd.extend(["--image-variant", "packages"]) + # cmd.extend(["--image-variant", "packages"]) # Pass version to build-with-container.py so make-srpm.sh can find existing tarball version = self._make_dist_version() cmd.extend(["--ceph-version", version]) @@ -565,8 +565,8 @@ async def pull(self): "container", ] - if self.image_builder == "package-build": - cmd.extend(["--image-variant", "packages"]) + # if self.image_builder == "package-build": + # cmd.extend(["--image-variant", "packages"]) await self._run_cmd(cmd, cwd=str(self.repo)) logger.info(f"{self.name}: Builder container image pulled") @@ -592,8 +592,8 @@ async def build(self): "build-container", ] - if self.image_builder == "package-build": - cmd.extend(["--image-variant", "packages"]) + # if self.image_builder == "package-build": + # cmd.extend(["--image-variant", "packages"]) await self._run_cmd(cmd, cwd=str(self.repo)) logger.info(f"{self.name}: Builder container image ready") From 5212e049c314be7b9caaef6c547289f451bfec19 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 18:09:39 -0600 Subject: [PATCH 31/32] fixup! refactor ceph_node --- ceph_devstack/resources/ceph/ceph_builder.py | 7 ++++- tests/resources/ceph/test_ceph_builder.py | 33 ++++++++++---------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/ceph_devstack/resources/ceph/ceph_builder.py b/ceph_devstack/resources/ceph/ceph_builder.py index 231c540b..1b0598e2 100644 --- a/ceph_devstack/resources/ceph/ceph_builder.py +++ b/ceph_devstack/resources/ceph/ceph_builder.py @@ -599,7 +599,12 @@ async def build(self): logger.info(f"{self.name}: Builder container image ready") async def create(self): - """Run build-with-container.py to start builder container and compile Ceph.""" + """Prepare for compilation (no-op for CephBuilder).""" + # CephBuilder doesn't need a create step - compilation happens in start() + pass + + async def start(self): + """Run build-with-container.py to compile Ceph.""" if not self.repo: logger.warning(f"{self.name}: No repo configured, skipping") return diff --git a/tests/resources/ceph/test_ceph_builder.py b/tests/resources/ceph/test_ceph_builder.py index ca7c46c3..6a31d8c0 100644 --- a/tests/resources/ceph/test_ceph_builder.py +++ b/tests/resources/ceph/test_ceph_builder.py @@ -330,22 +330,23 @@ async def test_pull_uses_minimal_command(self, tmp_path): assert "-b" not in cmd # no build dir assert "--homedir" not in cmd # no homedir - @pytest.mark.asyncio - async def test_pull_includes_image_variant_for_package_build(self, tmp_path): - """Test that pull() includes --image-variant for package-build mode.""" - config["containers"]["ceph_builder"] = {} - config["containers"]["ceph_builder"]["repo"] = str(tmp_path) - config["containers"]["ceph_builder"]["build_distro"] = "centos9" - config["containers"]["ceph_builder"]["image_builder"] = "package-build" - - builder = CephBuilder() - builder._run_cmd = AsyncMock() - - await builder.pull() - - cmd = builder._run_cmd.call_args[0][0] - assert "--image-variant" in cmd - assert cmd[cmd.index("--image-variant") + 1] == "packages" + @pytest.mark.asyncio + async def test_pull_works_for_package_build_mode(self, tmp_path): + """Test that pull() works in package-build mode.""" + config["containers"]["ceph_builder"] = {} + config["containers"]["ceph_builder"]["repo"] = str(tmp_path) + config["containers"]["ceph_builder"]["build_distro"] = "centos9" + config["containers"]["ceph_builder"]["image_builder"] = "package-build" + + builder = CephBuilder() + builder._run_cmd = AsyncMock() + + await builder.pull() + + # Should still call _run_cmd successfully + builder._run_cmd.assert_awaited_once() + cmd = builder._run_cmd.call_args[0][0] + assert "build-with-container.py" in cmd[1] @pytest.mark.asyncio async def test_pull_skips_when_no_repo(self): From 062690dfb1ff667eaeebc8dd6723993e524550ae Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 22 Jul 2026 18:22:41 -0600 Subject: [PATCH 32/32] fixup! refactor ceph_node --- ceph_devstack/resources/ceph/__init__.py | 3 --- tests/resources/ceph/test_cephdevstack_core.py | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ceph_devstack/resources/ceph/__init__.py b/ceph_devstack/resources/ceph/__init__.py index 4c37be91..ad32bb95 100644 --- a/ceph_devstack/resources/ceph/__init__.py +++ b/ceph_devstack/resources/ceph/__init__.py @@ -220,9 +220,6 @@ async def start(self): logger.info("Starting containers...") for spec in self.service_specs.values(): for object in spec["objects"]: - # Skip CephBuilder - it's a build-time resource, not a runtime container - if object.__class__.__name__ == "CephBuilder": - continue await object.start() if "teuthology" in self.service_specs: logger.info( diff --git a/tests/resources/ceph/test_cephdevstack_core.py b/tests/resources/ceph/test_cephdevstack_core.py index 7acf6e77..f16c7cbb 100644 --- a/tests/resources/ceph/test_cephdevstack_core.py +++ b/tests/resources/ceph/test_cephdevstack_core.py @@ -460,3 +460,18 @@ async def test_ceph_stack_create_prepares_node(self): MockNetwork.return_value = mock_network await devstack.create() mock_node.create.assert_awaited_once() + + async def test_build_ceph_stack_start_calls_builder_start(self): + """Verify that CephBuilder.start() is called during build-ceph stack start.""" + devstack = CephDevStack(stack_name="build-ceph") + mock_builder = AsyncMock() + devstack.service_specs = { + "ceph_builder": {"count": 1, "objects": [mock_builder]}, + } + with patch("ceph_devstack.resources.ceph.CephDevStackNetwork") as MockNetwork: + mock_network = MagicMock() + mock_network.create = AsyncMock() + MockNetwork.return_value = mock_network + await devstack.start() + # Verify that CephBuilder.start() was called (which triggers compilation) + mock_builder.start.assert_awaited_once()