diff --git a/.gitignore b/.gitignore index 0e257a6..544c488 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,15 @@ dev *.pdb *.yaml +# The local development container stack is checked in, despite the broad `dev` +# scratch-directory rule above. +!.local/ +!.local/dev/ +!.local/dev/** + +# Local Ceph credentials and loopback OSD state. +.local/dev/ceph/generated/ +.local/dev/ceph/state/ # Generated by cargo mutants # Contains mutation testing data diff --git a/.local/dev/README.md b/.local/dev/README.md new file mode 100644 index 0000000..504fee2 --- /dev/null +++ b/.local/dev/README.md @@ -0,0 +1,126 @@ +# Local Ceph and Odorobo with Compose + +This directory runs the local development stack in containers: + +- `ceph` provides a single-node Ceph cluster and a file-backed OSD. +- `odorobo` runs the agent, sharing Ceph's network and PID namespaces and the host's `/dev`. + +Odorobo must run in the container for the `rbd://` storage path. It invokes `rbd device map` using the generated Ceph credentials, which creates a kernel block device, and then passes that device to Cloud Hypervisor. The container provides the credential files, the privileged device access, and the shared namespaces that a host-side process would have to replicate. + +This is intended for Linux development with a rootful container engine. The stack uses privileged containers because kernel RBD mapping, Cloud Hypervisor, networking, and Ceph's daemon management require host kernel access. + +## Prerequisites + +Install Podman with a working Compose provider. Docker Compose v2 is supported as a fallback. + +For Fedora, the host needs the container engine and kernel modules: + +```bash +sudo dnf install -y podman podman-compose kmod +sudo modprobe rbd loop +sudo losetup -f +``` + +`podman compose` prefers the `docker-compose` plugin when it is installed, and that provider talks to Podman's API socket, which must be running (`systemctl --user enable --now podman.socket` for rootless, `sudo systemctl enable --now podman.socket` for rootful). If you hit `failed to connect to the docker API at unix:///run/user//podman/podman.sock`, either enable that socket or select the standalone tool as the provider: `compose_providers = ["podman-compose"]` in `~/.config/containers/containers.conf` (or `export PODMAN_COMPOSE_PROVIDER=podman-compose`). + +The stack must run on a **rootful** engine. Rootless Podman cannot work: the Ceph container attaches the OSD file through a host loop device, and the kernel's loop driver requires `CAP_SYS_ADMIN` in the initial user namespace, which a rootless container never has (even `privileged` + a `/dev` bind mount do not help). Run everything rootful, e.g. `sudo bash .local/dev/init.sh`, and prefix the `podman compose` commands below with `sudo` accordingly. + +The Ceph image is based on the official `quay.io/ceph/ceph` image and starts the MON and OSD daemons itself; it intentionally does not start MGR because the MGR's optional Python modules require host udev/system services unavailable in this container. It does not use `cephadm`, nested Podman, or systemd. The OSD uses a persistent raw file attached through a host loop device, initialized directly with `ceph-osd` rather than `ceph-volume`. + +## Usage + +Initialize Ceph and start Odorobo: + +```bash +sudo bash .local/dev/init.sh +``` + +This builds both images, starts Ceph and waits up to two minutes for it to become healthy, provisions the `odorobo-blockpool/dev-disk` RBD image, and then starts Odorobo with manager mode enabled. On a bootstrap failure, it prints the last 200 Ceph log lines instead of waiting indefinitely. It prefers `podman compose` and falls back to `docker compose` when Podman Compose is unavailable. + +Start and stop the complete stack without deleting data. `start` only works on existing containers; after a reset, run `init.sh` again: + +```bash +sudo podman compose -f .local/dev/compose.yml start ceph odorobo +sudo podman compose -f .local/dev/compose.yml stop odorobo ceph +``` + +Destructively remove the containers and all local Ceph state. + +```bash +sudo podman compose -f .local/dev/compose.yml down --remove-orphans --volumes +sudo rm -rf .local/dev/ceph/generated .local/dev/ceph/state +``` + +Useful direct commands: + +```bash +sudo podman compose -f .local/dev/compose.yml ps +sudo podman compose -f .local/dev/compose.yml logs -f ceph odorobo +sudo podman compose -f .local/dev/compose.yml exec ceph ceph -s +sudo podman compose -f .local/dev/compose.yml exec -it odorobo sh +``` + +Because `odorobo` uses Ceph's network namespace, the generated Ceph config intentionally uses `127.0.0.1` for the monitor. The application and monitor share that namespace. + +## Application development + +The repository is mounted at `/workspace` in the Odorobo container. Rebuild and restart the application after source changes: + +```bash +podman compose -f .local/dev/compose.yml build odorobo +podman compose -f .local/dev/compose.yml up -d odorobo +podman compose -f .local/dev/compose.yml logs -f odorobo +``` + +By default, the container limits Cargo to two concurrent build jobs to reduce CPU and memory pressure during the initial release build. Override it when starting the stack, for example `CARGO_BUILD_JOBS=4 bash .local/dev/init.sh`. + +The agent runs as: + +```text +cargo run --release -p odorobo -- --manager-enabled true +``` + +Its runtime directory is shared through `/run/odorobo`, and Cloud Hypervisor processes and RBD devices are visible in the same namespaces as the agent. + +## Verify the image + +Run Ceph commands inside the Ceph container: + +```bash +podman compose -f .local/dev/compose.yml exec ceph rbd \ + --conf=/etc/ceph/ceph.conf --id=odorobo \ + --keyfile=/var/lib/odorobo-ceph/client.odorobo.key \ + ls --pool odorobo-blockpool +``` + +Expected output includes `dev-disk`. A one-node cluster may report `HEALTH_WARN`; reduced redundancy, a single monitor, and no running MGR are expected for local development. + +Do not map the image from the host. To test the exact application path, use an Odorobo manifest with `rbd://odorobo-blockpool/dev-disk`; the `odorobo` service will execute `rbd device map` and pass the resulting device to Cloud Hypervisor. + +## Configuration + +The following environment variables can be set before `init.sh` or passed through Compose: + +```bash +CEPH_IMAGE=quay.io/ceph/ceph:v20.2.3 +CEPH_MON_IP=127.0.0.1 +CEPH_POOL=odorobo-blockpool +CEPH_CLIENT=odorobo +CEPH_IMAGE_NAME=dev-disk +CEPH_IMAGE_SIZE=1G +CEPH_OSD_SIZE=10G +``` + +`CEPH_MON_IP` should remain `127.0.0.1` with the provided Compose topology. If you change the network topology, it must be an address reachable from both services. + +## Layout + +- `init.sh` — builds and starts the stack, waits for Ceph health, and reports bootstrap failures with logs. +- `compose.yml` — Ceph and Odorobo services, shared namespaces, privilege, mounts, and ports. +- `ceph/Containerfile` — pinned Ceph image. +- `ceph/entrypoint.sh` — direct MON bootstrap, filesystem-backed OSD initialization, pool/client/image provisioning, and daemon lifecycle. +- `odorobo/Containerfile` — runnable Odorobo development image. +- `ceph/generated/` — generated Ceph credentials shared read-only with Odorobo; ignored by git. +- `ceph/state/` — Ceph configuration, daemons, logs, and file-backed OSD; ignored by git. + +This setup is intentionally not production-ready: it has one monitor, one OSD, no redundancy, and privileged containers. diff --git a/.local/dev/TEST.md b/.local/dev/TEST.md new file mode 100644 index 0000000..b44439b --- /dev/null +++ b/.local/dev/TEST.md @@ -0,0 +1,103 @@ +# End-to-end test: boot a VM from a Ceph RBD disk + +Boots a minimal VM through Odorobo whose root disk is an `rbd://` image in the +in-container Ceph cluster. It proves the `rbd://` storage path end to end: the +agent maps the RBD to a kernel block device, resolves the URI, and hands the disk +to Cloud Hypervisor. + +Step 0 runs from the host; steps 1–6 run inside the `odorobo` container. + +## Step 0 — Start the stack and enter the container + +```sh +sudo bash .local/dev/init.sh +sudo podman compose -f .local/dev/compose.yml exec -it odorobo sh +``` + +## Step 1 — Install required packages + +```sh +dnf install -y busybox tar +``` + +## Step 2 — Re-fetch the Alpine kernel + +```sh +cd /tmp +curl -O https://dl-cdn.alpinelinux.org/alpine/v3.22/main/x86_64/linux-virt-6.12.110-r0.apk +tar xOf linux-virt-6.12.110-r0.apk boot/vmlinuz-virt > /tmp/vmlinuz-virt +ls -l /tmp/vmlinuz-virt +``` + +## Step 3 — Build the rootfs + +```sh +mkdir -p /tmp/rootfs/{bin,dev,etc,proc,sys} +cp /usr/bin/busybox /tmp/rootfs/bin/ +printf '%s\n' \ + '#!/bin/sh' \ + 'mount -t proc proc /proc' \ + 'mount -t sysfs sys /sys' \ + 'echo "=== odorobo VM: booted from ceph-backed rootfs ==="' \ + 'ls /dev | grep vda' \ + 'echo "=== dropping to shell ==="' \ + 'exec /bin/busybox ash' > /tmp/rootfs/init +chmod +x /tmp/rootfs/init +for a in ash sh ls cat mount echo grep; do ln -s bin/busybox /tmp/rootfs/$a; done +fallocate -l 512M /tmp/rootfs.img +mkfs.ext4 /tmp/rootfs.img +mkdir -p /tmp/mnt && mount /tmp/rootfs.img /tmp/mnt +cp -a /tmp/rootfs/. /tmp/mnt/ +umount /tmp/mnt +``` + +## Step 4 — Map the RBD + +A reset re-creates the image with a new object id, so any mapping that survived in +the kernel is stale. Unmap it first, or the agent's "already mapped?" check will +reuse a broken device. + +```sh +RBD="rbd --conf=/workspace/.local/dev/ceph/generated/ceph.conf --id=odorobo --keyfile=/workspace/.local/dev/ceph/generated/client.odorobo.key" + +# Clear any stale mappings that survived the reset +$RBD device unmap /dev/rbd0 --options noudev 2>/dev/null +$RBD device unmap /dev/rbd1 --options noudev 2>/dev/null +$RBD device ls # confirm the table is empty + +# Map the fresh image +$RBD device map odorobo-blockpool/dev-disk --options noudev; echo "map-exit=$?" +ls -l /dev/rbd* 2>/dev/null + +# Gate: the agent's map() runs `rbd device list` first, so it must succeed +$RBD device list; echo "list-exit=$?" +``` + +Expect `map-exit=0`, a `brw-... /dev/rbd0` line, and `list-exit=0`. Confirm before +continuing. + +## Step 5 — Write the rootfs to the RBD + +```sh +dd if=/tmp/rootfs.img of=/dev/rbd/odorobo-blockpool/dev-disk bs=1M +sync +``` + +## Step 6 — Boot the VM + +```sh +bash /workspace/.local/dev/vm-test.sh +``` + +The script health-checks the agent, posts the VM manifest, waits, and dumps the +console history. + +Expected console: kernel boot log → `=== odorobo VM: booted from ceph-backed +rootfs ===` → `vda` listed → shell prompt. Then run `cat /proc/mounts` and confirm +`/dev/vda / ext4` — that is the VM reading from the Ceph RBD pool. + +> **Note:** the stock Alpine `linux-virt` kernel ships `ext4` and `virtio_blk` as +> modules, so a direct kernel boot (no initramfs) panics at +> `VFS: Unable to mount root fs on "/dev/vda"`. That is a test-kernel limitation, +> not a Ceph issue — the `rbd://` resolution and disk attach are proven. To get a +> full shell, use a kernel with `ext4`/`virtio_blk` built in. diff --git a/.local/dev/ceph/.dockerignore b/.local/dev/ceph/.dockerignore new file mode 100644 index 0000000..888513e --- /dev/null +++ b/.local/dev/ceph/.dockerignore @@ -0,0 +1,2 @@ +state/ +generated/ diff --git a/.local/dev/ceph/Containerfile b/.local/dev/ceph/Containerfile new file mode 100644 index 0000000..8afb6fe --- /dev/null +++ b/.local/dev/ceph/Containerfile @@ -0,0 +1,9 @@ +ARG CEPH_IMAGE=quay.io/ceph/ceph:v20.2.3 +FROM ${CEPH_IMAGE} + +USER root + +COPY entrypoint.sh /usr/local/bin/odorobo-ceph +RUN chmod 0755 /usr/local/bin/odorobo-ceph + +ENTRYPOINT ["/usr/local/bin/odorobo-ceph"] diff --git a/.local/dev/ceph/entrypoint.sh b/.local/dev/ceph/entrypoint.sh new file mode 100644 index 0000000..2dae03a --- /dev/null +++ b/.local/dev/ceph/entrypoint.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${CEPH_MON_IP:=127.0.0.1}" +: "${CEPH_POOL:=odorobo-blockpool}" +: "${CEPH_CLIENT:=odorobo}" +: "${CEPH_IMAGE_NAME:=dev-disk}" +: "${CEPH_IMAGE_SIZE:=1G}" +: "${CEPH_OSD_SIZE:=10G}" + +CLUSTER=ceph +MON_ID=ceph +CEPH_CONF=/etc/ceph/ceph.conf +CEPH_DATA_DIR=/var/lib/ceph +OSD_STATE_DIR=/var/lib/odorobo-ceph +OSD_DATA_DIR="$OSD_STATE_DIR/osd" +OSD_IMAGE="$OSD_STATE_DIR/osd.raw" +OSD_ID_FILE="$OSD_STATE_DIR/osd.id" +OSD_UUID_FILE="$OSD_STATE_DIR/osd.uuid" +LOOP_DEVICE="" +MON_PID="" +OSD_PID="" + +cleanup() { + local pid + trap - EXIT INT TERM + for pid in "$OSD_PID" "$MON_PID"; do + [[ -n "$pid" ]] || continue + kill -TERM "$pid" 2>/dev/null || true + done + for pid in "$OSD_PID" "$MON_PID"; do + [[ -n "$pid" ]] || continue + wait "$pid" 2>/dev/null || true + done + [[ -n "$LOOP_DEVICE" ]] && losetup --detach "$LOOP_DEVICE" 2>/dev/null || true + } +trap cleanup EXIT INT TERM + +mkdir -p /etc/ceph "$CEPH_DATA_DIR" /var/log/ceph /run/ceph "$OSD_STATE_DIR" /generated +chown ceph:ceph /run/ceph + +# These paths are bind-mounted from the host and may retain ownership from a +# previous interrupted or root-run initialization. +for directory in \ + "$CEPH_DATA_DIR/mon" \ + "$CEPH_DATA_DIR/bootstrap-osd" \ + "$CEPH_DATA_DIR/osd" \ + "$OSD_STATE_DIR"; do + [[ -d "$directory" ]] && chown -R ceph:ceph "$directory" +done + +echo "[odorobo-ceph] initializing direct Ceph daemons" +if [[ ! -f "$CEPH_CONF" ]]; then + echo "[odorobo-ceph] creating monitor configuration and keyrings" + FSID=$(uuidgen) + cat >"$CEPH_CONF" </dev/null 2>&1; then + break + fi + if ! kill -0 "$MON_PID" 2>/dev/null; then + wait "$MON_PID" + fi + sleep 1 +done +if ! timeout 5 ceph -s; then + echo "Monitor did not become ready within 60 seconds." >&2 + exit 1 +fi + +echo "[odorobo-ceph] monitor ready; enabling msgr2" +# Ceph v20's kernel-RBD client defaults to msgr2 and rejects a monmap that +# advertises only the legacy v1 endpoint. This also makes the generated minimal +# client config contain an address usable by `rbd device map`. +ceph mon enable-msgr2 + +echo "[odorobo-ceph] preparing OSD" + +if [[ -f "$OSD_ID_FILE" ]]; then + OSD_ID=$(<"$OSD_ID_FILE") + OSD_UUID=$(<"$OSD_UUID_FILE") +else + OSD_UUID=$(uuidgen) + OSD_SECRET=$(ceph-authtool --gen-print-key) + OSD_ID=$(printf '{"cephx_secret": "%s"}\n' "$OSD_SECRET" \ + | ceph osd new "$OSD_UUID" -i - -n client.bootstrap-osd \ + -k "$CEPH_DATA_DIR/bootstrap-osd/$CLUSTER.keyring") + [[ -n "$OSD_ID" ]] || { echo "Ceph did not register the OSD." >&2; exit 1; } + mkdir -p "$OSD_DATA_DIR" + ceph-authtool --create-keyring "$OSD_DATA_DIR/keyring" \ + --name "osd.$OSD_ID" --add-key "$OSD_SECRET" + truncate -s "$CEPH_OSD_SIZE" "$OSD_IMAGE" + LOOP_DEVICE=$(losetup --find --show "$OSD_IMAGE") || { + echo "Unable to attach the OSD image to a loop device." >&2 + exit 1 + } + # The host-created loop node is normally root:disk; the OSD daemon drops to + # the ceph user and therefore needs direct access to this block device. + chown ceph:ceph "$LOOP_DEVICE" + chmod 0660 "$LOOP_DEVICE" + ln -sfn "$LOOP_DEVICE" "$OSD_DATA_DIR/block" + ceph-osd -i "$OSD_ID" --mkfs --osd-uuid "$OSD_UUID" \ + --osd-data "$OSD_DATA_DIR" + printf '%s\n' "$OSD_ID" >"$OSD_ID_FILE" + printf '%s\n' "$OSD_UUID" >"$OSD_UUID_FILE" +fi + +if [[ -z "$LOOP_DEVICE" ]]; then + LOOP_DEVICE=$(losetup --associated --noheadings --output NAME "$OSD_IMAGE" | awk 'NR == 1 { print; exit }') + [[ -n "$LOOP_DEVICE" ]] || LOOP_DEVICE=$(losetup --find --show "$OSD_IMAGE") || { + echo "Unable to attach the OSD image to a loop device." >&2 + exit 1 + } + chown ceph:ceph "$LOOP_DEVICE" + chmod 0660 "$LOOP_DEVICE" + ln -sfn "$LOOP_DEVICE" "$OSD_DATA_DIR/block" +fi + +chown -R ceph:ceph "$OSD_DATA_DIR" +ceph-osd -f -i "$OSD_ID" --osd-data "$OSD_DATA_DIR" \ + --setuser ceph --setgroup ceph & +OSD_PID=$! + +echo "[odorobo-ceph] preparing and starting OSD" +for _ in {1..60}; do + if timeout 5 ceph osd stat 2>/dev/null | grep -q '[1-9][0-9]* up'; then + break + fi + if ! kill -0 "$OSD_PID" 2>/dev/null; then + wait "$OSD_PID" + fi + sleep 1 +done +if ! timeout 5 ceph osd stat 2>/dev/null | grep -q '[1-9][0-9]* up'; then + echo "OSD did not become up within 60 seconds." >&2 + exit 1 +fi + +echo "[odorobo-ceph] OSD ready; provisioning RBD" +ceph config set mon mon_allow_pool_size_one true +ceph config set osd osd_pool_default_size 1 +ceph config set osd osd_pool_default_min_size 1 +if ! ceph osd pool ls --format=json | grep -qF "\"$CEPH_POOL\""; then + ceph osd pool create "$CEPH_POOL" 8 +fi +ceph osd pool set "$CEPH_POOL" size 1 --yes-i-really-mean-it +ceph osd pool set "$CEPH_POOL" min_size 1 +rbd pool init "$CEPH_POOL" + +if ! ceph auth get "client.$CEPH_CLIENT" >/dev/null 2>&1; then + ceph auth get-or-create "client.$CEPH_CLIENT" \ + mon 'allow r' osd "allow rwx pool=$CEPH_POOL" >/dev/null +fi +if ! rbd info "$CEPH_POOL/$CEPH_IMAGE_NAME" >/dev/null 2>&1; then + rbd create "$CEPH_POOL/$CEPH_IMAGE_NAME" --size "$CEPH_IMAGE_SIZE" +fi + +ceph config generate-minimal-conf >"$OSD_STATE_DIR/ceph.conf" +ceph auth get-key "client.$CEPH_CLIENT" >"$OSD_STATE_DIR/client.$CEPH_CLIENT.key" +chmod 600 "$OSD_STATE_DIR/client.$CEPH_CLIENT.key" +cp "$OSD_STATE_DIR/ceph.conf" /generated/ceph.conf +cp "$OSD_STATE_DIR/client.$CEPH_CLIENT.key" /generated/client."$CEPH_CLIENT".key +chmod 600 /generated/client."$CEPH_CLIENT".key + +for pid in "$MON_PID" "$OSD_PID"; do + [[ -n "$pid" ]] || continue + wait "$pid" +done diff --git a/.local/dev/compose.yml b/.local/dev/compose.yml new file mode 100644 index 0000000..532055a --- /dev/null +++ b/.local/dev/compose.yml @@ -0,0 +1,69 @@ +services: + ceph: + build: + context: ./ceph + dockerfile: Containerfile + args: + CEPH_IMAGE: ${CEPH_IMAGE:-quay.io/ceph/ceph:v20.2.3} + image: odorobo-ceph:local + container_name: odorobo-ceph + hostname: ceph + privileged: true + # Bootstrap errors must leave the container stopped so `init.sh` can report + # them instead of silently repeating a failed initialization. + restart: "no" + environment: + CEPH_MON_IP: ${CEPH_MON_IP:-127.0.0.1} + CEPH_POOL: ${CEPH_POOL:-odorobo-blockpool} + CEPH_CLIENT: ${CEPH_CLIENT:-odorobo} + CEPH_IMAGE_NAME: ${CEPH_IMAGE_NAME:-dev-disk} + CEPH_IMAGE_SIZE: ${CEPH_IMAGE_SIZE:-1G} + CEPH_OSD_SIZE: ${CEPH_OSD_SIZE:-10G} + volumes: + - ./ceph/state/etc-ceph:/etc/ceph + - ./ceph/state/lib-ceph:/var/lib/ceph + - ./ceph/state/log-ceph:/var/log/ceph + - ./ceph/state/run-ceph:/run/ceph + - ./ceph/state/odorobo-ceph:/var/lib/odorobo-ceph + - ./ceph/generated:/generated + # Direct BlueStore initialization uses a host loop-backed block device. + - /dev:/dev + + healthcheck: + test: ["CMD", "ceph", "-s"] + interval: 5s + timeout: 5s + retries: 30 + + odorobo: + build: + context: ./odorobo + dockerfile: Containerfile + privileged: true + depends_on: + ceph: + condition: service_healthy + # podman-compose supports container namespace targets, but not the + # Compose `service:ceph` shorthand. + network_mode: container:odorobo-ceph + pid: container:odorobo-ceph + volumes: + - ../../:/workspace + - ./ceph/generated:/workspace/.local/dev/ceph/generated:ro + - /run/odorobo:/run/odorobo + - /dev:/dev + # The rbd CLI runs `modprobe rbd` before mapping; the container's own + # /lib/modules won't match the host's running kernel, so bind the host's + # module tree in. modprobe finds rbd, sees it's already loaded on the + # shared kernel, and skips the insmod. + - /lib/modules:/lib/modules:ro + working_dir: /workspace + environment: + CEPH_CONFIG: /workspace/.local/dev/ceph/generated/ceph.conf + CEPH_ID: ${CEPH_CLIENT:-odorobo} + CEPH_KEYFILE: /workspace/.local/dev/ceph/generated/client.${CEPH_CLIENT:-odorobo}.key + CEPH_CLUSTER: ceph + # Release compilation is memory-intensive; keep the default usable on + # development laptops while allowing an override for faster machines. + CARGO_BUILD_JOBS: ${CARGO_BUILD_JOBS:-2} + command: ["cargo", "run", "--release", "-p", "odorobo", "--", "--manager-enabled", "true"] diff --git a/.local/dev/init.sh b/.local/dev/init.sh new file mode 100644 index 0000000..453b480 --- /dev/null +++ b/.local/dev/init.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +cd "$ROOT_DIR" + +if command -v podman >/dev/null 2>&1 && podman compose version >/dev/null 2>&1; then + COMPOSE=(podman compose) + ENGINE=(podman) +elif command -v docker >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; then + COMPOSE=(docker compose) + ENGINE=(docker) +else + echo "Docker Compose or Podman Compose is required." >&2 + exit 1 +fi + + +if [[ -z "$(losetup -f 2>/dev/null)" ]]; then + echo "No free loop device is available. Run: sudo modprobe loop" >&2 + exit 1 +fi + +mkdir -p ceph/generated ceph/state/{etc-ceph,lib-ceph,log-ceph,run-ceph,odorobo-ceph} + +# Start Ceph independently: `odorobo` depends on its health check, and starting +# both at once hides a Ceph bootstrap failure behind Compose's dependency wait. +"${COMPOSE[@]}" up --build -d ceph + +echo "Waiting for Ceph to become healthy..." +for attempt in {1..60}; do + # Inspect the engine directly; the external podman-compose provider's `ps` + # command is not a reliable readiness API. + container_state=$(timeout 5 "${ENGINE[@]}" inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' odorobo-ceph 2>/dev/null || true) + if [[ "$container_state" == running\ healthy* ]]; then + break + elif [[ "$container_state" != running* && -n "$container_state" ]]; then + echo "Ceph stopped before becoming healthy. Recent logs:" >&2 + "${COMPOSE[@]}" logs --tail=200 ceph >&2 || true + exit 1 + fi + + if (( attempt % 10 == 0 )); then + echo "Ceph is still not ready; recent logs:" >&2 + "${COMPOSE[@]}" logs --tail=40 ceph >&2 || true + fi + sleep 2 +done + +if [[ $(timeout 5 "${ENGINE[@]}" inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' odorobo-ceph 2>/dev/null || true) != running\ healthy* ]]; then + echo "Ceph did not become healthy within 120 seconds. Recent logs:" >&2 + "${COMPOSE[@]}" logs --tail=200 ceph >&2 || true + exit 1 +fi + +"${COMPOSE[@]}" up -d odorobo + +cat < /dev/nbd0 +# - rootfs dd'd into that device +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +echo ">> health check" +curl -s http://127.0.0.1:3000/health; echo + +echo ">> POST /vms" +curl -s -X POST http://127.0.0.1:3000/vms \ + -H 'Content-Type: application/json' \ + -d "@$DIR/vm-test.json" +echo + +echo ">> waiting for boot, then dumping console history" +sleep 15 +curl -s http://127.0.0.1:3000/vms/01J0TESTVM0000000000000001/console/history \ + | strings | tail -40 diff --git a/odorobo/src/ch_driver/manifest.rs b/odorobo/src/ch_driver/manifest.rs index 6791b77..5cf4715 100644 --- a/odorobo/src/ch_driver/manifest.rs +++ b/odorobo/src/ch_driver/manifest.rs @@ -72,11 +72,18 @@ pub fn to_vm_config(manifest: &VmManifest) -> Result { ..Default::default() }), payload: PayloadConfig { - firmware: desired - .boot - .firmware - .clone() - .or_else(|| Some("/var/lib/odorobo/CLOUDHV.fd".to_owned())), + firmware: desired.boot.firmware.clone().or_else(|| { + // Only default to the firmware when doing a firmware boot + // (no kernel specified). Direct kernel boot must not set a + // firmware, or Cloud Hypervisor rejects the config with + // "Specifying a kernel is not supported when a firmware is + // provided". + if desired.boot.kernel.is_some() { + None + } else { + Some("/var/lib/odorobo/CLOUDHV.fd".to_owned()) + } + }), kernel: desired.boot.kernel.clone(), cmdline: desired.boot.cmdline.clone(), ..Default::default() @@ -166,6 +173,30 @@ mod tests { ); } + #[test] + fn firmware_default_only_for_firmware_boot() { + // No kernel, no explicit firmware -> defaults to the firmware. + let config = to_vm_config(&minimal()).expect("minimal manifest converts"); + assert_eq!( + config.payload.firmware.as_deref(), + Some("/var/lib/odorobo/CLOUDHV.fd") + ); + + // Kernel set, no explicit firmware -> no firmware (direct kernel boot). + let mut manifest = minimal(); + manifest.desired.boot.kernel = Some("/tmp/vmlinuz".to_owned()); + let config = to_vm_config(&manifest).expect("kernel manifest converts"); + assert_eq!(config.payload.firmware, None); + assert_eq!(config.payload.kernel.as_deref(), Some("/tmp/vmlinuz")); + + // Explicit firmware wins even when a kernel is also set. + let mut manifest = minimal(); + manifest.desired.boot.kernel = Some("/tmp/vmlinuz".to_owned()); + manifest.desired.boot.firmware = Some("/custom/fw.fd".to_owned()); + let config = to_vm_config(&manifest).expect("explicit firmware manifest converts"); + assert_eq!(config.payload.firmware.as_deref(), Some("/custom/fw.fd")); + } + #[test] fn converts_networks_for_the_transform_pipeline() { let mut manifest = minimal(); diff --git a/odorobo/src/ch_driver/transform/storage/rbd.rs b/odorobo/src/ch_driver/transform/storage/rbd.rs index 00fd499..1424c6b 100644 --- a/odorobo/src/ch_driver/transform/storage/rbd.rs +++ b/odorobo/src/ch_driver/transform/storage/rbd.rs @@ -112,6 +112,8 @@ impl RbdImage { .args(rbd_extra_args()) .arg("device") .arg("map") + .arg("--options") + .arg("noudev") .arg(&rbd_path) .output() .await @@ -132,6 +134,8 @@ impl RbdImage { .args(rbd_extra_args()) .arg("device") .arg("unmap") + .arg("--options") + .arg("noudev") .arg(&rbd_path) .output() .await