Requirements

nvidia.com/GH100_H100_SXM5_80GB = 8
mellanox.com/cx7_ib_pf          = 8
nvidia.com/gpu                  = 0   # expected: the worker requests the passthrough name, not nvidia.com/gpu

spec:
  configuration:
    permittedHostDevices:
      pciHostDevices:
        - pciVendorSelector: "10DE:2330"          # NVIDIA H100 SXM5 80GB
          resourceName: nvidia.com/GH100_H100_SXM5_80GB
          externalResourceProvider: true          # advertised by the KubeVirt GPU device plugin
        - pciVendorSelector: "15B3:1021"          # Mellanox ConnectX-7 IB PF
          resourceName: mellanox.com/cx7_ib_pf
          externalResourceProvider: false

kubectl -n <workload-namespace> create secret generic ghcr-pull-cred \
  --from-literal=accessKeyId='<github-username>' \
  --from-literal=secretKey='<github-PAT>' \
  --from-literal=DOCKER_REGISTRY='ghcr.io' \
  --from-literal=DOCKER_USERNAME='<github-username>' \
  --from-literal=DOCKER_TOKEN='<github-PAT>'

kubectl apply -f setup/cfg-create.yml

apiVersion: v1
kind: ConfigMap
metadata:
  name: bootstrap-cm
  namespace: <workload-namespace>   # e.g. cosmic-ac

data:
  docker-bootstrap.sh: |
    #!/bin/bash
    set -eu

    # Helper: wait for block device
    wait_for_dev() { for i in {1..60}; do [ -b "$1" ] && return 0; sleep 1; done; echo "ERROR: $1 not found"; lsblk -b; exit 1; }

    # Wait out cloud-init / unattended-upgrades / multi-node IB apt holding dpkg.
    for _ in $(seq 1 300); do
      fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || break
      sleep 1
    done
    for _ in 1 2 3 4 5; do
      DEBIAN_FRONTEND=noninteractive apt-get update -y \
        && DEBIAN_FRONTEND=noninteractive apt-get install -y curl ca-certificates docker.io parted e2fsprogs util-linux jq \
        && break
      sleep 10
    done
    dpkg -s docker.io >/dev/null 2>&1 || { echo "ERROR: apt install failed"; exit 1; }
    systemctl enable --now docker

    # Setup PVC disk
    ROOT_DISK="$(findmnt -n -o SOURCE / | sed 's/[0-9]*$//' | xargs basename)"
    [ -f /etc/default/docker-bootstrap ] && source /etc/default/docker-bootstrap
    MNT="${MNT:-}"
    DISK_SERIAL="${DISK_SERIAL:-}"
    ENV_FILE="/etc/default/docker-bootstrap.env"

    if [ -n "${MNT}" ]; then
      # A team storage volume is addressed by its virtio serial. Size alone cannot
      # identify it, and picking the wrong disk would reformat somebody's data.
      if [ -n "${DISK_SERIAL}" ]; then
        BY_ID="/dev/disk/by-id/virtio-${DISK_SERIAL}"
        wait_for_dev "${BY_ID}"
        DISK="$(readlink -f "${BY_ID}")"
        [ -b "${DISK}" ] || { echo "ERROR: ${BY_ID} did not resolve to a block device"; lsblk -b; exit 1; }
      else
        # Legacy instances created before DISK_SERIAL: take the largest non-root
        # disk. A provisioner may hand back slightly less than the requested size
        # (KubeVirt itself tolerates up to 10 %), so no absolute floor is applied
        # beyond ruling out the cloud-init disk.
        PVC_DISK_NAME="$(lsblk -b -dn -o NAME,TYPE,SIZE | awk -v r="${ROOT_DISK}" '$2=="disk" && $1!=r && $1!~/^(sr|loop)/ && $3>=1073741824 {print $1,$3}' | sort -k2,2nr | head -n1 | awk '{print $1}')"
        [ -z "${PVC_DISK_NAME}" ] && { echo "ERROR: Could not find a data disk to mount at ${MNT}"; lsblk -b; exit 1; }
        DISK="/dev/${PVC_DISK_NAME}"
      fi

      PART="${DISK}1"
      wait_for_dev "${DISK}"

      # A reattached volume already carries its GPT label and ext4 filesystem, so
      # both of these are skipped and the existing data is preserved.
      [ ! -b "${PART}" ] && { parted -s "${DISK}" mklabel gpt mkpart primary ext4 1MiB 100%; udevadm settle || true; }
      wait_for_dev "${PART}"
      blkid "${PART}" | grep -q 'TYPE=' || mkfs.ext4 -F "${PART}"

      mkdir -p "${MNT}"
      UUID="$(blkid -s UUID -o value "${PART}")"
      grep -q "${UUID}" /etc/fstab || echo "UUID=${UUID} ${MNT} ext4 defaults,nofail 0 2" >> /etc/fstab
      mount -a
    else
      echo "MNT not set, skipping external disk setup"
    fi

    # Docker login and pull
    SECRET_MNT="/mnt/registrycred"
    mkdir -p "${SECRET_MNT}"
    if ! mountpoint -q "${SECRET_MNT}"; then
      wait_for_dev /dev/sr0
      mount -t iso9660 -o ro /dev/sr0 "${SECRET_MNT}"
    else
      echo "${SECRET_MNT} already mounted, reusing"
    fi

    REGISTRY="$(cat "${SECRET_MNT}/DOCKER_REGISTRY")"
    USERNAME="$(cat "${SECRET_MNT}/DOCKER_USERNAME")"
    DOCKER_CONFIG="/run/docker-config"
    mkdir -p "${DOCKER_CONFIG}" && chmod 700 "${DOCKER_CONFIG}"

    cat "${SECRET_MNT}/DOCKER_TOKEN" | docker login "${REGISTRY}" -u "${USERNAME}" --password-stdin

    IMAGE_FULL="${IMG:-${USERNAME}/cosmicac-wrk-agent-instance:latest}"
    NESTED_IMG="${NESTED_IMG:-}"

    # Retry each pull up to 3 times (transient registry/network failures)
    pull_with_retry() {
      local img="$1" attempt=1
      until docker pull "${img}"; do
        if [ "${attempt}" -ge 3 ]; then
          echo "ERROR: docker pull failed for ${img} after ${attempt} attempts"
          return 1
        fi
        echo "WARN: docker pull failed for ${img} (attempt ${attempt}/3), retrying..."
        attempt=$((attempt + 1))
        sleep $((attempt * 2))
      done
    }

    # Pull agent + nested images in parallel (each with up to 3 retries)
    IMAGES=("${IMAGE_FULL}")
    [ -n "${NESTED_IMG}" ] && [ "${NESTED_IMG}" != "${IMAGE_FULL}" ] && IMAGES+=("${NESTED_IMG}")

    PIDS=()
    for img in "${IMAGES[@]}"; do
      pull_with_retry "${img}" & PIDS+=($!)
    done
    ec=0; for p in "${PIDS[@]}"; do wait "$p" || ec=1; done
    [ "$ec" -eq 0 ] || { echo "ERROR: image pull failed"; exit 1; }

    docker logout "${REGISTRY}" || true
    rm -rf "${DOCKER_CONFIG}" && umount "${SECRET_MNT}" || true

    # Run container if not already running
    CONTAINER_NAME="${NAME:-cosmicac-container}"
    if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
      echo "Container ${CONTAINER_NAME} already running"
    elif docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
      # Created/Exited from a prior failed start — start it (systemd Restart retries if this fails).
      docker start "${CONTAINER_NAME}"
    elif [ -n "${IMG}" ]; then
      # Host agent FIFO lives under /var/run/cosmicac (bind-mounted into agent).
      mkdir -p /var/run/cosmicac/host-agent
      if [ -x /usr/local/bin/cosmicac-host-net-agent.sh ]; then
        systemctl daemon-reload 2>/dev/null || true
        # Overlay iface (if any) is discovered into /etc/default/cosmicac-overlay by cloud-init.
        systemctl enable --now cosmicac-host-net-agent.service 2>/dev/null \
          || nohup env HOST_AGENT_DIR=/var/run/cosmicac/host-agent /usr/local/bin/cosmicac-host-net-agent.sh >/var/log/cosmicac-host-net-agent.log 2>&1 &
        # Wait briefly so cmd_in.pipe exists before the agent container starts.
        for i in $(seq 1 40); do
          [ -p /var/run/cosmicac/host-agent/cmd_in.pipe ] && break
          sleep 0.25
        done
      fi

      # Inference: nested container gets GPUs; --gpus all on agent can leave it Created.
      BASE_CMD="docker run -d --name ${CONTAINER_NAME} --init --restart unless-stopped --ipc=host"
      [ -z "${NESTED_IMG:-}" ] && BASE_CMD="${BASE_CMD} --gpus all"
      [ -n "${MNT}" ] && BASE_CMD="${BASE_CMD} -v ${MNT}:${MNT}"
      # Nested docker (INFERENCE_MODE=docker) needs host docker socket + host-agent FIFOs.
      # OVERLAY_IFACE is appended to the env file by cosmicac-setup-overlay-nic.sh on multi-node guests.
      if [ -n "${NESTED_IMG}" ]; then
        mkdir -p /var/run/cosmicac/host-agent
        BASE_CMD="${BASE_CMD} --network host -v /var/run/docker.sock:/var/run/docker.sock -v /var/run/cosmicac:/var/run/cosmicac -e HOST_AGENT_DIR=/var/run/cosmicac/host-agent"
      fi

      if [ "${JOB_TYPE:-}" = "GPU_CONTAINER" ]; then
        BASE_CMD="${BASE_CMD} --privileged --cap-add=NET_ADMIN --cap-add=SYS_ADMIN --device /dev/net/tun"
        # Host-agent FIFOs only; GPU_CONTAINER keeps bridge networking and no docker socket.
        BASE_CMD="${BASE_CMD} -v /var/run/cosmicac:/var/run/cosmicac -e HOST_AGENT_DIR=/var/run/cosmicac/host-agent"
      fi

      [ -f "${ENV_FILE}" ] && BASE_CMD="${BASE_CMD} --env-file ${ENV_FILE}"

      eval "${BASE_CMD} ${IMG}"
    fi
    # Still Created/Exited → fail unit so Restart=on-failure re-enters the docker start branch.
    if [ -n "${IMG:-}" ] && ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
      echo "ERROR: ${CONTAINER_NAME} not running"; docker ps -a --filter "name=^${CONTAINER_NAME}$" || true; exit 1
    fi

    echo "Bootstrap completed successfully."
  cosmicac-host-net-agent.sh: |
    #!/bin/bash
    # CosmicAC host agent — runs on the VMI host (NOT in the agent container).
    # Container talks over FIFO: <op>|<reply_fifo>[|<arg>]
    # ops: ping | ip | ib | gpu | cpu
    set -eu
    D="${HOST_AGENT_DIR:-/var/run/cosmicac/host-agent}"
    C="${D}/cmd_in.pipe"
    IB_SYS=/sys/class/infiniband
    GPU_QUERY=index,uuid,utilization.gpu,utilization.memory,memory.used,memory.total,temperature.gpu,power.draw
    PROC_STAT="${HOST_AGENT_PROC_STAT:-/proc/stat}"
    # Gap between the two /proc/stat reads op_cpu diffs.
    CPU_SAMPLE_SEC="${HOST_AGENT_CPU_SAMPLE_SEC:-0.25}"
    T="${HOST_AGENT_TIMEOUT_SEC:-8}"
    # How long op_ip waits for DHCP on the overlay NIC (agent boots right after netplan).
    IP_WAIT_SEC="${HOST_AGENT_IP_WAIT_SEC:-30}"
    # Cap concurrent handle() subshells so a flood of FIFO lines cannot leak jobs.
    MAX_JOBS="${HOST_AGENT_MAX_JOBS:-8}"

    # OVERLAY_IFACE env → /etc/default/cosmicac-overlay → non-default-route NIC (prefer one with IPv4).
    resolve_overlay_iface () {
      [ -n "${OVERLAY_IFACE:-}" ] && [ "$OVERLAY_IFACE" != auto ] && { echo "$OVERLAY_IFACE"; return; }
      if [ -f /etc/default/cosmicac-overlay ]; then
        # shellcheck disable=SC1091
        . /etc/default/cosmicac-overlay
      fi
      [ -n "${OVERLAY_IFACE:-}" ] && [ "$OVERLAY_IFACE" != auto ] && { echo "$OVERLAY_IFACE"; return; }

      local p i cands=()
      p=$(ip -4 route show default 2>/dev/null | awk '{print $5; exit}' || true)
      shopt -s nullglob
      for i in /sys/class/net/*/device; do
        i=$(basename "$(dirname "$i")")
        case $i in lo|wg*|docker*|br-*|veth*|virbr*|tun*|tap*) continue;; esac
        [ "$i" = "${p:-}" ] && continue
        cands+=("$i")
      done
      for i in "${cands[@]+"${cands[@]}"}"; do
        ip -4 -o addr show dev "$i" 2>/dev/null | grep -q . && { echo "$i"; return; }
      done
      [ "${#cands[@]}" -gt 0 ] && { echo "${cands[0]}"; return; }
      return 1
    }

    mkdir -p "$D" && chmod 0777 "$D"
    rm -f "$C" && mkfifo "$C" && chmod 666 "$C"

    log () { printf '%s %s\n' "$(date -Is)" "$*" >&2; }
    # JSON-encode a string (includes quotes; jq escapes controls like \r/\n/\t).
    jesc () { printf '%s' "${1:-}" | jq -Rs .; }
    ok () { printf '{"ok":true,"op":"%s","data":%s}\n' "$1" "$2"; }
    fail () { printf '{"ok":false,"op":"%s","error":%s}\n' "$1" "$(jesc "$2")"; }

    op_ping () { ok ping '{}'; }

    op_ip () {
      local iface="${1:-}" ip n=0
      [ -z "$iface" ] && iface="$(resolve_overlay_iface || true)"
      [ -n "${iface:-}" ] || { fail ip 'no overlay iface'; return; }
      while [ "$n" -lt "$IP_WAIT_SEC" ]; do
        ip="$(timeout "$T" ip -4 -o addr show dev "$iface" 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -n1 || true)"
        [ -n "${ip:-}" ] && break
        sleep 1
        n=$((n + 1))
      done
      [ -n "${ip:-}" ] && ok ip "{\"ip\":\"$ip\",\"iface\":$(jesc "$iface")}" || fail ip "no ipv4 on ${iface}"
    }

    op_ib () {
      [ -d "$IB_SYS" ] || { ok ib '{"present":false,"hcaPrefix":null,"hcas":[]}'; return; }
      local names=() n p=null
      for n in "$IB_SYS"/*; do [ -e "$n" ] && names+=("$(basename "$n")"); done
      for n in "${names[@]+"${names[@]}"}"; do case "$n" in ibp*) p='"ibp"'; break;; esac; done
      [ "$p" = null ] && for n in "${names[@]+"${names[@]}"}"; do case "$n" in mlx5_*) p='"mlx5"'; break;; esac; done
      local arr='[' f=1
      for n in "${names[@]+"${names[@]}"}"; do [ $f -eq 1 ] || arr+=','; f=0; arr+="\"$n\""; done
      arr+=']'
      ok ib "{\"present\":$([ ${#names[@]} -gt 0 ] && echo true || echo false),\"hcaPrefix\":$p,\"hcas\":$arr}"
    }

    op_cpu () {
      local a b pct
      [ -r "$PROC_STAT" ] || { fail cpu "cannot read ${PROC_STAT}"; return; }
      a="$(awk '/^cpu /{print; exit}' "$PROC_STAT")"
      sleep "$CPU_SAMPLE_SEC"
      b="$(awk '/^cpu /{print; exit}' "$PROC_STAT")"
      pct="$(printf '%s\n%s\n' "$a" "$b" | awk '
        NR==1 { for (i = 2; i <= 9 && i <= NF; i++) t1 += $i; i1 = $5 + $6 }
        NR==2 { for (i = 2; i <= 9 && i <= NF; i++) t2 += $i; i2 = $5 + $6 }
        END {
          dt = t2 - t1; di = i2 - i1
          if (dt <= 0) exit 1
          p = (dt - di) / dt * 100
          if (p < 0) p = 0
          if (p > 100) p = 100
          printf "%.2f", p
        }')" || { fail cpu 'no cpu time elapsed between samples'; return; }
      [ -n "${pct:-}" ] || { fail cpu 'cpu sample unavailable'; return; }
      ok cpu "{\"percent\":${pct},\"cores\":$(nproc 2>/dev/null || echo null)}"
    }

    op_gpu () {
      local csv
      command -v nvidia-smi >/dev/null 2>&1 || { fail gpu 'nvidia-smi not found on host'; return; }
      csv="$(timeout "$T" nvidia-smi --query-gpu="$GPU_QUERY" --format=csv,noheader,nounits 2>&1 || true)"
      [ -n "${csv:-}" ] || { fail gpu 'no nvidia-smi output'; return; }
      ok gpu "{\"csv\":$(jesc "$csv")}"
    }

    handle () {
      local line="$1" op reply arg i=0 base
      IFS='|' read -r op reply arg <<<"$line" || true
      [ -n "${op:-}" ] && [ -n "${reply:-}" ] || return 0
      # Reply FIFO must be a direct child of $D named resp_*.pipe (no ../ or nested paths).
      base="$(basename -- "$reply")"
      case "$base" in resp_*.pipe) ;; *) log "reject $reply"; return 0 ;; esac
      [ "$reply" = "${D}/${base}" ] || { log "reject $reply"; return 0; }
      while [ ! -p "$reply" ] && [ "$i" -lt 50 ]; do sleep 0.05; i=$((i + 1)); done
      [ -p "$reply" ] || { log "missing $reply"; return 0; }
      {
        case "$op" in
          ping) op_ping ;;
          ip) op_ip "${arg:-}" ;;
          ib) op_ib ;;
          gpu) op_gpu ;;
          cpu) op_cpu ;;
          *) fail "$op" 'unknown op' ;;
        esac
      } >"$reply" 2>/dev/null || true
      rm -f "$reply"
    }

    running_jobs () { jobs -pr 2>/dev/null | grep -c . || true; }

    # Reap exited children so the jobs table does not grow unbounded.
    reap_finished () {
      local pid running
      running=" $(jobs -pr 2>/dev/null | tr '\n' ' ') "
      for pid in $(jobs -p 2>/dev/null); do
        case "$running" in
          *" $pid "*) ;;
          *) wait "$pid" 2>/dev/null || true ;;
        esac
      done
    }

    wait_for_slot () {
      reap_finished
      while [ "$(running_jobs)" -ge "$MAX_JOBS" ]; do
        wait -n 2>/dev/null || { sleep 0.05; wait 2>/dev/null || true; }
        reap_finished
      done
    }

    log "listening on $C (max ${MAX_JOBS} concurrent handlers)"
    while IFS= read -r line || [ -n "${line:-}" ]; do
      [ -z "${line:-}" ] && continue
      wait_for_slot
      ( handle "$line" ) &
    done <>"$C"
  cosmicac-host-net-agent.service: |
    [Unit]
    Description=CosmicAC host agent (FIFO IPC for inference container)
    After=network-online.target
    Wants=network-online.target
    Before=docker-bootstrap.service

    [Service]
    Type=simple
    Environment=HOST_AGENT_DIR=/var/run/cosmicac/host-agent
    EnvironmentFile=-/etc/default/cosmicac-overlay
    ExecStart=/usr/local/bin/cosmicac-host-net-agent.sh
    Restart=always
    RestartSec=2

    [Install]
    WantedBy=multi-user.target
  docker-bootstrap.service: |
    [Unit]
    Description=Deferred Docker Bootstrap (login + pull + run)
    Wants=network-online.target cosmicac-host-net-agent.service
    After=network-online.target multi-user.target cosmicac-host-net-agent.service apt-daily.service apt-daily-upgrade.service
    StartLimitIntervalSec=600
    StartLimitBurst=3

    [Service]
    Type=oneshot
    ExecStart=/usr/local/bin/docker-bootstrap.sh
    RemainAfterExit=yes
    Restart=on-failure
    RestartSec=20

    [Install]
    WantedBy=multi-user.target

kubectl get configmap bootstrap-cm -n <workload-namespace>   # must exist

# Nodes Ready + control plane healthy; k8s version supported by KubeVirt/CDI
kubectl get nodes -o wide
kubectl version -o json | jq -r '.serverVersion.gitVersion'

# Per GPU node: GPU (and IB) advertised under the name the worker requests
kubectl get nodes -o json | jq -r '
  .items[] | .metadata.name + "  gpu=" +
  (.status.allocatable["nvidia.com/GH100_H100_SXM5_80GB"] // "0") + "  ib=" +
  (.status.allocatable["mellanox.com/cx7_ib_pf"] // "0")'

# KubeVirt + CDI healthy; HostDevices feature gate on
kubectl -n kubevirt get kubevirt kubevirt -o jsonpath='{.status.observedKubeVirtVersion}{"\n"}'
kubectl -n kubevirt get kubevirt kubevirt -o jsonpath='{.spec.configuration.developerConfiguration.featureGates}{"\n"}'  # includes HostDevices
kubectl -n cdi get cdi cdi -o jsonpath='{.status.observedVersion}{"\n"}'

# KubeVirt GPU device plugin advertising the passthrough resource
kubectl -n kube-system get ds nvidia-kubevirt-gpu-dp-daemonset

# permittedHostDevices includes your GPU/IB
kubectl -n kubevirt get kubevirt kubevirt -o jsonpath='{.spec.configuration.permittedHostDevices}{"\n"}' | jq .

# A working local-path StorageClass exists (need not be the default)
kubectl get sc

# Workload-namespace prerequisites
kubectl -n <workload-namespace> get secret ghcr-pull-cred
kubectl -n <workload-namespace> get configmap bootstrap-cm

# Device-admission smoke test: a VMI that CLAIMS a full GPU is admitted + scheduled.
# NOTE: with no boot disk this only proves device admission + scheduling, NOT a
# working guest. For a real boot test, add a DataVolume root disk.
cat <<'EOF' | kubectl apply -f -
apiVersion: kubevirt.io/v1
kind: VirtualMachineInstance
metadata:
  name: gpu-smoke
  namespace: default
spec:
  domain:
    devices:
      gpus:
        - deviceName: nvidia.com/GH100_H100_SXM5_80GB
          name: gpu0
    resources:
      requests: { memory: 8Gi }
  volumes: []
EOF
kubectl get vmi gpu-smoke -w     # reaches Scheduled/Running (device admitted), then:
kubectl delete vmi gpu-smoke

# Multus / Whereabouts / OVS daemonsets Ready in kube-system
kubectl -n kube-system get ds | grep -iE 'multus|whereabouts|ovs'

# NAD CRD present
kubectl get crd network-attachment-definitions.k8s.cni.cncf.io

# Subcluster manager SA + RBAC present
kubectl -n kube-system get sa subcluster-manager-sa
kubectl get clusterrole subcluster-manager-role
kubectl get clusterrolebinding subcluster-manager-binding

# LoadBalancer can allocate an external IP for UDP services (provider-specific)