diff --git a/Makefile b/Makefile index 8b93389c1..6205f2a87 100644 --- a/Makefile +++ b/Makefile @@ -6,10 +6,16 @@ TAR_REPRODUCIBLE = tar --mtime="@${SOURCE_DATE_EPOCH}" --sort=name --owner=0 --g all: bin manpages +# bcvk is bind-mounted and ran inside the container it starts. We build it statically +# so that it does not depend on the image's loader or its libc. An explicit --target +# is required because proc-macros cannot be built statically. +CARGO_BUILD_TARGET := $(shell rustc -vV | sed -n 's/^host: //p') + .PHONY: bin bin: cargo check --workspace - cargo build --release + RUSTFLAGS="-C target-feature=+crt-static" cargo build --release --target $(CARGO_BUILD_TARGET) + install -D -m755 target/$(CARGO_BUILD_TARGET)/release/bcvk target/release/bcvk # Generate man pages from markdown sources MAN8_SOURCES := $(wildcard docs/src/man/*.md) diff --git a/crates/kit/scripts/entrypoint.sh b/crates/kit/scripts/entrypoint.sh deleted file mode 100644 index 06474ff2f..000000000 --- a/crates/kit/scripts/entrypoint.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -set -euo pipefail - -SELFEXE=/run/selfexe - -# Check for required binaries early -if ! command -v bwrap &>/dev/null; then - echo "Error: bwrap (bubblewrap) is currently required in the target container image" >&2 - exit 1 -fi - -# Shell script library -init_tmproot() { - if test -d /run/inner-shared; then return 0; fi - # Should have been created by podman when initializing - # the bind mount - cd /run/tmproot - - # Create essential symlinks - ln -sf usr/bin bin - ln -sf usr/lib lib - ln -sf usr/lib64 lib64 - ln -sf usr/sbin sbin - mkdir -p {etc,var,dev,proc,run,sys,tmp} - # Ensure we have /etc/passwd as ssh-keygen wants it for bad reasons - systemd-sysusers --root $(pwd) &>/dev/null - - # Copy DNS configuration from container's /etc/resolv.conf (configured by podman --dns) - # into the bwrap namespace so QEMU's slirp can use it for DNS resolution - if [ -f /etc/resolv.conf ]; then - cp /etc/resolv.conf /run/tmproot/etc/resolv.conf - fi - - # Shared directory between containers - mkdir /run/inner-shared -} - -BWRAP_ARGS=( - --bind /run/tmproot / - --proc /proc - --dev-bind /dev /dev - --bind /var/tmp /var/tmp - --tmpfs /run - --tmpfs /tmp - --bind /run/inner-shared /run/inner-shared -) - -# Pass ALL arguments to container-entrypoint -# Default to "run-ephemeral" if no args -if [[ $# -eq 0 ]]; then - set -- "run-ephemeral" - # Initialize environment - init_tmproot -else - # Other commands should wait for the other process - # to create the temp root - while test '!' -d /run/inner-shared; do sleep 0.1; done -fi - -# Check systemd version from the container image (not host) -export SYSTEMD_VERSION=$(systemctl --version 2>/dev/null) - -# Execute with proper environment passing -# Set up signal handlers that will cleanly exit on INT or TERM -trap 'kill -TERM $BWRAP_PID 2>/dev/null; exit 0' INT TERM - -# Run bwrap in background so we can handle signals; xref -# https://github.com/containers/bubblewrap/pull/586 -# But probably really we should switch to systemd -bwrap --as-pid-1 --unshare-pid "${BWRAP_ARGS[@]}" --bind /run /run -- ${SELFEXE} container-entrypoint "$@" & -BWRAP_PID=$! - -# Wait for bwrap to complete -wait $BWRAP_PID -EXIT_CODE=$? - -# Exit with the same code as bwrap -exit $EXIT_CODE diff --git a/crates/kit/src/main.rs b/crates/kit/src/main.rs index a593feddb..43c30d3b5 100644 --- a/crates/kit/src/main.rs +++ b/crates/kit/src/main.rs @@ -47,6 +47,8 @@ mod run_ephemeral; #[cfg(target_os = "linux")] mod run_ephemeral_ssh; #[cfg(target_os = "linux")] +mod sandbox; +#[cfg(target_os = "linux")] mod ssh; #[cfg(target_os = "linux")] mod status_monitor; @@ -272,6 +274,16 @@ fn main() -> Result<(), Report> { let cli = Cli::parse(); + #[cfg(target_os = "linux")] + if let Commands::ContainerEntrypoint(opts) = &cli.command { + if matches!( + opts.command, + container_entrypoint::ContainerCommands::RunEphemeral + ) { + sandbox::setup()?; + } + } + #[cfg(target_os = "linux")] let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() diff --git a/crates/kit/src/run_ephemeral.rs b/crates/kit/src/run_ephemeral.rs index aa7ad7657..6765bbcb1 100644 --- a/crates/kit/src/run_ephemeral.rs +++ b/crates/kit/src/run_ephemeral.rs @@ -15,7 +15,7 @@ //! The execution follows this chain: //! 1. **Host Process**: `bcvk run-ephemeral` invoked on host //! 2. **Container Launch**: Podman privileged container with KVM and host mounts -//! 3. **Namespace Setup**: bwrap creates isolated namespace with hybrid rootfs +//! 3. **Namespace Setup**: the entrypoint changes root to the hybrid rootfs //! 4. **Binary Re-execution**: Same binary re-executes with `container-entrypoint` //! 5. **VM Launch**: QEMU starts with VirtioFS root and additional mounts //! @@ -28,11 +28,10 @@ //! - `/run/selfexe`: The bcvk binary itself (for re-execution) //! - `/run/source-image`: Target container image via `--mount=type=image` //! - `/run/hostusr`: Host `/usr` directory (read-only, for QEMU/tools) -//! - `/var/lib/bcvk/entrypoint`: Embedded entrypoint.sh script //! - Handles real-time output streaming for `--execute` commands //! -//! ### Phase 2: Hybrid Rootfs Creation (entrypoint.sh) -//! The entrypoint script creates a hybrid root filesystem at `/run/tmproot`: +//! ### Phase 2: Hybrid Rootfs Creation (`sandbox`) +//! The container entrypoint creates a hybrid root filesystem at `/run/tmproot`: //! ```text //! /run/tmproot/ //! ├── usr/ → bind mount to /run/hostusr (host binaries) @@ -41,12 +40,12 @@ //! └── [other dirs created empty for container compatibility] //! ``` //! -//! ### Phase 3: Namespace Isolation (bwrap) -//! Uses bubblewrap to create isolated namespace: +//! ### Phase 3: Namespace Isolation (`sandbox`) +//! Unshares a mount namespace and changes root to the hybrid root: //! - New mount namespace with `/run/tmproot` as root //! - Shared `/run/inner-shared` for virtiofsd socket communication -//! - Proper `/proc`, `/dev`, `/tmp` mounts -//! - Re-executes binary: `bwrap ... -- /run/selfexe container-entrypoint` +//! - Proper `/proc` and `/dev` mounts +//! - Runs as `/run/selfexe container-entrypoint` //! //! ### Phase 4: VM Execution (`run_impl`) //! - Runs inside the container after namespace setup @@ -88,7 +87,7 @@ //! - Ensures perfect fidelity of user options across process boundaries use std::fs::File; -use std::io::{BufWriter, IsTerminal, Seek, Write}; +use std::io::{IsTerminal, Seek}; use std::os::unix::process::CommandExt; use std::process::{Command, Stdio}; @@ -104,7 +103,7 @@ use serde::{Deserialize, Serialize}; use tokio::io::AsyncReadExt; use tracing::{debug, warn}; -const ENTRYPOINT: &str = "/var/lib/bcvk/entrypoint"; +pub(crate) const SELFEXE: &str = "/run/selfexe"; /// Get default vCPU count (number of available processors, or 2 as fallback) pub fn default_vcpus() -> u32 { @@ -630,10 +629,7 @@ fn read_host_dns_servers() -> Option> { /// Launch privileged container with QEMU+KVM for ephemeral VM, spawning as subprocess. /// Returns the container ID instead of executing the command. pub fn run_detached(opts: RunEphemeralOpts) -> Result { - let (mut cmd, temp_dir, _journal_fds) = prepare_run_command_with_temp(opts)?; - - // Leak the tempdir to keep it alive for the entire container lifetime. - std::mem::forget(temp_dir); + let (mut cmd, _journal_fds) = prepare_run_command(opts)?; debug!("Podman command: {:?}", cmd); let output = cmd.output().context("Failed to execute podman command")?; @@ -661,23 +657,22 @@ pub fn run(opts: RunEphemeralOpts) -> Result<()> { } } - let (mut cmd, _temp_dir, _journal_fds) = prepare_run_command_with_temp(opts)?; + let (mut cmd, _journal_fds) = prepare_run_command(opts)?; - // Keep _temp_dir and _journal_fds alive until exec replaces our process. + // Keep _journal_fds alive until exec replaces our process. // The journal fds (if any) are inherited across execve and reach podman // via --preserve-fd; podman in turn passes them into the container. return Err(cmd.exec()).context("execve"); } -/// Returns `(cmd, tempdir, journal_fds)` where `journal_fds` holds open file +/// Returns `(cmd, journal_fds)` where `journal_fds` holds open file /// descriptors for `journal.json` and `journal-initrd.json` (when /// `--log-dir=journal=…` was requested). The caller must keep them alive until /// podman exits so the fds are not closed prematurely. -fn prepare_run_command_with_temp( +fn prepare_run_command( opts: RunEphemeralOpts, ) -> Result<( std::process::Command, - tempfile::TempDir, Vec>, )> { debug!("Running QEMU inside hybrid container for {}", opts.image); @@ -693,22 +688,6 @@ fn prepare_run_command_with_temp( debug!("Image {} supports Ignition", opts.image); } - let script = include_str!("../scripts/entrypoint.sh"); - - let td = tempfile::tempdir()?; - let td_path = td.path().to_str().unwrap(); - - let entrypoint_path = &format!("{}/entrypoint", td_path); - { - let f = File::create(entrypoint_path)?; - let mut f = BufWriter::new(f); - f.write_all(script.as_bytes())?; - use std::{fs::Permissions, os::unix::fs::PermissionsExt}; - let f = f.into_inner()?; - let perms = Permissions::from_mode(0o755); - f.set_permissions(perms)?; - } - let self_exe = std::env::current_exe()?; let self_exe = self_exe.as_str()?; @@ -847,9 +826,7 @@ fn prepare_run_command_with_temp( // library locations in the mounted /usr "/etc/ld.so.cache:/run/tmproot/etc/ld.so.cache:ro", "-v", - &format!("{}:{}", entrypoint_path, ENTRYPOINT), - "-v", - &format!("{self_exe}:/run/selfexe:ro"), + &format!("{self_exe}:{SELFEXE}:ro"), // Since we run as init by default "--stop-signal=SIGKILL", // And bind mount in the pristine image (without any mounts on top) @@ -1022,10 +999,14 @@ fn prepare_run_command_with_temp( cmd.args(["-e", &format!("BOOTC_DISK_FILES={}", disk_specs)]); } - let entrypoint = opts.debug_entrypoint.as_deref().unwrap_or(ENTRYPOINT); - cmd.args(["--", &opts.image, entrypoint]); + cmd.args(["--", &opts.image]); + if let Some(entrypoint) = opts.debug_entrypoint.as_deref() { + cmd.arg(entrypoint); + } else { + cmd.args([SELFEXE, "container-entrypoint", "run-ephemeral"]); + } - Ok((cmd, td, journal_fds)) + Ok((cmd, journal_fds)) } /// Process --mount-disk-file specs: parse file:name format, create sparse files if needed (2x image size), @@ -1208,8 +1189,6 @@ fn parse_service_exit_code(status_content: &str) -> Result { fn check_required_container_binaries() -> Result<()> { // systemctl: used for checking cloud-init and other systemd operations // objcopy: for UKI kernel extraction (when using UKI images) - // NOTE: bwrap is checked earlier in entrypoint.sh, not here, because by the - // time run_impl() executes we're already inside the bwrap namespace let required_binaries = ["systemctl", "objcopy"]; let mut missing = Vec::new(); @@ -1299,14 +1278,11 @@ pub(crate) async fn run_impl(opts: RunEphemeralOpts) -> Result<()> { let status_writer = StatusWriter::new("/run/supervisor-status.json"); status_writer.update_state(SupervisorState::WaitingForSystemd)?; - // Check systemd version from the container image - let systemd_version = { - Some(std::env::var("SYSTEMD_VERSION")?) - .filter(|v| !v.is_empty()) - .as_deref() - .map(systemd::SystemdVersion::from_version_output) - .transpose()? - }; + // Read before the root change, so this is the container image's systemd, not + // the host's. + let systemd_version = crate::sandbox::systemd_version() + .map(systemd::SystemdVersion::from_version_output) + .transpose()?; debug!("Container image systemd version: {systemd_version:?}"); // Check if we need to handle cloud-init @@ -1833,7 +1809,7 @@ Options= // Check if disk file exists and is accessible if !Utf8Path::new(&disk_file).exists() { return Err(eyre!( - "Disk file does not exist in bwrap namespace: {} (serial: {})", + "Disk file does not exist in the container: {} (serial: {})", disk_file, serial )); @@ -1987,7 +1963,7 @@ Options= }); } - // DNS is configured via podman --dns flags (see prepare_run_command_with_temp) + // DNS is configured via podman --dns flags (see prepare_run_command) // This fixes DNS resolution issues when QEMU runs inside containers. // QEMU's slirp reads /etc/resolv.conf from the container's network namespace, // and podman properly sets it up using --dns instead of relying on bridge DNS. diff --git a/crates/kit/src/run_ephemeral_ssh.rs b/crates/kit/src/run_ephemeral_ssh.rs index 07c594d41..0bedfa5f5 100644 --- a/crates/kit/src/run_ephemeral_ssh.rs +++ b/crates/kit/src/run_ephemeral_ssh.rs @@ -161,7 +161,8 @@ fn spawn_status_monitor(container_name: &str) -> Result { "exec", "--", container_name, - "/var/lib/bcvk/entrypoint", + crate::run_ephemeral::SELFEXE, + "container-entrypoint", "monitor-status", ]); // SAFETY: This API is safe to call in a forked child. diff --git a/crates/kit/src/sandbox.rs b/crates/kit/src/sandbox.rs new file mode 100644 index 000000000..157dcb4c6 --- /dev/null +++ b/crates/kit/src/sandbox.rs @@ -0,0 +1,148 @@ +//! Container-side setup for the VM supervisor. +//! +//! bcvk runs QEMU and virtiofsd from the host's `/usr`, which podman bind-mounts +//! into the container at `/run/tmproot/usr`. Those processes need that hybrid +//! tree as their root, so the supervisor assembles it, makes it this process's +//! root with pivot_root(2), and only then execs anything out of it. + +use std::path::Path; +use std::process::Command; + +use color_eyre::eyre::{self, eyre, Context as _}; +use color_eyre::Result; +use rustix::mount::{ + mount, mount_bind_recursive, mount_change, unmount, MountFlags, MountPropagationFlags, + UnmountFlags, +}; +use rustix::process::pivot_root; +use rustix::thread::{unshare_unsafe, UnshareFlags}; +use tracing::debug; + +pub const TMPROOT: &str = "/run/tmproot"; + +/// Holds the virtiofsd sockets, shared with processes outside this namespace. +const SOCKETS: &str = "/run/inner-shared"; + +/// The target image's systemd version, read before the root change puts the +/// host's `/usr` in place of the image's. +static SYSTEMD_VERSION: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Assemble the hybrid root and make it this process's root. +/// +/// Only the VM supervisor calls this. Anything else arrives later through +/// `podman exec`, which joins the supervisor's namespaces and so is already in +/// the hybrid root. +pub fn setup() -> Result<()> { + init_tmproot().context("Assembling hybrid root")?; + let _ = SYSTEMD_VERSION.set(read_systemd_version()); + enter(TMPROOT) +} + +/// The target image's systemd version output, if it reported one. +pub fn systemd_version() -> Option<&'static str> { + SYSTEMD_VERSION + .get() + .map(String::as_str) + .filter(|v| !v.is_empty()) +} + +fn init_tmproot() -> Result<()> { + let root = Path::new(TMPROOT); + + for (target, source) in [ + ("bin", "usr/bin"), + ("lib", "usr/lib"), + ("lib64", "usr/lib64"), + ("sbin", "usr/sbin"), + ] { + let target = root.join(target); + std::os::unix::fs::symlink(source, &target) + .with_context(|| format!("Creating {target:?}"))?; + } + for dir in ["etc", "var", "var/tmp", "dev", "proc", "run", "sys", "tmp"] { + std::fs::create_dir_all(root.join(dir))?; + } + + // ssh-keygen wants /etc/passwd to exist. + let st = Command::new("systemd-sysusers") + .arg("--root") + .arg(root) + .output() + .context("Running systemd-sysusers")?; + eyre::ensure!( + st.status.success(), + "systemd-sysusers failed: {}", + String::from_utf8_lossy(&st.stderr).trim() + ); + + // QEMU's user-mode networking resolves DNS with the resolv.conf podman + // wrote for the container, which is outside the new root. + if Path::new("/etc/resolv.conf").exists() { + std::fs::copy("/etc/resolv.conf", root.join("etc/resolv.conf"))?; + } + + std::fs::create_dir(SOCKETS)?; + Ok(()) +} + +/// Ask the image's systemctl for its version. An image that cannot report one +/// yields an empty string, which callers treat as unknown. +fn read_systemd_version() -> String { + Command::new("systemctl") + .arg("--version") + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).into_owned()) + .unwrap_or_default() +} + +fn enter(newroot: &str) -> Result<()> { + let root = Path::new(newroot); + if !root.join("usr").exists() { + return Err(eyre!( + "{newroot}/usr does not exist: the container was not set up by bcvk" + )); + } + + // A new mount namespace applies to the calling thread only, so this has to + // run before the tokio runtime exists. + // + // SAFETY: unshare is unsafe only for UnshareFlags::FILES, where one thread + // can be left unable to use another's file descriptors. + #[allow(unsafe_code)] + unsafe { unshare_unsafe(UnshareFlags::NEWNS) }.context("Unsharing mount namespace")?; + + // Keep these mounts out of the container's mount namespace, which is the + // one `podman exec` joins. + mount_change( + "/", + MountPropagationFlags::REC | MountPropagationFlags::DOWNSTREAM, + ) + .context("Making / slave")?; + + // pivot_root(2) requires the new root to be a mount point of its own. The + // bind has to be recursive, or podman's mount of the host /usr at + // /usr is left behind and the new root has no binaries in it. + mount_bind_recursive(newroot, newroot).context("Binding new root onto itself")?; + + let proc = root.join("proc"); + mount("proc", &proc, "proc", MountFlags::empty(), None).context("Mounting /proc")?; + + // /run is shared rather than private: the virtiofsd sockets, the status + // file the monitor watches, and the mounted source image all live there and + // are reached from outside this namespace. + for (source, target) in [("/dev", "dev"), ("/var/tmp", "var/tmp"), ("/run", "run")] { + mount_bind_recursive(source, root.join(target)) + .with_context(|| format!("Binding {source}"))?; + } + + // Passing "." for both arguments avoids needing a put_old directory. + std::env::set_current_dir(newroot)?; + pivot_root(".", ".").context("pivot_root")?; + unmount(".", UnmountFlags::DETACH).context("Detaching old root")?; + std::env::set_current_dir("/")?; + + debug!("Root is now {newroot}"); + Ok(()) +} diff --git a/docs/src/installation.md b/docs/src/installation.md index 8b10a3863..26a5a41b0 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -34,8 +34,6 @@ Optional: For `bcvk ephemeral` operations, the bootc container images you run must contain: - systemctl (systemd) -- objcopy (binutils) -- bwrap (bubblewrap) - ssh, ssh-keygen (openssh-clients) ## Development Binaries