ephemeral: Explain when podman can't see bcvk's filesystem

crates/kit/src/main.rs

@@ -41,6 +41,8 @@ mod libvirt_upload_disk;
 #[allow(dead_code)]
 mod podman;
 #[cfg(target_os = "linux")]
+mod podman_hint;
+#[cfg(target_os = "linux")]
 mod qemu;
 #[cfg(target_os = "linux")]
 mod run_ephemeral;

crates/kit/src/podman_hint.rs

@@ -0,0 +1,112 @@
+//! Explain podman failures caused by podman not seeing bcvk's filesystem.
+//!
+//! To launch a VM, bcvk asks podman to bind-mount paths from its own
+//! filesystem, including its own binary. When podman runs somewhere else,
+//! e.g. bcvk runs in a toolbox or distrobox with podman forwarded to the host,
+//! or podman is a remote client, those paths are resolved on podman's side,
+//! and one that only exists on bcvk's side fails with an obscure
+//! `statfs /usr/bin/bcvk: no such file or directory`.
+//!
+//! See <https://github.com/bootc-dev/bcvk/issues/5> and bcvk-ephemeral-run(8)
+//! for the full explanation.
+
+use camino::Utf8Path;
+use color_eyre::eyre::Report;
+use color_eyre::Section;
+
+/// How podman reports a bind mount source that doesn't exist, e.g.
+/// `Error: statfs /usr/bin/bcvk: no such file or directory`.
+const PODMAN_STATFS_PREFIX: &str = "statfs ";
+const ENOENT_SUFFIX: &str = ": no such file or directory";
+
+/// The bind mount source that podman reported missing, if any.
+fn missing_bind_source(stderr: &str) -> Option<&Utf8Path> {
+    stderr.lines().find_map(|line| {
+        let (_, rest) = line.split_once(PODMAN_STATFS_PREFIX)?;
+        let path = rest.trim_end().strip_suffix(ENOENT_SUFFIX)?;
+        Some(Utf8Path::new(path))
+    })
+}
+
+/// Wrap `err` with an explanation if podman's `stderr` says a bind mount
+/// source is missing, but `exists` says bcvk can see it.
+fn with_hint_if(err: Report, stderr: &str, exists: impl Fn(&Utf8Path) -> bool) -> Report {
+    let Some(path) = missing_bind_source(stderr).filter(|p| exists(p)) else {
+        return err;
+    };
+    err.wrap_err(format!(
+        "podman could not find {path}, but bcvk can see it: podman is not seeing bcvk's filesystem"
+    ))
+    .note(
+        "This happens when bcvk runs in a toolbox or distrobox container and podman is \
+         forwarded to the host, or when podman is a remote client. bcvk needs to run \
+         where podman runs.",
+    )
+    .suggestion(
+        "Install and run bcvk on the host, alongside podman. \
+         See bcvk-ephemeral-run(8) for details.",
+    )
+}
+
+/// Add guidance to the error for a failed podman invocation if podman could
+/// not find a bind mount source that bcvk can see.
+pub(crate) fn with_podman_failure_hint(err: Report, stderr: &str) -> Report {
+    with_hint_if(err, stderr, |p| p.try_exists().unwrap_or(false))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_missing_bind_source() {
+        let cases = [
+            (
+                "Error: statfs /usr/bin/bcvk: no such file or directory\n",
+                Some("/usr/bin/bcvk"),
+            ),
+            (
+                "some warning\nError: statfs /a b/c: no such file or directory  \n",
+                Some("/a b/c"),
+            ),
+            ("Error: statfs /x: permission denied\n", None),
+            ("Error: short-name resolution enforced\n", None),
+            ("", None),
+        ];
+        for (stderr, expected) in cases {
+            assert_eq!(
+                missing_bind_source(stderr),
+                expected.map(Utf8Path::new),
+                "{stderr:?}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_with_hint_if() {
+        const STATFS: &str = "Error: statfs /usr/bin/bcvk: no such file or directory\n";
+        const OTHER: &str = "Error: short-name resolution enforced\n";
+        // (stderr, whether bcvk can see the path, whether a hint is added)
+        let cases = [
+            (STATFS, true, true),
+            (STATFS, false, false),
+            (OTHER, true, false),
+        ];
+        for (stderr, exists, hinted) in cases {
+            let msg = format!("Podman command failed: {stderr}");
+            let r = with_hint_if(color_eyre::eyre::eyre!(msg.clone()), stderr, |_| exists);
+            let chain: Vec<String> = r.chain().map(|e| e.to_string()).collect();
+            if hinted {
+                assert_eq!(chain.len(), 2, "{chain:?}");
+                assert!(chain[0].contains("/usr/bin/bcvk"), "{chain:?}");
+                // The note and suggestion sections are only recorded with
+                // color_eyre's handler, which unit tests can't reliably
+                // install (eyre locks in its default hook on first use).
+                // The original podman error is kept as the cause.
+                assert_eq!(chain[1], msg);
+            } else {
+                assert_eq!(chain, [msg], "{stderr:?} exists={exists}");
+            }
+        }
+    }
+}

crates/kit/src/run_ephemeral.rs

@@ -647,7 +647,8 @@ pub fn run_detached(opts: RunEphemeralOpts) -> Result<String> {
     let output = cmd.output().context("Failed to execute podman command")?;
     if !output.status.success() {
         let stderr = String::from_utf8_lossy(&output.stderr);
-        return Err(color_eyre::eyre::eyre!("Podman command failed: {}", stderr));
+        let err = eyre!("Podman command failed: {}", stderr);
+        return Err(crate::podman_hint::with_podman_failure_hint(err, &stderr));
     }
 
     // Return the container ID from stdout

docs/src/installation.md

@@ -65,6 +65,41 @@ Inside a clone of the repo:
 cargo install --locked --path crates/kit
 ```
 
+## Toolbox, distrobox and remote podman
+
+bcvk is designed to be installed on the host, alongside podman and QEMU.
+To launch a VM, it asks podman to bind-mount the bcvk binary itself and
+the host's `/usr` (which provides QEMU and virtiofsd) into a new
+container, so podman must see the same filesystem as bcvk.
+
+When bcvk runs inside a [toolbox](https://containertoolbx.org/) or
+[distrobox](https://distrobox.it/) container with podman installed in
+that container too, podman sees bcvk's paths and the error described
+below doesn't occur, though nested podman has limitations of its own
+(for example around networking). However, podman is often forwarded to
+the host instead (for example via a `flatpak-spawn --host podman`
+wrapper or the podman socket), and the same applies to a remote podman
+client. Paths that bcvk passes to podman are then resolved on the host,
+so a bcvk binary installed only in the toolbox cannot be found, and
+podman fails with an error like `statfs /usr/bin/bcvk: no such file or
+directory`. bcvk explains this in its error when it sees that podman
+could not find a path that bcvk itself can see.
+
+The simplest fix is to install bcvk on the host and run it there. From
+inside a toolbox you can still invoke the host's copy with
+`flatpak-spawn --host bcvk ...`, and from a distrobox with
+`distrobox-host-exec bcvk ...`.
+
+Beware of version skew: if different bcvk binaries exist at the same
+path on the host and in the toolbox (e.g. `/usr/bin/bcvk` installed from
+packages in both), the error above doesn't happen, but the host's copy is
+the one podman mounts into the VM's container. The bcvk you ran and the
+one that sets up the VM are then different versions, which can fail in
+confusing ways. Keep them in sync, or install bcvk only on the host.
+
+See [#5](https://github.com/bootc-dev/bcvk/issues/5) for discussion of
+better support for this case.
+
 ## Platform Support
 
 - Linux: Supported

docs/src/man/bcvk-ephemeral-run.md

@@ -371,6 +371,33 @@ Virtiofsd logs are helpful for:
 - Understanding file handle support warnings
 - Investigating mount-related errors
 
+## Podman Can't See bcvk's Filesystem
+
+To launch a VM, bcvk asks podman to bind-mount paths from its own
+filesystem, including its own binary. This requires podman to see the
+same filesystem as bcvk, which fails with an error like:
+
+    Podman command failed: Error: statfs /usr/bin/bcvk: no such file or directory
+
+This happens when bcvk runs inside a toolbox or distrobox container and
+podman is forwarded to the host instead of running in the container (for
+example via a `flatpak-spawn --host podman` wrapper or the podman
+socket), or when podman is a remote client. Paths that bcvk passes to
+podman are then resolved on podman's side, so a bcvk binary installed
+only in the toolbox can't be found there. bcvk detects this and adds a
+hint to the error explaining it.
+
+The fix is to install bcvk on the host alongside podman and QEMU and
+run it there. From a toolbox, invoke the host's copy with
+`flatpak-spawn --host bcvk ...`; from a distrobox, with
+`distrobox-host-exec bcvk ...`.
+
+Beware of version skew: if bcvk binaries exist at the same path on both
+the host and in the toolbox (e.g. both installed from packages), this
+error doesn't occur, but podman mounts the host's copy into the VM's
+container, which can differ in version from the one you ran. Keep them
+in sync, or install bcvk only on the host.
+
 # SEE ALSO
 
 **bcvk**(8)