diff --git a/alioth-cli/src/boot/boot.rs b/alioth-cli/src/boot/boot.rs index 9e10a2c8..64907718 100644 --- a/alioth-cli/src/boot/boot.rs +++ b/alioth-cli/src/boot/boot.rs @@ -31,7 +31,7 @@ use alioth::hv::{CocoSpec, HvSpec, Hypervisor}; use alioth::loader::{Executable, PayloadSpec}; use alioth::mem::{MemBackend, MemSpec}; #[cfg(target_os = "linux")] -use alioth::vfio::{VfioCdevSpec, VfioContainerSpec, VfioGroupSpec, VfioIoasSpec}; +use alioth::vfio::{VfioCdevSpec, VfioContainerSpec, VfioGroupSpec, VfioIoasSpec, VfioUserSpec}; #[cfg(target_os = "linux")] use alioth::virtio::DeviceId; use alioth::virtio::dev::balloon::BalloonSpec; @@ -182,6 +182,10 @@ pub struct BootArgs { #[arg(long, help(help_text::("Add a new VFIO container.")))] vfio_container: Vec, + #[cfg(target_os = "linux")] + #[arg(long, help(help_text::("Assign a vfio-user device to the guest.")))] + vfio_user: Vec, + #[arg(long)] #[arg(long, help(help_text::("Add a VirtIO balloon device.")))] balloon: Option, @@ -358,6 +362,11 @@ fn parse_args(mut args: BootArgs, objects: HashMap<&str, &str>) -> Result(hypervisor: &H, spec: VmSpec) -> Result, ali for (index, cdev_spec) in spec.vfio_cdev.into_iter().enumerate() { vm.add_vfio_cdev(format!("vfio-{index}").into(), cdev_spec)?; } + #[cfg(target_os = "linux")] + for (index, user_spec) in spec.vfio_user.into_iter().enumerate() { + vm.add_vfio_user_dev(format!("vfio-user-{index}").into(), user_spec)?; + } #[cfg(target_os = "linux")] for container_spec in spec.vfio_container.into_iter() { diff --git a/alioth-cli/src/boot/boot_test.rs b/alioth-cli/src/boot/boot_test.rs index a29bb889..98cd0a50 100644 --- a/alioth-cli/src/boot/boot_test.rs +++ b/alioth-cli/src/boot/boot_test.rs @@ -212,6 +212,8 @@ fn test_parse_args() { container: Some("gpu_container".into()), devices: vec!["0000:06:0d.0".into(), "0000:06:0d.1".into()], }], + #[cfg(target_os = "linux")] + vfio_user: vec![], }; assert_eq!(spec, want); } diff --git a/alioth-cli/src/boot/config.rs b/alioth-cli/src/boot/config.rs index 1be32663..a2c5c0c3 100644 --- a/alioth-cli/src/boot/config.rs +++ b/alioth-cli/src/boot/config.rs @@ -21,7 +21,7 @@ use alioth::device::console::ConsoleSpec; use alioth::device::fw_cfg::FwCfgItemSpec; use alioth::loader::PayloadSpec; #[cfg(target_os = "linux")] -use alioth::vfio::{VfioCdevSpec, VfioContainerSpec, VfioGroupSpec, VfioIoasSpec}; +use alioth::vfio::{VfioCdevSpec, VfioContainerSpec, VfioGroupSpec, VfioIoasSpec, VfioUserSpec}; use alioth::virtio::dev::balloon::BalloonSpec; use alioth::virtio::dev::blk::BlkFileSpec; use alioth::virtio::dev::entropy::EntropySpec; @@ -119,4 +119,6 @@ pub struct VmSpec { pub vfio_group: Vec, #[cfg(target_os = "linux")] pub vfio_container: Vec, + #[cfg(target_os = "linux")] + pub vfio_user: Vec, } diff --git a/alioth/src/sys/linux/vfio.rs b/alioth/src/sys/linux/vfio.rs index 4d7b7edb..97894f24 100644 --- a/alioth/src/sys/linux/vfio.rs +++ b/alioth/src/sys/linux/vfio.rs @@ -13,6 +13,7 @@ // limitations under the License. use bitfield::bitfield; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::sys::ioctl::ioctl_io; use crate::{ @@ -80,7 +81,7 @@ consts! { } #[repr(C)] -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, KnownLayout, Immutable, FromBytes, IntoBytes)] pub struct VfioRegionInfo { pub argsz: u32, pub flags: VfioRegionInfoFlag, @@ -116,7 +117,7 @@ consts! { } #[repr(C)] -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, KnownLayout, Immutable, FromBytes, IntoBytes)] pub struct VfioIrqInfo { pub argsz: u32, pub flags: VfioIrqInfoFlag, diff --git a/alioth/src/vfio/cdev.rs b/alioth/src/vfio/cdev.rs index 1c3c53d1..3a86f405 100644 --- a/alioth/src/vfio/cdev.rs +++ b/alioth/src/vfio/cdev.rs @@ -13,25 +13,26 @@ // limitations under the License. use std::fmt::Debug; -use std::fs::{File, OpenOptions}; +use std::fs::OpenOptions; use std::mem::size_of; -use std::os::fd::AsRawFd; +use std::os::fd::{AsRawFd, BorrowedFd, OwnedFd}; use std::path::Path; use std::sync::Arc; use snafu::ResultExt; use crate::sys::vfio::{ - VfioDeviceAttachIommufdPt, VfioDeviceBindIommufd, VfioDeviceDetachIommufdPt, - vfio_device_attach_iommufd_pt, vfio_device_bind_iommufd, vfio_device_detach_iommufd_pt, + VfioDeviceAttachIommufdPt, VfioDeviceBindIommufd, VfioDeviceDetachIommufdPt, VfioDeviceInfo, + VfioIrqInfo, VfioRegionInfo, vfio_device_attach_iommufd_pt, vfio_device_bind_iommufd, + vfio_device_detach_iommufd_pt, }; -use crate::vfio::device::Device; +use crate::vfio::device::{Device, VfioIoDevice}; use crate::vfio::iommu::Ioas; use crate::vfio::{Result, error}; #[derive(Debug)] pub struct Cdev { - fd: File, + io_dev: VfioIoDevice, ioas: Option>, } @@ -44,7 +45,8 @@ impl Cdev { .context(error::AccessDevice { path: path.as_ref(), })?; - Ok(Cdev { fd, ioas: None }) + let io_dev = VfioIoDevice::new(fd)?; + Ok(Cdev { io_dev, ioas: None }) } } @@ -55,13 +57,13 @@ impl Cdev { iommufd: ioas.iommu.fd.as_raw_fd(), ..Default::default() }; - unsafe { vfio_device_bind_iommufd(&self.fd, &bind) }?; + unsafe { vfio_device_bind_iommufd(self.io_dev.fd(), &bind) }?; let attach = VfioDeviceAttachIommufdPt { argsz: size_of::() as u32, pt_id: ioas.id, ..Default::default() }; - unsafe { vfio_device_attach_iommufd_pt(&self.fd, &attach) }?; + unsafe { vfio_device_attach_iommufd_pt(self.io_dev.fd(), &attach) }?; self.ioas.replace(ioas); Ok(()) } @@ -69,27 +71,71 @@ impl Cdev { pub fn detach_iommu_ioas(&mut self) -> Result<()> { if self.ioas.is_none() { return Ok(()); - }; + } let detach = VfioDeviceDetachIommufdPt { argsz: size_of::() as u32, flags: 0, }; - unsafe { vfio_device_detach_iommufd_pt(&self.fd, &detach) }?; + unsafe { vfio_device_detach_iommufd_pt(self.io_dev.fd(), &detach) }?; self.ioas = None; Ok(()) } } impl Device for Cdev { - fn fd(&self) -> &File { - &self.fd + fn get_info(&self) -> Result { + self.io_dev.get_info() + } + + fn get_region_info(&self, index: u32) -> Result { + self.io_dev.get_region_info(index) + } + + fn get_irq_info(&self, index: u32) -> Result { + self.io_dev.get_irq_info(index) + } + + fn reset(&self) -> Result<()> { + self.io_dev.reset() + } + + fn set_irq_eventfd( + &self, + index: u32, + start: u32, + eventfds: &[Option>], + ) -> Result<()> { + self.io_dev.set_irq_eventfd(index, start, eventfds) + } + + fn disable_irq(&self, index: u32) -> Result<()> { + self.io_dev.disable_irq(index) + } + + fn read_region(&self, region: &VfioRegionInfo, offset: u64, buf: &mut [u8]) -> Result<()> { + self.io_dev.read_region(region, offset, buf) + } + + fn write_region(&self, region: &VfioRegionInfo, offset: u64, buf: &[u8]) -> Result<()> { + self.io_dev.write_region(region, offset, buf) + } + + fn get_region_mmap(&self, index: u32) -> Result> { + self.io_dev.get_region_mmap(index) + } + + fn get_dma_buf_fd(&self, index: u32, offset: u64, size: usize) -> Result { + self.io_dev.get_dma_buf_fd(index, offset, size) } } impl Drop for Cdev { fn drop(&mut self) { if let Err(e) = self.detach_iommu_ioas() { - log::error!("Cdev-{}: detaching ioas: {e:?}", self.fd.as_raw_fd()) + log::error!( + "Cdev-{}: detaching ioas: {e:?}", + self.io_dev.fd().as_raw_fd() + ) } } } diff --git a/alioth/src/vfio/container.rs b/alioth/src/vfio/container.rs index 0d57c4db..01cf3ecd 100644 --- a/alioth/src/vfio/container.rs +++ b/alioth/src/vfio/container.rs @@ -32,7 +32,7 @@ use crate::vfio::{Result, error}; #[derive(Debug)] pub struct Container { fd: File, - iommu: Mutex>, + pub(super) iommu: Mutex>, } impl Container { diff --git a/alioth/src/vfio/device.rs b/alioth/src/vfio/device.rs index d31ecbfe..a915c98b 100644 --- a/alioth/src/vfio/device.rs +++ b/alioth/src/vfio/device.rs @@ -15,57 +15,104 @@ use std::fmt::Debug; use std::fs::File; use std::mem::size_of; -use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; +use std::os::fd::{AsRawFd, BorrowedFd, FromRawFd, OwnedFd}; use std::os::unix::fs::FileExt; +use crate::errors::BoxTrace; use crate::mem; use crate::sys::vfio::{ DeviceFeature, VfioDeviceFeature, VfioDeviceFeatureDmaBuf, VfioDeviceFeatureFlag, - VfioDeviceInfo, VfioIrqInfo, VfioIrqSet, VfioIrqSetData, VfioIrqSetFlag, VfioPciIrq, - VfioRegionDmaRange, VfioRegionInfo, vfio_device_feature, vfio_device_get_info, + VfioDeviceInfo, VfioIrqInfo, VfioIrqSet, VfioIrqSetData, VfioIrqSetFlag, VfioRegionDmaRange, + VfioRegionInfo, VfioRegionInfoFlag, vfio_device_feature, vfio_device_get_info, vfio_device_get_irq_info, vfio_device_get_region_info, vfio_device_reset, vfio_device_set_irqs, }; use crate::vfio::Result; pub trait Device: Debug + Send + Sync + 'static { - fn fd(&self) -> &File; + fn get_info(&self) -> Result; + fn get_region_info(&self, index: u32) -> Result; + fn get_irq_info(&self, index: u32) -> Result; + fn reset(&self) -> Result<()>; - fn get_info(&self) -> Result { - let mut device_info = VfioDeviceInfo { - argsz: size_of::() as u32, - ..Default::default() + fn set_irq_eventfd( + &self, + index: u32, + start: u32, + eventfds: &[Option>], + ) -> Result<()>; + + fn disable_irq(&self, index: u32) -> Result<()>; + + fn read_region(&self, region: &VfioRegionInfo, offset: u64, buf: &mut [u8]) -> Result<()>; + fn write_region(&self, region: &VfioRegionInfo, offset: u64, buf: &[u8]) -> Result<()>; + + fn get_region_mmap(&self, index: u32) -> Result>; + + fn get_dma_buf_fd(&self, index: u32, offset: u64, size: usize) -> Result; + + // Helper methods for single-value read/write + fn read(&self, region: &VfioRegionInfo, offset: u64, size: u8) -> mem::Result { + let mut bytes = [0u8; 8]; + let Some(buf) = bytes.get_mut(0..size as usize) else { + log::error!( + "vfio: invalid read: index = {}, offset = {offset:#x}, size = {size:#x}", + region.index + ); + return Ok(0); }; - unsafe { vfio_device_get_info(self.fd(), &mut device_info) }?; - Ok(device_info) + self.read_region(region, offset, buf) + .box_trace(crate::mem::error::Mmio)?; + Ok(u64::from_ne_bytes(bytes)) } - fn get_dma_buf_fd(&self, index: u32, offset: u64, size: usize) -> Result { - let req = VfioDeviceFeature { - argsz: size_of::>>() as u32, - flags: VfioDeviceFeatureFlag::new(DeviceFeature::DMA_BUF, true, false, false), - data: VfioDeviceFeatureDmaBuf { - region_index: index, - open_flags: (libc::O_RDWR | libc::O_CLOEXEC) as u32, - flags: 0, - nr_ranges: 1, - dma_ranges: [VfioRegionDmaRange { - offset, - length: size as u64, - }], - }, + fn write(&self, region: &VfioRegionInfo, offset: u64, size: u8, val: u64) -> mem::Result<()> { + let bytes = val.to_ne_bytes(); + let Some(buf) = bytes.get(..size as usize) else { + log::error!( + "vfio: invalid write: index = {}, offset = {offset:#x}, size = {size:#x}, val = {val:#x}", + region.index + ); + return Ok(()); }; - let fd = unsafe { vfio_device_feature(self.fd(), &req) }?; - Ok(unsafe { OwnedFd::from_raw_fd(fd) }) + self.write_region(region, offset, buf) + .box_trace(crate::mem::error::Mmio)?; + Ok(()) + } +} + +#[derive(Debug)] +pub struct VfioIoDevice { + fd: File, +} + +impl VfioIoDevice { + pub fn new(fd: File) -> Result { + Ok(Self { fd }) + } + + pub fn fd(&self) -> &File { + &self.fd + } +} + +impl Device for VfioIoDevice { + fn get_info(&self) -> Result { + let mut info = VfioDeviceInfo { + argsz: size_of::() as u32, + ..Default::default() + }; + unsafe { vfio_device_get_info(&self.fd, &mut info) }?; + Ok(info) } fn get_region_info(&self, index: u32) -> Result { - let mut region_config = VfioRegionInfo { + let mut region = VfioRegionInfo { argsz: size_of::() as u32, index, ..Default::default() }; - unsafe { vfio_device_get_region_info(self.fd(), &mut region_config) }?; - Ok(region_config) + unsafe { vfio_device_get_region_info(&self.fd, &mut region) }?; + Ok(region) } fn get_irq_info(&self, index: u32) -> Result { @@ -74,55 +121,85 @@ pub trait Device: Debug + Send + Sync + 'static { index, ..Default::default() }; - unsafe { vfio_device_get_irq_info(self.fd(), &mut irq_info) }?; + unsafe { vfio_device_get_irq_info(&self.fd, &mut irq_info) }?; Ok(irq_info) } - fn set_irqs(&self, irq: &VfioIrqSet) -> Result<()> { - unsafe { vfio_device_set_irqs(self.fd(), irq) }?; + fn reset(&self) -> Result<()> { + unsafe { vfio_device_reset(&self.fd) }?; Ok(()) } - fn disable_all_irqs(&self, index: VfioPciIrq) -> Result<()> { - let vfio_irq_disable_all = VfioIrqSet { + fn set_irq_eventfd( + &self, + index: u32, + start: u32, + eventfds: &[Option>], + ) -> Result<()> { + let mut raw_fds = [-1; 2048]; + for (raw_fd, eventfd) in raw_fds.iter_mut().zip(eventfds) { + *raw_fd = eventfd.map(|fd| fd.as_raw_fd()).unwrap_or(-1); + } + let irq_set = VfioIrqSet { + argsz: (size_of::>() + eventfds.len() * size_of::()) as u32, + flags: VfioIrqSetFlag::DATA_EVENTFD | VfioIrqSetFlag::ACTION_TRIGGER, + index, + start, + count: eventfds.len() as u32, + data: VfioIrqSetData { eventfds: raw_fds }, + }; + unsafe { vfio_device_set_irqs(&self.fd, &irq_set) }?; + Ok(()) + } + + fn disable_irq(&self, index: u32) -> Result<()> { + let irq_set = VfioIrqSet { argsz: size_of::>() as u32, flags: VfioIrqSetFlag::DATA_NONE | VfioIrqSetFlag::ACTION_TRIGGER, - index: index.raw(), + index, start: 0, count: 0, data: VfioIrqSetData { eventfds: [] }, }; - self.set_irqs(&vfio_irq_disable_all) + unsafe { vfio_device_set_irqs(&self.fd, &irq_set) }?; + Ok(()) } - fn reset(&self) -> Result<()> { - unsafe { vfio_device_reset(self.fd()) }?; + fn read_region(&self, region: &VfioRegionInfo, offset: u64, buf: &mut [u8]) -> Result<()> { + self.fd.read_exact_at(buf, region.offset + offset)?; Ok(()) } - fn read(&self, offset: u64, size: u8) -> mem::Result { - let mut bytes = [0u8; 8]; - let Some(buf) = bytes.get_mut(0..size as usize) else { - log::error!( - "vfio-{}: invalid read: offset = {offset:#x}, size = {size:#x}", - self.fd().as_raw_fd() - ); - return Ok(0); - }; - self.fd().read_exact_at(buf, offset)?; - Ok(u64::from_ne_bytes(bytes)) + fn write_region(&self, region: &VfioRegionInfo, offset: u64, buf: &[u8]) -> Result<()> { + self.fd.write_all_at(buf, region.offset + offset)?; + Ok(()) } - fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result<()> { - let bytes = val.to_ne_bytes(); - let Some(buf) = bytes.get(..size as usize) else { - log::error!( - "vfio-{}: invalid write: offset = {offset:#x}, size = {size:#x}, val = {val:#x}", - self.fd().as_raw_fd() - ); - return Ok(()); + fn get_region_mmap(&self, index: u32) -> Result> { + let region_info = self.get_region_info(index)?; + if region_info.flags.contains(VfioRegionInfoFlag::MMAP) { + Ok(Some((self.fd.try_clone()?.into(), region_info.offset))) + } else { + Ok(None) + } + } + + fn get_dma_buf_fd(&self, index: u32, offset: u64, size: usize) -> Result { + let req = VfioDeviceFeature { + argsz: size_of::>>() as u32, + flags: VfioDeviceFeatureFlag::new(DeviceFeature::DMA_BUF, true, false, false), + data: VfioDeviceFeatureDmaBuf { + region_index: index, + open_flags: (libc::O_RDWR | libc::O_CLOEXEC) as u32, + flags: 0, + nr_ranges: 1, + dma_ranges: [VfioRegionDmaRange { + offset, + length: size as u64, + }], + }, }; - self.fd().write_all_at(buf, offset)?; - Ok(()) + let fd = unsafe { vfio_device_feature(&self.fd, &req) }?; + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) } } diff --git a/alioth/src/vfio/group.rs b/alioth/src/vfio/group.rs index 6b35ca51..860f095e 100644 --- a/alioth/src/vfio/group.rs +++ b/alioth/src/vfio/group.rs @@ -14,17 +14,18 @@ use std::ffi::CString; use std::fs::File; -use std::os::fd::{AsRawFd, FromRawFd}; +use std::os::fd::{AsRawFd, BorrowedFd, FromRawFd, OwnedFd}; use std::path::Path; use std::sync::Arc; use snafu::ResultExt; use crate::sys::vfio::{ - VfioIommu, vfio_group_get_device_fd, vfio_group_set_container, vfio_group_unset_container, + VfioDeviceInfo, VfioIommu, VfioIrqInfo, VfioRegionInfo, vfio_group_get_device_fd, + vfio_group_set_container, vfio_group_unset_container, }; use crate::vfio::container::Container; -use crate::vfio::device::Device; +use crate::vfio::device::{Device, VfioIoDevice}; use crate::vfio::{Result, error}; #[derive(Debug)] @@ -72,23 +73,66 @@ impl Drop for Group { #[derive(Debug)] pub struct DevFd { - fd: File, - _group: Arc, + pub(super) io_dev: VfioIoDevice, + pub(super) _group: Arc, } impl DevFd { pub fn new(group: Arc, id: &str) -> Result { let id_c = CString::new(id).unwrap(); let fd = unsafe { vfio_group_get_device_fd(&group.fd, id_c.as_ptr()) }?; + let file = unsafe { File::from_raw_fd(fd) }; + let io_dev = VfioIoDevice::new(file)?; Ok(DevFd { - fd: unsafe { File::from_raw_fd(fd) }, + io_dev, _group: group, }) } } impl Device for DevFd { - fn fd(&self) -> &File { - &self.fd + fn get_info(&self) -> Result { + self.io_dev.get_info() + } + + fn get_region_info(&self, index: u32) -> Result { + self.io_dev.get_region_info(index) + } + + fn get_irq_info(&self, index: u32) -> Result { + self.io_dev.get_irq_info(index) + } + + fn reset(&self) -> Result<()> { + self.io_dev.reset() + } + + fn set_irq_eventfd( + &self, + index: u32, + start: u32, + eventfds: &[Option>], + ) -> Result<()> { + self.io_dev.set_irq_eventfd(index, start, eventfds) + } + + fn disable_irq(&self, index: u32) -> Result<()> { + self.io_dev.disable_irq(index) + } + + fn read_region(&self, region: &VfioRegionInfo, offset: u64, buf: &mut [u8]) -> Result<()> { + self.io_dev.read_region(region, offset, buf) + } + + fn write_region(&self, region: &VfioRegionInfo, offset: u64, buf: &[u8]) -> Result<()> { + self.io_dev.write_region(region, offset, buf) + } + + fn get_region_mmap(&self, index: u32) -> Result> { + self.io_dev.get_region_mmap(index) + } + + fn get_dma_buf_fd(&self, index: u32, offset: u64, size: usize) -> Result { + self.io_dev.get_dma_buf_fd(index, offset, size) } } diff --git a/alioth/src/vfio/pci.rs b/alioth/src/vfio/pci.rs index df2e15d1..eabb39e0 100644 --- a/alioth/src/vfio/pci.rs +++ b/alioth/src/vfio/pci.rs @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::cmp::min; -use std::iter::zip; +use std::cmp::{max, min}; +use std::fs::File; use std::mem::size_of; use std::ops::Range; -use std::os::fd::{AsFd, AsRawFd, OwnedFd}; -use std::os::unix::fs::FileExt; +use std::os::fd::{AsFd, OwnedFd}; use std::sync::Arc; use std::sync::atomic::AtomicU64; @@ -41,8 +40,7 @@ use crate::pci::config::{ }; use crate::pci::{self, Pci, PciBar}; use crate::sys::vfio::{ - VfioDeviceInfoFlag, VfioIrqSet, VfioIrqSetData, VfioIrqSetFlag, VfioPciIrq, VfioPciRegion, - VfioRegionInfo, VfioRegionInfoFlag, + VfioDeviceInfoFlag, VfioPciIrq, VfioPciRegion, VfioRegionInfo, VfioRegionInfoFlag, }; use crate::vfio::device::Device; use crate::vfio::{Result, error}; @@ -52,36 +50,29 @@ fn round_up_range(range: Range) -> Range { (align_down!(range.start, 12))..(align_up!(range.end, 12)) } -fn create_mapped_bar_pages( - dev: &VfioDev, - region: &VfioRegionInfo, +fn create_mapped_bar_pages( + fd: &OwnedFd, + region_flags: VfioRegionInfoFlag, offset: u64, size: usize, + dma_buf: Option, ) -> Result<(ArcMemPages, Option)> { - let dma_buf = match dev.dev.get_dma_buf_fd(region.index, offset, size) { - Ok(fd) => Some(fd), - Err(e) => { - log::warn!("{}: failed to get dma buf fd: {e:?}", dev.name); - None - } - }; - let mut prot = 0; - if region.flags.contains(VfioRegionInfoFlag::READ) { + if region_flags.contains(VfioRegionInfoFlag::READ) { prot |= PROT_READ; } - if region.flags.contains(VfioRegionInfoFlag::WRITE) { + if region_flags.contains(VfioRegionInfoFlag::WRITE) { prot |= PROT_WRITE; } - let dev_fd = dev.dev.fd().try_clone()?; - let dev_fd_offset = region.offset + offset; - let mapped_pages = ArcMemPages::from_file(dev_fd, dev_fd_offset as i64, size, prot)?; + let cloned_fd = fd.try_clone()?; + let file = File::from(cloned_fd); + let mapped_pages = ArcMemPages::from_file(file, offset as i64, size, prot)?; Ok((mapped_pages, dma_buf)) } fn create_splitted_bar_region( dev: Arc>, - region_info: &VfioRegionInfo, + region_info: Arc, table_range: Range, pba_range: Range, msix_table: Arc>, @@ -92,20 +83,55 @@ where M: MsiSender, D: Device, { + let mmap = dev.dev.get_region_mmap(region_info.index)?; + + let create_device_range = |offset: usize, size: usize| -> Result { + if let Some((fd, mmap_offset)) = &mmap { + let dma_buf = match dev + .dev + .get_dma_buf_fd(region_info.index, offset as u64, size) + { + Ok(fd) => Some(fd), + Err(e) => { + log::warn!("{}: failed to get dma buf fd: {e:?}", dev.name); + None + } + }; + let (pages, dma_buf) = create_mapped_bar_pages( + fd, + region_info.flags, + mmap_offset + offset as u64, + size, + dma_buf, + )?; + Ok(MemRange::DevMem { pages, dma_buf }) + } else { + let pth = PthBarRegion { + cdev: dev.clone(), + size, + index: region_info.index, + offset: offset as u64, + region: region_info.clone(), + }; + Ok(MemRange::Emulated(Arc::new(pth))) + } + }; + let table_pages = round_up_range(table_range.clone()); let pba_pages = round_up_range(pba_range.clone()); - let (excluded_page1, excluded_page2) = if table_pages.clone().eq(0..0) { + let (excluded_page1, excluded_page2) = if table_pages == (0..0) { (0..0, pba_pages) - } else if pba_pages.clone().eq(0..0) { + } else if pba_pages == (0..0) { (0..0, table_pages) - } else if table_pages.start <= pba_pages.start && table_pages.end >= pba_pages.start { - (0..0, table_pages.start..pba_pages.end) - } else if pba_pages.start <= table_pages.start && pba_pages.end >= table_pages.start { - (0..0, pba_pages.start..table_pages.end) - } else if table_pages.end < pba_pages.start { + } else if table_pages.end <= pba_pages.start { (table_pages, pba_pages) - } else { + } else if pba_pages.end <= table_pages.start { (pba_pages, table_pages) + } else { + ( + 0..0, + min(table_pages.start, pba_pages.start)..max(table_pages.end, pba_pages.end), + ) }; let mut region = MemRegion { callbacks: Mutex::new(vec![]), @@ -116,8 +142,9 @@ where ranges: vec![], }; if excluded_page1.start > 0 { - let (pages, dma_buf) = create_mapped_bar_pages(&dev, region_info, 0, excluded_page1.start)?; - region.ranges.push(MemRange::DevMem { pages, dma_buf }); + region + .ranges + .push(create_device_range(0, excluded_page1.start)?); } if excluded_page1.end - excluded_page1.start > 0 { region.ranges.push(MemRange::Emulated(Arc::new(MsixBarMmio { @@ -127,19 +154,16 @@ where pba: Arc::new([]), pba_range: pba_range.clone(), cdev: dev.clone(), - cdev_offset: region_info.offset, + region: region_info.clone(), region_start: excluded_page1.start, region_size: excluded_page1.end - excluded_page1.start, }))); } if excluded_page2.start - excluded_page1.end > 0 { - let (pages, dma_buf) = create_mapped_bar_pages( - &dev, - region_info, - excluded_page1.end as u64, + region.ranges.push(create_device_range( + excluded_page1.end, excluded_page2.start - excluded_page1.end, - )?; - region.ranges.push(MemRange::DevMem { pages, dma_buf }); + )?); } if excluded_page2.end - excluded_page2.start > 0 { region.ranges.push(MemRange::Emulated(Arc::new(MsixBarMmio { @@ -149,27 +173,24 @@ where pba: Arc::new([]), pba_range, cdev: dev.clone(), - cdev_offset: region_info.offset, + region: region_info.clone(), region_start: excluded_page2.start, region_size: excluded_page2.end - excluded_page2.start, }))); } if excluded_page2.end < region_info.size as usize { - let (pages, dma_buf) = create_mapped_bar_pages( - &dev, - region_info, - excluded_page2.end as u64, + region.ranges.push(create_device_range( + excluded_page2.end, region_info.size as usize - excluded_page2.end, - )?; - region.ranges.push(MemRange::DevMem { pages, dma_buf }); + )?); } Ok(region) } -fn create_mappable_bar_region( +fn create_bar_region( cdev: Arc>, index: u32, - region_info: &VfioRegionInfo, + region_info: Arc, msix_cap: Option<&MsixCap>, msix_table: Arc>, msi_sender: Arc, @@ -229,7 +250,8 @@ where #[derive(Debug)] struct PthConfigArea { - offset: u64, // offset to dev + offset: u64, // offset in config space + region: Arc, size: u64, dev: Arc>, } @@ -243,11 +265,13 @@ where } fn read(&self, offset: u64, size: u8) -> mem::Result { - self.dev.dev.read(self.offset + offset, size) + self.dev.dev.read(&self.region, self.offset + offset, size) } fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result { - self.dev.dev.write(self.offset + offset, size, val)?; + self.dev + .dev + .write(&self.region, self.offset + offset, size, val)?; Ok(Action::None) } } @@ -314,7 +338,9 @@ where pub struct PthBarRegion { cdev: Arc>, size: usize, + index: u32, offset: u64, + region: Arc, } impl Mmio for PthBarRegion @@ -326,19 +352,23 @@ where } fn read(&self, offset: u64, size: u8) -> mem::Result { + let addr = self.offset + offset; log::trace!( - "{}: emulated read at {offset:#x}, size={size}", - self.cdev.name + "{}: emulated read at region {}, offset {addr:#x}, size={size}", + self.cdev.name, + self.index ); - self.cdev.dev.read(self.offset + offset, size) + self.cdev.dev.read(&self.region, addr, size) } fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result { + let addr = self.offset + offset; log::trace!( - "{}: emulated write at {offset:#x}, val={val:#x}, size={size}", - self.cdev.name + "{}: emulated write at region {}, offset {addr:#x}, val={val:#x}, size={size}", + self.cdev.name, + self.index ); - self.cdev.dev.write(self.offset + offset, size, val)?; + self.cdev.dev.write(&self.region, addr, size, val)?; Ok(Action::None) } } @@ -391,18 +421,19 @@ where let msi_sender = Arc::new(msi_sender); - let region_config = cdev.dev.get_region_info(VfioPciRegion::CONFIG.raw())?; + let region_config = Arc::new(cdev.dev.get_region_info(VfioPciRegion::CONFIG.raw())?); let pci_command = Command::IO | Command::MEM | Command::BUS_MASTER | Command::INTX_DISABLE; cdev.dev.write( - region_config.offset + CommonHeader::OFFSET_COMMAND as u64, + ®ion_config, + CommonHeader::OFFSET_COMMAND as u64, CommonHeader::SIZE_COMMAND as u8, pci_command.bits() as _, )?; let mut buf = vec![0u32; region_config.size as usize >> 2]; let buf = buf.as_mut_bytes(); - cdev.dev.fd().read_at(buf, region_config.offset)?; + cdev.dev.read_region(®ion_config, 0, buf)?; let (mut dev_header, _) = DeviceHeader::read_from_prefix(buf).unwrap(); let header_type = dev_header.common.header_type.raw() & !(1 << 7); @@ -424,32 +455,38 @@ where log::error!("{}: invalid cap offset: {cap_offset:#x}", cdev.name); break; }; - let (cap_header, _) = PciCapHdr::ref_from_prefix(cap_buf).unwrap(); + let Ok((cap_header, _)) = PciCapHdr::ref_from_prefix(cap_buf) else { + log::error!( + "{}: invalid cap header at offset: {cap_offset:#x}", + cdev.name + ); + break; + }; if cap_header.id == PciCapId::MSIX { - let Ok((mut c, _)) = MsixCap::read_from_prefix(cap_buf) else { + if let Ok((mut c, _)) = MsixCap::read_from_prefix(cap_buf) { + c.control.set_enabled(false); + c.control.set_masked(false); + msix_info = Some((cap_offset, c.clone())); + } else { log::error!( "{}: MSIX capability is at an invalid offset: {cap_offset:#x}", cdev.name ); - continue; - }; - c.control.set_enabled(false); - c.control.set_masked(false); - msix_info = Some((cap_offset, c.clone())); + } } else if cap_header.id == PciCapId::MSI { - let Ok((mut c, _)) = MsiCapHdr::read_from_prefix(cap_buf) else { + if let Ok((mut c, _)) = MsiCapHdr::read_from_prefix(cap_buf) { + log::info!("{}: MSI cap header: {c:#x?}", cdev.name); + c.control.set_enable(false); + c.control.set_ext_msg_data_cap(true); + let multi_msg_cap = min(5, c.control.multi_msg_cap()); + c.control.set_multi_msg_cap(multi_msg_cap); + msi_info = Some((cap_offset, c)); + } else { log::error!( "{}: MSI capability is at an invalid offset: {cap_offset:#x}", cdev.name ); - continue; - }; - log::info!("{}: MSI cap header: {c:#x?}", cdev.name); - c.control.set_enable(false); - c.control.set_ext_msg_data_cap(true); - let multi_msg_cap = min(5, c.control.multi_msg_cap()); - c.control.set_multi_msg_cap(multi_msg_cap); - msi_info = Some((cap_offset, c)); + } } cap_offset = cap_header.next as usize; } @@ -473,19 +510,12 @@ where .map(|_| msi_sender.create_irqfd()) .collect::, _>>()?; - let mut eventfds = [-1; 32]; - for (fd, irqfd) in zip(&mut eventfds, &irqfds) { - *fd = irqfd.as_fd().as_raw_fd(); + let mut eventfds = vec![]; + for irqfd in &irqfds { + eventfds.push(Some(irqfd.as_fd())); } - let set_eventfd = VfioIrqSet { - argsz: (size_of::>() + size_of::() * count) as u32, - flags: VfioIrqSetFlag::DATA_EVENTFD | VfioIrqSetFlag::ACTION_TRIGGER, - index: VfioPciIrq::MSI.raw(), - start: 0, - count: count as u32, - data: VfioIrqSetData { eventfds }, - }; - cdev.dev.set_irqs(&set_eventfd)?; + cdev.dev + .set_irq_eventfd(VfioPciIrq::MSI.raw(), 0, &eventfds)?; let mut msi_cap_mmio = MsiCapMmio::new(hdr.control, irqfds); msi_cap_mmio.set_next(hdr.header.next); @@ -500,9 +530,10 @@ where extra_areas.add( area_end, Box::new(PthConfigArea { - offset: region_config.offset + area_end, + offset: area_end, size: offset - area_end, dev: cdev.clone(), + region: region_config.clone(), }), )?; } @@ -513,9 +544,10 @@ where extra_areas.add( area_end, Box::new(PthConfigArea { - offset: region_config.offset + area_end, + offset: area_end, size: region_config.size - area_end, dev: cdev.clone(), + region: region_config.clone(), }), )?; } @@ -542,29 +574,18 @@ where let bar_vals = config_header.bars(); for index in VfioPciRegion::BAR0.raw()..=VfioPciRegion::BAR5.raw() { - let region_info = cdev.dev.get_region_info(index)?; + let region_info = Arc::new(cdev.dev.get_region_info(index)?); if region_info.size == 0 { continue; } - let region = if region_info.flags.contains(VfioRegionInfoFlag::MMAP) { - create_mappable_bar_region( - cdev.clone(), - index, - ®ion_info, - msix_cap.as_ref(), - msix_table.clone(), - msi_sender.clone(), - )? - } else { - MemRegion::with_emulated( - Arc::new(PthBarRegion { - cdev: cdev.clone(), - size: region_info.size as usize, - offset: region_info.offset, - }), - MemRegionType::Hidden, - ) - }; + let region = create_bar_region( + cdev.clone(), + index, + region_info, + msix_cap.as_ref(), + msix_table.clone(), + msi_sender.clone(), + )?; let index = index as usize; let bar_val = bar_vals[index]; if bar_val & BAR_IO == BAR_IO { @@ -594,7 +615,7 @@ where let is_irqfd = |e| matches!(e, &MsixTableMmioEntry::IrqFd(_)); if self.msix_table.entries.read().iter().any(is_irqfd) { let dev = &self.config.dev; - if let Err(e) = dev.dev.disable_all_irqs(VfioPciIrq::MSIX) { + if let Err(e) = dev.dev.disable_irq(VfioPciIrq::MSIX.raw()) { log::error!("{}: failed to disable MSIX IRQs: {e:?}", dev.name) } } @@ -616,7 +637,7 @@ where pba: Arc<[AtomicU64]>, // TODO pba_range: Range, cdev: Arc>, - cdev_offset: u64, + region: Arc, region_start: usize, region_size: usize, } @@ -658,26 +679,20 @@ where // subindex for the first time. // As long as the following set_irqs() succeeds, we can safely ignore // the error here. - let _ = self.cdev.dev.disable_all_irqs(VfioPciIrq::MSIX); + let _ = self.cdev.dev.disable_irq(VfioPciIrq::MSIX.raw()); - let mut eventfds = [-1; 2048]; + let mut eventfds = vec![None; entries.len()]; let mut count = 0; - for (index, (entry, fd)) in std::iter::zip(entries.iter(), &mut eventfds).enumerate() { - let MsixTableMmioEntry::IrqFd(irqfd) = entry else { - continue; - }; - count = index + 1; - *fd = irqfd.as_fd().as_raw_fd(); + for (index, (entry, fd)) in entries.iter().zip(&mut eventfds).enumerate() { + if let MsixTableMmioEntry::IrqFd(irqfd) = entry { + *fd = Some(irqfd.as_fd()); + count = index + 1; + } } - let vfio_irq_set_eventfd = VfioIrqSet { - argsz: (size_of::>() + size_of::() * count) as u32, - flags: VfioIrqSetFlag::DATA_EVENTFD | VfioIrqSetFlag::ACTION_TRIGGER, - index: VfioPciIrq::MSIX.raw(), - start: 0, - count: count as u32, - data: VfioIrqSetData { eventfds }, - }; - self.cdev.dev.set_irqs(&vfio_irq_set_eventfd) + eventfds.truncate(count); + self.cdev + .dev + .set_irq_eventfd(VfioPciIrq::MSIX.raw(), 0, &eventfds) } } @@ -701,7 +716,7 @@ where Ok(0) } else { log::trace!("{name}: emulated BAR read at {offset:#x}, size={size}",); - self.cdev.dev.read(self.cdev_offset + offset as u64, size) + self.cdev.dev.read(&self.region, offset as u64, size) } } @@ -722,8 +737,12 @@ where log::trace!("{name}: emulated BAR write at {offset:#x}, size={size}, val={val:#x}",); self.cdev .dev - .write(self.cdev_offset + offset as u64, size, val)?; + .write(&self.region, offset as u64, size, val)?; } Ok(Action::None) } } + +#[cfg(test)] +#[path = "pci_test.rs"] +mod tests; diff --git a/alioth/src/vfio/pci_test.rs b/alioth/src/vfio/pci_test.rs new file mode 100644 index 00000000..b46051cc --- /dev/null +++ b/alioth/src/vfio/pci_test.rs @@ -0,0 +1,1058 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::os::fd::{AsRawFd, BorrowedFd, OwnedFd, RawFd}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use assert_matches::assert_matches; +use parking_lot::{Mutex, RwLock}; +use zerocopy::{FromBytes, IntoBytes}; + +use crate::device::Pause; +use crate::hv::tests::TestMsiSender; +use crate::mem::MemRange; +use crate::mem::emulated::{Action, Mmio}; +use crate::pci::cap::{ + MsiCapHdr, MsiMsgCtrl, MsixCap, MsixCapOffset, MsixMsgCtrl, PciCapHdr, PciCapId, +}; +use crate::pci::config::{ + BAR_IO, BAR_MEM32, Command, CommonHeader, DeviceHeader, HeaderType, PciConfig, Status, +}; +use crate::pci::{Pci, PciBar}; +use crate::sys::vfio::{ + VfioDeviceInfo, VfioDeviceInfoFlag, VfioIrqInfo, VfioPciIrq, VfioPciRegion, VfioRegionInfo, + VfioRegionInfoFlag, +}; +use crate::vfio::device::Device; +use crate::vfio::pci::{PthBarRegion, VfioPciDev}; +use crate::vfio::{Error, Result}; + +type RegionMap = Arc)>>>; +type IrqMap = Arc>>; +type MmapFdMap = Arc>>; +type IrqEventFdList = Arc>)>>>; + +#[derive(Debug, Clone)] +struct MockVfioDevice { + info: VfioDeviceInfo, + regions: RegionMap, + irqs: IrqMap, + mmap_fds: MmapFdMap, + resets: Arc, + irq_eventfds: IrqEventFdList, + disabled_irqs: Arc>>, +} + +impl Default for MockVfioDevice { + fn default() -> Self { + MockVfioDevice { + info: VfioDeviceInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioDeviceInfoFlag::PCI | VfioDeviceInfoFlag::RESET, + num_regions: 9, + num_irqs: 4, + cap_offset: 0, + pad: 0, + }, + regions: Arc::new(RwLock::new(HashMap::new())), + irqs: Arc::new(RwLock::new(HashMap::new())), + mmap_fds: Arc::new(RwLock::new(HashMap::new())), + resets: Arc::new(AtomicUsize::new(0)), + irq_eventfds: Arc::new(Mutex::new(Vec::new())), + disabled_irqs: Arc::new(Mutex::new(Vec::new())), + } + } +} + +impl Device for MockVfioDevice { + fn get_info(&self) -> Result { + Ok(self.info.clone()) + } + + fn get_region_info(&self, index: u32) -> Result { + let regions = self.regions.read(); + let (info, _) = regions + .get(&index) + .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::NotFound))?; + Ok(info.clone()) + } + + fn get_irq_info(&self, index: u32) -> Result { + let irqs = self.irqs.read(); + let info = irqs + .get(&index) + .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::NotFound))?; + Ok(info.clone()) + } + + fn reset(&self) -> Result<()> { + self.resets.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn set_irq_eventfd( + &self, + index: u32, + start: u32, + eventfds: &[Option>], + ) -> Result<()> { + let raw_fds = eventfds + .iter() + .map(|fd| fd.as_ref().map(|f| f.as_raw_fd())) + .collect(); + self.irq_eventfds.lock().push((index, start, raw_fds)); + Ok(()) + } + + fn disable_irq(&self, index: u32) -> Result<()> { + self.disabled_irqs.lock().push(index); + Ok(()) + } + + fn read_region(&self, region: &VfioRegionInfo, offset: u64, buf: &mut [u8]) -> Result<()> { + let regions = self.regions.read(); + let (_, data) = regions + .get(®ion.index) + .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::NotFound))?; + let offset = offset as usize; + let end = offset + buf.len(); + if end > data.len() { + return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof).into()); + } + buf.copy_from_slice(&data[offset..end]); + Ok(()) + } + + fn write_region(&self, region: &VfioRegionInfo, offset: u64, buf: &[u8]) -> Result<()> { + let mut regions = self.regions.write(); + let (_, data) = regions + .get_mut(®ion.index) + .ok_or_else(|| std::io::Error::from(std::io::ErrorKind::NotFound))?; + let offset = offset as usize; + let end = offset + buf.len(); + if end > data.len() { + return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof).into()); + } + data[offset..end].copy_from_slice(buf); + Ok(()) + } + + fn get_region_mmap(&self, index: u32) -> Result> { + let mmap_fds = self.mmap_fds.read(); + if let Some((fd, offset)) = mmap_fds.get(&index) { + let cloned = fd.try_clone()?; + Ok(Some((cloned, *offset))) + } else { + Ok(None) + } + } + + fn get_dma_buf_fd(&self, _index: u32, _offset: u64, _size: usize) -> Result { + Err(std::io::Error::from(std::io::ErrorKind::Unsupported).into()) + } +} + +fn create_mock_pci_config_space() -> Vec { + let mut config = vec![0u8; 4096]; + + let header = DeviceHeader { + common: CommonHeader { + vendor: 0x8086, + device: 0x1234, + command: Command::empty(), + status: Status::CAP, + revision: 0x01, + prog_if: 0, + subclass: 0x00, + class: 0x02, // Network controller + cache_line_size: 0, + latency_timer: 0, + header_type: HeaderType::DEVICE, + bist: 0, + }, + bars: [0; 6], + cardbus_cis_pointer: 0, + subsystem_vendor: 0x8086, + subsystem: 0x5678, + expansion_rom: 0, + capability_pointer: 0x40, + reserved: [0; 7], + intx_line: 0, + intx_pin: 1, + min_gnt: 0, + max_lat: 0, + }; + config[0..64].copy_from_slice(header.as_bytes()); + + // MSI-X capability at offset 0x40 + let msix_cap = MsixCap { + header: PciCapHdr { + id: PciCapId::MSIX, + next: 0, + }, + control: MsixMsgCtrl::new(4), // 4 entries + table_offset: MsixCapOffset::new(0, 0), // BAR 0, offset 0 + pba_offset: MsixCapOffset::new(0x800, 0), // BAR 0, offset 0x800 + }; + config[0x40..0x40 + std::mem::size_of::()].copy_from_slice(msix_cap.as_bytes()); + + config +} + +#[test] +fn test_vfio_pci_dev_creation_and_config() { + let mock = MockVfioDevice::default(); + let config_bytes = create_mock_pci_config_space(); + + // Setup Config region (Region 7) + mock.regions.write().insert( + VfioPciRegion::CONFIG.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::CONFIG.raw(), + cap_offset: 0, + size: 4096, + offset: 0, + }, + config_bytes, + ), + ); + + // Setup BAR 0 (Region 0) + mock.regions.write().insert( + VfioPciRegion::BAR0.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::BAR0.raw(), + cap_offset: 0, + size: 0x10000, + offset: 0x1_0000, + }, + vec![0u8; 0x10000], + ), + ); + + // Setup other BARs as size 0 + for bar_idx in 1..=5 { + mock.regions.write().insert( + bar_idx, + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::empty(), + index: bar_idx, + cap_offset: 0, + size: 0, + offset: 0, + }, + vec![], + ), + ); + } + + let msi_sender = TestMsiSender::default(); + let dev = VfioPciDev::new(Arc::from("test-vfio-pci"), mock.clone(), msi_sender).unwrap(); + + // Verify name + assert_eq!(dev.name(), "test-vfio-pci"); + + // Verify device reset was called + assert_eq!(mock.resets.load(Ordering::SeqCst), 1); + + // Verify PCI config reads + let config = dev.config(); + assert_matches!(config.read(0x00, 2), Ok(0x8086)); // Vendor ID + assert_matches!(config.read(0x02, 2), Ok(0x1234)); // Device ID + assert_matches!(config.read(0x08, 1), Ok(0x01)); // Revision + assert_matches!(config.read(0x0a, 2), Ok(0x0200)); // Class / Subclass + + // Verify command register was initialized with INTX_DISABLE on backend + let cmd_val = { + let regions = mock.regions.read(); + let config_data = ®ions[&VfioPciRegion::CONFIG.raw()].1; + u16::from_le_bytes(config_data[4..6].try_into().unwrap()) + }; + let expected_cmd = Command::IO | Command::MEM | Command::BUS_MASTER | Command::INTX_DISABLE; + assert_eq!(cmd_val, expected_cmd.bits()); + + // Config space write and read (extra config area) + assert_matches!(config.write(0x100, 4, 0xcafe_babe), Ok(Action::None)); + assert_matches!(config.read(0x100, 4), Ok(0xcafe_babe)); + + // Reset via Pci trait + assert_matches!(dev.reset(), Ok(())); + assert_eq!(mock.resets.load(Ordering::SeqCst), 2); +} + +#[test] +fn test_vfio_pci_dev_unsupported_header_type() { + let mock = MockVfioDevice::default(); + let mut config_bytes = create_mock_pci_config_space(); + // Set header type to PCI-to-PCI bridge (0x01) + config_bytes[0x0e] = 0x01; + + mock.regions.write().insert( + VfioPciRegion::CONFIG.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::CONFIG.raw(), + cap_offset: 0, + size: 4096, + offset: 0, + }, + config_bytes, + ), + ); + + let res = VfioPciDev::new(Arc::from("test-vfio"), mock, TestMsiSender::default()); + assert_matches!(res, Err(Error::NotSupportedHeader { ty: 1, .. })); +} + +#[test] +fn test_vfio_pci_dev_msix_bar_and_irqfd() { + let mock = MockVfioDevice::default(); + let config_bytes = create_mock_pci_config_space(); + + mock.regions.write().insert( + VfioPciRegion::CONFIG.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::CONFIG.raw(), + cap_offset: 0, + size: 4096, + offset: 0, + }, + config_bytes, + ), + ); + + // Setup BAR 0 with 0x10000 bytes + let bar0_data = vec![0u8; 0x10000]; + mock.regions.write().insert( + VfioPciRegion::BAR0.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::BAR0.raw(), + cap_offset: 0, + size: 0x10000, + offset: 0x1_0000, + }, + bar0_data, + ), + ); + + for bar_idx in 1..=5 { + mock.regions.write().insert( + bar_idx, + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::empty(), + index: bar_idx, + cap_offset: 0, + size: 0, + offset: 0, + }, + vec![], + ), + ); + } + + let msi_sender = TestMsiSender::default(); + let dev = VfioPciDev::new(Arc::from("test-vfio"), mock.clone(), msi_sender).unwrap(); + + let PciBar::Mem(bar0) = &dev.config.header.bars[0] else { + panic!("expected Mem BAR for BAR 0"); + }; + + // BAR 0 should have Emulated range for MSI-X table & PBA (page 0) and Emulated/DevMem for remainder + let MemRange::Emulated(msix_bar_mmio) = &bar0.ranges[0] else { + panic!("expected Emulated range for MSI-X table in BAR 0"); + }; + + // Test writing to MSI-X table: entry 0 + // addr_lo = 0xfee0_0000, addr_hi = 0, data = 0x20, control = 0 (unmasked) + assert_matches!(msix_bar_mmio.write(0x00, 4, 0xfee0_0000), Ok(Action::None)); + assert_matches!(msix_bar_mmio.write(0x04, 4, 0), Ok(Action::None)); + assert_matches!(msix_bar_mmio.write(0x08, 4, 0x20), Ok(Action::None)); + // Unmasking triggers irqfd enablement + assert_matches!(msix_bar_mmio.write(0x0c, 4, 0), Ok(Action::None)); + + // Verify MSI-X eventfd was set on mock device + let events = mock.irq_eventfds.lock().clone(); + assert!(!events.is_empty()); + assert_eq!(events.last().unwrap().0, VfioPciIrq::MSIX.raw()); + assert_eq!(events.last().unwrap().1, 0); // start = 0 + assert_eq!(events.last().unwrap().2.len(), 1); // 1 active entry + + // Test reading back from MSI-X table + assert_matches!(msix_bar_mmio.read(0x00, 4), Ok(0xfee0_0000)); + assert_matches!(msix_bar_mmio.read(0x08, 4), Ok(0x20)); + + // Test reading / writing PBA area (offset 0x800) + assert_matches!(msix_bar_mmio.read(0x800, 4), Ok(0)); + assert_matches!(msix_bar_mmio.write(0x800, 4, 0), Ok(Action::None)); + + // Test reading / writing emulated region outside table/PBA within page 0 (e.g. offset 0x900) + assert_matches!(msix_bar_mmio.write(0x900, 4, 0x1122_3344), Ok(Action::None)); + assert_matches!(msix_bar_mmio.read(0x900, 4), Ok(0x1122_3344)); + + // Test reset disables active MSI-X IRQs + assert_matches!(dev.reset(), Ok(())); + let disabled = mock.disabled_irqs.lock().clone(); + assert!(disabled.contains(&VfioPciIrq::MSIX.raw())); +} + +#[test] +fn test_vfio_pci_dev_msi_only() { + let mock = MockVfioDevice::default(); + let mut config = vec![0u8; 4096]; + + let header = DeviceHeader { + common: CommonHeader { + vendor: 0x10ec, + device: 0x8168, + status: Status::CAP, + header_type: HeaderType::DEVICE, + ..Default::default() + }, + capability_pointer: 0x50, + ..Default::default() + }; + config[0..64].copy_from_slice(header.as_bytes()); + + // MSI capability at offset 0x50 (Id: 0x05, next: 0) + let mut msi_cap = MsiCapHdr { + header: PciCapHdr { + id: PciCapId::MSI, + next: 0, + }, + control: MsiMsgCtrl(0), + }; + msi_cap.control.set_multi_msg_cap(2); // 4 messages + config[0x50..0x50 + std::mem::size_of::()].copy_from_slice(msi_cap.as_bytes()); + + mock.regions.write().insert( + VfioPciRegion::CONFIG.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::CONFIG.raw(), + cap_offset: 0, + size: 4096, + offset: 0, + }, + config, + ), + ); + + for bar_idx in 0..=5 { + mock.regions.write().insert( + bar_idx, + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::empty(), + index: bar_idx, + cap_offset: 0, + size: 0, + offset: 0, + }, + vec![], + ), + ); + } + + let msi_sender = TestMsiSender::default(); + let dev = VfioPciDev::new(Arc::from("test-msi"), mock.clone(), msi_sender).unwrap(); + + // Verify MSI irq eventfds were registered + let events = mock.irq_eventfds.lock().clone(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].0, VfioPciIrq::MSI.raw()); + assert_eq!(events[0].2.len(), 4); // 4 messages + + // Verify MSI capability MMIO at offset 0x50 + let config = dev.config(); + assert_matches!(config.read(0x50, 1), Ok(val) if val == PciCapId::MSI.raw() as u64); +} + +#[test] +fn test_vfio_pci_dev_both_msi_and_msix() { + let mock = MockVfioDevice::default(); + let mut config = vec![0u8; 4096]; + + let header = DeviceHeader { + common: CommonHeader { + vendor: 0x1af4, + device: 0x1000, + status: Status::CAP, + header_type: HeaderType::DEVICE, + ..Default::default() + }, + capability_pointer: 0x40, + ..Default::default() + }; + config[0..64].copy_from_slice(header.as_bytes()); + + // MSI at 0x40 -> points to MSI-X at 0x60 + let msi_cap = MsiCapHdr { + header: PciCapHdr { + id: PciCapId::MSI, + next: 0x60, + }, + control: MsiMsgCtrl(0), + }; + config[0x40..0x40 + std::mem::size_of::()].copy_from_slice(msi_cap.as_bytes()); + + // MSI-X at 0x60 -> next 0 + let msix_cap = MsixCap { + header: PciCapHdr { + id: PciCapId::MSIX, + next: 0, + }, + control: MsixMsgCtrl::new(4), + table_offset: MsixCapOffset::new(0, 0), + pba_offset: MsixCapOffset::new(0x1000, 0), + }; + config[0x60..0x60 + std::mem::size_of::()].copy_from_slice(msix_cap.as_bytes()); + + mock.regions.write().insert( + VfioPciRegion::CONFIG.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::CONFIG.raw(), + cap_offset: 0, + size: 4096, + offset: 0, + }, + config, + ), + ); + + mock.regions.write().insert( + VfioPciRegion::BAR0.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::BAR0.raw(), + cap_offset: 0, + size: 0x10000, + offset: 0, + }, + vec![0u8; 0x10000], + ), + ); + + for bar_idx in 1..=5 { + mock.regions.write().insert( + bar_idx, + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::empty(), + index: bar_idx, + cap_offset: 0, + size: 0, + offset: 0, + }, + vec![], + ), + ); + } + + let dev = VfioPciDev::new(Arc::from("test-both"), mock, TestMsiSender::default()).unwrap(); + + let config = dev.config(); + // MSI at 0x40 was masked with NullCap (id reads 0, next reads 0x60) + assert_matches!(config.read(0x40, 1), Ok(0)); + assert_matches!(config.read(0x41, 1), Ok(0x60)); + // MSI-X at 0x60 is active + assert_matches!(config.read(0x60, 1), Ok(val) if val == PciCapId::MSIX.raw() as u64); +} + +#[test] +fn test_vfio_pci_dev_bar_types_and_splitting() { + let mock = MockVfioDevice::default(); + let mut config = create_mock_pci_config_space(); + + // Set BAR 0 as Mem32, BAR 1 as IO BAR + let (mut header, _) = DeviceHeader::read_from_prefix(&config).unwrap(); + header.bars[0] = BAR_MEM32; + header.bars[1] = BAR_IO; + config[0..64].copy_from_slice(header.as_bytes()); + + mock.regions.write().insert( + VfioPciRegion::CONFIG.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::CONFIG.raw(), + cap_offset: 0, + size: 4096, + offset: 0, + }, + config, + ), + ); + + // BAR 0: 0x10000 bytes with table at 0x1000 (page 1) and PBA at 0x3000 (page 3) + mock.regions.write().insert( + VfioPciRegion::BAR0.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::BAR0.raw(), + cap_offset: 0, + size: 0x10000, + offset: 0, + }, + vec![0u8; 0x10000], + ), + ); + + // BAR 1: IO space BAR (0x100 bytes) + mock.regions.write().insert( + VfioPciRegion::BAR1.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::BAR1.raw(), + cap_offset: 0, + size: 0x100, + offset: 0, + }, + vec![0u8; 0x100], + ), + ); + + for bar_idx in 2..=5 { + mock.regions.write().insert( + bar_idx, + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::empty(), + index: bar_idx, + cap_offset: 0, + size: 0, + offset: 0, + }, + vec![], + ), + ); + } + + let dev = VfioPciDev::new(Arc::from("test-bars"), mock, TestMsiSender::default()).unwrap(); + + // BAR 0 should be PciBar::Mem with 3 ranges (page 0 emulated table/PBA, and remaining range) + assert_matches!(dev.config.header.bars[0], PciBar::Mem(_)); + + // BAR 1 should be PciBar::Io + assert_matches!(dev.config.header.bars[1], PciBar::Io(_)); +} + +#[test] +fn test_pth_bar_region_mmio() { + let mock = MockVfioDevice::default(); + mock.regions.write().insert( + 0, + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: 0, + cap_offset: 0, + size: 0x1000, + offset: 0x1000, + }, + vec![0u8; 0x1000], + ), + ); + + let vfio_dev = Arc::new(crate::vfio::pci::VfioDev { + name: Arc::from("test-dev"), + dev: mock.clone(), + flags: VfioDeviceInfoFlag::empty(), + }); + + let region_info = mock.regions.read()[&0].0.clone(); + let pth = PthBarRegion { + cdev: vfio_dev, + size: 0x1000, + index: 0, + offset: 0, + region: Arc::new(region_info), + }; + + assert_eq!(pth.size(), 0x1000); + assert_matches!(pth.write(0x10, 4, 0x1234_5678), Ok(Action::None)); + assert_matches!(pth.read(0x10, 4), Ok(0x1234_5678)); +} + +#[test] +fn test_vfio_pci_dev_mmap_bar() { + let mock = MockVfioDevice::default(); + let mut config = create_mock_pci_config_space(); + // Clear MSI-X capability so BAR 0 is not split + let (mut header, _) = DeviceHeader::read_from_prefix(&config).unwrap(); + header.common.status = Status::empty(); + header.capability_pointer = 0; + config[0..64].copy_from_slice(header.as_bytes()); + + mock.regions.write().insert( + VfioPciRegion::CONFIG.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::CONFIG.raw(), + cap_offset: 0, + size: 4096, + offset: 0, + }, + config, + ), + ); + + // Create temp file for mmap + let tmp_file = tempfile::tempfile().unwrap(); + tmp_file.set_len(0x10000).unwrap(); + mock.mmap_fds.write().insert(0, (tmp_file.into(), 0)); + + // BAR 0 with MMAP flag + mock.regions.write().insert( + VfioPciRegion::BAR0.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ + | VfioRegionInfoFlag::WRITE + | VfioRegionInfoFlag::MMAP, + index: VfioPciRegion::BAR0.raw(), + cap_offset: 0, + size: 0x10000, + offset: 0, + }, + vec![0u8; 0x10000], + ), + ); + + for bar_idx in 1..=5 { + mock.regions.write().insert( + bar_idx, + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::empty(), + index: bar_idx, + cap_offset: 0, + size: 0, + offset: 0, + }, + vec![], + ), + ); + } + + let dev = VfioPciDev::new(Arc::from("test-mmap"), mock, TestMsiSender::default()).unwrap(); + assert_eq!(dev.config.header.bars.len(), 6); + let PciBar::Mem(bar0) = &dev.config.header.bars[0] else { + panic!("expected Mem BAR"); + }; + assert_matches!(bar0.ranges[0], MemRange::DevMem { .. }); +} + +#[test] +fn test_vfio_pci_dev_bar_mem64() { + let mock = MockVfioDevice::default(); + let mut config = create_mock_pci_config_space(); + + let (mut header, _) = DeviceHeader::read_from_prefix(&config).unwrap(); + header.bars[0] = crate::pci::config::BAR_MEM64; + header.bars[1] = 0; + config[0..64].copy_from_slice(header.as_bytes()); + + mock.regions.write().insert( + VfioPciRegion::CONFIG.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::CONFIG.raw(), + cap_offset: 0, + size: 4096, + offset: 0, + }, + config, + ), + ); + + mock.regions.write().insert( + VfioPciRegion::BAR0.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::BAR0.raw(), + cap_offset: 0, + size: 0x20000, + offset: 0, + }, + vec![0u8; 0x20000], + ), + ); + + for bar_idx in 1..=5 { + mock.regions.write().insert( + bar_idx, + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::empty(), + index: bar_idx, + cap_offset: 0, + size: 0, + offset: 0, + }, + vec![], + ), + ); + } + + let dev = VfioPciDev::new(Arc::from("test-bar64"), mock, TestMsiSender::default()).unwrap(); + assert_matches!(dev.config.header.bars[0], PciBar::Mem(_)); + assert_matches!(dev.config.header.bars[1], PciBar::Empty); +} + +#[test] +fn test_vfio_pci_cap_parsing_malformed() { + let mock = MockVfioDevice::default(); + let mut config = create_mock_pci_config_space(); + + // Cap pointer points beyond config space (offset 0x5000 in 4096-byte config) + let (mut header, _) = DeviceHeader::read_from_prefix(&config).unwrap(); + header.capability_pointer = 0xff; // 0xff > 64 and beyond valid cap header + config[0..64].copy_from_slice(header.as_bytes()); + + mock.regions.write().insert( + VfioPciRegion::CONFIG.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::CONFIG.raw(), + cap_offset: 0, + size: 4096, + offset: 0, + }, + config, + ), + ); + + for bar_idx in 0..=5 { + mock.regions.write().insert( + bar_idx, + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::empty(), + index: bar_idx, + cap_offset: 0, + size: 0, + offset: 0, + }, + vec![], + ), + ); + } + + // Should complete successfully without infinite loop or panic + let dev = VfioPciDev::new( + Arc::from("test-malformed-cap"), + mock, + TestMsiSender::default(), + ) + .unwrap(); + assert_eq!(dev.name(), "test-malformed-cap"); +} + +#[test] +fn test_vfio_pci_pause_resume_reset_and_pth_config() { + let mock = MockVfioDevice::default(); + let config = create_mock_pci_config_space(); + + mock.regions.write().insert( + VfioPciRegion::CONFIG.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::CONFIG.raw(), + cap_offset: 0, + size: 4096, + offset: 0, + }, + config, + ), + ); + + for bar_idx in 0..=5 { + mock.regions.write().insert( + bar_idx, + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::empty(), + index: bar_idx, + cap_offset: 0, + size: 0, + offset: 0, + }, + vec![], + ), + ); + } + + let dev = VfioPciDev::new( + Arc::from("test-pause-resume"), + mock, + TestMsiSender::default(), + ) + .unwrap(); + assert_eq!(dev.name(), "test-pause-resume"); + assert_matches!(Pause::pause(&dev), Err(_)); + assert_matches!(Pause::resume(&dev), Err(_)); + assert_matches!(dev.reset(), Ok(())); + let cfg = dev.config(); + assert_eq!(cfg.get_header().data.read().get_bar(0), (0, 0)); + + let pth_config = &dev.config; + assert_matches!(pth_config.reset(), Ok(())); +} + +#[test] +fn test_vfio_pci_disjoint_and_reversed_msix_bar() { + let mock = MockVfioDevice::default(); + let mut config = create_mock_pci_config_space(); + + // Table at 0x20000, PBA at 0x10000 on BAR 0 of size 0x40000 + let msix_cap = MsixCap { + header: PciCapHdr { + id: PciCapId::MSIX, + next: 0, + }, + control: MsixMsgCtrl::new(3), // 4 entries + table_offset: MsixCapOffset::new(0x20000, 0), // BAR 0, offset 0x20000 + pba_offset: MsixCapOffset::new(0x10000, 0), // BAR 0, offset 0x10000 + }; + config[0x40..0x40 + std::mem::size_of::()].copy_from_slice(msix_cap.as_bytes()); + + mock.regions.write().insert( + VfioPciRegion::CONFIG.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::CONFIG.raw(), + cap_offset: 0, + size: 4096, + offset: 0, + }, + config, + ), + ); + + mock.regions.write().insert( + VfioPciRegion::BAR0.raw(), + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: VfioPciRegion::BAR0.raw(), + cap_offset: 0, + size: 0x40000, + offset: 0, + }, + vec![0u8; 0x40000], + ), + ); + + for bar_idx in 1..=5 { + mock.regions.write().insert( + bar_idx, + ( + VfioRegionInfo { + argsz: std::mem::size_of::() as u32, + flags: VfioRegionInfoFlag::empty(), + index: bar_idx, + cap_offset: 0, + size: 0, + offset: 0, + }, + vec![], + ), + ); + } + + let dev = VfioPciDev::new( + Arc::from("test-disjoint-bar"), + mock.clone(), + TestMsiSender::default(), + ) + .unwrap(); + let PciBar::Mem(bar0) = &dev.config.header.bars[0] else { + panic!("expected Mem BAR"); + }; + + // BAR 0 should be split: [DevMem, Emulated(PBA), DevMem, Emulated(Table), DevMem] + assert_eq!(bar0.ranges.len(), 5); + + let MemRange::Emulated(table_mmio) = &bar0.ranges[3] else { + panic!("expected Emulated table mmio"); + }; + assert!(table_mmio.size() > 0); + + // 1. Write masked entry (control = 1) -> does not activate irqfd + table_mmio.write(0x20000 - 0x20000, 4, 0xfee0_0000).unwrap(); // addr_lo + table_mmio.write(0x20004 - 0x20000, 4, 0x0).unwrap(); // addr_hi + table_mmio.write(0x20008 - 0x20000, 4, 0x40).unwrap(); // data + table_mmio.write(0x2000c - 0x20000, 4, 0x1).unwrap(); // control (masked) + + // 2. Unmask entry (control = 0) -> activates irqfd + table_mmio.write(0x2000c - 0x20000, 4, 0x0).unwrap(); + assert_eq!(mock.irq_eventfds.lock().len(), 1); + + // 3. Write again while already IrqFd (control = 0) + table_mmio.write(0x2000c - 0x20000, 4, 0x0).unwrap(); +} diff --git a/alioth/src/vfio/user/bindings.rs b/alioth/src/vfio/user/bindings.rs new file mode 100644 index 00000000..e321c712 --- /dev/null +++ b/alioth/src/vfio/user/bindings.rs @@ -0,0 +1,130 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bitfield::bitfield; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::sys::vfio::{VfioDeviceInfoFlag, VfioIrqSetFlag}; +use crate::{bitflags, consts}; + +consts! { + pub struct VfioUserCmd(u16) { + VERSION = 1; + DMA_MAP = 2; + DMA_UNMAP = 3; + DEVICE_GET_INFO = 4; + DEVICE_GET_REGION_INFO = 5; + DEVICE_GET_REGION_IO_FDS = 6; + DEVICE_GET_IRQ_INFO = 7; + DEVICE_SET_IRQS = 8; + REGION_READ = 9; + REGION_WRITE = 10; + DMA_READ = 11; + DMA_WRITE = 12; + DEVICE_RESET = 13; + REGION_WRITE_MULTI = 15; + DEVICE_FEATURE = 16; + MIG_DATA_READ = 17; + MIG_DATA_WRITE = 18; + } +} + +consts! { + pub struct VfioUserMessageType(u8) { + COMMAND = 0; + REPLY = 1; + } +} + +bitfield! { + #[derive(Copy, Clone, Default, IntoBytes, FromBytes, Immutable, KnownLayout)] + pub struct VfioUserHeaderFlag(u32); + impl Debug; + impl new; + pub u8, from into VfioUserMessageType, ty, set_ty: 3, 0; + pub no_reply, set_no_reply: 4; + pub error, set_error: 5; +} + +#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, Immutable, Default)] +#[repr(C)] +pub struct VfioUserHeader { + pub msg_id: u16, + pub cmd: VfioUserCmd, + pub msg_size: u32, + pub flags: VfioUserHeaderFlag, + pub error_no: u32, +} + +#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, Immutable)] +#[repr(C)] +pub struct VfioUserVersion { + pub major: u16, + pub minor: u16, +} + +#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, Immutable, Default)] +#[repr(C)] +pub struct VfioUserDeviceInfo { + pub argsz: u32, + pub flags: VfioDeviceInfoFlag, + pub num_regions: u32, + pub num_irqs: u32, +} + +#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, Immutable, Default)] +#[repr(C)] +pub struct VfioUserIrqSet { + pub argsz: u32, + pub flags: VfioIrqSetFlag, + pub index: u32, + pub start: u32, + pub count: u32, +} + +#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, Immutable, Default)] +#[repr(C)] +pub struct VfioUserRegionAccess { + pub offset: u64, + pub region: u32, + pub count: u32, +} + +bitflags! { + pub struct VfioUserDmaMapFlag(u32) { + READ = 1 << 0; + WRITE = 1 << 1; + MMAP = 1 << 2; + FILE_IO = 1 << 3; + } +} + +#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, Immutable, Default)] +#[repr(C)] +pub struct VfioUserDmaMap { + pub argsz: u32, + pub flags: VfioUserDmaMapFlag, + pub offset: u64, + pub addr: u64, + pub size: u64, +} + +#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, Immutable, Default)] +#[repr(C)] +pub struct VfioUserDmaUnmap { + pub argsz: u32, + pub flags: u32, + pub addr: u64, + pub size: u64, +} diff --git a/alioth/src/vfio/user/conn.rs b/alioth/src/vfio/user/conn.rs new file mode 100644 index 00000000..2423168c --- /dev/null +++ b/alioth/src/vfio/user/conn.rs @@ -0,0 +1,255 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::{IoSlice, IoSliceMut}; +use std::mem::size_of; +use std::os::fd::{BorrowedFd, OwnedFd}; +use std::os::unix::net::UnixStream; +use std::sync::atomic::{AtomicU16, Ordering}; + +use parking_lot::Mutex; +use zerocopy::{FromBytes, IntoBytes}; + +use crate::errors::BoxTrace; +use crate::mem; +use crate::mem::mapped::ArcMemPages; +use crate::utils::uds::{recv_msg_with_fds, send_msg_with_fds}; +use crate::vfio::{Result, error}; + +use super::bindings::*; + +#[derive(Debug)] +pub struct VfioUserSession { + stream: Mutex, + next_msg_id: AtomicU16, +} + +impl VfioUserSession { + pub fn new(stream: UnixStream) -> Self { + VfioUserSession { + stream: Mutex::new(stream), + next_msg_id: AtomicU16::new(0), + } + } + + fn alloc_msg_id(&self) -> u16 { + self.next_msg_id.fetch_add(1, Ordering::AcqRel) + } + + pub fn transact( + &self, + cmd: VfioUserCmd, + req_slices: &[IoSlice<'_>], + req_fds: &[BorrowedFd<'_>], + resp_buf: &mut [u8], + resp_fds: &mut [Option], + ) -> Result { + let msg_id = self.alloc_msg_id(); + let mut stream = self.stream.lock(); + + let req_payload_size: usize = req_slices.iter().map(|s| s.len()).sum(); + let total_req_size = size_of::() + req_payload_size; + + let header = VfioUserHeader { + msg_id, + cmd, + msg_size: total_req_size as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::COMMAND, false, false), + error_no: 0, + }; + + let mut send_slices = Vec::with_capacity(req_slices.len() + 1); + send_slices.push(IoSlice::new(header.as_bytes())); + send_slices.extend_from_slice(req_slices); + + send_msg_with_fds(&stream, &send_slices, req_fds)?; + + let mut reply_header = VfioUserHeader::default(); + let mut reply_header_slice = [IoSliceMut::new(reply_header.as_mut_bytes())]; + + let bytes_read = recv_msg_with_fds(&stream, &mut reply_header_slice, resp_fds)?; + let header_size = size_of::(); + if bytes_read < header_size { + use std::io::Read; + stream.read_exact(&mut reply_header.as_mut_bytes()[bytes_read..header_size])?; + } + + let reply_msg_id = reply_header.msg_id; + let reply_flags = reply_header.flags; + let reply_error_no = reply_header.error_no; + let reply_msg_size = reply_header.msg_size; + + if reply_msg_id != msg_id { + return error::VfioUser { + msg: format!("msg_id mismatch: expected {}, got {}", msg_id, reply_msg_id), + } + .fail(); + } + if reply_flags.ty() != VfioUserMessageType::REPLY { + return error::VfioUser { + msg: format!( + "unexpected message flags: {:?} (expected reply type)", + reply_flags + ), + } + .fail(); + } + + if reply_flags.error() { + return Err(std::io::Error::from_raw_os_error(reply_error_no as i32).into()); + } + + if (reply_msg_size as usize) < header_size { + return error::VfioUser { + msg: format!( + "invalid reply message size: {} (must be at least {})", + reply_msg_size, header_size + ), + } + .fail(); + } + let payload_size = reply_msg_size as usize - header_size; + if payload_size > 0 { + if resp_buf.len() < payload_size { + return error::VfioUser { + msg: format!( + "response buffer too small: need {}, got {}", + payload_size, + resp_buf.len() + ), + } + .fail(); + } + let read_buf = &mut resp_buf[..payload_size]; + use std::io::Read; + stream.read_exact(read_buf)?; + } + + Ok(reply_header) + } + + pub fn negotiate_version(&self) -> Result<()> { + let version_hdr = VfioUserVersion { major: 0, minor: 2 }; + let caps_str = "{\"capabilities\":{\"max_fds\":32,\"max_data_xfer_size\":1048576}}\0"; + let caps_bytes = caps_str.as_bytes(); + + let req_slices = [ + IoSlice::new(version_hdr.as_bytes()), + IoSlice::new(caps_bytes), + ]; + + let mut resp_buf = vec![0u8; 8192]; + let mut resp_fds = vec![]; + + let reply = self.transact( + VfioUserCmd::VERSION, + &req_slices, + &[], + &mut resp_buf, + &mut resp_fds, + )?; + + let payload_size = reply.msg_size as usize - size_of::(); + if payload_size < size_of::() { + return error::VfioUser { + msg: format!("version reply payload too small: {}", payload_size), + } + .fail(); + } + + let Ok((version_reply, _)) = VfioUserVersion::read_from_prefix(&resp_buf[..payload_size]) + else { + return error::VfioUser { + msg: "failed to parse version reply".to_string(), + } + .fail(); + }; + let server_major = version_reply.major; + let server_minor = version_reply.minor; + log::debug!( + "vfio-user server version: {}.{}", + server_major, + server_minor + ); + if server_major != 0 { + return error::VfioUser { + msg: format!("unsupported server major version: {}", server_major), + } + .fail(); + } + + let caps_reply_size = payload_size - size_of::(); + if caps_reply_size > 0 { + let caps_reply = &resp_buf[size_of::()..payload_size]; + log::debug!( + "vfio-user server capabilities: {}", + String::from_utf8_lossy(caps_reply) + ); + } + + Ok(()) + } + + pub fn dma_map(&self, gpa: u64, pages: &ArcMemPages) -> mem::Result<()> { + let Some((fd, offset)) = pages.fd() else { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Memory is not backed by an FD", + ) + .into()); + }; + + let map = VfioUserDmaMap { + argsz: size_of::() as u32, + flags: VfioUserDmaMapFlag::READ | VfioUserDmaMapFlag::WRITE, + offset, + addr: gpa, + size: pages.size(), + }; + + let mut resp_fds = vec![]; + let ret = self.transact( + VfioUserCmd::DMA_MAP, + &[IoSlice::new(map.as_bytes())], + &[fd], + &mut [], + &mut resp_fds, + ); + + ret.box_trace(mem::error::ChangeLayout)?; + Ok(()) + } + + pub fn dma_unmap(&self, gpa: u64, pages: &ArcMemPages) -> mem::Result<()> { + let unmap = VfioUserDmaUnmap { + argsz: size_of::() as u32, + flags: 0, + addr: gpa, + size: pages.size(), + }; + + let mut resp = VfioUserDmaUnmap::default(); + let mut resp_fds = vec![]; + let ret = self.transact( + VfioUserCmd::DMA_UNMAP, + &[IoSlice::new(unmap.as_bytes())], + &[], + resp.as_mut_bytes(), + &mut resp_fds, + ); + + ret.box_trace(mem::error::ChangeLayout)?; + Ok(()) + } +} diff --git a/alioth/src/vfio/user/device.rs b/alioth/src/vfio/user/device.rs new file mode 100644 index 00000000..d1e5f7f6 --- /dev/null +++ b/alioth/src/vfio/user/device.rs @@ -0,0 +1,399 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::Debug; +use std::io::IoSlice; +use std::mem::size_of; +use std::os::fd::{BorrowedFd, OwnedFd}; +use std::sync::Arc; + +use parking_lot::RwLock; +use zerocopy::{FromBytes, IntoBytes}; + +use crate::mem; +use crate::mem::LayoutChanged; +use crate::mem::mapped::ArcMemPages; +use crate::sys::vfio::{ + VfioDeviceInfo, VfioDeviceInfoFlag, VfioIrqInfo, VfioIrqSetFlag, VfioRegionInfo, + VfioRegionInfoFlag, +}; +use crate::vfio::device::Device; +use crate::vfio::{Result, error}; + +use super::bindings::*; +use super::conn::VfioUserSession; + +#[derive(Debug)] +pub struct VfioUserDevice { + session: Arc, + num_regions: u32, + num_irqs: u32, + flags: VfioDeviceInfoFlag, + regions: RwLock>, + region_fds: RwLock>>, +} + +impl VfioUserDevice { + pub fn new(session: Arc) -> Result { + let req = VfioUserDeviceInfo { + argsz: size_of::() as u32, + ..Default::default() + }; + let mut resp = VfioUserDeviceInfo::default(); + let mut resp_fds = vec![]; + session.transact( + VfioUserCmd::DEVICE_GET_INFO, + &[IoSlice::new(req.as_bytes())], + &[], + resp.as_mut_bytes(), + &mut resp_fds, + )?; + + let num_regions = resp.num_regions; + let num_irqs = resp.num_irqs; + let flags = resp.flags; + + let mut regions = Vec::with_capacity(num_regions as usize); + let mut region_fds = Vec::with_capacity(num_regions as usize); + for _ in 0..num_regions { + regions.push(VfioRegionInfo::default()); + region_fds.push(None); + } + + let dev = VfioUserDevice { + session, + num_regions, + num_irqs, + flags, + regions: RwLock::new(regions), + region_fds: RwLock::new(region_fds), + }; + + dev.fetch_regions()?; + + Ok(dev) + } + + fn fetch_regions(&self) -> Result<()> { + let mut regions = self.regions.write(); + let mut region_fds = self.region_fds.write(); + + for index in 0..self.num_regions { + let req = VfioRegionInfo { + argsz: size_of::() as u32, + index, + ..Default::default() + }; + let mut resp = VfioRegionInfo::default(); + let mut resp_fds = Vec::new(); + resp_fds.push(None); + + self.session.transact( + VfioUserCmd::DEVICE_GET_REGION_INFO, + &[IoSlice::new(req.as_bytes())], + &[], + resp.as_mut_bytes(), + &mut resp_fds, + )?; + + regions[index as usize] = VfioRegionInfo { + argsz: size_of::() as u32, + flags: resp.flags, + index: resp.index, + cap_offset: resp.cap_offset, + size: resp.size, + offset: resp.offset, + }; + + if let Some(fd) = resp_fds[0].take() { + log::debug!("vfio-user: got FD for region {}: {:?}", index, fd); + region_fds[index as usize] = Some(fd); + } + } + Ok(()) + } +} + +impl Device for VfioUserDevice { + fn get_info(&self) -> Result { + Ok(VfioDeviceInfo { + argsz: size_of::() as u32, + flags: self.flags, + num_regions: self.num_regions, + num_irqs: self.num_irqs, + cap_offset: 0, + pad: 0, + }) + } + + fn get_region_info(&self, index: u32) -> Result { + if index >= self.num_regions { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "index out of range", + ) + .into()); + } + Ok(self.regions.read()[index as usize].clone()) + } + + fn get_irq_info(&self, index: u32) -> Result { + let req = VfioIrqInfo { + argsz: size_of::() as u32, + index, + ..Default::default() + }; + let mut resp = VfioIrqInfo::default(); + let mut resp_fds = vec![]; + self.session.transact( + VfioUserCmd::DEVICE_GET_IRQ_INFO, + &[IoSlice::new(req.as_bytes())], + &[], + resp.as_mut_bytes(), + &mut resp_fds, + )?; + + Ok(VfioIrqInfo { + argsz: size_of::() as u32, + flags: resp.flags, + index: resp.index, + count: resp.count, + }) + } + + fn reset(&self) -> Result<()> { + let mut resp_fds = vec![]; + self.session + .transact(VfioUserCmd::DEVICE_RESET, &[], &[], &mut [], &mut resp_fds)?; + Ok(()) + } + + fn set_irq_eventfd( + &self, + index: u32, + start: u32, + eventfds: &[Option>], + ) -> Result<()> { + let mut send_fds = vec![]; + let mut fd_indices = vec![]; + for fd in eventfds { + if let Some(f) = fd { + fd_indices.push(send_fds.len() as i32); + send_fds.push(*f); + } else { + fd_indices.push(-1); + } + } + + let irq_set = VfioUserIrqSet { + argsz: (size_of::() + fd_indices.len() * size_of::()) as u32, + flags: VfioIrqSetFlag::DATA_EVENTFD | VfioIrqSetFlag::ACTION_TRIGGER, + index, + start, + count: eventfds.len() as u32, + }; + + let req_slices = [ + IoSlice::new(irq_set.as_bytes()), + IoSlice::new(fd_indices.as_bytes()), + ]; + + let mut resp_fds = vec![]; + self.session.transact( + VfioUserCmd::DEVICE_SET_IRQS, + &req_slices, + &send_fds, + &mut [], + &mut resp_fds, + )?; + Ok(()) + } + + fn disable_irq(&self, index: u32) -> Result<()> { + let irq_set = VfioUserIrqSet { + argsz: size_of::() as u32, + flags: VfioIrqSetFlag::DATA_NONE | VfioIrqSetFlag::ACTION_TRIGGER, + index, + start: 0, + count: 0, + }; + let mut resp_fds = vec![]; + self.session.transact( + VfioUserCmd::DEVICE_SET_IRQS, + &[IoSlice::new(irq_set.as_bytes())], + &[], + &mut [], + &mut resp_fds, + )?; + Ok(()) + } + + fn read_region(&self, region: &VfioRegionInfo, offset: u64, buf: &mut [u8]) -> Result<()> { + let req = VfioUserRegionAccess { + offset, + region: region.index, + count: buf.len() as u32, + }; + let needed_size = size_of::() + buf.len(); + let mut stack_buf = [0u8; 128]; + let mut heap_buf; + let resp_buf = if needed_size <= 128 { + &mut stack_buf[..needed_size] + } else { + heap_buf = vec![0u8; needed_size]; + &mut heap_buf[..] + }; + + let mut resp_fds = vec![]; + self.session.transact( + VfioUserCmd::REGION_READ, + &[IoSlice::new(req.as_bytes())], + &[], + resp_buf, + &mut resp_fds, + )?; + let Ok((resp, _)) = VfioUserRegionAccess::read_from_prefix(&resp_buf[..]) else { + return error::VfioUser { + msg: "failed to parse read region response".to_string(), + } + .fail(); + }; + let count = resp.count; + if count != buf.len() as u32 { + return error::VfioUser { + msg: format!( + "read region truncated: expected {}, got {}", + buf.len(), + count + ), + } + .fail(); + } + buf.copy_from_slice(&resp_buf[size_of::()..]); + Ok(()) + } + + fn write_region(&self, region: &VfioRegionInfo, offset: u64, buf: &[u8]) -> Result<()> { + let req = VfioUserRegionAccess { + offset, + region: region.index, + count: buf.len() as u32, + }; + let req_slices = [IoSlice::new(req.as_bytes()), IoSlice::new(buf)]; + let mut resp = VfioUserRegionAccess::default(); + let mut resp_fds = vec![]; + self.session.transact( + VfioUserCmd::REGION_WRITE, + &req_slices, + &[], + resp.as_mut_bytes(), + &mut resp_fds, + )?; + let written = resp.count; + if written != buf.len() as u32 { + return error::VfioUser { + msg: format!( + "write region truncated: expected {}, got {}", + buf.len(), + written + ), + } + .fail(); + } + Ok(()) + } + + fn get_region_mmap(&self, index: u32) -> Result> { + if index >= self.num_regions { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "index out of range", + ) + .into()); + } + let regions = self.regions.read(); + let region_info = ®ions[index as usize]; + if region_info.flags.contains(VfioRegionInfoFlag::MMAP) { + let fds = self.region_fds.read(); + if let Some(fd) = &fds[index as usize] { + let cloned_fd = fd.try_clone()?; + Ok(Some((cloned_fd, region_info.offset))) + } else { + Err( + std::io::Error::new(std::io::ErrorKind::NotFound, "No FD for mappable region") + .into(), + ) + } + } else { + Ok(None) + } + } + + fn get_dma_buf_fd(&self, _index: u32, _offset: u64, _size: usize) -> Result { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "dma-buf is not supported in vfio-user", + ) + .into()) + } +} + +#[derive(Debug)] +pub struct UpdateVfioUserMapping { + session: Arc, +} + +impl UpdateVfioUserMapping { + pub fn new(session: Arc) -> Self { + UpdateVfioUserMapping { session } + } +} + +impl LayoutChanged for UpdateVfioUserMapping { + fn ram_added(&self, gpa: u64, pages: &ArcMemPages) -> mem::Result<()> { + self.session.dma_map(gpa, pages) + } + + fn ram_removed(&self, gpa: u64, pages: &ArcMemPages) -> mem::Result<()> { + self.session.dma_unmap(gpa, pages) + } + + fn dev_mem_added( + &self, + gpa: u64, + pages: &ArcMemPages, + _: Option, + ) -> mem::Result<()> { + if pages.fd().is_none() { + log::warn!( + "vfio-user: dev_mem_added: no fd for pages at gpa {:#x}, skipping mapping", + gpa + ); + return Ok(()); + } + self.session.dma_map(gpa, pages) + } + + fn dev_mem_removed( + &self, + gpa: u64, + pages: &ArcMemPages, + _: Option, + ) -> mem::Result<()> { + if pages.fd().is_none() { + return Ok(()); + } + self.ram_removed(gpa, pages) + } +} diff --git a/alioth/src/vfio/user/user.rs b/alioth/src/vfio/user/user.rs new file mode 100644 index 00000000..44b90402 --- /dev/null +++ b/alioth/src/vfio/user/user.rs @@ -0,0 +1,21 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod bindings; +pub mod conn; +pub mod device; + +#[cfg(test)] +#[path = "user_test.rs"] +mod tests; diff --git a/alioth/src/vfio/user/user_test.rs b/alioth/src/vfio/user/user_test.rs new file mode 100644 index 00000000..38d0bce2 --- /dev/null +++ b/alioth/src/vfio/user/user_test.rs @@ -0,0 +1,432 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::{IoSlice, IoSliceMut, Read}; +use std::os::fd::{AsFd, OwnedFd}; +use std::os::unix::net::UnixStream; +use std::sync::Arc; +use std::thread; + +use assert_matches::assert_matches; +use zerocopy::{FromBytes, IntoBytes}; + +use crate::mem::LayoutChanged; +use crate::mem::mapped::ArcMemPages; +use crate::sys::vfio::{ + VfioDeviceInfoFlag, VfioIrqInfo, VfioIrqInfoFlag, VfioRegionInfo, VfioRegionInfoFlag, +}; +use crate::utils::uds::{recv_msg_with_fds, send_msg_with_fds}; +use crate::vfio::Error; +use crate::vfio::device::Device; +use crate::vfio::user::bindings::{ + VfioUserCmd, VfioUserDeviceInfo, VfioUserDmaUnmap, VfioUserHeader, VfioUserHeaderFlag, + VfioUserMessageType, VfioUserRegionAccess, VfioUserVersion, +}; +use crate::vfio::user::conn::VfioUserSession; +use crate::vfio::user::device::{UpdateVfioUserMapping, VfioUserDevice}; + +fn handle_vfio_user_server(stream: &UnixStream) { + let dummy_file = tempfile::tempfile().unwrap(); + dummy_file.set_len(0x1000).unwrap(); + let mmap_fd: OwnedFd = dummy_file.into(); + + let mut header_buf = [0u8; size_of::()]; + let mut payload_buf = [0u8; 4096]; + let mut recv_fds = [const { None }; 32]; + + loop { + for fd in &mut recv_fds { + *fd = None; + } + let mut header_slice = [IoSliceMut::new(&mut header_buf)]; + let bytes = match recv_msg_with_fds(stream, &mut header_slice, &mut recv_fds) { + Ok(0) => break, // EOF + Ok(b) => b, + Err(e) => { + eprintln!("server recv_msg error: {e:?}"); + break; + } + }; + if bytes < size_of::() { + let mut s = stream; + if let Err(e) = s.read_exact(&mut header_buf[bytes..]) { + eprintln!("server read_exact header error: {e:?}"); + break; + } + } + let (req_header, _) = VfioUserHeader::read_from_prefix(&header_buf).unwrap(); + let payload_size = req_header.msg_size as usize - size_of::(); + if payload_size > 0 { + let mut s = stream; + if let Err(e) = s.read_exact(&mut payload_buf[..payload_size]) { + eprintln!("server read_exact payload error: {e:?}"); + break; + } + } + + match req_header.cmd { + VfioUserCmd::VERSION => { + let reply_version = VfioUserVersion { major: 0, minor: 2 }; + let cap_str = b"{\"capabilities\":{\"max_msg_fds\":32}}\0"; + let reply_size = + size_of::() + size_of::() + cap_str.len(); + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: VfioUserCmd::VERSION, + msg_size: reply_size as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, false), + error_no: 0, + }; + let slices = [ + IoSlice::new(reply_hdr.as_bytes()), + IoSlice::new(reply_version.as_bytes()), + IoSlice::new(cap_str), + ]; + send_msg_with_fds(stream, &slices, &[]).unwrap(); + } + VfioUserCmd::DEVICE_GET_INFO => { + let dev_info = VfioUserDeviceInfo { + argsz: size_of::() as u32, + flags: VfioDeviceInfoFlag::PCI | VfioDeviceInfoFlag::RESET, + num_regions: 2, + num_irqs: 2, + }; + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: VfioUserCmd::DEVICE_GET_INFO, + msg_size: (size_of::() + size_of::()) + as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, false), + error_no: 0, + }; + let slices = [ + IoSlice::new(reply_hdr.as_bytes()), + IoSlice::new(dev_info.as_bytes()), + ]; + send_msg_with_fds(stream, &slices, &[]).unwrap(); + } + VfioUserCmd::DEVICE_GET_REGION_INFO => { + let (reg_req, _) = VfioRegionInfo::read_from_prefix(&payload_buf).unwrap(); + let (reg_info, fd_to_send) = if reg_req.index == 0 { + ( + VfioRegionInfo { + argsz: size_of::() as u32, + flags: VfioRegionInfoFlag::READ + | VfioRegionInfoFlag::WRITE + | VfioRegionInfoFlag::MMAP, + index: 0, + cap_offset: 0, + size: 0x1000, + offset: 0, + }, + Some(mmap_fd.as_fd()), + ) + } else { + ( + VfioRegionInfo { + argsz: size_of::() as u32, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + index: 1, + cap_offset: 0, + size: 0x1000, + offset: 0, + }, + None, + ) + }; + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: VfioUserCmd::DEVICE_GET_REGION_INFO, + msg_size: (size_of::() + size_of::()) as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, false), + error_no: 0, + }; + let slices = [ + IoSlice::new(reply_hdr.as_bytes()), + IoSlice::new(reg_info.as_bytes()), + ]; + if let Some(fd) = fd_to_send { + send_msg_with_fds(stream, &slices, &[fd]).unwrap(); + } else { + send_msg_with_fds(stream, &slices, &[]).unwrap(); + } + } + VfioUserCmd::DEVICE_GET_IRQ_INFO => { + let (irq_req, _) = VfioIrqInfo::read_from_prefix(&payload_buf).unwrap(); + let irq_info = VfioIrqInfo { + argsz: size_of::() as u32, + flags: VfioIrqInfoFlag::EVENTFD, + index: irq_req.index, + count: 4, + }; + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: VfioUserCmd::DEVICE_GET_IRQ_INFO, + msg_size: (size_of::() + size_of::()) as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, false), + error_no: 0, + }; + let slices = [ + IoSlice::new(reply_hdr.as_bytes()), + IoSlice::new(irq_info.as_bytes()), + ]; + send_msg_with_fds(stream, &slices, &[]).unwrap(); + } + VfioUserCmd::DEVICE_RESET => { + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: VfioUserCmd::DEVICE_RESET, + msg_size: size_of::() as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, false), + error_no: 0, + }; + let slices = [IoSlice::new(reply_hdr.as_bytes())]; + send_msg_with_fds(stream, &slices, &[]).unwrap(); + } + VfioUserCmd::DEVICE_SET_IRQS => { + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: VfioUserCmd::DEVICE_SET_IRQS, + msg_size: size_of::() as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, false), + error_no: 0, + }; + let slices = [IoSlice::new(reply_hdr.as_bytes())]; + send_msg_with_fds(stream, &slices, &[]).unwrap(); + } + VfioUserCmd::REGION_READ => { + let (access_req, _) = VfioUserRegionAccess::read_from_prefix(&payload_buf).unwrap(); + let access_resp = VfioUserRegionAccess { + offset: access_req.offset, + region: access_req.region, + count: access_req.count, + }; + let data = vec![0xaa; access_req.count as usize]; + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: VfioUserCmd::REGION_READ, + msg_size: (size_of::() + + size_of::() + + data.len()) as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, false), + error_no: 0, + }; + let slices = [ + IoSlice::new(reply_hdr.as_bytes()), + IoSlice::new(access_resp.as_bytes()), + IoSlice::new(&data), + ]; + send_msg_with_fds(stream, &slices, &[]).unwrap(); + } + VfioUserCmd::REGION_WRITE => { + let (access_req, _) = VfioUserRegionAccess::read_from_prefix(&payload_buf).unwrap(); + let access_resp = VfioUserRegionAccess { + offset: access_req.offset, + region: access_req.region, + count: access_req.count, + }; + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: VfioUserCmd::REGION_WRITE, + msg_size: (size_of::() + size_of::()) + as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, false), + error_no: 0, + }; + let slices = [ + IoSlice::new(reply_hdr.as_bytes()), + IoSlice::new(access_resp.as_bytes()), + ]; + send_msg_with_fds(stream, &slices, &[]).unwrap(); + } + VfioUserCmd::DMA_MAP => { + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: VfioUserCmd::DMA_MAP, + msg_size: size_of::() as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, false), + error_no: 0, + }; + let slices = [IoSlice::new(reply_hdr.as_bytes())]; + send_msg_with_fds(stream, &slices, &[]).unwrap(); + } + VfioUserCmd::DMA_UNMAP => { + let (unmap_req, _) = VfioUserDmaUnmap::read_from_prefix(&payload_buf).unwrap(); + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: VfioUserCmd::DMA_UNMAP, + msg_size: (size_of::() + size_of::()) as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, false), + error_no: 0, + }; + let slices = [ + IoSlice::new(reply_hdr.as_bytes()), + IoSlice::new(unmap_req.as_bytes()), + ]; + send_msg_with_fds(stream, &slices, &[]).unwrap(); + } + _ => { + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: req_header.cmd, + msg_size: size_of::() as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, true), + error_no: libc::ENOSYS as u32, + }; + let slices = [IoSlice::new(reply_hdr.as_bytes())]; + send_msg_with_fds(stream, &slices, &[]).unwrap(); + } + } + } +} + +#[test] +fn test_vfio_user_device_full_lifecycle() { + let (client, server) = UnixStream::pair().unwrap(); + let server_handle = thread::spawn(move || { + handle_vfio_user_server(&server); + }); + + let session = Arc::new(VfioUserSession::new(client)); + let dev = VfioUserDevice::new(session.clone()).unwrap(); + + // 1. Test get_info + let info = dev.get_info().unwrap(); + assert_eq!(info.num_regions, 2); + assert_eq!(info.num_irqs, 2); + + // 2. Test get_region_info + let reg0 = dev.get_region_info(0).unwrap(); + assert_eq!(reg0.size, 0x1000); + assert!(reg0.flags.contains(VfioRegionInfoFlag::MMAP)); + + let reg1 = dev.get_region_info(1).unwrap(); + assert_eq!(reg1.size, 0x1000); + assert!(!reg1.flags.contains(VfioRegionInfoFlag::MMAP)); + + // 3. Test get_region_mmap + let mmap0 = dev.get_region_mmap(0).unwrap(); + assert!(mmap0.is_some()); + let mmap1 = dev.get_region_mmap(1).unwrap(); + assert!(mmap1.is_none()); + + // 4. Test get_irq_info + let irq_info = dev.get_irq_info(0).unwrap(); + assert_eq!(irq_info.count, 4); + + // 5. Test read_region and write_region + let mut read_buf = [0u8; 16]; + dev.read_region(®0, 0, &mut read_buf).unwrap(); + assert_eq!(read_buf, [0xaa; 16]); + + let write_buf = [0x55; 16]; + dev.write_region(®0, 0, &write_buf).unwrap(); + + // 6. Test set_irq_eventfd and disable_irq + let eventfd_file = tempfile::tempfile().unwrap(); + let eventfd_borrowed = eventfd_file.as_fd(); + dev.set_irq_eventfd(0, 0, &[Some(eventfd_borrowed)]) + .unwrap(); + dev.disable_irq(0).unwrap(); + + // 7. Test reset + dev.reset().unwrap(); + + // 8. Test get_dma_buf_fd (unsupported) + assert_matches!(dev.get_dma_buf_fd(0, 0, 0x1000), Err(_)); + + // 9. Test DMA mapping and UpdateVfioUserMapping + let arc_anon = ArcMemPages::from_memfd(c"test_mem", 0x2000, None).unwrap(); + + let updater = UpdateVfioUserMapping::new(session.clone()); + // RAM add / remove + assert_matches!(updater.ram_added(0x1000_0000, &arc_anon), Ok(())); + assert_matches!(updater.ram_removed(0x1000_0000, &arc_anon), Ok(())); + + // Dev mem add / remove + assert_matches!(updater.dev_mem_added(0x2000_0000, &arc_anon, None), Ok(())); + assert_matches!( + updater.dev_mem_removed(0x2000_0000, &arc_anon, None), + Ok(()) + ); + + drop(updater); + drop(dev); + drop(session); + server_handle.join().unwrap(); +} + +#[test] +fn test_vfio_user_version_server_mismatch() { + let (client, server) = UnixStream::pair().unwrap(); + let server_handle = thread::spawn(move || { + let mut header_buf = [0u8; size_of::()]; + let mut recv_fds = [const { None }; 32]; + let mut header_slice = [IoSliceMut::new(&mut header_buf)]; + recv_msg_with_fds(&server, &mut header_slice, &mut recv_fds).unwrap(); + let (req_header, _) = VfioUserHeader::read_from_prefix(&header_buf).unwrap(); + + // Server sends major version 1 (mismatch) + let reply_version = VfioUserVersion { major: 1, minor: 0 }; + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: VfioUserCmd::VERSION, + msg_size: (size_of::() + size_of::()) as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, false), + error_no: 0, + }; + let slices = [ + IoSlice::new(reply_hdr.as_bytes()), + IoSlice::new(reply_version.as_bytes()), + ]; + send_msg_with_fds(&server, &slices, &[]).unwrap(); + }); + + let session = VfioUserSession::new(client); + let res = session.negotiate_version(); + assert_matches!(res, Err(Error::VfioUser { .. })); + + server_handle.join().unwrap(); +} + +#[test] +fn test_vfio_user_server_error_response() { + let (client, server) = UnixStream::pair().unwrap(); + let server_handle = thread::spawn(move || { + let mut header_buf = [0u8; size_of::()]; + let mut recv_fds = [const { None }; 32]; + let mut header_slice = [IoSliceMut::new(&mut header_buf)]; + recv_msg_with_fds(&server, &mut header_slice, &mut recv_fds).unwrap(); + let (req_header, _) = VfioUserHeader::read_from_prefix(&header_buf).unwrap(); + + // Server replies with ERROR flag and EINVAL + let reply_hdr = VfioUserHeader { + msg_id: req_header.msg_id, + cmd: req_header.cmd, + msg_size: size_of::() as u32, + flags: VfioUserHeaderFlag::new(VfioUserMessageType::REPLY, false, true), + error_no: libc::EINVAL as u32, + }; + let slices = [IoSlice::new(reply_hdr.as_bytes())]; + send_msg_with_fds(&server, &slices, &[]).unwrap(); + }); + + let session = VfioUserSession::new(client); + let mut resp_buf = [0u8; 64]; + let mut resp_fds = [const { None }; 32]; + let res = session.transact(VfioUserCmd::VERSION, &[], &[], &mut resp_buf, &mut resp_fds); + assert_matches!(res, Err(Error::System { .. })); + + server_handle.join().unwrap(); +} diff --git a/alioth/src/vfio/vfio.rs b/alioth/src/vfio/vfio.rs index 0005adaa..f0daddc4 100644 --- a/alioth/src/vfio/vfio.rs +++ b/alioth/src/vfio/vfio.rs @@ -18,6 +18,8 @@ pub mod device; pub mod group; pub mod iommu; pub mod pci; +#[path = "user/user.rs"] +pub mod user; use std::path::Path; @@ -47,6 +49,8 @@ pub enum Error { NotSupportedHeader { ty: u8 }, #[snafu(display("Setting container iommu to {new:?}, but it already has {current:?}"))] SetContainerIommu { current: VfioIommu, new: VfioIommu }, + #[snafu(display("vfio-user protocol error: {msg}"))] + VfioUser { msg: String }, } pub type Result = std::result::Result; @@ -85,3 +89,13 @@ pub struct VfioContainerSpec { /// Path to the vfio device. [default: /dev/vfio/vfio] pub dev_vfio: Option>, } + +#[derive(Debug, PartialEq, Eq, Deserialize, Help)] +pub struct VfioUserSpec { + /// Path to the vfio-user UNIX domain socket. + pub socket: Box, +} + +#[cfg(test)] +#[path = "vfio_test.rs"] +mod tests; diff --git a/alioth/src/vfio/vfio_test.rs b/alioth/src/vfio/vfio_test.rs new file mode 100644 index 00000000..b8f67e05 --- /dev/null +++ b/alioth/src/vfio/vfio_test.rs @@ -0,0 +1,345 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::os::fd::{BorrowedFd, OwnedFd}; +use std::path::Path; +use std::sync::Arc; + +use assert_matches::assert_matches; +use parking_lot::Mutex; + +use crate::mem::LayoutChanged; +use crate::mem::mapped::ArcMemPages; +use crate::sys::vfio::{ + VfioDeviceInfo, VfioIommu, VfioIrqInfo, VfioRegionInfo, VfioRegionInfoFlag, +}; +use crate::vfio::cdev::Cdev; +use crate::vfio::container::{Container, UpdateContainerMapping}; +use crate::vfio::device::{Device, VfioIoDevice}; +use crate::vfio::group::Group; +use crate::vfio::iommu::{Iommu, UpdateIommuIoas}; +use crate::vfio::{ + Error, Result, VfioCdevSpec, VfioContainerSpec, VfioGroupSpec, VfioIoasSpec, VfioUserSpec, +}; + +#[derive(Debug, Default)] +struct MemoryDevice { + data: Mutex>, +} + +impl Device for MemoryDevice { + fn get_info(&self) -> Result { + Ok(VfioDeviceInfo::default()) + } + + fn get_region_info(&self, index: u32) -> Result { + Ok(VfioRegionInfo { + index, + size: self.data.lock().len() as u64, + ..Default::default() + }) + } + + fn get_irq_info(&self, _index: u32) -> Result { + Ok(VfioIrqInfo::default()) + } + + fn reset(&self) -> Result<()> { + Ok(()) + } + + fn set_irq_eventfd( + &self, + _index: u32, + _start: u32, + _eventfds: &[Option>], + ) -> Result<()> { + Ok(()) + } + + fn disable_irq(&self, _index: u32) -> Result<()> { + Ok(()) + } + + fn read_region(&self, _region: &VfioRegionInfo, offset: u64, buf: &mut [u8]) -> Result<()> { + let data = self.data.lock(); + let offset = offset as usize; + buf.copy_from_slice(&data[offset..offset + buf.len()]); + Ok(()) + } + + fn write_region(&self, _region: &VfioRegionInfo, offset: u64, buf: &[u8]) -> Result<()> { + let mut data = self.data.lock(); + let offset = offset as usize; + data[offset..offset + buf.len()].copy_from_slice(buf); + Ok(()) + } + + fn get_region_mmap(&self, _index: u32) -> Result> { + Ok(None) + } + + fn get_dma_buf_fd(&self, _index: u32, _offset: u64, _size: usize) -> Result { + Err(std::io::Error::from(std::io::ErrorKind::Unsupported).into()) + } +} + +#[test] +fn test_device_read_write_helpers() { + let dev = MemoryDevice { + data: Mutex::new(vec![0u8; 64]), + }; + let region = VfioRegionInfo { + index: 0, + size: 64, + ..Default::default() + }; + + // 1-byte read/write + assert_matches!(dev.write(®ion, 0, 1, 0xab), Ok(())); + assert_matches!(dev.read(®ion, 0, 1), Ok(0xab)); + + // 2-byte read/write + assert_matches!(dev.write(®ion, 2, 2, 0x1234), Ok(())); + assert_matches!(dev.read(®ion, 2, 2), Ok(0x1234)); + + // 4-byte read/write + assert_matches!(dev.write(®ion, 4, 4, 0xdead_beef), Ok(())); + assert_matches!(dev.read(®ion, 4, 4), Ok(0xdead_beef)); + + // 8-byte read/write + assert_matches!(dev.write(®ion, 8, 8, 0x0123_4567_89ab_cdef), Ok(())); + assert_matches!(dev.read(®ion, 8, 8), Ok(0x0123_4567_89ab_cdef)); + + // Invalid size (> 8 bytes) + assert_matches!(dev.write(®ion, 0, 16, 0), Ok(())); + assert_matches!(dev.read(®ion, 0, 16), Ok(0)); +} + +#[test] +fn test_vfio_io_device_file_access() { + let mut tmp = tempfile::tempfile().unwrap(); + use std::io::Write; + tmp.write_all(&[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]) + .unwrap(); + + let io_dev = VfioIoDevice::new(tmp).unwrap(); + assert!(io_dev.fd().metadata().is_ok()); + + let region = VfioRegionInfo { + index: 0, + size: 8, + offset: 0, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + ..Default::default() + }; + + let mut buf = [0u8; 4]; + io_dev.read_region(®ion, 2, &mut buf).unwrap(); + assert_eq!(buf, [0x33, 0x44, 0x55, 0x66]); + + io_dev.write_region(®ion, 0, &[0xaa, 0xbb]).unwrap(); + io_dev.read_region(®ion, 0, &mut buf).unwrap(); + assert_eq!(buf, [0xaa, 0xbb, 0x33, 0x44]); +} + +#[test] +fn test_vfio_specs_deserialization() { + // VfioCdevSpec + let cdev_aco = "path=/dev/vfio/devices/vfio0,ioas=my_ioas"; + let cdev_spec: VfioCdevSpec = serde_aco::from_arg(cdev_aco).unwrap(); + assert_eq!(cdev_spec.path.to_str().unwrap(), "/dev/vfio/devices/vfio0"); + assert_eq!(cdev_spec.ioas.as_deref(), Some("my_ioas")); + + // VfioIoasSpec + let ioas_aco = "name=ioas0,dev_iommu=/dev/iommu"; + let ioas_spec: VfioIoasSpec = serde_aco::from_arg(ioas_aco).unwrap(); + assert_eq!(&*ioas_spec.name, "ioas0"); + assert_eq!(ioas_spec.dev_iommu.unwrap().to_str().unwrap(), "/dev/iommu"); + + // VfioGroupSpec + let group_aco = "path=/dev/vfio/12,devices=0000:06:0d.0,container=c0"; + let group_spec: VfioGroupSpec = serde_aco::from_arg(group_aco).unwrap(); + assert_eq!(group_spec.path.to_str().unwrap(), "/dev/vfio/12"); + assert_eq!(group_spec.devices.len(), 1); + assert_eq!(&*group_spec.devices[0], "0000:06:0d.0"); + assert_eq!(group_spec.container.as_deref(), Some("c0")); + + // VfioContainerSpec + let container_aco = "name=c0,dev_vfio=/dev/vfio/vfio"; + let container_spec: VfioContainerSpec = serde_aco::from_arg(container_aco).unwrap(); + assert_eq!(&*container_spec.name, "c0"); + assert_eq!( + container_spec.dev_vfio.unwrap().to_str().unwrap(), + "/dev/vfio/vfio" + ); + + // VfioUserSpec + let user_aco = "socket=/tmp/vfio-user.sock"; + let user_spec: VfioUserSpec = serde_aco::from_arg(user_aco).unwrap(); + assert_eq!(user_spec.socket.to_str().unwrap(), "/tmp/vfio-user.sock"); +} + +#[test] +fn test_container_and_group_errors_and_drop() { + // Non-existent path returns AccessDevice + assert_matches!( + Container::new("/nonexistent/path/vfio"), + Err(Error::AccessDevice { .. }) + ); + assert_matches!( + Group::new(Path::new("/nonexistent/path/group")), + Err(Error::AccessDevice { .. }) + ); + assert_matches!( + Cdev::new("/nonexistent/path/cdev"), + Err(Error::AccessDevice { .. }) + ); + assert_matches!( + Iommu::new("/nonexistent/path/iommu"), + Err(Error::AccessDevice { .. }) + ); + + // Group detach when not attached returns Ok(()) + let tmp_group = tempfile::NamedTempFile::new().unwrap(); + let mut group = Group::new(tmp_group.path()).unwrap(); + assert_matches!(group.detach(), Ok(())); + + // Cdev detach when not attached returns Ok(()) + let tmp_cdev = tempfile::NamedTempFile::new().unwrap(); + let mut cdev = Cdev::new(tmp_cdev.path()).unwrap(); + assert_matches!(cdev.detach_iommu_ioas(), Ok(())); +} + +#[test] +fn test_container_set_iommu_mismatch() { + let tmp_file = tempfile::NamedTempFile::new().unwrap(); + let container = Container::new(tmp_file.path()).unwrap(); + // Simulate container already having TYPE1 + *container.iommu.lock() = Some(VfioIommu::TYPE1); + + // Setting same IOMMU returns Ok(()) + assert_matches!(container.set_iommu(VfioIommu::TYPE1), Ok(())); + + // Setting different IOMMU returns SetContainerIommu error + assert_matches!( + container.set_iommu(VfioIommu::TYPE1_V2), + Err(Error::SetContainerIommu { + current: VfioIommu::TYPE1, + new: VfioIommu::TYPE1_V2, + .. + }) + ); +} + +#[test] +fn test_layout_changed_callbacks() { + let arc_anon = ArcMemPages::from_anonymous(0x2000, None, None).unwrap(); + + // UpdateContainerMapping dev_mem callbacks without real vfio fd return ioctl error + let container = + Arc::new(Container::new(tempfile::NamedTempFile::new().unwrap().path()).unwrap()); + let container_updater = UpdateContainerMapping { container }; + assert_matches!( + container_updater.dev_mem_added(0x1000, &arc_anon, None), + Err(_) + ); + assert_matches!( + container_updater.dev_mem_removed(0x1000, &arc_anon, None), + Err(_) + ); + + // UpdateIommuIoas dev_mem callbacks without real iommu fd return ioctl error + let iommu = Arc::new(Iommu::new(tempfile::NamedTempFile::new().unwrap().path()).unwrap()); + // Create Ioas with dummy id 0 + let ioas = Arc::new(crate::vfio::iommu::Ioas { iommu, id: 0 }); + let iommu_updater = UpdateIommuIoas { ioas }; + assert_matches!(iommu_updater.dev_mem_added(0x1000, &arc_anon, None), Err(_)); + assert_matches!( + iommu_updater.dev_mem_removed(0x1000, &arc_anon, None), + Err(_) + ); +} + +#[test] +fn test_cdev_and_dev_fd_device_impl() { + let tmp = tempfile::NamedTempFile::new().unwrap(); + use std::io::Write; + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(tmp.path()) + .unwrap(); + file.write_all(&[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]) + .unwrap(); + + let cdev = Cdev::new(tmp.path()).unwrap(); + let region = VfioRegionInfo { + index: 0, + size: 8, + offset: 0, + flags: VfioRegionInfoFlag::READ | VfioRegionInfoFlag::WRITE, + ..Default::default() + }; + + let mut buf = [0u8; 4]; + cdev.read_region(®ion, 2, &mut buf).unwrap(); + assert_eq!(buf, [0x33, 0x44, 0x55, 0x66]); + + cdev.write_region(®ion, 0, &[0xaa, 0xbb]).unwrap(); + cdev.read_region(®ion, 0, &mut buf).unwrap(); + assert_eq!(buf, [0xaa, 0xbb, 0x33, 0x44]); + + assert_matches!(cdev.get_info(), Err(_)); + assert_matches!(cdev.get_region_info(0), Err(_)); + assert_matches!(cdev.get_irq_info(0), Err(_)); + assert_matches!(cdev.reset(), Err(_)); + assert_matches!(cdev.set_irq_eventfd(0, 0, &[None]), Err(_)); + assert_matches!(cdev.disable_irq(0), Err(_)); + assert_matches!(cdev.get_region_mmap(0), Err(_)); + assert_matches!(cdev.get_dma_buf_fd(0, 0, 0x1000), Err(_)); + + // Test DevFd + let group = Arc::new(Group::new(tmp.path()).unwrap()); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(tmp.path()) + .unwrap(); + let io_dev = VfioIoDevice::new(file).unwrap(); + let dev_fd = crate::vfio::group::DevFd { + io_dev, + _group: group, + }; + assert_matches!(dev_fd.get_info(), Err(_)); + assert_matches!(dev_fd.get_region_info(0), Err(_)); + assert_matches!(dev_fd.get_irq_info(0), Err(_)); + assert_matches!(dev_fd.reset(), Err(_)); + assert_matches!(dev_fd.set_irq_eventfd(0, 0, &[None]), Err(_)); + assert_matches!(dev_fd.disable_irq(0), Err(_)); + assert_matches!(dev_fd.read_region(®ion, 0, &mut buf), Ok(())); + assert_matches!(dev_fd.write_region(®ion, 0, &[0x11, 0x22]), Ok(())); + assert_matches!(dev_fd.get_region_mmap(0), Err(_)); + assert_matches!(dev_fd.get_dma_buf_fd(0, 0, 0x1000), Err(_)); +} + +#[test] +fn test_ioas_methods() { + let tmp = tempfile::NamedTempFile::new().unwrap(); + let iommu = Arc::new(Iommu::new(tmp.path()).unwrap()); + let ioas = crate::vfio::iommu::Ioas { iommu, id: 1 }; + + assert_matches!(ioas.map(0x1000, 0x2000, 0x1000), Err(_)); + assert_matches!(ioas.unmap(0x2000, 0x1000), Err(_)); + assert_matches!(ioas.reset(), Err(_)); +} diff --git a/alioth/src/vm/vm.rs b/alioth/src/vm/vm.rs index 30397715..b853ab42 100644 --- a/alioth/src/vm/vm.rs +++ b/alioth/src/vm/vm.rs @@ -64,7 +64,14 @@ use crate::vfio::iommu::{Ioas, Iommu, UpdateIommuIoas}; #[cfg(target_os = "linux")] use crate::vfio::pci::VfioPciDev; #[cfg(target_os = "linux")] -use crate::vfio::{VfioCdevSpec, VfioContainerSpec, VfioGroupSpec, VfioIoasSpec}; +use crate::vfio::user::conn::VfioUserSession; +#[cfg(target_os = "linux")] +use crate::vfio::user::device::{UpdateVfioUserMapping, VfioUserDevice}; +#[cfg(target_os = "linux")] +use crate::vfio::{VfioCdevSpec, VfioContainerSpec, VfioGroupSpec, VfioIoasSpec, VfioUserSpec}; +#[cfg(target_os = "linux")] +use std::os::unix::net::UnixStream; + use crate::virtio::dev::{DevSpec, Virtio, VirtioDevice}; use crate::virtio::pci::VirtioPciDevice; @@ -373,6 +380,28 @@ where Ok(()) } + #[cfg(target_os = "linux")] + pub fn add_vfio_user_dev(&self, name: Arc, spec: VfioUserSpec) -> Result<(), Error> { + let stream = UnixStream::connect(&spec.socket).map_err(crate::vfio::Error::from)?; + + let session = Arc::new(VfioUserSession::new(stream)); + session.negotiate_version()?; + + let dev = VfioUserDevice::new(session.clone())?; + + let bdf = self.ctx.board.pci_bus.reserve(None).unwrap(); + let msi_sender = self.ctx.board.vm.create_msi_sender( + #[cfg(target_arch = "aarch64")] + u32::from(bdf.0), + )?; + let dev = VfioPciDev::new(name.clone(), dev, msi_sender)?; + self.add_pci_dev(Some(bdf), Arc::new(dev))?; + + let update = Box::new(UpdateVfioUserMapping::new(session.clone())); + self.ctx.board.memory.register_change_callback(update)?; + Ok(()) + } + pub fn add_vfio_container(&self, spec: VfioContainerSpec) -> Result, Error> { let mut containers = self.vfio_containers.lock(); if containers.contains_key(&spec.name) {