From c1e145294d2b67aee91a22e4785e5fa87f32bad9 Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Tue, 14 Jan 2020 17:45:21 +0800 Subject: [PATCH 01/14] Remove vm-memory dependency Use u64 for guest memory address type since this can make vm-device independent from on vm-memory. Signed-off-by: Jing Liu --- Cargo.toml | 1 - src/lib.rs | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 16e2bc5..42ef5bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,4 +6,3 @@ repository = "https://github.com/rust-vmm/vm-device" license = "Apache-2.0" [dependencies] -vm-memory = { git = "https://github.com/rust-vmm/vm-memory" } diff --git a/src/lib.rs b/src/lib.rs index 9ef255d..7d2eaa1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,10 +3,6 @@ //! rust-vmm device model. -extern crate vm_memory; - -use vm_memory::GuestAddress; - pub mod resources; /// IO Addresses. @@ -16,7 +12,7 @@ pub enum IoAddress { Pio(u16), /// Memory mapped I/O address. - Mmio(GuestAddress), + Mmio(u64), } /// Device IO trait. From 06b9e004ecddcaeb13649e03dd2460eaafb9e57f Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Tue, 14 Jan 2020 17:40:50 +0800 Subject: [PATCH 02/14] Improve DeviceIo interface Change DeviceIo interface parameters to base and offset, so that devices with several IO ranges can use it to locate right range. Signed-off-by: Jing Liu --- src/lib.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7d2eaa1..6bde461 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,9 +21,11 @@ pub enum IoAddress { /// The VMM will then dispatch IO (PIO or MMIO) VM exits by calling into the /// registered devices read or write method from this trait. pub trait DeviceIo: Send { - /// Read from the guest physical address `addr` to `data`. - fn read(&mut self, addr: IoAddress, data: &mut [u8]); + /// Read from guest physical address `base + offset` of the registered + /// device to `data`. + fn read(&mut self, base: IoAddress, offset: IoAddress, data: &mut [u8]); - /// Write `data` to the guest physical address `addr`. - fn write(&mut self, addr: IoAddress, data: &[u8]); + /// Write `data` to the guest physical address `base + offset` of the + /// registered device. + fn write(&mut self, base: IoAddress, offset: IoAddress, data: &[u8]); } From d4664f87fd4ad472bef4b1d38695b67b3cb08fcb Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Tue, 14 Jan 2020 15:05:13 +0100 Subject: [PATCH 03/14] lib: Improve DeviceIo methods documentation As suggested from https://github.com/rust-vmm/vm-device/pull/18#discussion_r366238131 Suggested-by: Andreea Florescu Signed-off-by: Samuel Ortiz --- src/lib.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 6bde461..b60ba5d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,11 +21,10 @@ pub enum IoAddress { /// The VMM will then dispatch IO (PIO or MMIO) VM exits by calling into the /// registered devices read or write method from this trait. pub trait DeviceIo: Send { - /// Read from guest physical address `base + offset` of the registered - /// device to `data`. + /// Read from the guest physical address `base`, starting at `offset`. + /// Result is placed in `data`. fn read(&mut self, base: IoAddress, offset: IoAddress, data: &mut [u8]); - /// Write `data` to the guest physical address `base + offset` of the - /// registered device. + /// Write `data` to the guest physical address `base`, starting from `offset`. fn write(&mut self, base: IoAddress, offset: IoAddress, data: &[u8]); } From 3d51c9e19f505ade6a1f75eef07e5df45b03b8c9 Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Tue, 17 Dec 2019 19:32:42 +0800 Subject: [PATCH 04/14] Update rust-vmm-ci Signed-off-by: Jing Liu --- rust-vmm-ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-vmm-ci b/rust-vmm-ci index bb1cd14..c309d06 160000 --- a/rust-vmm-ci +++ b/rust-vmm-ci @@ -1 +1 @@ -Subproject commit bb1cd14d2c164b4f699b08c885c06a02fbe3f7b0 +Subproject commit c309d0627bde6b07db91201dd8b47007841c100a From 6aae2024a9dff840c22c62a9a748057466ce34f3 Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Mon, 20 Jan 2020 18:21:10 +0800 Subject: [PATCH 05/14] Make DeviceIo internal mutability In order to get a real multiple threads handling to enhance performance, the DeviceIo trait need adopt interior mutability pattern. Signed-off-by: Jing Liu --- src/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b60ba5d..b0b272d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,11 +20,13 @@ pub enum IoAddress { /// register itself against the different IO type ranges it handles. /// The VMM will then dispatch IO (PIO or MMIO) VM exits by calling into the /// registered devices read or write method from this trait. +/// The DeviceIo trait adopts the interior mutability pattern +/// so we can get a real multiple threads handling. pub trait DeviceIo: Send { /// Read from the guest physical address `base`, starting at `offset`. /// Result is placed in `data`. - fn read(&mut self, base: IoAddress, offset: IoAddress, data: &mut [u8]); + fn read(&self, base: IoAddress, offset: IoAddress, data: &mut [u8]); /// Write `data` to the guest physical address `base`, starting from `offset`. - fn write(&mut self, base: IoAddress, offset: IoAddress, data: &[u8]); + fn write(&self, base: IoAddress, offset: IoAddress, data: &[u8]); } From ef21cf17f135fc70bad560b0a591aadea08c045f Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Thu, 31 Oct 2019 22:36:42 +0800 Subject: [PATCH 06/14] Add IO manager support Based on resources definition, this adds device IO manager to manage all devices IO ranges. Signed-off-by: Jing Liu --- src/device_manager.rs | 108 ++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 2 files changed, 109 insertions(+) create mode 100644 src/device_manager.rs diff --git a/src/device_manager.rs b/src/device_manager.rs new file mode 100644 index 0000000..1d858c1 --- /dev/null +++ b/src/device_manager.rs @@ -0,0 +1,108 @@ +// Copyright © 2019 Intel Corporation. All Rights Reserved. +// SPDX-License-Identifier: (Apache-2.0 OR BSD-3-Clause) + +//! System level device management. +//! +//! [IoManager](struct.IoManager.html) is respondsible for managing +//! all devices of virtual machine, registering IO resources callback, +//! unregistering devices and helping VM IO exit handling. +//！ +//！VMM would be responsible for getting device resource request, ask +//! vm_allocator to allocate the resources, ask vm_device to register the +//! devices IO ranges, and finally set resources to virtual device. + +use crate::resources::Resource; +use crate::DeviceIo; + +use std::collections::btree_map::BTreeMap; +use std::result; +use std::sync::Arc; + +/// Error type for `IoManager` usage. +#[derive(Debug)] +pub enum Error { + /// The inserting device overlaps with a current device. + DeviceOverlap, +} + +/// Simplify the `Result` type. +pub type Result = result::Result; + +/// System IO manager serving for all devices management and VM exit handling. +#[derive(Default)] +pub struct IoManager { + /// Range mapping for VM exit pio operations. + pio_bus: BTreeMap<(u16, u16), Arc>, + /// Range mapping for VM exit mmio operations. + mmio_bus: BTreeMap<(u64, u64), Arc>, +} + +impl IoManager { + /// Create an default IoManager with empty IO member. + pub fn new() -> Self { + IoManager::default() + } + /// Register a new device IO with its allocated resources. + /// VMM is responsible for providing the allocated resources to virtual device. + /// + /// # Arguments + /// + /// * `device`: device instance object to be registered + /// * `resources`: resources that this device owns, might include + /// port I/O and memory-mapped I/O ranges, irq number, etc. + pub fn register_device_io( + &mut self, + device: Arc, + resources: &[Resource], + ) -> Result<()> { + // Register and mark device resources + // The resources addresses being registered are sucessfully allocated before. + for (idx, res) in resources.iter().enumerate() { + match *res { + Resource::PioAddressRange { base, size } => { + if self.pio_bus.insert((base, size), device.clone()).is_some() { + // Unregister registered resources. + self.unregister_device_io(&resources[0..idx]) + .expect("failed to unregister devices"); + + return Err(Error::DeviceOverlap); + } + } + Resource::MmioAddressRange { base, size } => { + if self.mmio_bus.insert((base, size), device.clone()).is_some() { + // Unregister registered resources. + self.unregister_device_io(&resources[0..idx]) + .expect("failed to unregister devices"); + + return Err(Error::DeviceOverlap); + } + } + _ => continue, + } + } + Ok(()) + } + + /// Unregister a device from `IoManager`, e.g. users specified removing. + /// VMM pre-fetches the resources e.g. dev.get_assigned_resources() + /// VMM is responsible for freeing the resources. + /// + /// # Arguments + /// + /// * `resources`: resources that this device owns, might include + /// port I/O and memory-mapped I/O ranges, irq number, etc. + pub fn unregister_device_io(&mut self, resources: &[Resource]) -> Result<()> { + for res in resources.iter() { + match *res { + Resource::PioAddressRange { base, size } => { + self.pio_bus.remove(&(base, size)); + } + Resource::MmioAddressRange { base, size } => { + self.mmio_bus.remove(&(base, size)); + } + _ => continue, + } + } + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index b0b272d..8620bfb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ //! rust-vmm device model. +pub mod device_manager; pub mod resources; /// IO Addresses. From 372bedf94bbc5cf5485cc10d5832c23b0b92a694 Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Sat, 23 Nov 2019 00:58:29 +0800 Subject: [PATCH 07/14] Add read and write operations handling IO manager is responsible for handling IO operation when VMExit. It works out the specific device according to the address range and hand over to DeviceIo trait. Signed-off-by: Jing Liu --- src/device_manager.rs | 142 +++++++++++++++++++++++++++++++++++++++--- src/lib.rs | 52 ++++++++++++++++ 2 files changed, 187 insertions(+), 7 deletions(-) diff --git a/src/device_manager.rs b/src/device_manager.rs index 1d858c1..11471a8 100644 --- a/src/device_manager.rs +++ b/src/device_manager.rs @@ -12,8 +12,9 @@ //! devices IO ranges, and finally set resources to virtual device. use crate::resources::Resource; -use crate::DeviceIo; +use crate::{DeviceIo, IoAddress, IoSize}; +use std::cmp::{Ord, Ordering, PartialEq, PartialOrd}; use std::collections::btree_map::BTreeMap; use std::result; use std::sync::Arc; @@ -23,18 +24,62 @@ use std::sync::Arc; pub enum Error { /// The inserting device overlaps with a current device. DeviceOverlap, + /// The device doesn't exist. + NoDevice, } /// Simplify the `Result` type. pub type Result = result::Result; +// Structure describing an IO range. +#[derive(Debug, Copy, Clone)] +struct IoRange { + base: IoAddress, + size: IoSize, +} + +impl IoRange { + fn new_pio_range(base: u16, size: u16) -> Self { + IoRange { + base: IoAddress::Pio(base), + size: IoSize::Pio(size), + } + } + fn new_mmio_range(base: u64, size: u64) -> Self { + IoRange { + base: IoAddress::Mmio(base), + size: IoSize::Mmio(size), + } + } +} + +impl Eq for IoRange {} + +impl PartialEq for IoRange { + fn eq(&self, other: &IoRange) -> bool { + self.base == other.base + } +} + +impl Ord for IoRange { + fn cmp(&self, other: &IoRange) -> Ordering { + self.base.cmp(&other.base) + } +} + +impl PartialOrd for IoRange { + fn partial_cmp(&self, other: &IoRange) -> Option { + self.base.partial_cmp(&other.base) + } +} + /// System IO manager serving for all devices management and VM exit handling. #[derive(Default)] pub struct IoManager { /// Range mapping for VM exit pio operations. - pio_bus: BTreeMap<(u16, u16), Arc>, + pio_bus: BTreeMap>, /// Range mapping for VM exit mmio operations. - mmio_bus: BTreeMap<(u64, u64), Arc>, + mmio_bus: BTreeMap>, } impl IoManager { @@ -60,7 +105,11 @@ impl IoManager { for (idx, res) in resources.iter().enumerate() { match *res { Resource::PioAddressRange { base, size } => { - if self.pio_bus.insert((base, size), device.clone()).is_some() { + if self + .pio_bus + .insert(IoRange::new_pio_range(base, size), device.clone()) + .is_some() + { // Unregister registered resources. self.unregister_device_io(&resources[0..idx]) .expect("failed to unregister devices"); @@ -69,7 +118,11 @@ impl IoManager { } } Resource::MmioAddressRange { base, size } => { - if self.mmio_bus.insert((base, size), device.clone()).is_some() { + if self + .mmio_bus + .insert(IoRange::new_mmio_range(base, size), device.clone()) + .is_some() + { // Unregister registered resources. self.unregister_device_io(&resources[0..idx]) .expect("failed to unregister devices"); @@ -95,14 +148,89 @@ impl IoManager { for res in resources.iter() { match *res { Resource::PioAddressRange { base, size } => { - self.pio_bus.remove(&(base, size)); + self.pio_bus.remove(&IoRange::new_pio_range(base, size)); } Resource::MmioAddressRange { base, size } => { - self.mmio_bus.remove(&(base, size)); + self.mmio_bus.remove(&IoRange::new_mmio_range(base, size)); } _ => continue, } } Ok(()) } + + fn get_entry(&self, addr: IoAddress) -> Option<(&IoRange, &Arc)> { + match addr { + IoAddress::Pio(a) => self + .pio_bus + .range(..=&IoRange::new_pio_range(a, 0)) + .nth_back(0), + IoAddress::Mmio(a) => self + .mmio_bus + .range(..=&IoRange::new_mmio_range(a, 0)) + .nth_back(0), + } + } + + // Return the Device mapped `addr` and the base address. + fn get_device(&self, addr: IoAddress) -> Option<(&Arc, IoAddress)> { + if let Some((range, dev)) = self.get_entry(addr) { + if (addr.raw_value() - range.base.raw_value()) < range.size.raw_value() { + return Some((dev, range.base)); + } + } + None + } + + /// A helper function handling PIO read command during VM exit. + /// The virtual device itself provides mutable ability and thead-safe protection. + /// + /// Return error if failed to get the device. + pub fn pio_read(&self, addr: u16, data: &mut [u8]) -> Result<()> { + if let Some((device, base)) = self.get_device(IoAddress::Pio(addr)) { + device.read(base, IoAddress::Pio(addr - (base.raw_value() as u16)), data); + Ok(()) + } else { + Err(Error::NoDevice) + } + } + + /// A helper function handling PIO write command during VM exit. + /// The virtual device itself provides mutable ability and thead-safe protection. + /// + /// Return error if failed to get the device. + pub fn pio_write(&self, addr: u16, data: &[u8]) -> Result<()> { + if let Some((device, base)) = self.get_device(IoAddress::Pio(addr)) { + device.write(base, IoAddress::Pio(addr - (base.raw_value() as u16)), data); + Ok(()) + } else { + Err(Error::NoDevice) + } + } + + /// A helper function handling MMIO read command during VM exit. + /// The virtual device itself provides mutable ability and thead-safe protection. + /// + /// Return error if failed to get the device. + pub fn mmio_read(&self, addr: u64, data: &mut [u8]) -> Result<()> { + if let Some((device, base)) = self.get_device(IoAddress::Mmio(addr)) { + device.read(base, IoAddress::Mmio(addr - base.raw_value()), data); + Ok(()) + } else { + Err(Error::NoDevice) + } + } + + /// A helper function handling MMIO write command during VM exit. + /// The virtual device itself provides mutable ability and thead-safe protection. + /// + /// Return error if failed to get the device. + pub fn mmio_write(&self, addr: u64, data: &[u8]) -> Result<()> { + if let Some((device, base)) = self.get_device(IoAddress::Mmio(addr)) { + device.write(base, IoAddress::Mmio(addr - base.raw_value()), data); + Ok(()) + } else { + Err(Error::NoDevice) + } + } } diff --git a/src/lib.rs b/src/lib.rs index 8620bfb..dc09ddb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,9 +3,31 @@ //! rust-vmm device model. +use std::cmp::{Ord, Ordering, PartialOrd}; + pub mod device_manager; pub mod resources; +// IO Size. +#[derive(Debug, Copy, Clone)] +enum IoSize { + // Port I/O size. + Pio(u16), + + // Memory mapped I/O size. + Mmio(u64), +} + +impl IoSize { + // Get the raw value as u64 to make operation simple. + fn raw_value(&self) -> u64 { + match *self { + IoSize::Pio(p) => u64::from(p), + IoSize::Mmio(m) => m, + } + } +} + /// IO Addresses. #[derive(Debug, Copy, Clone)] pub enum IoAddress { @@ -16,6 +38,36 @@ pub enum IoAddress { Mmio(u64), } +impl IoAddress { + // Get the raw value of IO Address to make operation simple. + fn raw_value(&self) -> u64 { + match *self { + IoAddress::Pio(p) => u64::from(p), + IoAddress::Mmio(m) => m, + } + } +} + +impl Eq for IoAddress {} + +impl PartialEq for IoAddress { + fn eq(&self, other: &IoAddress) -> bool { + self.raw_value() == other.raw_value() + } +} + +impl Ord for IoAddress { + fn cmp(&self, other: &IoAddress) -> Ordering { + self.raw_value().cmp(&other.raw_value()) + } +} + +impl PartialOrd for IoAddress { + fn partial_cmp(&self, other: &IoAddress) -> Option { + self.raw_value().partial_cmp(&other.raw_value()) + } +} + /// Device IO trait. /// A device supporting memory based I/O should implement this trait, then /// register itself against the different IO type ranges it handles. From 66166a385ab2e6a41cb2fc879c439fc299d6b6f6 Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Sat, 23 Nov 2019 02:51:49 +0800 Subject: [PATCH 08/14] Add unit tests Unit tests for IO manager. Signed-off-by: Jing Liu --- coverage_config.json | 2 +- src/device_manager.rs | 122 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/coverage_config.json b/coverage_config.json index 03bbeba..c466612 100644 --- a/coverage_config.json +++ b/coverage_config.json @@ -1,5 +1,5 @@ { - "coverage_score": 75.8, + "coverage_score": 78.7, "exclude_path": "", "crate_features": "" } diff --git a/src/device_manager.rs b/src/device_manager.rs index 11471a8..bbbb857 100644 --- a/src/device_manager.rs +++ b/src/device_manager.rs @@ -234,3 +234,125 @@ impl IoManager { } } } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + const PIO_ADDRESS_SIZE: u16 = 4; + const PIO_ADDRESS_BASE: u16 = 0x40; + const MMIO_ADDRESS_SIZE: u64 = 0x8765_4321; + const MMIO_ADDRESS_BASE: u64 = 0x1234_5678; + const LEGACY_IRQ: u32 = 4; + const CONFIG_DATA: u32 = 0x1234; + + struct DummyDevice { + config: Mutex, + } + + impl DummyDevice { + fn new(config: u32) -> Self { + DummyDevice { + config: Mutex::new(config), + } + } + } + + impl DeviceIo for DummyDevice { + fn read(&self, _base: IoAddress, _offset: IoAddress, data: &mut [u8]) { + if data.len() > 4 { + return; + } + for (idx, iter) in data.iter_mut().enumerate() { + let config = self.config.lock().expect("failed to acquire lock"); + *iter = (*config >> (idx * 8) & 0xff) as u8; + } + } + + fn write(&self, _base: IoAddress, _offset: IoAddress, data: &[u8]) { + let mut config = self.config.lock().expect("failed to acquire lock"); + *config = u32::from(data[0]) & 0xff; + } + } + + #[test] + fn test_register_unregister_device_io() { + let mut io_mgr = IoManager::new(); + let dummy = DummyDevice::new(0); + let dum = Arc::new(dummy); + + let mut resource: Vec = Vec::new(); + let mmio = Resource::MmioAddressRange { + base: MMIO_ADDRESS_BASE, + size: MMIO_ADDRESS_SIZE, + }; + let irq = Resource::LegacyIrq(LEGACY_IRQ); + + resource.push(mmio); + resource.push(irq); + + assert!(io_mgr.register_device_io(dum.clone(), &resource).is_ok()); + assert!(io_mgr.unregister_device_io(&resource).is_ok()) + } + + #[test] + fn test_mmio_read_write() { + let mut io_mgr: IoManager = Default::default(); + let dum = Arc::new(DummyDevice::new(CONFIG_DATA)); + let mut resource: Vec = Vec::new(); + + let mmio = Resource::MmioAddressRange { + base: MMIO_ADDRESS_BASE, + size: MMIO_ADDRESS_SIZE, + }; + resource.push(mmio); + assert!(io_mgr.register_device_io(dum.clone(), &resource).is_ok()); + + let mut data = [0; 4]; + assert!(io_mgr.mmio_read(MMIO_ADDRESS_BASE, &mut data).is_ok()); + assert_eq!(data, [0x34, 0x12, 0, 0]); + + assert!(io_mgr + .mmio_read(MMIO_ADDRESS_BASE + MMIO_ADDRESS_SIZE, &mut data) + .is_err()); + + data = [0; 4]; + assert!(io_mgr.mmio_write(MMIO_ADDRESS_BASE, &data).is_ok()); + assert_eq!(*dum.config.lock().unwrap(), 0); + + assert!(io_mgr + .mmio_write(MMIO_ADDRESS_BASE + MMIO_ADDRESS_SIZE, &data) + .is_err()); + } + + #[test] + fn test_pio_read_write() { + let mut io_mgr: IoManager = Default::default(); + let dum = Arc::new(DummyDevice::new(CONFIG_DATA)); + let mut resource: Vec = Vec::new(); + + let pio = Resource::PioAddressRange { + base: PIO_ADDRESS_BASE, + size: PIO_ADDRESS_SIZE, + }; + resource.push(pio); + assert!(io_mgr.register_device_io(dum.clone(), &resource).is_ok()); + + let mut data = [0; 4]; + assert!(io_mgr.pio_read(PIO_ADDRESS_BASE, &mut data).is_ok()); + assert_eq!(data, [0x34, 0x12, 0, 0]); + + assert!(io_mgr + .pio_read(PIO_ADDRESS_BASE + PIO_ADDRESS_SIZE, &mut data) + .is_err()); + + data = [0; 4]; + assert!(io_mgr.pio_write(PIO_ADDRESS_BASE, &data).is_ok()); + assert_eq!(*dum.config.lock().unwrap(), 0); + + assert!(io_mgr + .pio_write(PIO_ADDRESS_BASE + PIO_ADDRESS_SIZE, &data) + .is_err()); + } +} From 3daea681b4f840f273d5e8780f2a88d59487dac4 Mon Sep 17 00:00:00 2001 From: Jing Liu Date: Wed, 18 Dec 2019 20:54:32 +0800 Subject: [PATCH 09/14] Append missing tests for resources Append missing tests for resources and fix some typo. Signed-off-by: Jing Liu --- coverage_config.json | 2 +- src/resources.rs | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/coverage_config.json b/coverage_config.json index c466612..a9e9a75 100644 --- a/coverage_config.json +++ b/coverage_config.json @@ -1,5 +1,5 @@ { - "coverage_score": 78.7, + "coverage_score": 79.9, "exclude_path": "", "crate_features": "" } diff --git a/src/resources.rs b/src/resources.rs index 5ae37dd..8a74ea8 100644 --- a/src/resources.rs +++ b/src/resources.rs @@ -401,7 +401,7 @@ mod tests { assert_eq!(align, 0x1000); assert_eq!(size, 0x2000); } else { - panic!("Pio resource constraint is invalid."); + panic!("Mmio resource constraint is invalid."); } if let ResourceConstraint::MmioAddress { range, align, size } = @@ -411,7 +411,24 @@ mod tests { assert_eq!(align, 0x2000); assert_eq!(size, 0x2000); } else { - panic!("Pio resource constraint is invalid."); + panic!("Mmio resource constraint is invalid."); + } + + if let ResourceConstraint::LegacyIrq { irq } = + ResourceConstraint::new_legacy_irq(Some(0x123)) + { + assert_eq!(irq, Some(0x123)); + } else { + panic!("IRQ resource constraint is invalid."); + } + + if let ResourceConstraint::KvmMemSlot { slot, size } = + ResourceConstraint::new_kvm_mem_slot(0x1000, Some(0x2000)) + { + assert_eq!(slot, Some(0x2000)); + assert_eq!(size, 0x1000); + } else { + panic!("KVM slot resource constraint is invalid."); } } } From 8d7f8b363d96b1d0780f9d3e3efd5a3544d503fb Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Tue, 17 Dec 2019 18:01:30 +0100 Subject: [PATCH 10/14] interrupt: Initial interrupt manager traits Signed-off-by: Samuel Ortiz --- src/interrupt/mod.rs | 140 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 2 files changed, 141 insertions(+) create mode 100644 src/interrupt/mod.rs diff --git a/src/interrupt/mod.rs b/src/interrupt/mod.rs new file mode 100644 index 0000000..344c3cc --- /dev/null +++ b/src/interrupt/mod.rs @@ -0,0 +1,140 @@ +// Copyright (C) 2019 Alibaba Cloud. All rights reserved. +// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// Copyright © 2019 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause + +//! Traits and Structs to manage interrupt sources for devices. +//! +//! In system programming, an interrupt is a signal to the processor emitted by hardware or +//! software indicating an event that needs immediate attention. An interrupt alerts the processor +//! to a high-priority condition requiring the interruption of the current code the processor is +//! executing. The processor responds by suspending its current activities, saving its state, and +//! executing a function called an interrupt handler (or an interrupt service routine, ISR) to deal +//! with the event. This interruption is temporary, and, after the interrupt handler finishes, +//! unless handling the interrupt has emitted a fatal error, the processor resumes normal +//! activities. +//! +//! Hardware interrupts are used by devices to communicate that they require attention from the +//! operating system, or a bare-metal program running on the CPU if there are no OSes. The act of +//! initiating a hardware interrupt is referred to as an interrupt request (IRQ). Different devices +//! are usually associated with different interrupts using a unique value associated with each +//! interrupt. This makes it possible to know which hardware device caused which interrupts. +//! These interrupt values are often called IRQ lines, or just interrupt lines. +//! +//! Nowadays, IRQ lines is not the only mechanism to deliver device interrupts to processors. +//! MSI [(Message Signaled Interrupt)](https://en.wikipedia.org/wiki/Message_Signaled_Interrupts) +//! is another commonly used alternative in-band method of signaling an interrupt, using special +//! in-band messages to replace traditional out-of-band assertion of dedicated interrupt lines. +//! While more complex to implement in a device, message signaled interrupts have some significant +//! advantages over pin-based out-of-band interrupt signaling. Message signaled interrupts are +//! supported in PCI bus since its version 2.2, and in later available PCI Express bus. Some non-PCI +//! architectures also use message signaled interrupts. +//! +//! While IRQ is a term commonly used by Operating Systems when dealing with hardware +//! interrupts, the IRQ numbers managed by OSes are independent of the ones managed by VMM. +//! For simplicity sake, the term `Interrupt Source` is used instead of IRQ to represent both pin-based +//! interrupts and MSI interrupts. +//! +//! A device may support multiple types of interrupts, and each type of interrupt may support one +//! or multiple interrupt sources. For example, a PCI device may support: +//! * Legacy Irq: exactly one interrupt source. +//! * PCI MSI Irq: 1,2,4,8,16,32 interrupt sources. +//! * PCI MSIx Irq: 2^n(n=0-11) interrupt sources. +//! +//! A distinct Interrupt Source Identifier (ISID) will be assigned to each interrupt source. +//! An ID allocator will be used to allocate and free Interrupt Source Identifiers for devices. +//! To decouple the vm-device crate from the ID allocator, the vm-device crate doesn't take the +//! responsibility to allocate/free Interrupt Source IDs but only makes use of assigned IDs. +//! +//! The overall flow to deal with interrupts is: +//! * The VMM creates an interrupt manager +//! * The VMM creates a device manager, passing on an reference to the interrupt manager +//! * The device manager passes on an reference to the interrupt manager to all registered devices +//! * The guest kernel loads drivers for virtual devices +//! * The guest device driver determines the type and number of interrupts needed, and update the +//! device configuration +//! * The virtual device backend requests the interrupt manager to create an interrupt group +//! according to guest configuration information + +use std::fs::File; +use std::sync::Arc; + +/// Reuse std::io::Result to simplify interoperability among crates. +pub type Result = std::io::Result; + +/// Data type to store an interrupt source identifier. +pub type InterruptIndex = u32; + +/// Data type to store an interrupt source type. +pub type InterruptType = u32; + +pub const PIN_IRQ: InterruptType = 0; +pub const PCI_MSI_IRQ: InterruptType = 1; + +/// Trait to manage interrupt sources for virtual device backends. +/// +/// The InterruptManager implementations should protect itself from concurrent accesses internally, +/// so it could be invoked from multi-threaded context. +pub trait InterruptManager { + /// Create an [InterruptSourceGroup](trait.InterruptSourceGroup.html) object to manage + /// interrupt sources for a virtual device + /// + /// An [InterruptSourceGroup](trait.InterruptSourceGroup.html) object manages all interrupt + /// sources of the same type for a virtual device. + /// + /// # Arguments + /// * interrupt_type: type of interrupt source. + /// * base: base Interrupt Source ID to be managed by the group object. + /// * count: number of Interrupt Sources to be managed by the group object. + fn create_group( + &self, + interrupt_type: InterruptType, + base: InterruptIndex, + count: InterruptIndex, + ) -> Result>>; + + /// Destroy an [InterruptSourceGroup](trait.InterruptSourceGroup.html) object created by + /// [create_group()](trait.InterruptManager.html#tymethod.create_group). + /// + /// Assume the caller takes the responsibility to disable all interrupt sources of the group + /// before calling destroy_group(). This assumption helps to simplify InterruptSourceGroup + /// implementations. + fn destroy_group(&self, group: Arc>) -> Result<()>; +} + +#[derive(Copy, Clone, Debug, Default)] +pub struct MsiIrqSourceConfig { + /// High address to delivery message signaled interrupt. + pub high_addr: u32, + /// Low address to delivery message signaled interrupt. + pub low_addr: u32, + /// Data to write to delivery message signaled interrupt. + pub data: u32, +} + +pub trait InterruptSourceGroup: Send + Sync { + /// Enable the interrupt sources in the group to generate interrupts. + fn enable(&self) -> Result<()>; + + /// Disable the interrupt sources in the group to generate interrupts. + fn disable(&self) -> Result<()>; + + /// Inject an interrupt from this interrupt source into the guest. + fn trigger(&self, index: InterruptIndex) -> Result<()>; + + /// Returns an interrupt notifier from this interrupt. + /// An interrupt notifier allows for external components and processes + /// to inject interrupts into a guest, by writing to the file returned + /// by this method. + fn notifier(&self, index: InterruptIndex) -> Option; +} + +pub trait InterruptSourceGroupMsi: Send + Sync + InterruptSourceGroup { + /// Update the interrupt source group configuration. + /// + /// # Arguments + /// * index: sub-index into the group. + /// * config: configuration data for the interrupt source. + fn update(&self, index: InterruptIndex, config: &MsiIrqSourceConfig) -> Result<()>; +} diff --git a/src/lib.rs b/src/lib.rs index dc09ddb..235e4da 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,6 +6,7 @@ use std::cmp::{Ord, Ordering, PartialOrd}; pub mod device_manager; +pub mod interrupt; pub mod resources; // IO Size. From a649808e9a9a64690bf5129a0748d9f556b846d2 Mon Sep 17 00:00:00 2001 From: Andreea Florescu Date: Fri, 27 Dec 2019 14:58:46 +0200 Subject: [PATCH 11/14] interrupts: documentation improvements Wrapped lines at 100 chars. Added more details about the InterruptType. Signed-off-by: Andreea Florescu --- src/interrupt/mod.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/interrupt/mod.rs b/src/interrupt/mod.rs index 344c3cc..e2bf249 100644 --- a/src/interrupt/mod.rs +++ b/src/interrupt/mod.rs @@ -28,13 +28,13 @@ //! in-band messages to replace traditional out-of-band assertion of dedicated interrupt lines. //! While more complex to implement in a device, message signaled interrupts have some significant //! advantages over pin-based out-of-band interrupt signaling. Message signaled interrupts are -//! supported in PCI bus since its version 2.2, and in later available PCI Express bus. Some non-PCI -//! architectures also use message signaled interrupts. +//! supported in PCI bus since its version 2.2, and in later available PCI Express bus. Some +//! non-PCI architectures also use message signaled interrupts. //! //! While IRQ is a term commonly used by Operating Systems when dealing with hardware //! interrupts, the IRQ numbers managed by OSes are independent of the ones managed by VMM. -//! For simplicity sake, the term `Interrupt Source` is used instead of IRQ to represent both pin-based -//! interrupts and MSI interrupts. +//! For simplicity sake, the term `Interrupt Source` is used instead of IRQ to represent both +//! pin-based interrupts and MSI interrupts. //! //! A device may support multiple types of interrupts, and each type of interrupt may support one //! or multiple interrupt sources. For example, a PCI device may support: @@ -67,6 +67,9 @@ pub type Result = std::io::Result; pub type InterruptIndex = u32; /// Data type to store an interrupt source type. +/// +/// The interrupt source type is a slim wrapper so that the `InterruptManager` +/// can be implemented in external, non rust-vmm crates. pub type InterruptType = u32; pub const PIN_IRQ: InterruptType = 0; @@ -124,6 +127,7 @@ pub trait InterruptSourceGroup: Send + Sync { fn trigger(&self, index: InterruptIndex) -> Result<()>; /// Returns an interrupt notifier from this interrupt. + /// /// An interrupt notifier allows for external components and processes /// to inject interrupts into a guest, by writing to the file returned /// by this method. From 5aca2b9dd13d0d8f22777eef3d9d3310fc394eeb Mon Sep 17 00:00:00 2001 From: Andreea Florescu Date: Fri, 27 Dec 2019 15:02:31 +0200 Subject: [PATCH 12/14] Added default implementation for interrupt traits In the InterruptSourceGroup trait the only method that is going to be implemented for every interrupt type is the trigger method. We can make it easier for simple VMMs (like the ones using only legacy IRQs) to implement this trait by provided no-op implementation for all the other trait methods. Signed-off-by: Andreea Florescu --- src/interrupt/mod.rs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/interrupt/mod.rs b/src/interrupt/mod.rs index e2bf249..010054f 100644 --- a/src/interrupt/mod.rs +++ b/src/interrupt/mod.rs @@ -118,10 +118,18 @@ pub struct MsiIrqSourceConfig { pub trait InterruptSourceGroup: Send + Sync { /// Enable the interrupt sources in the group to generate interrupts. - fn enable(&self) -> Result<()>; + fn enable(&self) -> Result<()> { + // Not all interrupt sources can be enabled. + // To accommodate this, we can have a no-op here. + Ok(()) + } /// Disable the interrupt sources in the group to generate interrupts. - fn disable(&self) -> Result<()>; + fn disable(&self) -> Result<()> { + // Not all interrupt sources can be disabled. + // To accommodate this, we can have a no-op here. + Ok(()) + } /// Inject an interrupt from this interrupt source into the guest. fn trigger(&self, index: InterruptIndex) -> Result<()>; @@ -131,7 +139,12 @@ pub trait InterruptSourceGroup: Send + Sync { /// An interrupt notifier allows for external components and processes /// to inject interrupts into a guest, by writing to the file returned /// by this method. - fn notifier(&self, index: InterruptIndex) -> Option; + #[allow(unused_variables)] + fn notifier(&self, index: InterruptIndex) -> Option { + // One use case of the notifier is to implement vhost user backends. + // For all other implementations we can just return None here. + None + } } pub trait InterruptSourceGroupMsi: Send + Sync + InterruptSourceGroup { From 841bc20fd5d3f69780240dc21b084af2392c87f3 Mon Sep 17 00:00:00 2001 From: Andreea Florescu Date: Fri, 27 Dec 2019 18:03:36 +0200 Subject: [PATCH 13/14] interrupts: specify mutability of self in traits Most methods in the traits need to mutate the state. The inital idea was that save mutability can be achieved by wrapping the trait implementers in a Mutex. I find this to be misleading when reading the code. The proposal is to use mut where needed unless there is a good reason not to. Signed-off-by: Andreea Florescu --- src/interrupt/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/interrupt/mod.rs b/src/interrupt/mod.rs index 010054f..bbff298 100644 --- a/src/interrupt/mod.rs +++ b/src/interrupt/mod.rs @@ -91,7 +91,7 @@ pub trait InterruptManager { /// * base: base Interrupt Source ID to be managed by the group object. /// * count: number of Interrupt Sources to be managed by the group object. fn create_group( - &self, + &mut self, interrupt_type: InterruptType, base: InterruptIndex, count: InterruptIndex, @@ -103,7 +103,7 @@ pub trait InterruptManager { /// Assume the caller takes the responsibility to disable all interrupt sources of the group /// before calling destroy_group(). This assumption helps to simplify InterruptSourceGroup /// implementations. - fn destroy_group(&self, group: Arc>) -> Result<()>; + fn destroy_group(&mut self, group: Arc>) -> Result<()>; } #[derive(Copy, Clone, Debug, Default)] @@ -118,14 +118,14 @@ pub struct MsiIrqSourceConfig { pub trait InterruptSourceGroup: Send + Sync { /// Enable the interrupt sources in the group to generate interrupts. - fn enable(&self) -> Result<()> { + fn enable(&mut self) -> Result<()> { // Not all interrupt sources can be enabled. // To accommodate this, we can have a no-op here. Ok(()) } /// Disable the interrupt sources in the group to generate interrupts. - fn disable(&self) -> Result<()> { + fn disable(&mut self) -> Result<()> { // Not all interrupt sources can be disabled. // To accommodate this, we can have a no-op here. Ok(()) @@ -153,5 +153,5 @@ pub trait InterruptSourceGroupMsi: Send + Sync + InterruptSourceGroup { /// # Arguments /// * index: sub-index into the group. /// * config: configuration data for the interrupt source. - fn update(&self, index: InterruptIndex, config: &MsiIrqSourceConfig) -> Result<()>; + fn update(&mut self, index: InterruptIndex, config: &MsiIrqSourceConfig) -> Result<()>; } From f886e49cb0c994213a6b0dd85214c8a4c81771de Mon Sep 17 00:00:00 2001 From: Samuel Ortiz Date: Tue, 28 Jan 2020 16:31:12 +0100 Subject: [PATCH 14/14] interrupt: InterruptSourceGroup extension We extend the trait to support the mask/unmask operations. We also merge the MSI interrupt source group trait into the main one, which leads to using interrupt source type and config enums instead of type wrappers. Signed-off-by: Samuel Ortiz --- src/interrupt/mod.rs | 68 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/src/interrupt/mod.rs b/src/interrupt/mod.rs index bbff298..d696224 100644 --- a/src/interrupt/mod.rs +++ b/src/interrupt/mod.rs @@ -67,13 +67,15 @@ pub type Result = std::io::Result; pub type InterruptIndex = u32; /// Data type to store an interrupt source type. -/// -/// The interrupt source type is a slim wrapper so that the `InterruptManager` -/// can be implemented in external, non rust-vmm crates. -pub type InterruptType = u32; - -pub const PIN_IRQ: InterruptType = 0; -pub const PCI_MSI_IRQ: InterruptType = 1; +#[derive(Copy, Clone, Debug)] +pub enum InterruptSourceType { + /// Legacy Pin-based Interrupt. + /// On x86 platforms, legacy interrupts are routed through 8259 PICs and/or IOAPICs. + LegacyIrq, + /// Message Signaled Interrupt (PCI MSI/PCI MSIx). + /// Some non-PCI devices (like HPET on x86) make use of generic MSI in platform specific ways. + PciMsiIrq, +} /// Trait to manage interrupt sources for virtual device backends. /// @@ -92,7 +94,7 @@ pub trait InterruptManager { /// * count: number of Interrupt Sources to be managed by the group object. fn create_group( &mut self, - interrupt_type: InterruptType, + interrupt_type: InterruptSourceType, base: InterruptIndex, count: InterruptIndex, ) -> Result>>; @@ -106,6 +108,15 @@ pub trait InterruptManager { fn destroy_group(&mut self, group: Arc>) -> Result<()>; } +/// Configuration data for legacy interrupts. +/// +/// On x86 platforms, legacy interrupts means those interrupts routed through PICs or IOAPICs. +#[derive(Copy, Clone, Debug)] +pub struct LegacyIrqSourceConfig {} + +/// Configuration data for MSI/MSI-X interrupts. +/// +/// On x86 platforms, these interrupts are vectors delivered directly to the LAPIC. #[derive(Copy, Clone, Debug, Default)] pub struct MsiIrqSourceConfig { /// High address to delivery message signaled interrupt. @@ -116,6 +127,15 @@ pub struct MsiIrqSourceConfig { pub data: u32, } +/// Configuration data for an interrupt source. +#[derive(Copy, Clone, Debug)] +pub enum InterruptSourceConfig { + /// Configuration data for Legacy interrupts. + LegacyIrq(LegacyIrqSourceConfig), + /// Configuration data for PciMsi, PciMsix and generic MSI interrupts. + MsiIrq(MsiIrqSourceConfig), +} + pub trait InterruptSourceGroup: Send + Sync { /// Enable the interrupt sources in the group to generate interrupts. fn enable(&mut self) -> Result<()> { @@ -132,6 +152,9 @@ pub trait InterruptSourceGroup: Send + Sync { } /// Inject an interrupt from this interrupt source into the guest. + /// + /// # Arguments + /// * index: sub-index into the group. fn trigger(&self, index: InterruptIndex) -> Result<()>; /// Returns an interrupt notifier from this interrupt. @@ -145,13 +168,36 @@ pub trait InterruptSourceGroup: Send + Sync { // For all other implementations we can just return None here. None } -} -pub trait InterruptSourceGroupMsi: Send + Sync + InterruptSourceGroup { /// Update the interrupt source group configuration. /// /// # Arguments /// * index: sub-index into the group. /// * config: configuration data for the interrupt source. - fn update(&mut self, index: InterruptIndex, config: &MsiIrqSourceConfig) -> Result<()>; + fn update(&self, _index: InterruptIndex, _config: InterruptSourceConfig) -> Result<()> { + // Interrupt source group update is a typical MSI + // related operation. Most legacy interrupts don't + // support that. + Ok(()) + } + + /// Mask an interrupt from this interrupt source group. + /// + /// # Arguments + /// * index: sub-index into the group. + fn mask(&self, _index: InterruptIndex) -> Result<()> { + // Not all interrupt sources can be disabled. + // To accommodate this, we can have a no-op here. + Ok(()) + } + + /// Unmask an interrupt from this interrupt source group. + /// + /// # Arguments + /// * index: sub-index into the group. + fn unmask(&self, _index: InterruptIndex) -> Result<()> { + // Not all interrupt sources can be disabled. + // To accommodate this, we can have a no-op here. + Ok(()) + } }