diff --git a/api/ruxfeat/Cargo.toml b/api/ruxfeat/Cargo.toml index a7c423aff..3ac96db34 100644 --- a/api/ruxfeat/Cargo.toml +++ b/api/ruxfeat/Cargo.toml @@ -93,6 +93,9 @@ log-level-trace = ["axlog/log-level-trace"] tty = ["ruxhal/tty", "ruxruntime/tty", "alloc", "irq"] +# inter-VM communication on hvisor +ivc = ["alloc", "ruxhal/ivc"] + [dependencies] ruxruntime = { path = "../../modules/ruxruntime" } ruxhal = { path = "../../modules/ruxhal" } diff --git a/modules/ruxconfig/defconfig.toml b/modules/ruxconfig/defconfig.toml index bda9cc8df..dc2022033 100644 --- a/modules/ruxconfig/defconfig.toml +++ b/modules/ruxconfig/defconfig.toml @@ -43,3 +43,6 @@ smp = "1" # Maximum number of keys per thread. pthread-key-max = "1024" + +# Maximum number of inter vm-communication zones +ivc-zones = "2" \ No newline at end of file diff --git a/modules/ruxhal/Cargo.toml b/modules/ruxhal/Cargo.toml index 622cbc01b..98eb2677f 100644 --- a/modules/ruxhal/Cargo.toml +++ b/modules/ruxhal/Cargo.toml @@ -24,7 +24,7 @@ default = [] musl = [] signal = [] virtio_console = ["driver_console", "driver_virtio", "driver_virtio/console", "driver_common", "virtio-drivers", "axalloc", "lazy_static", "alloc", "virtio_hal"] - +ivc = ["axalloc"] [dependencies] log = "0.4" diff --git a/modules/ruxhal/src/ivc.rs b/modules/ruxhal/src/ivc.rs new file mode 100644 index 000000000..5115365de --- /dev/null +++ b/modules/ruxhal/src/ivc.rs @@ -0,0 +1,208 @@ +/* Copyright (c) [2024] [Syswonder Community] +* [Ruxos] is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* http://license.coscl.org.cn/MulanPSL2 +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +*/ + +use core::alloc::Layout; +use core::ptr::NonNull; +use axalloc::global_allocator; +use memory_addr::{PhysAddr, VirtAddr}; +use ruxconfig::IVC_ZONES; +use crate::mem::{direct_virt_to_phys, phys_to_virt}; + +pub const CONFIG_MAX_IVC_CONFIGS: usize = 0x2; +pub const HVISOR_HC_IVC_INFO: usize = 0x5; +pub const HVISOR_HC_IVC_INFO_ALIGN: usize = 0x8; +pub const HVISOR_HC_IVC_INFO_SIZE: usize = 56; +pub const __PA: usize = 0xffff_0000_0000_0000; + +#[repr(C)] +#[derive(Debug)] +struct IvCInfo { + /// The number of IVC shared memory + len: u64, + /// Control Table IPA + ivc_ct_ipas: [u64; IVC_ZONES], + /// Share memory IPA + ivc_shmem_ipas: [u64; IVC_ZONES], + /// IVC id; the ivc id of zones that communicate with each other have to be the same + ivc_ids: [u32; IVC_ZONES], + /// irq number + ivc_irqs: [u32; IVC_ZONES], +} + +#[repr(C)] +#[derive(Debug)] +struct ControlTable { + ivc_id: u32, + max_peers: u32, + rw_sec_size: u32, + out_sec_size: u32, + peer_id: u32, + ipi_invoke: u32, +} + +/// This module provides a way to establish a communication channel with a hypervisor (hvisor) +/// for virtual machine (VM) communication using shared memory. Each VM can have two communication +/// regions, and the region to be used for communication is specified during the connection setup. +/// +/// The communication is handled through the following steps: +/// +/// 1. **Connection Setup**: The `connect()` function allocates memory for communication structures +/// and invokes a hypervisor call (`hvc`) to retrieve necessary information about the IVC +/// (Inter-VM Communication) and control tables. The specific communication region to be used +/// is determined by the parameter passed to `connect()`. This process sets up the shared memory +/// and prepares the communication channel. +/// 2. **Message Sending**: The `send_message()` function writes a message to the shared memory area +/// specified by the hypervisor. The message is written to a predefined memory location, and +/// the control table is updated to notify the target VM of the message. +/// 3. **Connection Teardown**: The `close()` function frees the allocated memory and closes the +/// communication channel, cleaning up resources to prevent memory leaks. +/// +/// The module relies on a `GlobalAllocator` for memory management and uses raw pointers and unsafe +/// Rust operations to interact with memory addresses provided by the hypervisor. It is critical +/// that the communication sequence follows the correct order: connect -> send_message -> close. +/// +/// # Example +/// ``` +/// let mut conn = Connection::new(); +/// if let Err(e) = conn.connect(0) { // Choose the first communication region (0) +/// info!("Error connecting: {}", e); +/// return; +/// } +/// if let Err(e) = conn.send_message("Hello from zone1 ruxos!") { +/// error!("Error sending message: {}", e); +/// } +/// if let Err(e) = conn.close() { +/// error!("Error closing connection: {}", e); +/// } +/// ``` + +pub fn ivc_example() { + let mut conn = Connection::new(); + + // Establish the connection + if let Err(e) = conn.connect(0) { + info!("Error connecting: {}", e); + return; + } + + // Send the message + if let Err(e) = conn.send_message("Hello from zone1 ruxos!4 ") { + info!("Error sending message: {}", e); + } + + // Close the connection + if let Err(e) = conn.close() { + info!("Error closing connection: {}", e); + } +} + +pub struct Connection { + ivc_info: Option, + control_table: Option>, +} + +impl Connection { + pub fn new() -> Self { + debug!("Connection created."); + Connection { + ivc_info: None, + control_table: None, + } + } + + pub fn connect(&mut self, communication_zone: usize) -> Result<(), &'static str> { + let alloc_size = HVISOR_HC_IVC_INFO_SIZE; + let align = HVISOR_HC_IVC_INFO_ALIGN; + let layout = Layout::from_size_align(alloc_size, align).unwrap(); + + let ptr = global_allocator().alloc(layout).expect("Memory allocation failed!"); + + let vpa_ivcinfo = VirtAddr::from(ptr.as_ptr() as usize); + // convert the virtual address to physical address, to use hvc on physical address + let pa_ivcinfo: PhysAddr = direct_virt_to_phys(vpa_ivcinfo); + debug!("The memory address of the IVC Info: VA: 0x{:x}, IPA: 0x{:x}", vpa_ivcinfo.as_usize(), pa_ivcinfo.as_usize()); + + ivc_hvc_call(HVISOR_HC_IVC_INFO as u32, pa_ivcinfo.as_usize(), HVISOR_HC_IVC_INFO_SIZE); + debug!("ivc_hvc_call finished."); + + let ivc_info_ptr = ptr.as_ptr() as *const IvCInfo; + let ivc_info: IvCInfo = unsafe { ivc_info_ptr.read() }; + self.ivc_info = Some(ivc_info); + + global_allocator().dealloc(ptr, layout); + + let ivc_info = self.ivc_info.as_ref().unwrap(); + let pa_control_table = PhysAddr::from(ivc_info.ivc_ct_ipas[communication_zone] as usize); + // convert the physical address to virtual address to use it in the kernel + let vpa_control_table: VirtAddr = phys_to_virt(pa_control_table); + self.control_table = NonNull::new(vpa_control_table.as_ptr() as *mut ControlTable); + + info!("IVC Connection established."); + Ok(()) + } + + pub fn send_message(&mut self, message: &str) -> Result<(), &'static str> { + let ivc_info = self.ivc_info.as_ref().ok_or("Not connected")?; + let mut control_table_ptr = self.control_table.ok_or("Not connected")?; + + // Safely get an immutable reference to the ControlTable for reading out_sec_size + let control_table = unsafe { control_table_ptr.as_ref() }; + + // Suppose we are zone1, we are to send message to zone0. + // Therefore use the out_sec_size field of the ControlTable struct (typically 0x1000). + let offset = control_table.out_sec_size as u64; + let vpa_share_memory_zone1 = phys_to_virt(PhysAddr::from((ivc_info.ivc_shmem_ipas[0] + offset) as usize)); + + write_to_address(vpa_share_memory_zone1, message)?; + info!("Message written to shared memory: {}", message); + + // Get a mutable reference to ControlTable to modify ipi_invoke + let control_table = unsafe { control_table_ptr.as_mut() }; + debug!("Ipi_invoke reset to inform Zone0 linux."); + control_table.ipi_invoke = 0x0; + + Ok(()) + } + + pub fn close(&mut self) -> Result<(), &'static str> { + if self.ivc_info.is_none() { + return Err("Not connected"); + } + self.ivc_info = None; + self.control_table = None; + info!("IVC Connection closed."); + Ok(()) + } +} + +/// ivc "hvc" method call +fn ivc_hvc_call(func: u32, arg0: usize, arg1: usize) -> usize { + let ret; + unsafe { + core::arch::asm!( + "hvc #0", + inlateout("x0") func as usize => ret, + in("x1") arg0, + in("x2") arg1, + options(nostack) + ); + info!("Ivc call: func: {:x}, arg0: 0x{:x}, arg1: 0x{:x}, ret: 0x{:x}", func, arg0, arg1, ret); + } + ret +} + +fn write_to_address(addr: VirtAddr, data: &str) -> Result<(), &'static str> { + unsafe { + let ptr = addr.as_usize() as *mut u8; + ptr.copy_from_nonoverlapping(data.as_ptr(), data.len()); + // Add a null terminator to the end of the string + *ptr.add(data.len()) = 0; + } + Ok(()) +} diff --git a/modules/ruxhal/src/lib.rs b/modules/ruxhal/src/lib.rs index 584fa54ba..4b4383449 100644 --- a/modules/ruxhal/src/lib.rs +++ b/modules/ruxhal/src/lib.rs @@ -50,6 +50,8 @@ mod platform; pub mod time; pub mod trap; pub mod virtio; +#[cfg(feature = "ivc")] +pub mod ivc; #[cfg(feature = "tls")] pub mod tls; @@ -77,10 +79,6 @@ pub mod misc { pub use super::platform::misc::*; } -/// Inter-VM communication. -pub mod ivc { - pub use super::platform::ivc::*; -} /// Multi-core operations. #[cfg(feature = "smp")] diff --git a/modules/ruxhal/src/platform/aarch64_common/ivc.rs b/modules/ruxhal/src/platform/aarch64_common/ivc.rs deleted file mode 100644 index 743651aad..000000000 --- a/modules/ruxhal/src/platform/aarch64_common/ivc.rs +++ /dev/null @@ -1,218 +0,0 @@ -/* Copyright (c) [2024] [Syswonder Community] - * [Ruxos] is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * http://license.coscl.org.cn/MulanPSL2 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - */ - - use core::alloc::Layout; - use core::ptr; - use core::ptr::NonNull; - use axalloc:: {global_allocator, GlobalAllocator}; - - pub const CONFIG_MAX_IVC_CONFIGS: usize = 0x2; - pub const HVISOR_HC_IVC_INFO: usize = 0x5; - pub const HVISOR_HC_IVC_INFO_ALIGN: usize = 0x8; - pub const HVISOR_HC_IVC_INFO_SIZE: usize = 56; - pub const __PA: usize = 0xffff_0000_0000_0000; - - #[repr(C)] - #[derive(Debug)] - struct IvCInfo { - len: u64, // The number of IVC shared memory - ivc_ct_ipas: [u64; CONFIG_MAX_IVC_CONFIGS], // Control Table IPA - ivc_shmem_ipas: [u64; CONFIG_MAX_IVC_CONFIGS], // Share memory IPA - ivc_ids: [u32; CONFIG_MAX_IVC_CONFIGS], // IVC id; the ivc id of zones that communicate with each other have to be the same - ivc_irqs: [u32; CONFIG_MAX_IVC_CONFIGS], // irq number - } - - #[repr(C)] - #[derive(Debug)] - struct ControlTable { - ivc_id: u32, - max_peers: u32, - rw_sec_size: u32, - out_sec_size: u32, - peer_id: u32, - ipi_invoke: u32, - } - - /// This module provides a way to establish a communication channel with a hypervisor (hvisor) - /// for virtual machine (VM) communication using shared memory. Each VM can have two communication - /// regions, and the region to be used for communication is specified during the connection setup. - /// - /// The communication is handled through the following steps: - /// - /// 1. **Connection Setup**: The `connect()` function allocates memory for communication structures - /// and invokes a hypervisor call (`hvc`) to retrieve necessary information about the IVC - /// (Inter-VM Communication) and control tables. The specific communication region to be used - /// is determined by the parameter passed to `connect()`. This process sets up the shared memory - /// and prepares the communication channel. - /// 2. **Message Sending**: The `send_message()` function writes a message to the shared memory area - /// specified by the hypervisor. The message is written to a predefined memory location, and - /// the control table is updated to notify the target VM of the message. - /// 3. **Connection Teardown**: The `close()` function frees the allocated memory and closes the - /// communication channel, cleaning up resources to prevent memory leaks. - /// - /// The module relies on a `GlobalAllocator` for memory management and uses raw pointers and unsafe - /// Rust operations to interact with memory addresses provided by the hypervisor. It is critical - /// that the communication sequence follows the correct order: connect -> send_message -> close. - /// - /// # Example - /// ``` - /// let mut conn = Connection::new(); - /// if let Err(e) = conn.connect(0) { // Choose the first communication region (0) - /// info!("Error connecting: {}", e); - /// return; - /// } - /// if let Err(e) = conn.send_message("Hello from VM!") { - /// error!("Error sending message: {}", e); - /// } - /// if let Err(e) = conn.close() { - /// error!("Error closing connection: {}", e); - /// } - /// ``` - - pub fn ivc_example() { - let mut conn = Connection::new(); - - // Establish the connection - if let Err(e) = conn.connect(0) { - info!("Error connecting: {}", e); - return; - } - - // Send the message - if let Err(e) = conn.send_message("Hello from VM!") { - info!("Error sending message: {}", e); - } - - // Close the connection - if let Err(e) = conn.close() { - info!("Error closing connection: {}", e); - } - } - - pub struct Connection<'a> { - // Reference to the allocator - allocator: &'a GlobalAllocator, - ivc_info_ptr: *mut IvCInfo, - control_table_ptr: *mut ControlTable, - } - - impl<'a> Connection<'a> { - - pub fn new() -> Self { - // Get the reference to the global allocator - let allocator = global_allocator(); - debug!("Connection created."); - Connection { - allocator, - ivc_info_ptr: ptr::null_mut(), - control_table_ptr: ptr::null_mut(), - } - } - - pub fn connect(&mut self, communication_zone: usize) -> Result<(), &'static str> { - let alloc_size = HVISOR_HC_IVC_INFO_SIZE; - let align = HVISOR_HC_IVC_INFO_ALIGN; - let layout = Layout::from_size_align(alloc_size, align).unwrap(); - - let ptr = self.allocator.alloc(layout).expect("Memory allocation failed!"); - self.ivc_info_ptr = ptr.as_ptr() as *mut IvCInfo; - - let vpa_ivcinfo = self.ivc_info_ptr as usize; - let pa_ivcinfo: usize = vpa_ivcinfo - __PA; - - debug!("The memory address of the IVC Info: VA: 0x{:x}, IPA: 0x{:x}", vpa_ivcinfo, pa_ivcinfo); - - ivc_hvc_call(HVISOR_HC_IVC_INFO as u32, pa_ivcinfo, HVISOR_HC_IVC_INFO_SIZE); - debug!("ivc_hvc_call finished."); - - // Safety: At this point we know ivc_info_ptr is valid and allocated - let ivc_info: &IvCInfo = unsafe { &*self.ivc_info_ptr }; - - let control_table_ptr = (ivc_info.ivc_ct_ipas[communication_zone] + __PA as u64) as *mut ControlTable; - self.control_table_ptr = control_table_ptr; - - info!("IVC Connection established."); - Ok(()) - } - - pub fn send_message(&self, message: &str) -> Result<(), &'static str> { - if self.ivc_info_ptr.is_null() { - return Err("Not connected"); - } - - let ivc_info: &IvCInfo = unsafe { &*self.ivc_info_ptr }; - - let control_table: &ControlTable = unsafe { &*self.control_table_ptr }; - - // Suppose we are zone1, we are to send message to zone0. - // Therefore use the out_sec_size field of the ControlTable struct (typically 0x1000). - let offset = control_table.out_sec_size as u64; - let address: u64 = ivc_info.ivc_shmem_ipas[0] + offset + __PA as u64; - - write_to_address(address, message)?; - info!("Message written to shared memory: {}", message); - - // Modify the control table to inform Zone0 Linux - let control_table: &mut ControlTable = unsafe { &mut *self.control_table_ptr }; - debug!("Ipi_invoke reset to inform Zone0 linux."); - control_table.ipi_invoke = 0x0; - - Ok(()) - } - - - pub fn close(&mut self) -> Result<(), &'static str> { - if self.ivc_info_ptr.is_null() { - return Err("Not connected"); - } - // free the memory - let layout = Layout::from_size_align(HVISOR_HC_IVC_INFO_SIZE, HVISOR_HC_IVC_INFO_ALIGN).unwrap(); - self.allocator.dealloc(unsafe { NonNull::new_unchecked(self.ivc_info_ptr as *mut u8) }, layout); - info!("IVC Connection closed."); - Ok(()) - } - } - - /// ivc "hvc" method call - fn ivc_hvc_call(func: u32, arg0: usize, arg1: usize) -> usize { - let ret; - unsafe { - core::arch::asm!( - "hvc #4856", - inlateout("x0") func as usize => ret, - in("x1") arg0, - in("x2") arg1, - options(nostack) - ); - info!("Ivc call: func: {:x}, arg0: 0x{:x}, arg1: 0x{:x}, ret: 0x{:x}", func, arg0, arg1, ret); - } - ret - } - - fn write_to_address(addr: u64, data: &str) -> Result<(), &'static str> { - let len = data.len(); - - if addr == 0 { - return Err("Invalid address: 0x0"); - } - - unsafe { - let ptr = addr as *mut u8; - - for (i, &byte) in data.as_bytes().iter().enumerate() { - let target_ptr = ptr.add(i); - *target_ptr = byte; - } - - let null_ptr = ptr.add(len); - *null_ptr = 0; - } - - Ok(()) - } \ No newline at end of file diff --git a/modules/ruxhal/src/platform/aarch64_common/mod.rs b/modules/ruxhal/src/platform/aarch64_common/mod.rs index 707f0fb57..933847f3d 100644 --- a/modules/ruxhal/src/platform/aarch64_common/mod.rs +++ b/modules/ruxhal/src/platform/aarch64_common/mod.rs @@ -21,5 +21,3 @@ pub mod pl011; #[cfg(feature = "rtc")] pub mod pl031; - -pub mod ivc; \ No newline at end of file diff --git a/modules/ruxhal/src/platform/aarch64_qemu_virt/mod.rs b/modules/ruxhal/src/platform/aarch64_qemu_virt/mod.rs index b380f7a35..b15270403 100644 --- a/modules/ruxhal/src/platform/aarch64_qemu_virt/mod.rs +++ b/modules/ruxhal/src/platform/aarch64_qemu_virt/mod.rs @@ -34,10 +34,6 @@ pub mod misc { pub use crate::platform::aarch64_common::psci::system_off as terminate; } -pub mod ivc { - pub use crate::platform::aarch64_common::ivc::*; -} - extern "C" { fn exception_vector_base(); fn rust_main(cpu_id: usize, dtb: usize);