From d4d30afc39a537e6809378654fbfc95d83911b17 Mon Sep 17 00:00:00 2001 From: HeartLinked Date: Sat, 7 Dec 2024 22:21:19 +0800 Subject: [PATCH] add ivc feature: inter vm communication on hvisor --- api/ruxfeat/Cargo.toml | 3 + modules/ruxconfig/defconfig.toml | 3 + modules/ruxhal/Cargo.toml | 4 +- modules/ruxhal/src/ivc.rs | 194 ++++++++++++++++++ modules/ruxhal/src/lib.rs | 3 + .../ruxhal/src/platform/aarch64_common/ivc.rs | 24 +++ .../ruxhal/src/platform/aarch64_common/mod.rs | 3 + .../src/platform/aarch64_qemu_virt/mod.rs | 5 + platforms/aarch64-qemu-virt.toml | 2 + 9 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 modules/ruxhal/src/ivc.rs create mode 100644 modules/ruxhal/src/platform/aarch64_common/ivc.rs diff --git a/api/ruxfeat/Cargo.toml b/api/ruxfeat/Cargo.toml index a7c423aff..3ac96db34 100644 --- a/api/ruxfeat/Cargo.toml +++ b/api/ruxfeat/Cargo.toml @@ -93,6 +93,9 @@ log-level-trace = ["axlog/log-level-trace"] tty = ["ruxhal/tty", "ruxruntime/tty", "alloc", "irq"] +# inter-VM communication on hvisor +ivc = ["alloc", "ruxhal/ivc"] + [dependencies] ruxruntime = { path = "../../modules/ruxruntime" } ruxhal = { path = "../../modules/ruxhal" } diff --git a/modules/ruxconfig/defconfig.toml b/modules/ruxconfig/defconfig.toml index bda9cc8df..dc2022033 100644 --- a/modules/ruxconfig/defconfig.toml +++ b/modules/ruxconfig/defconfig.toml @@ -43,3 +43,6 @@ smp = "1" # Maximum number of keys per thread. pthread-key-max = "1024" + +# Maximum number of inter vm-communication zones +ivc-zones = "2" \ No newline at end of file diff --git a/modules/ruxhal/Cargo.toml b/modules/ruxhal/Cargo.toml index 521b91a09..98eb2677f 100644 --- a/modules/ruxhal/Cargo.toml +++ b/modules/ruxhal/Cargo.toml @@ -14,7 +14,7 @@ repository = "https://github.com/syswonder/ruxos/tree/main/modules/ruxhal" [features] virtio_hal = ["driver_virtio","virtio-drivers","axalloc"] smp = [] -alloc = [] +alloc = ["axalloc"] fp_simd = [] paging = ["axalloc", "page_table"] irq = [] @@ -24,7 +24,7 @@ default = [] musl = [] signal = [] virtio_console = ["driver_console", "driver_virtio", "driver_virtio/console", "driver_common", "virtio-drivers", "axalloc", "lazy_static", "alloc", "virtio_hal"] - +ivc = ["axalloc"] [dependencies] log = "0.4" diff --git a/modules/ruxhal/src/ivc.rs b/modules/ruxhal/src/ivc.rs new file mode 100644 index 000000000..9257aa4b2 --- /dev/null +++ b/modules/ruxhal/src/ivc.rs @@ -0,0 +1,194 @@ +/* Copyright (c) [2024] [Syswonder Community] +* [Ruxos] is licensed under Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* http://license.coscl.org.cn/MulanPSL2 +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +*/ + +use core::alloc::Layout; +use core::ptr::NonNull; +use axalloc::global_allocator; +use memory_addr::{PhysAddr, VirtAddr}; +use ruxconfig::IVC_ZONES; +use crate::mem::{direct_virt_to_phys, phys_to_virt}; +use crate::platform::ivc::ivc_hvc_call; + +pub const CONFIG_MAX_IVC_CONFIGS: usize = 0x2; +pub const HVISOR_HC_IVC_INFO: usize = 0x5; +pub const HVISOR_HC_IVC_INFO_ALIGN: usize = 0x8; +pub const HVISOR_HC_IVC_INFO_SIZE: usize = 56; +pub const __PA: usize = 0xffff_0000_0000_0000; + +#[repr(C)] +#[derive(Debug)] +struct IvCInfo { + /// The number of IVC shared memory + len: u64, + /// Control Table IPA + ivc_ct_ipas: [u64; IVC_ZONES], + /// Share memory IPA + ivc_shmem_ipas: [u64; IVC_ZONES], + /// IVC id; the ivc id of zones that communicate with each other have to be the same + ivc_ids: [u32; IVC_ZONES], + /// irq number + ivc_irqs: [u32; IVC_ZONES], +} + +#[repr(C)] +#[derive(Debug)] +struct ControlTable { + ivc_id: u32, + max_peers: u32, + rw_sec_size: u32, + out_sec_size: u32, + peer_id: u32, + ipi_invoke: u32, +} + +/// This module provides a way to establish a communication channel with a hypervisor (hvisor) +/// for virtual machine (VM) communication using shared memory. Each VM can have two communication +/// regions, and the region to be used for communication is specified during the connection setup. +/// +/// The communication is handled through the following steps: +/// +/// 1. **Connection Setup**: The `connect()` function allocates memory for communication structures +/// and invokes a hypervisor call (`hvc`) to retrieve necessary information about the IVC +/// (Inter-VM Communication) and control tables. The specific communication region to be used +/// is determined by the parameter passed to `connect()`. This process sets up the shared memory +/// and prepares the communication channel. +/// 2. **Message Sending**: The `send_message()` function writes a message to the shared memory area +/// specified by the hypervisor. The message is written to a predefined memory location, and +/// the control table is updated to notify the target VM of the message. +/// 3. **Connection Teardown**: The `close()` function frees the allocated memory and closes the +/// communication channel, cleaning up resources to prevent memory leaks. +/// +/// The module relies on a `GlobalAllocator` for memory management and uses raw pointers and unsafe +/// Rust operations to interact with memory addresses provided by the hypervisor. It is critical +/// that the communication sequence follows the correct order: connect -> send_message -> close. +/// +/// # Example +/// ``` +/// let mut conn = Connection::new(); +/// if let Err(e) = conn.connect(0) { // Choose the first communication region (0) +/// info!("Error connecting: {}", e); +/// return; +/// } +/// if let Err(e) = conn.send_message("Hello from zone1 ruxos!") { +/// error!("Error sending message: {}", e); +/// } +/// if let Err(e) = conn.close() { +/// error!("Error closing connection: {}", e); +/// } +/// ``` + +pub fn ivc_example() { + let mut conn = Connection::new(); + + // Establish the connection + if let Err(e) = conn.connect(0) { + info!("Error connecting: {}", e); + return; + } + + // Send the message + if let Err(e) = conn.send_message("Hello from zone1 ruxos!4 ") { + info!("Error sending message: {}", e); + } + + // Close the connection + if let Err(e) = conn.close() { + info!("Error closing connection: {}", e); + } +} + +pub struct Connection { + ivc_info: Option, + control_table: Option>, +} + +impl Connection { + pub fn new() -> Self { + debug!("Connection created."); + Connection { + ivc_info: None, + control_table: None, + } + } + + pub fn connect(&mut self, communication_zone: usize) -> Result<(), &'static str> { + let alloc_size = HVISOR_HC_IVC_INFO_SIZE; + let align = HVISOR_HC_IVC_INFO_ALIGN; + let layout = Layout::from_size_align(alloc_size, align).unwrap(); + + let ptr = global_allocator().alloc(layout).expect("Memory allocation failed!"); + + let vpa_ivcinfo = VirtAddr::from(ptr.as_ptr() as usize); + // convert the virtual address to physical address, to use hvc on physical address + let pa_ivcinfo: PhysAddr = direct_virt_to_phys(vpa_ivcinfo); + debug!("The memory address of the IVC Info: VA: 0x{:x}, IPA: 0x{:x}", vpa_ivcinfo.as_usize(), pa_ivcinfo.as_usize()); + + ivc_hvc_call(HVISOR_HC_IVC_INFO as u32, pa_ivcinfo.as_usize(), HVISOR_HC_IVC_INFO_SIZE); + debug!("ivc_hvc_call finished."); + + let ivc_info_ptr = ptr.as_ptr() as *const IvCInfo; + let ivc_info: IvCInfo = unsafe { ivc_info_ptr.read() }; + self.ivc_info = Some(ivc_info); + + global_allocator().dealloc(ptr, layout); + + let ivc_info = self.ivc_info.as_ref().unwrap(); + let pa_control_table = PhysAddr::from(ivc_info.ivc_ct_ipas[communication_zone] as usize); + // convert the physical address to virtual address to use it in the kernel + let vpa_control_table: VirtAddr = phys_to_virt(pa_control_table); + self.control_table = NonNull::new(vpa_control_table.as_ptr() as *mut ControlTable); + + info!("IVC Connection established."); + Ok(()) + } + + pub fn send_message(&mut self, message: &str) -> Result<(), &'static str> { + let ivc_info = self.ivc_info.as_ref().ok_or("Not connected")?; + let mut control_table_ptr = self.control_table.ok_or("Not connected")?; + + // Safely get an immutable reference to the ControlTable for reading out_sec_size + let control_table = unsafe { control_table_ptr.as_ref() }; + + // Suppose we are zone1, we are to send message to zone0. + // Therefore use the out_sec_size field of the ControlTable struct (typically 0x1000). + let offset = control_table.out_sec_size as u64; + let vpa_share_memory_zone1 = phys_to_virt(PhysAddr::from((ivc_info.ivc_shmem_ipas[0] + offset) as usize)); + + write_to_address(vpa_share_memory_zone1, message)?; + info!("Message written to shared memory: {}", message); + + // Get a mutable reference to ControlTable to modify ipi_invoke + let control_table = unsafe { control_table_ptr.as_mut() }; + debug!("Ipi_invoke reset to inform Zone0 linux."); + control_table.ipi_invoke = 0x0; + + Ok(()) + } + + pub fn close(&mut self) -> Result<(), &'static str> { + if self.ivc_info.is_none() { + return Err("Not connected"); + } + self.ivc_info = None; + self.control_table = None; + info!("IVC Connection closed."); + Ok(()) + } +} + +/// Writes the given data to the specified virtual address. +fn write_to_address(addr: VirtAddr, data: &str) -> Result<(), &'static str> { + unsafe { + let ptr = addr.as_usize() as *mut u8; + ptr.copy_from_nonoverlapping(data.as_ptr(), data.len()); + // Add a null terminator to the end of the string + *ptr.add(data.len()) = 0; + } + Ok(()) +} diff --git a/modules/ruxhal/src/lib.rs b/modules/ruxhal/src/lib.rs index d0c0fd94c..4b4383449 100644 --- a/modules/ruxhal/src/lib.rs +++ b/modules/ruxhal/src/lib.rs @@ -50,6 +50,8 @@ mod platform; pub mod time; pub mod trap; pub mod virtio; +#[cfg(feature = "ivc")] +pub mod ivc; #[cfg(feature = "tls")] pub mod tls; @@ -77,6 +79,7 @@ pub mod misc { pub use super::platform::misc::*; } + /// Multi-core operations. #[cfg(feature = "smp")] pub mod mp { diff --git a/modules/ruxhal/src/platform/aarch64_common/ivc.rs b/modules/ruxhal/src/platform/aarch64_common/ivc.rs new file mode 100644 index 000000000..775b0e8ea --- /dev/null +++ b/modules/ruxhal/src/platform/aarch64_common/ivc.rs @@ -0,0 +1,24 @@ +/* Copyright (c) [2024] [Syswonder Community] + * [Ruxos] is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +/// ivc "hvc" method call on aarch64 hvisor +pub fn ivc_hvc_call(func: u32, arg0: usize, arg1: usize) -> usize { + let ret; + unsafe { + core::arch::asm!( + "hvc #0", + inlateout("x0") func as usize => ret, + in("x1") arg0, + in("x2") arg1, + options(nostack) + ); + info!("Ivc call: func: {:x}, arg0: 0x{:x}, arg1: 0x{:x}, ret: 0x{:x}", func, arg0, arg1, ret); + } + ret +} \ No newline at end of file diff --git a/modules/ruxhal/src/platform/aarch64_common/mod.rs b/modules/ruxhal/src/platform/aarch64_common/mod.rs index 933847f3d..53b7dd58c 100644 --- a/modules/ruxhal/src/platform/aarch64_common/mod.rs +++ b/modules/ruxhal/src/platform/aarch64_common/mod.rs @@ -21,3 +21,6 @@ pub mod pl011; #[cfg(feature = "rtc")] pub mod pl031; + +#[cfg(feature = "ivc")] +pub mod ivc; diff --git a/modules/ruxhal/src/platform/aarch64_qemu_virt/mod.rs b/modules/ruxhal/src/platform/aarch64_qemu_virt/mod.rs index b15270403..9d3fa0a67 100644 --- a/modules/ruxhal/src/platform/aarch64_qemu_virt/mod.rs +++ b/modules/ruxhal/src/platform/aarch64_qemu_virt/mod.rs @@ -34,6 +34,11 @@ pub mod misc { pub use crate::platform::aarch64_common::psci::system_off as terminate; } +#[cfg(feature = "ivc")] +pub mod ivc { + pub use crate::platform::aarch64_common::ivc::*; +} + extern "C" { fn exception_vector_base(); fn rust_main(cpu_id: usize, dtb: usize); diff --git a/platforms/aarch64-qemu-virt.toml b/platforms/aarch64-qemu-virt.toml index aae7690b7..fa58cb0e2 100644 --- a/platforms/aarch64-qemu-virt.toml +++ b/platforms/aarch64-qemu-virt.toml @@ -9,6 +9,7 @@ family = "aarch64-qemu-virt" phys-memory-base = "0x4000_0000" # Size of the whole physical memory. # phys-memory-size = "0x800_0000" # 128M +# phys-memory-size = "0x3000_0000" # 0.75G, same as zone1 linux on Hvisor # phys-memory-size = "0x4000_0000" # 1G phys-memory-size = "0x8000_0000" # 2G # phys-memory-size = "0xc000_0000" # 3G @@ -34,6 +35,7 @@ mmio-regions = [ ["0x1000_0000", "0x2eff_0000"], # PCI memory ranges (ranges 1: 32-bit MMIO space) ["0x40_1000_0000", "0x1000_0000"], # PCI config space ["0x0901_0000", "0x1000"], # RTC space + ["0x7000_0000", "0x4_0000"] # Hvisor IVC ] # VirtIO MMIO regions with format (`base_paddr`, `size`). virtio-mmio-regions = [