Merge pull request #77 from HKalbasi/dpdk-2311

Update dpdk

.github/workflows/ci.yml

@@ -11,6 +11,14 @@ env:
 
 jobs:
   build:
+    strategy:
+      fail-fast: false
+      matrix:
+        dpdk_version:
+          - "20.11"
+          - "23.11"
+    env:
+      DPDK_VERSION: ${{ matrix.dpdk_version }}
     runs-on: ubuntu-latest
     steps:
     - name: Check out repo
@@ -24,22 +32,22 @@ jobs:
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
-        python-version: 3.7
+        python-version: 3.11
     - name: Install pip
       run: |
           python -m pip install --upgrade pip
     - name: Install pyelftools
       run: |
           pip3 install pyelftools
-    - name: Install libpcap (Linux)
+    - name: Install apt dependencies (Linux)
       run: |
           sudo apt-get update
-          sudo apt-get install libpcap-dev
+          sudo apt-get install libpcap-dev libnuma-dev
     - name: DPDK
       run: |
             echo "Compiling DPDK..." ;
             export STABLE=
-            export VERSION=20.11
+            export VERSION=${{ matrix.dpdk_version }}
             export PKG_CONFIG_PATH=$(pwd)/dpdk-$STABLE$VERSION/install/lib/x86_64-linux-gnu/pkgconfig/ ;
             export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(pwd)/dpdk-$STABLE$VERSION/install/lib/x86_64-linux-gnu/ ;
             export DPDK_PATH=$(pwd)/dpdk-$STABLE$VERSION/install/
@@ -58,9 +66,6 @@ jobs:
             echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH" >> $GITHUB_ENV
             echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV
             echo "DPDK_PATH=$DPDK_PATH" >> $GITHUB_ENV
-    - name: Remove LLVM and Clang 13/14
-      run: sudo apt remove --auto-remove clang-14 llvm-14 && sudo apt remove --auto-remove clang-13 llvm-13;
-
     - name: Clippy retina-core (no mlx5)
       run: cargo clippy --manifest-path core/Cargo.toml --no-default-features -- --deny warnings
     - name: Unit test retina-core (no mlx5)

INSTALL.md

@@ -24,7 +24,7 @@ sudo apt install build-essential meson pkg-config libnuma-dev python3-pyelftools
 ```
 
 ## Building and Installing DPDK
-Retina currently requires [**DPDK 21.08**](https://core.dpdk.org/download/). The latest LTS release (21.11) contains breaking API changes while 20.11 LTS has a bug that causes inaccurate packet drop metrics on some NICs.
+Retina currently requires [**DPDK 20.11 or 21.08 or 23.11**](https://core.dpdk.org/download/). Note that 20.11 LTS has a bug that causes inaccurate packet drop metrics on some NICs.
 
 ### System Configuration
 To get high performance from DPDK applications, we recommend the following system configuration steps. More details from the DPDK docs can be found [here](https://doc.dpdk.org/guides/linux_gsg/nic_perf_intel_platform.html).
@@ -63,15 +63,16 @@ sudo /etc/init.d/openibd restart
 This may update the firmware on your NIC, a reboot should complete the update if necessary.
 
 ### Install DPDK from source
-We recommend a local DPDK install from source. Download version 21.08 from the [DPDK downloads page](http://core.dpdk.org/download/):
+We recommend a local DPDK install from source. Download version 23.11 (or desired version) from the [DPDK downloads page](http://core.dpdk.org/download/):
 ```sh
-wget http://fast.dpdk.org/rel/dpdk-21.08.tar.xz
-tar xJf dpdk-21.08.tar.xz
+wget http://fast.dpdk.org/rel/dpdk-23.11.tar.xz
+tar xJf dpdk-23.11.tar.xz
 ```
 
-Set environment variables:
+Set environment variables (For changing the version, set `DPDK_VERSION` properly):
 ```sh
-export DPDK_PATH=/path/to/dpdk/dpdk-21.08
+export DPDK_PATH=/path/to/dpdk/dpdk-23.11
+export DPDK_VERSION=23.11
 export LD_LIBRARY_PATH=$DPDK_PATH/lib/x86_64-linux-gnu
 export PKG_CONFIG_PATH=$LD_LIBRARY_PATH/pkgconfig
 ```
@@ -128,13 +129,9 @@ sudo env LD_LIBRARY_PATH=$LD_LIBRARY_PATH RUST_LOG=error ./target/release/my_app
 
 #### Troubleshooting: Bindgen
 
-Retina uses [bindgen](https://docs.rs/bindgen/latest/bindgen/) to generate bindings to DPDK functions implemented in C. As of 06/2024, we have encountered issues when using bindgen with clang/llvm >13, apparently due to introduced APIs for SIMD intrinsics.
+Retina uses [bindgen](https://docs.rs/bindgen/latest/bindgen/) to generate bindings to DPDK functions implemented in C.
 
-If you are using clang and building Retina fails with an error such as the below, downgrade clang/llvm to <=13.
-
-```sh
-error: invalid conversion between vector type '__m128i' (vector of 2 'long long' values) and integer type 'int' of different size
-```
+For newer versions of DPDK, bindgen requires us to use `.clang_macro_fallback()` to access certain RSS constants. This requires clang/llvm >=13.
 
 ## Testing Retina (Offline) on a VM
 

core/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [build-dependencies]
-bindgen = "0.69.4"
+bindgen = "0.70.1"
 cc = "1.0.79"
 
 [dependencies]

core/build.rs

@@ -8,11 +8,34 @@ fn main() {
     // modified from https://github.com/deeptir18/cornflakes/blob/master/cornflakes-libos/build.rs
 
     println!("cargo:rerun-if-env-changed=DPDK_PATH");
+    println!("cargo:rerun-if-env-changed=DPDK_VERSION");
     println!("cargo:rerun-if-changed=build.rs");
     println!("cargo:rerun-if-changed=src/dpdk/inline.c");
     let cargo_manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
     let cargo_dir = Path::new(&cargo_manifest_dir);
 
+    println!("cargo:rustc-check-cfg=cfg(dpdk_ge_2011)");
+    println!("cargo:rustc-check-cfg=cfg(dpdk_ge_2108)");
+    println!("cargo:rustc-check-cfg=cfg(dpdk_ge_2111)");
+    println!("cargo:rustc-check-cfg=cfg(dpdk_ge_2311)");
+    let dpdk_version = env::var("DPDK_VERSION").expect("Set DPDK_VERSION env variable");
+
+    if !["20.11", "21.08", "23.11"].contains(&dpdk_version.as_str()) {
+        println!("Unsupported dpdk version");
+        exit(1);
+    }
+
+    println!("cargo:rustc-cfg=dpdk_ge_2011");
+    if dpdk_version != "20.11" {
+        println!("cargo:rustc-cfg=dpdk_ge_2108");
+        if dpdk_version != "21.08" {
+            println!("cargo:rustc-cfg=dpdk_ge_2111");
+            if dpdk_version != "21.11" {
+                println!("cargo:rustc-cfg=dpdk_ge_2311");
+            }
+        }
+    }
+
     let out_dir_s = env::var("OUT_DIR").unwrap();
     let out_dir = Path::new(&out_dir_s);
     let load_lib_path_s = env::var("LD_LIBRARY_PATH").unwrap();
@@ -92,6 +115,7 @@ fn main() {
         .join("dpdk_headers.h");
     let bindings = builder
         .header(headers_file.to_str().unwrap())
+        .clang_macro_fallback()
         // mark as opaque per bindgen bug on packed+aligned structs:
         // https://github.com/rust-lang/rust-bindgen/issues/1538
         .opaque_type(r"rte_arp_ipv4|rte_arp_hdr")

core/src/dpdk/mod.rs

@@ -9,6 +9,53 @@ include!(concat!(env!("OUT_DIR"), "/dpdk.rs"));
 
 use std::os::raw::{c_char, c_int, c_uint, c_void};
 
+#[cfg(dpdk_ge_2311)]
+impl std::fmt::Debug for rte_gtp_psc_generic_hdr {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("rte_gtp_psc_generic_hdr")
+            .field("ext_hdr_len", &self.ext_hdr_len)
+            .field("_bitfield_align_1", &self._bitfield_align_1)
+            .field("_bitfield_1", &self._bitfield_1)
+            .field("data", &self.data)
+            .finish()
+    }
+}
+
+impl rte_ipv4_hdr {
+    #[cfg(not(dpdk_ge_2311))]
+    pub fn set_version_ihl(&mut self, version_ihl: u8) {
+        self.version_ihl = version_ihl;
+    }
+
+    #[cfg(dpdk_ge_2311)]
+    pub fn set_version_ihl(&mut self, version_ihl: u8) {
+        self.__bindgen_anon_1.version_ihl = version_ihl;
+    }
+}
+
+/*
+ * DPDK v23.11 uses RTE_BIT64(x) macro to define RSS values, so we use
+ * bindgen clang_macro_fallback to access them.
+ * There may be a bug in clang_macro_fallback impacting values >u32_max:
+ * https://github.com/rust-lang/rust-bindgen/issues/2944
+ * This is okay for the RSS values we use here, but it should be consulted
+ * before adding new RSS offload types.
+ */
+#[cfg(dpdk_ge_2311)]
+pub use rte_eth_fc_mode_RTE_ETH_FC_NONE as rte_eth_fc_mode_RTE_FC_NONE;
+#[cfg(dpdk_ge_2311)]
+pub use rte_eth_rx_mq_mode_RTE_ETH_MQ_RX_RSS as rte_eth_rx_mq_mode_ETH_MQ_RX_RSS;
+#[cfg(dpdk_ge_2311)]
+pub use RTE_ETH_RETA_GROUP_SIZE as RTE_RETA_GROUP_SIZE;
+#[cfg(dpdk_ge_2311)]
+pub use RTE_ETH_RSS_IP as ETH_RSS_IP;
+#[cfg(dpdk_ge_2311)]
+pub use RTE_ETH_RSS_TCP as ETH_RSS_TCP;
+#[cfg(dpdk_ge_2311)]
+pub use RTE_ETH_RSS_UDP as ETH_RSS_UDP;
+#[cfg(dpdk_ge_2311)]
+pub use RTE_ETH_VLAN_STRIP_OFFLOAD as DEV_RX_OFFLOAD_VLAN_STRIP;
+
 #[link(name = "inlined")]
 extern "C" {
     fn rte_pktmbuf_free_(packet: *const rte_mbuf);

core/src/filter/hardware/flow_item.rs

@@ -94,8 +94,8 @@ impl FlowPattern {
                     "version_ihl" => match value {
                         Value::Int(i) => {
                             if let Ok(val) = u8::try_from(*i) {
-                                ipv4_spec.hdr.version_ihl = val;
-                                ipv4_mask.hdr.version_ihl = u8::MAX;
+                                ipv4_spec.hdr.set_version_ihl(val);
+                                ipv4_mask.hdr.set_version_ihl(u8::MAX);
                             } else {
                                 bail!(FilterError::InvalidRhsValue(value.to_string()))
                             }

core/src/port/mod.rs

@@ -19,11 +19,11 @@ use std::ptr;
 
 use anyhow::{bail, Result};
 
-const RSS_KEY_LEN: usize = 40;
-pub(crate) const SYMMETRIC_RSS_KEY: [u8; RSS_KEY_LEN] = [
+pub(crate) const SYMMETRIC_RSS_KEY: [u8; 52] = [
     0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
     0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
-    0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
+    0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
+    0x6D, 0x5A, 0x6D, 0x5A,
 ];
 const RSS_RETA_SIZE: usize = 512;
 
@@ -292,15 +292,22 @@ impl Port {
         // turn on RSS
         if dev_info.flow_type_rss_offloads != 0 {
             port_conf.rxmode.mq_mode = dpdk::rte_eth_rx_mq_mode_ETH_MQ_RX_RSS;
+            if ![40, 52].contains(&dev_info.hash_key_size) {
+                panic!("Unexpected hash key size {}", dev_info.hash_key_size);
+            }
             port_conf.rx_adv_conf.rss_conf.rss_key = SYMMETRIC_RSS_KEY.as_ptr() as *mut u8;
-            port_conf.rx_adv_conf.rss_conf.rss_key_len = RSS_KEY_LEN as u8;
+            port_conf.rx_adv_conf.rss_conf.rss_key_len = dev_info.hash_key_size;
             port_conf.rx_adv_conf.rss_conf.rss_hf =
                 (dpdk::ETH_RSS_IP | dpdk::ETH_RSS_TCP | dpdk::ETH_RSS_UDP) as u64
                     & dev_info.flow_type_rss_offloads;
         }
 
-        let max_rx_pkt_len = mtu_to_max_frame_len(mtu as u32);
-        port_conf.rxmode.max_rx_pkt_len = cmp::max(dpdk::RTE_ETHER_MAX_LEN, max_rx_pkt_len);
+        // In newer DPDKs setting only mtu with `dpdk::rte_eth_dev_set_mtu` is enough
+        #[cfg(not(dpdk_ge_2311))]
+        {
+            let max_rx_pkt_len = mtu_to_max_frame_len(mtu as u32);
+            port_conf.rxmode.max_rx_pkt_len = cmp::max(dpdk::RTE_ETHER_MAX_LEN, max_rx_pkt_len);
+        }
 
         // turns on VLAN stripping if supported
         if dev_info.rx_offload_capa & dpdk::DEV_RX_OFFLOAD_VLAN_STRIP as u64 != 0 {