eth: latter tuning

Cargo.lock

@@ -79,6 +79,7 @@ version = "0.0.0"
 dependencies = [
  "bit_field 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libregister 0.0.0",
+ "volatile-register 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]

armv7-none-eabihf.json

@@ -12,7 +12,7 @@
   "emit-debug-gdb-scripts": false,
   "env": "",
   "executables": true,
-  "features": "+v7,+vfp3,-d32,+thumb2,-neon",
+  "features": "+v7,+vfp3,-d32,+thumb2,-neon,+strict-align",
   "is-builtin": false,
   "linker": "rust-lld",
   "linker-flavor": "ld.lld",

experiments/src/main.rs

@@ -15,6 +15,7 @@ use libboard_zynq::{
     clocks::source::{ArmPll, ClockSource, IoPll},
     clocks::Clocks,
     print, println,
+    setup_l2cache,
     sdio::sd_card::SdCard,
     smoltcp::{
         iface::{EthernetInterfaceBuilder, NeighborCache, Routes},
@@ -82,6 +83,9 @@ pub fn main_core0() {
         clocks.cpu_2x(),
         clocks.cpu_1x()
     );
+    info!("Setup L2Cache");
+    setup_l2cache();
+    info!("L2Cache done");
 
     // commented out due to OCM full
     // let sd = libboard_zynq::sdio::SDIO::sdio0(true);
@@ -121,6 +125,7 @@ pub fn main_core0() {
     ddr.memtest();
     ram::init_alloc_ddr(&mut ddr);
 
+if false {
     #[cfg(dev)]
     for i in 0..=1 {
         let mut flash_io = flash.manual_mode(i);
@@ -174,14 +179,15 @@ pub fn main_core0() {
         }
     });
     core1.disable();
+}
 
     let eth = zynq::eth::Eth::default(HWADDR.clone());
     println!("Eth on");
 
-    const RX_LEN: usize = 4096;
+    const RX_LEN: usize = 8192;
     // Number of transmission buffers (minimum is two because with
     // one, duplicate packet transmission occurs)
-    const TX_LEN: usize = 4096;
+    const TX_LEN: usize = 8192;
     let eth = eth.start_rx(RX_LEN);
     let mut eth = eth.start_tx(TX_LEN);
 
@@ -211,7 +217,7 @@ pub fn main_core0() {
         while let Ok(stream) = TcpStream::accept(TCP_PORT, 0x10_0000, 0x10_0000).await {
             let stats_tx = stats_tx.clone();
             task::spawn(async move {
-                let tx_data = (0..=255).take(4096).collect::<alloc::vec::Vec<u8>>();
+                let tx_data = (0..=255).take(8192).collect::<alloc::vec::Vec<u8>>();
                 loop {
                     // const CHUNK_SIZE: usize = 65536;
                     // match stream.send((0..=255).cycle().take(CHUNK_SIZE)).await {

libboard_zynq/Cargo.toml

@@ -21,5 +21,5 @@ libcortex_a9 = { path = "../libcortex_a9" }
 
 [dependencies.smoltcp]
 version = "0.6"
-features = ["ethernet", "proto-ipv4", "proto-ipv6", "socket-tcp"]
+features = ["ethernet", "proto-ipv4", "socket-tcp"]
 default-features = false

libboard_zynq/src/eth/rx.rs

@@ -2,6 +2,8 @@ use core::ops::Deref;
 use alloc::{vec, vec::Vec};
 use libcortex_a9::{asm::*, cache::*, UncachedSlice};
 use libregister::*;
+use log::debug;
+use crate::l2cache;
 use super::Buffer;
 
 #[derive(Debug)]
@@ -81,8 +83,7 @@ impl DescList {
             entry.word1.write(
                 DescWord1::zeroed()
             );
-            // Flush buffer from cache, to be filled by the peripheral
+            l2cache().invalidate_slice(&mut buffer[..]);
-            // before next read
             dcci_slice(&buffer[..]);
         }
 
@@ -109,6 +110,9 @@ impl DescList {
             let word1 = entry.word1.read();
             let len = word1.frame_length_lsbs().into();
             let buffer = &mut self.buffers[self.next][0..len];
+            // // Invalidate caches for packet buffer
+            // l2cache().invalidate_slice(&mut buffer[..]);
+            // dcci_slice(&buffer[..]);
 
             self.next += 1;
             if self.next >= list_len {
@@ -117,8 +121,10 @@ impl DescList {
 
             let pkt = PktRef { entry, buffer };
             if word1.start_of_frame() && word1.end_of_frame() {
+                // debug!("pkt {}: {:08X}..{:08X}", len, &pkt.buffer[0] as *const _ as usize, &pkt.buffer[pkt.len()-1] as *const _ as usize);
                 Ok(Some(pkt))
             } else {
+                debug!("pkt trunc");
                 Err(Error::Truncated)
             }
         } else {
@@ -137,6 +143,7 @@ impl<'a> Drop for PktRef<'a> {
     fn drop(&mut self) {
         // Flush buffer from cache, to be filled by the peripheral
         // before next read
+        l2cache().invalidate_slice(self.buffer);
         dcci_slice(self.buffer);
 
         self.entry.word0.modify(|_, w| w.used(false));

libboard_zynq/src/eth/tx.rs

@@ -1,7 +1,9 @@
 use core::ops::{Deref, DerefMut};
 use alloc::{vec, vec::Vec};
-use libcortex_a9::{cache::dcc_slice, UncachedSlice};
+use libcortex_a9::{asm::dmb, cache::dcc_slice, UncachedSlice};
 use libregister::*;
+use log::{debug, warn};
+use crate::l2cache;
 use super::{Buffer, regs};
 
 /// Descriptor entry
@@ -94,8 +96,10 @@ impl DescList {
     }
 
     pub fn send<'s: 'p, 'p>(&'s mut self, regs: &'s mut regs::RegisterBlock, length: usize) -> Option<PktRef<'p>> {
+        // debug!("send {}", length);
         let list_len = self.list.len();
         let entry = &mut self.list[self.next];
+        dmb();
         if entry.word1.read().used() {
             let buffer = &mut self.buffers[self.next][0..length];
             entry.word1.write(DescWord1::zeroed()
@@ -113,6 +117,7 @@ impl DescList {
             Some(PktRef { entry, buffer, regs })
         } else {
             // Still in use by HW (sending too fast, ring exceeded)
+            warn!("tx ring overflow");
             None
         }
     }
@@ -128,10 +133,13 @@ pub struct PktRef<'a> {
 
 impl<'a> Drop for PktRef<'a> {
     fn drop(&mut self) {
-        // Write back all dirty cachelines of this buffer
+        // Write back all dirty cachelines of packet buffer
         dcc_slice(self.buffer);
+        l2cache().clean_slice(self.buffer);
 
         self.entry.word1.modify(|_, w| w.used(false));
+        dmb();
+        // dsb();
         if ! self.regs.tx_status.read().tx_go() {
             // Start TX if not already running
             self.regs.net_ctrl.modify(|_, w| w.start_tx(true));

libboard_zynq/src/lib.rs

@@ -21,3 +21,31 @@ pub mod timer;
 pub mod sdio;
 pub mod logger;
 pub mod ps7_init;
+
+pub use libcortex_a9::pl310::L2Cache;
+
+pub fn l2cache() -> L2Cache {
+    const PL310_BASEADDR: usize = 0xF8F02000;
+    L2Cache::new(PL310_BASEADDR)
+}
+
+pub fn setup_l2cache() {
+    slcr::RegisterBlock::unlocked(|slcr| {
+        assert_eq!(&slcr.unnamed1 as *const _ as u32, 0xF8000A1C);
+        unsafe { slcr.unnamed1.write(0x020202); }
+    });
+    
+    let mut l2 = l2cache();
+    use log::info;
+    info!("l2 aux={:08X}", l2.regs.aux_control.read());
+    // TODO: set prefetch
+
+    // Configure ZYNQ-specific latency
+    l2.set_tag_ram_latencies(1, 1, 1);
+    l2.set_data_ram_latencies(1, 2, 1);
+
+    l2.disable_interrupts();
+    l2.reset_interrupts();
+    l2.invalidate_all();
+    l2.enable();
+}

libboard_zynq/src/slcr.rs

@@ -229,15 +229,18 @@ pub struct RegisterBlock {
     pub lvl_shftr_en: LvlShftr,
     reserved18: [u32; 3],
     pub ocm_cfg: RW<u32>,
-    reserved19: [u32; 123],
+    reserved19: [u32; 66],
+    /// barely documented unnamed register to prepare L2 cache setup
+    pub unnamed1: RW<u32>,
+    reserved120: [u32; 56],
     pub gpiob_ctrl: GpiobCtrl,
     pub gpiob_cfg_cmos18: RW<u32>,
     pub gpiob_cfg_cmos25: RW<u32>,
     pub gpiob_cfg_cmos33: RW<u32>,
-    reserved20: [u32; 1],
+    reserved21: [u32; 1],
     pub gpiob_cfg_hstl: RW<u32>,
     pub gpiob_drvr_bias_ctrl: RW<u32>,
-    reserved21: [u32; 9],
+    reserved22: [u32; 9],
     pub ddriob_addr0: DdriobConfig,
     pub ddriob_addr1: DdriobConfig,
     pub ddriob_data0: DdriobConfig,

libcortex_a9/Cargo.toml

@@ -11,4 +11,5 @@ default = ["target_zc706"]
 
 [dependencies]
 bit_field = "0.10"
+volatile-register = "0.2"
 libregister = { path = "../libregister" }

libcortex_a9/src/cache.rs

@@ -197,7 +197,7 @@ pub unsafe fn dcimvac(addr: usize) {
     llvm_asm!("mcr p15, 0, $0, c7, c6, 1" :: "r" (addr) :: "volatile");
 }
 
-/// Data cache clean and invalidate for an object.
+/// Data cache invalidate for an object.
 pub unsafe fn dci<T>(object: &mut T) {
     let first_addr = object as *const _ as usize;
     let beyond_addr = (object as *const _ as usize) + core::mem::size_of_val(object);

libcortex_a9/src/lib.rs

@@ -15,5 +15,6 @@ mod uncached;
 mod fpu;
 pub use uncached::UncachedSlice;
 pub use fpu::enable_fpu;
+pub mod pl310;
 
 global_asm!(include_str!("exceptions.s"));

libcortex_a9/src/mmu.rs

@@ -158,7 +158,7 @@ impl L1Table {
                 global: true,
                 shareable: true,
                 access: AccessPermissions::FullAccess,
-                tex: 0b101,
+                tex: 0b111,
                 domain: 0b1111,
                 exec: true,
                 cacheable: true,

libcortex_a9/src/pl310/mod.rs

@@ -0,0 +1,166 @@
+//! L2 cache controller
+
+use libregister::RegisterW;
+use crate::asm::*;
+
+mod regs;
+
+const CACHE_LINE: usize = 0x20;
+const CACHE_LINE_MASK: usize = CACHE_LINE - 1;
+
+#[inline]
+fn cache_line_addrs(first_addr: usize, beyond_addr: usize) -> impl Iterator<Item = usize> {
+    let first_addr = first_addr & !CACHE_LINE_MASK;
+    let beyond_addr = (beyond_addr | CACHE_LINE_MASK) + 1;
+
+    (first_addr..beyond_addr).step_by(CACHE_LINE)
+}
+
+fn object_cache_line_addrs<T>(object: &T) -> impl Iterator<Item = usize> {
+    let first_addr = object as *const _ as usize;
+    let beyond_addr = (object as *const _ as usize) + core::mem::size_of_val(object);
+    cache_line_addrs(first_addr, beyond_addr)
+}
+
+fn slice_cache_line_addrs<T>(slice: &[T]) -> impl Iterator<Item = usize> {
+    let first_addr = &slice[0] as *const _ as usize;
+    let beyond_addr = (&slice[slice.len() - 1] as *const _ as usize) +
+        core::mem::size_of_val(&slice[slice.len() - 1]);
+    cache_line_addrs(first_addr, beyond_addr)
+}
+
+pub struct L2Cache {
+    pub regs: &'static mut regs::RegisterBlock,
+}
+
+impl L2Cache {
+    pub fn new(register_baseaddr: usize) -> Self {
+        let regs = unsafe {
+            regs::RegisterBlock::new_at(register_baseaddr)
+        };
+        L2Cache { regs }
+    }
+
+    pub fn set_tag_ram_latencies(&mut self, setup_lat: u8, rd_access_lat: u8, wr_access_lat: u8) {
+        self.regs.tag_ram_control.write(
+            regs::RamControl::zeroed()
+                .setup_lat(setup_lat)
+                .rd_access_lat(rd_access_lat)
+                .wr_access_lat(wr_access_lat)
+        );
+    }
+
+    pub fn set_data_ram_latencies(&mut self, setup_lat: u8, rd_access_lat: u8, wr_access_lat: u8) {
+        self.regs.data_ram_control.write(
+            regs::RamControl::zeroed()
+                .setup_lat(setup_lat)
+                .rd_access_lat(rd_access_lat)
+                .wr_access_lat(wr_access_lat)
+        );
+    }
+
+    pub fn disable_interrupts(&mut self) {
+        self.regs.int_mask.write(
+            regs::Interrupts::zeroed()
+                .ecntr(true)
+                .parrt(true)
+                .parrd(true)
+                .errwt(true)
+                .errwd(true)
+                .errrt(true)
+                .errrd(true)
+                .slverr(true)
+                .decerr(true)
+        );
+    }
+
+    pub fn reset_interrupts(&mut self) {
+        self.regs.int_clear.write(
+            regs::Interrupts::zeroed()
+                .ecntr(true)
+                .parrt(true)
+                .parrd(true)
+                .errwt(true)
+                .errwd(true)
+                .errrt(true)
+                .errrd(true)
+                .slverr(true)
+                .decerr(true)
+        );
+    }
+
+    pub fn invalidate_all(&mut self) {
+        unsafe { self.regs.inv_way.write(0xFFFF); }
+        unsafe { self.regs.cache_sync.write(1); }
+        while self.regs.cache_sync.read() != 0 {}
+    }
+
+    pub fn enable(&mut self) {
+        dmb();
+        self.regs.control.write(
+            regs::Control::zeroed()
+                .l2_enable(true)
+        );
+        dsb();
+    }
+
+    pub fn clean_invalidate<T>(&mut self, obj: &T) {
+        dmb();
+        for addr in object_cache_line_addrs(obj) {
+            unsafe {
+                self.regs.clean_inv_pa.write(addr as u32);
+            }
+        }
+        dsb();
+        unsafe { self.regs.cache_sync.write(1); }
+        while self.regs.cache_sync.read() != 0 {}
+    }
+
+    pub fn clean_invalidate_slice<T>(&mut self, slice: &[T]) {
+        dmb();
+        for addr in slice_cache_line_addrs(slice) {
+            unsafe {
+                self.regs.clean_inv_pa.write(addr as u32);
+            }
+        }
+        dsb();
+        unsafe { self.regs.cache_sync.write(1); }
+        while self.regs.cache_sync.read() != 0 {}
+    }
+
+    pub fn clean_slice<T>(&mut self, slice: &[T]) {
+        dmb();
+        for addr in slice_cache_line_addrs(slice) {
+            unsafe {
+                self.regs.clean_pa.write(addr as u32);
+            }
+        }
+        dsb();
+        unsafe { self.regs.cache_sync.write(1); }
+        while self.regs.cache_sync.read() != 0 {}
+    }
+
+    pub fn invalidate<T>(&mut self, obj: &mut T) {
+        dmb();
+        for addr in object_cache_line_addrs(obj) {
+            unsafe {
+                self.regs.inv_pa.write(addr as u32);
+            }
+        }
+        dsb();
+        unsafe { self.regs.cache_sync.write(1); }
+        while self.regs.cache_sync.read() != 0 {}
+    }
+
+    pub fn invalidate_slice<T>(&mut self, slice: &mut [T]) {
+        dmb();
+        for addr in slice_cache_line_addrs(slice) {
+            unsafe {
+                self.regs.inv_pa.write(addr as u32);
+            }
+        }
+        dsb();
+        unsafe { self.regs.cache_sync.write(1); }
+        while self.regs.cache_sync.read() != 0 {}
+    }
+}

libcortex_a9/src/pl310/regs.rs

@@ -0,0 +1,93 @@
+use volatile_register::{RO, WO, RW};
+use libregister::{register, register_bit, register_bits, RegisterW};
+
+#[repr(C)]
+pub struct RegisterBlock {
+    pub cache_id: RW<u32>,
+    pub cache_type: RW<u32>,
+    pub _unused1: [RO<u32>; 62],
+    pub control: Control,
+    pub aux_control: RW<u32>,
+    pub tag_ram_control: RamControl,
+    pub data_ram_control: RamControl,
+    pub _unused2: [RO<u32>; 60],
+    pub ev_counter_ctrl: RW<u32>,
+    pub ev_counter1_cfg: RW<u32>,
+    pub ev_counter2_cfg: RW<u32>,
+    pub ev_counter1: RW<u32>,
+    pub ev_counter2: RW<u32>,
+    pub int_mask: Interrupts,
+    pub int_mask_status: Interrupts,
+    pub int_raw_status: Interrupts,
+    pub int_clear: Interrupts,
+    pub _unused3: [RO<u32>; 323],
+    pub cache_sync: RW<u32>,
+    pub _unused4: [RO<u32>; 15],
+    pub inv_pa: RW<u32>,
+    pub _unused5: [RO<u32>; 2],
+    pub inv_way: RW<u32>,
+    pub _unused6: [RO<u32>; 12],
+    pub clean_pa: RW<u32>,
+    pub _unused7: [RO<u32>; 1],
+    pub clean_index: RW<u32>,
+    pub clean_way: RW<u32>,
+    pub _unused8: [RO<u32>; 12],
+    pub clean_inv_pa: RW<u32>,
+    pub _unused9: [RO<u32>; 1],
+    pub clean_inv_index: RW<u32>,
+    pub clean_inv_way: RW<u32>,
+    pub _unused10: [RO<u32>; 64],
+    pub d_lockdown0: RW<u32>,
+    pub i_lockdown0: RW<u32>,
+    pub d_lockdown1: RW<u32>,
+    pub i_lockdown1: RW<u32>,
+    pub d_lockdown2: RW<u32>,
+    pub i_lockdown2: RW<u32>,
+    pub d_lockdown3: RW<u32>,
+    pub i_lockdown3: RW<u32>,
+    pub d_lockdown4: RW<u32>,
+    pub i_lockdown4: RW<u32>,
+    pub d_lockdown5: RW<u32>,
+    pub i_lockdown5: RW<u32>,
+    pub d_lockdown6: RW<u32>,
+    pub i_lockdown6: RW<u32>,
+    pub d_lockdown7: RW<u32>,
+    pub i_lockdown7: RW<u32>,
+    pub _unused11: [RO<u32>; 4],
+    pub lock_line_en: RW<u32>,
+    pub unlock_way: RW<u32>,
+    pub _unused12: [RO<u32>; 170],
+    pub addr_filtering_start: RW<u32>,
+    pub addr_filtering_end: RW<u32>,
+    pub _unused13: [RO<u32>; 206],
+    pub debug_ctrl: RW<u32>,
+    pub _unused14: [RO<u32>; 7],
+    pub prefetch_ctrl: RW<u32>,
+    pub _unused15: [RO<u32>; 7],
+    pub power_ctrl: RW<u32>,
+}
+
+impl RegisterBlock {
+    pub unsafe fn new_at(baseaddr: usize) -> &'static mut Self {
+        &mut *(baseaddr as *mut _)
+    }
+}
+
+register!(control, Control, RW, u32);
+register_bit!(control, l2_enable, 0);
+
+register!(ram_control, RamControl, RW, u32);
+register_bits!(ram_control, setup_lat, u8, 0, 2);
+register_bits!(ram_control, rd_access_lat, u8, 4, 6);
+register_bits!(ram_control, wr_access_lat, u8, 8, 10);
+
+register!(interrupts, Interrupts, RW, u32);
+register_bit!(interrupts, ecntr, 0);
+register_bit!(interrupts, parrt, 1);
+register_bit!(interrupts, parrd, 2);
+register_bit!(interrupts, errwt, 3);
+register_bit!(interrupts, errwd, 4);
+register_bit!(interrupts, errrt, 5);
+register_bit!(interrupts, errrd, 6);
+register_bit!(interrupts, slverr, 7);
+register_bit!(interrupts, decerr, 8);

libcortex_a9/src/regs.rs

@@ -149,7 +149,6 @@ pub struct ACTLR;
 wrap_reg!(actlr);
 def_reg_r!(ACTLR, actlr::Read, "mrc p15, 0, $0, c1, c0, 1");
 def_reg_w!(ACTLR, actlr::Write, "mcr p15, 0, $0, c1, c0, 1");
-// SMP bit
 register_bit!(actlr, parity_on, 9);
 register_bit!(actlr, alloc_one_way, 8);
 register_bit!(actlr, excl, 7);

libcortex_a9/src/uncached.rs

@@ -25,9 +25,10 @@ impl<T> UncachedSlice<T> {
         for page_start in (start..(start + size)).step_by(L1_PAGE_SIZE) {
             L1Table::get()
                 .update(page_start as *const (), |l1_section| {
-                    l1_section.tex = 0b100;
+                    // Shareable Device
+                    l1_section.tex = 0b000;
                     l1_section.cacheable = false;
-                    l1_section.bufferable = false;
+                    l1_section.bufferable = true;
                 });
         }