diff --git a/artiq/firmware/libboard/cache.rs b/artiq/firmware/libboard/cache.rs new file mode 100644 index 000000000..c99e01cde --- /dev/null +++ b/artiq/firmware/libboard/cache.rs @@ -0,0 +1,44 @@ +use spr::{self, mfspr, mtspr}; +use csr; +use mem; + +pub fn flush_cpu_icache() { + unsafe { + let iccfgr = mfspr(spr::SPR_ICCFGR); + let ways = 1 << (iccfgr & spr::SPR_ICCFGR_NCW); + let set_size = 1 << ((iccfgr & spr::SPR_ICCFGR_NCS) >> 3); + let block_size = if iccfgr & spr::SPR_ICCFGR_CBS != 0 { 32 } else { 16 }; + let size = set_size * ways * block_size; + + let mut i = 0; + while i < size { + mtspr(spr::SPR_ICBIR, i); + i += block_size; + } + } +} + +pub fn flush_cpu_dcache() { + unsafe { + let dccfgr = mfspr(spr::SPR_DCCFGR); + let ways = 1 << (dccfgr & spr::SPR_ICCFGR_NCW); + let set_size = 1 << ((dccfgr & spr::SPR_DCCFGR_NCS) >> 3); + let block_size = if dccfgr & spr::SPR_DCCFGR_CBS != 0 { 32 } else { 16 }; + let size = set_size * ways * block_size; + + let mut i = 0; + while i < size { + mtspr(spr::SPR_DCBIR, i); + i += block_size; + } + } +} + +pub fn flush_l2_cache() { + unsafe { + for i in 0..2 * (csr::CONFIG_L2_SIZE as usize) / 4 { + let addr = mem::MAIN_RAM_BASE + i * 4; + asm!("l.lwz r0, 0($1)"::"r"(addr):"r0":"volatile") + } + } +} diff --git a/artiq/firmware/libboard/lib.rs b/artiq/firmware/libboard/lib.rs index 4c97cd7bc..2fb2cbec8 100644 --- a/artiq/firmware/libboard/lib.rs +++ b/artiq/firmware/libboard/lib.rs @@ -10,6 +10,7 @@ include!(concat!(env!("BUILDINC_DIRECTORY"), "/generated/mem.rs")); include!(concat!(env!("BUILDINC_DIRECTORY"), "/generated/csr.rs")); pub mod spr; pub mod irq; +pub mod cache; pub mod clock; pub mod uart; #[cfg(feature = "uart_console")] @@ -34,11 +35,6 @@ pub mod ad9154; #[cfg(feature = "uart_console")] pub use uart_console::Console; -extern { - pub fn flush_cpu_dcache(); - pub fn flush_l2_cache(); -} - pub fn ident(buf: &mut [u8]) -> &str { unsafe { let len = ptr::read_volatile(csr::IDENTIFIER_MEM_BASE); diff --git a/artiq/firmware/libboard/spr.rs b/artiq/firmware/libboard/spr.rs index 8d13c0a74..1df61fd2f 100644 --- a/artiq/firmware/libboard/spr.rs +++ b/artiq/firmware/libboard/spr.rs @@ -57,6 +57,28 @@ pub const SPR_ESR_BASE: u32 = SPRGROUP_SYS + 64; pub const SPR_ESR_LAST: u32 = SPRGROUP_SYS + 79; pub const SPR_GPR_BASE: u32 = SPRGROUP_SYS + 1024; +/* Data MMU group */ +pub const SPR_DMMUCR: u32 = SPRGROUP_DMMU + 0; +pub const SPR_DTLBEIR: u32 = SPRGROUP_DMMU + 2; + +/* Instruction MMU group */ +pub const SPR_IMMUCR: u32 = SPRGROUP_IMMU + 0; +pub const SPR_ITLBEIR: u32 = SPRGROUP_IMMU + 2; + +/* Data cache group */ +pub const SPR_DCCR: u32 = SPRGROUP_DC + 0; +pub const SPR_DCBPR: u32 = SPRGROUP_DC + 1; +pub const SPR_DCBFR: u32 = SPRGROUP_DC + 2; +pub const SPR_DCBIR: u32 = SPRGROUP_DC + 3; +pub const SPR_DCBWR: u32 = SPRGROUP_DC + 4; +pub const SPR_DCBLR: u32 = SPRGROUP_DC + 5; + +/* Instruction cache group */ +pub const SPR_ICCR: u32 = SPRGROUP_IC + 0; +pub const SPR_ICBPR: u32 = SPRGROUP_IC + 1; +pub const SPR_ICBIR: u32 = SPRGROUP_IC + 2; +pub const SPR_ICBLR: u32 = SPRGROUP_IC + 3; + // [snip] /* PIC group */ @@ -89,3 +111,80 @@ pub const SPR_SR_FO: u32 = 0x00008000; /* Fixed one */ pub const SPR_SR_SUMRA: u32 = 0x00010000; /* Supervisor SPR read access */ pub const SPR_SR_RES: u32 = 0x0ffe0000; /* Reserved */ pub const SPR_SR_CID: u32 = 0xf0000000; /* Context ID */ + +/* + * Bit definitions for Data Cache Control register + * + */ +pub const SPR_DCCR_EW: u32 = 0x000000ff; /* Enable ways */ + +/* + * Bit definitions for Insn Cache Control register + * + */ +pub const SPR_ICCR_EW: u32 = 0x000000ff; /* Enable ways */ + +/* + * Bit definitions for Data Cache Configuration Register + * + */ +pub const SPR_DCCFGR_NCW: u32 = 0x00000007; +pub const SPR_DCCFGR_NCS: u32 = 0x00000078; +pub const SPR_DCCFGR_CBS: u32 = 0x00000080; +pub const SPR_DCCFGR_CWS: u32 = 0x00000100; +pub const SPR_DCCFGR_CCRI: u32 = 0x00000200; +pub const SPR_DCCFGR_CBIRI: u32 = 0x00000400; +pub const SPR_DCCFGR_CBPRI: u32 = 0x00000800; +pub const SPR_DCCFGR_CBLRI: u32 = 0x00001000; +pub const SPR_DCCFGR_CBFRI: u32 = 0x00002000; +pub const SPR_DCCFGR_CBWBRI: u32 = 0x00004000; + +pub const SPR_DCCFGR_NCW_OFF: u32 = 0; +pub const SPR_DCCFGR_NCS_OFF: u32 = 3; +pub const SPR_DCCFGR_CBS_OFF: u32 = 7; + +/* + * Bit definitions for Instruction Cache Configuration Register + * + */ +pub const SPR_ICCFGR_NCW: u32 = 0x00000007; +pub const SPR_ICCFGR_NCS: u32 = 0x00000078; +pub const SPR_ICCFGR_CBS: u32 = 0x00000080; +pub const SPR_ICCFGR_CCRI: u32 = 0x00000200; +pub const SPR_ICCFGR_CBIRI: u32 = 0x00000400; +pub const SPR_ICCFGR_CBPRI: u32 = 0x00000800; +pub const SPR_ICCFGR_CBLRI: u32 = 0x00001000; + +pub const SPR_ICCFGR_NCW_OFF: u32 = 0; +pub const SPR_ICCFGR_NCS_OFF: u32 = 3; +pub const SPR_ICCFGR_CBS_OFF: u32 = 7; + +/* + * Bit definitions for Data MMU Configuration Register + * + */ +pub const SPR_DMMUCFGR_NTW: u32 = 0x00000003; +pub const SPR_DMMUCFGR_NTS: u32 = 0x0000001C; +pub const SPR_DMMUCFGR_NAE: u32 = 0x000000E0; +pub const SPR_DMMUCFGR_CRI: u32 = 0x00000100; +pub const SPR_DMMUCFGR_PRI: u32 = 0x00000200; +pub const SPR_DMMUCFGR_TEIRI: u32 = 0x00000400; +pub const SPR_DMMUCFGR_HTR: u32 = 0x00000800; + +pub const SPR_DMMUCFGR_NTW_OFF: u32 = 0; +pub const SPR_DMMUCFGR_NTS_OFF: u32 = 2; + +/* + * Bit definitions for Instruction MMU Configuration Register + * + */ +pub const SPR_IMMUCFGR_NTW: u32 = 0x00000003; +pub const SPR_IMMUCFGR_NTS: u32 = 0x0000001C; +pub const SPR_IMMUCFGR_NAE: u32 = 0x000000E0; +pub const SPR_IMMUCFGR_CRI: u32 = 0x00000100; +pub const SPR_IMMUCFGR_PRI: u32 = 0x00000200; +pub const SPR_IMMUCFGR_TEIRI: u32 = 0x00000400; +pub const SPR_IMMUCFGR_HTR: u32 = 0x00000800; + +pub const SPR_IMMUCFGR_NTW_OFF: u32 = 0; +pub const SPR_IMMUCFGR_NTS_OFF: u32 = 2; diff --git a/artiq/firmware/runtime/analyzer.rs b/artiq/firmware/runtime/analyzer.rs index 24b0d6520..9d4eac0a2 100644 --- a/artiq/firmware/runtime/analyzer.rs +++ b/artiq/firmware/runtime/analyzer.rs @@ -1,5 +1,5 @@ use std::io::{self, Write}; -use board::{self, csr}; +use board::{csr, cache}; use sched::{Io, TcpSocket}; use analyzer_proto::*; @@ -35,8 +35,8 @@ fn disarm() { unsafe { csr::rtio_analyzer::enable_write(0); while csr::rtio_analyzer::busy_read() != 0 {} - board::flush_cpu_dcache(); - board::flush_l2_cache(); + cache::flush_cpu_dcache(); + cache::flush_l2_cache(); } } diff --git a/artiq/firmware/runtime/mailbox.rs b/artiq/firmware/runtime/mailbox.rs index b573734ed..db1f5af47 100644 --- a/artiq/firmware/runtime/mailbox.rs +++ b/artiq/firmware/runtime/mailbox.rs @@ -1,7 +1,7 @@ use core::ptr::{read_volatile, write_volatile}; -use board; +use board::{mem, cache}; -const MAILBOX: *mut usize = board::mem::MAILBOX_BASE as *mut usize; +const MAILBOX: *mut usize = mem::MAILBOX_BASE as *mut usize; static mut LAST: usize = 0; pub unsafe fn send(data: usize) { @@ -23,7 +23,7 @@ pub fn receive() -> usize { 0 } else { if data != 0 { - board::flush_cpu_dcache() + cache::flush_cpu_dcache() } data } diff --git a/artiq/firmware/runtime/rpc_queue.rs b/artiq/firmware/runtime/rpc_queue.rs index 3155813eb..f19befcec 100644 --- a/artiq/firmware/runtime/rpc_queue.rs +++ b/artiq/firmware/runtime/rpc_queue.rs @@ -2,10 +2,10 @@ use core::ptr::{read_volatile, write_volatile}; use core::slice; -use board; +use board::{mem, cache}; -const SEND_MAILBOX: *mut usize = (board::mem::MAILBOX_BASE + 4) as *mut usize; -const RECV_MAILBOX: *mut usize = (board::mem::MAILBOX_BASE + 8) as *mut usize; +const SEND_MAILBOX: *mut usize = (mem::MAILBOX_BASE + 4) as *mut usize; +const RECV_MAILBOX: *mut usize = (mem::MAILBOX_BASE + 8) as *mut usize; const QUEUE_BEGIN: usize = 0x40400000; const QUEUE_END: usize = 0x407fff80; @@ -51,7 +51,7 @@ pub fn dequeue(f: F) -> Result debug_assert!(!empty()); unsafe { - board::flush_cpu_dcache(); + cache::flush_cpu_dcache(); let slice = slice::from_raw_parts_mut(read_volatile(RECV_MAILBOX) as *mut u8, QUEUE_CHUNK); f(slice).and_then(|x| { write_volatile(RECV_MAILBOX, next(read_volatile(RECV_MAILBOX)));