add ACP kernel initiator

Based on work by Chris Ballance
https://github.com/m-labs/artiq/issues/1167#issuecomment-427188287
M-Labs/artiq-zynq#55

Work-in-progress, only gateware part and build system, untested.

src/Makefile

@@ -9,7 +9,11 @@ all: ../build/firmware/armv7-none-eabihf/release/szl
 	mkdir -p ../build
 	python gateware/zc706.py -r ../build/pl.rs -V $(VARIANT)
 
-../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs $(shell find . -path ./szl -prune -o -print)
+../build/rustc-cfg: gateware/*
+	mkdir -p ../build
+	python gateware/zc706.py -c ../build/rustc-cfg -V $(VARIANT)
+
+../build/firmware/armv7-none-eabihf/release/runtime: ../build/pl.rs ../build/rustc-cfg $(shell find . -path ./szl -prune -o -print)
 	XBUILD_SYSROOT_PATH=`pwd`/../build/sysroot cargo xbuild --release -p runtime --target-dir ../build/firmware
 
 ../build/szl-payload.bin.lzma: ../build/firmware/armv7-none-eabihf/release/runtime

src/gateware/acpki.py

@@ -0,0 +1,260 @@
+from operator import attrgetter
+
+from migen import *
+from migen.genlib.cdc import MultiReg
+from migen_axi.interconnect import axi
+from misoc.interconnect.csr import *
+
+from artiq.gateware import rtio
+
+
+OUT_BURST_LEN = 4
+IN_BURST_LEN = 4
+
+
+class Engine(Module, AutoCSR):
+    def __init__(self, bus, user):
+        self.addr_base = CSRStorage(32)
+        self.trig_count = CSRStatus(32)
+        self.write_count = CSRStatus(32)
+
+        self.trigger_stb = Signal()
+
+        # Dout : Data received from CPU, output by DMA module
+        # Din : Data driven into DMA module, written into CPU
+        # When stb assert, index shows word being read/written, dout/din holds
+        # data
+        #
+        # Cycle:
+        # trigger_stb pulsed at start
+        # Then out_burst_len words are strobed out of dout
+        # Then, when din_ready is high, in_burst_len words are strobed in to din
+        self.dout_stb = Signal()
+        self.din_stb = Signal()
+        self.dout_index = Signal(max=16)
+        self.din_index = Signal(max=16)
+        self.din_ready = Signal()
+        self.dout = Signal(64)
+        self.din = Signal(64)
+
+        ###
+
+        self.sync += If(self.trigger_stb, self.trig_count.status.eq(self.trig_count.status+1))
+
+        self.comb += [
+            user.aruser.eq(0x1f),
+            user.awuser.eq(0x1f)
+        ]
+
+        ar, aw, w, r, b = attrgetter("ar", "aw", "w", "r", "b")(bus)
+
+        ### Read
+        self.comb += [
+            ar.addr.eq(self.addr_base.storage),
+            self.dout.eq(r.data),
+            r.ready.eq(1),
+            ar.burst.eq(axi.Burst.incr.value),
+            ar.len.eq(OUT_BURST_LEN-1), # Number of transfers in burst (0->1 transfer, 1->2 transfers...)
+            ar.size.eq(3), # Width of burst: 3 = 8 bytes = 64 bits
+            ar.cache.eq(0xf),
+        ]
+
+        # read control
+        self.submodules.read_fsm = read_fsm = FSM(reset_state="IDLE")
+        read_fsm.act("IDLE",
+            If(self.trigger_stb,
+                ar.valid.eq(1),
+                If(ar.ready,
+                    NextState("READ")
+                ).Else(
+                    NextState("READ_START")
+                )
+            )
+        )
+        read_fsm.act("READ_START",
+            ar.valid.eq(1),
+            If(ar.ready,
+                NextState("READ"),
+            )
+        )
+        read_fsm.act("READ",
+            ar.valid.eq(0),
+            If(r.last & r.valid,
+                NextState("IDLE")
+            )
+        )
+
+        self.sync += [
+            If(read_fsm.ongoing("IDLE"),
+                self.dout_index.eq(0)
+            ).Else(If(r.valid & read_fsm.ongoing("READ"),
+                    self.dout_index.eq(self.dout_index+1)
+                )
+            )
+        ]
+
+        self.comb += self.dout_stb.eq(r.valid & r.ready)
+
+        ### Write
+        self.comb += [
+            w.data.eq(self.din),
+            aw.addr.eq(self.addr_base.storage+32), # Write to next cache line
+            w.strb.eq(0xff),
+            aw.burst.eq(axi.Burst.incr.value),
+            aw.len.eq(IN_BURST_LEN-1), # Number of transfers in burst minus 1
+            aw.size.eq(3), # Width of burst: 3 = 8 bytes = 64 bits
+            aw.cache.eq(0xf),
+            b.ready.eq(1),
+        ]
+
+        # write control
+        self.submodules.write_fsm = write_fsm = FSM(reset_state="IDLE")
+        write_fsm.act("IDLE",
+            w.valid.eq(0),
+            aw.valid.eq(0),
+            If(self.trigger_stb,
+                aw.valid.eq(1),
+                If(aw.ready, # assumes aw.ready is not randomly deasserted
+                    NextState("DATA_WAIT")
+                ).Else(
+                    NextState("AW_READY_WAIT")
+                )
+            )
+        )
+        write_fsm.act("AW_READY_WAIT",
+            aw.valid.eq(1),
+            If(aw.ready,
+                NextState("DATA_WAIT"),
+            )
+        )
+        write_fsm.act("DATA_WAIT",
+            aw.valid.eq(0),
+            If(self.din_ready,
+                w.valid.eq(1),
+                NextState("WRITE")
+            )
+        )
+        write_fsm.act("WRITE",
+            w.valid.eq(1),
+            If(w.ready & w.last,
+                NextState("IDLE")
+            )
+        )
+
+        self.sync += If(w.ready & w.valid, self.write_count.status.eq(self.write_count.status+1))
+
+        self.sync += [
+            If(write_fsm.ongoing("IDLE"),
+                self.din_index.eq(0)
+            ),
+            If(w.ready & w.valid, self.din_index.eq(self.din_index+1))
+        ]
+
+        self.comb += [
+            w.last.eq(0),
+            If(self.din_index==aw.len, w.last.eq(1))
+        ]
+
+        self.comb += self.din_stb.eq(w.valid & w.ready)
+
+
+
+class KernelInitiator(Module, AutoCSR):
+    def __init__(self, tsc, bus, user, evento):
+        # Core is disabled upon reset to avoid spurious triggering if evento toggles from e.g. boot code.
+        self.enable = CSRStorage()
+
+        self.counter = CSRStatus(64)
+        self.counter_update = CSR()
+
+        self.submodules.engine = Engine(bus, user)
+        self.cri = rtio.cri.Interface()
+
+        ###
+
+        evento_stb = Signal()
+        evento_latched = Signal()
+        evento_latched_d = Signal()
+        self.specials += MultiReg(evento, evento_latched)
+        self.sync += evento_latched_d.eq(evento_latched)
+        self.comb += self.engine.trigger_stb.eq(self.enable.storage & (evento_latched != evento_latched_d))
+
+        cri = self.cri
+
+        cmd = Signal(8)
+        cmd_write = Signal()
+        cmd_read = Signal()
+        self.comb += [
+            cmd_write.eq(cmd == 0),
+            cmd_read.eq(cmd == 1)
+        ]
+
+        dout_cases = {}
+        dout_lw = Signal(32)
+        dout_hw = Signal(32)
+        self.comb += [
+            dout_lw.eq(self.engine.dout[:32]),
+            dout_hw.eq(self.engine.dout[32:])
+        ]
+        dout_cases[0] = [
+            cmd.eq(dout_lw[24:]),
+            cri.chan_sel.eq(dout_lw[:24]),
+            cri.o_address.eq(dout_hw[:16])
+        ]
+        dout_cases[1] = [
+            cri.o_timestamp.eq(self.engine.dout)
+        ]
+        dout_cases[2] = [cri.o_data.eq(self.engine.dout)] # only lowest 64 bits
+
+        self.sync += [
+            cri.cmd.eq(rtio.cri.commands["nop"]),
+            If(self.engine.dout_stb,
+                Case(self.engine.dout_index, dout_cases),
+                If(self.engine.dout_index == 2,
+                    If(cmd_write, cri.cmd.eq(rtio.cri.commands["write"])),
+                    If(cmd_read, cri.cmd.eq(rtio.cri.commands["read"]))
+                )
+            )
+        ]
+
+        # If input event, wait for response before allow input data to be
+        # sampled
+        # TODO: If output, wait for wait flag clear
+        RTIO_I_STATUS_WAIT_STATUS = 4
+        RTIO_O_STATUS_WAIT = 1
+
+        self.submodules.fsm = fsm = FSM(reset_state="IDLE")
+
+        fsm.act("IDLE",
+            If(self.engine.trigger_stb, NextState("WAIT_OUT_CYCLE"))
+        )
+        fsm.act("WAIT_OUT_CYCLE",
+            self.engine.din_ready.eq(0),
+            If(self.engine.dout_stb & (self.engine.dout_index == 3),
+                NextState("WAIT_READY")
+            )
+        )
+        fsm.act("WAIT_READY",
+            If(cmd_read & (cri.i_status & RTIO_I_STATUS_WAIT_STATUS == 0) \
+                | cmd_write & ~(cri.o_status & RTIO_O_STATUS_WAIT),
+                self.engine.din_ready.eq(1),
+                NextState("IDLE")
+            )
+        )
+
+        din_cases_cmdwrite = {
+            0: [self.engine.din.eq((1<<16) | cri.o_status)],
+            1: [self.engine.din.eq(0)],
+        }
+        din_cases_cmdread = {
+            0: [self.engine.din[:32].eq((1<<16) | cri.i_status), self.engine.din[32:].eq(cri.i_data)],
+            1: [self.engine.din.eq(cri.i_timestamp)]
+        }
+
+        self.comb += [
+            If(cmd_read, Case(self.engine.din_index, din_cases_cmdread)),
+            If(cmd_write, Case(self.engine.din_index, din_cases_cmdwrite)),
+        ]
+
+        # RTIO counter access
+        self.sync += If(self.counter_update.re, self.counter.status.eq(tsc.full_ts_cri))

src/gateware/zc706.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import argparse
+from operator import itemgetter
 
 from migen import *
 from migen.build.generic_platform import *
@@ -16,6 +17,7 @@ from artiq.gateware.rtio.phy import ttl_simple, ttl_serdes_7series, dds, spi2
 
 import dma
 import analyzer
+import acpki
 
 
 class RTIOCRG(Module, AutoCSR):
@@ -63,12 +65,18 @@ class RTIOCRG(Module, AutoCSR):
 
 
 class ZC706(SoCCore):
-    def __init__(self):
+    def __init__(self, acpki=False):
+        self.acpki = acpki
+        self.rustc_cfg = dict()
+
         platform = zc706.Platform()
         platform.toolchain.bitstream_commands.extend([
             "set_property BITSTREAM.GENERAL.COMPRESS True [current_design]",
         ])
-        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=self.__class__.__name__)
+        ident = self.__class__.__name__
+        if self.acpki:
+            ident = "acpki_" + ident
+        SoCCore.__init__(self, platform=platform, csr_data_width=32, ident=ident)
 
         platform.add_platform_command("create_clock -name clk_fpga_0 -period 8 [get_pins \"PS7/FCLKCLK[0]\"]")
         platform.add_platform_command("set_input_jitter clk_fpga_0 0.24")
@@ -84,9 +92,20 @@ class ZC706(SoCCore):
         self.submodules.rtio_tsc = rtio.TSC("async", glbl_fine_ts_width=3)
         self.submodules.rtio_core = rtio.Core(self.rtio_tsc, rtio_channels)
         self.csr_devices.append("rtio_core")
+
+        if self.acpki:
+            self.rustc_cfg["ki_impl"] = "csr"
             self.submodules.rtio = rtio.KernelInitiator(self.rtio_tsc, now64=True)
-        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
             self.csr_devices.append("rtio")
+        else:
+            self.rustc_cfg["ki_impl"] = "acp"
+            self.submodules.rtio = acpki.KernelInitiator(self.rtio_tsc,
+                                                         bus=self.ps7.s_axi_acp,
+                                                         user=self.ps7.s_axi_acp_user,
+                                                         evento=self.ps7.event.o)
+            self.csr_devices.append("rtio")
+
+        self.submodules.rtio_dma = dma.DMA(self.ps7.s_axi_hp0)
         self.csr_devices.append("rtio_dma")
 
         self.submodules.cri_con = rtio.CRIInterconnectShared(
@@ -103,8 +122,8 @@ class ZC706(SoCCore):
 
 
 class Simple(ZC706):
-    def __init__(self):
+    def __init__(self, **kwargs):
-        ZC706.__init__(self)
+        ZC706.__init__(self, **kwargs)
 
         platform = self.platform
 
@@ -134,8 +153,8 @@ class NIST_CLOCK(ZC706):
     """
     NIST clock hardware, with old backplane and 11 DDS channels
     """
-    def __init__(self):
+    def __init__(self, **kwargs):
-        ZC706.__init__(self)
+        ZC706.__init__(self, **kwargs)
 
         platform = self.platform
         platform.add_extension(nist_clock.fmc_adapter_io)
@@ -188,8 +207,8 @@ class NIST_QC2(ZC706):
     NIST QC2 hardware, as used in Quantum I and Quantum II, with new backplane
     and 24 DDS channels.  Two backplanes are used.
     """
-    def __init__(self):
+    def __init__(self, **kwargs):
-        ZC706.__init__(self)
+        ZC706.__init__(self, **kwargs)
 
         platform = self.platform
         platform.add_extension(nist_qc2.fmc_adapter_io)
@@ -242,31 +261,48 @@ def write_csr_file(soc, filename):
             soc.get_csr_regions(), soc.get_csr_groups(), soc.get_constants()))
 
 
+def write_rustc_cfg_file(soc, filename):
+    with open(filename, "w") as f:
+        for k, v in sorted(soc.rustc_cfg.items(), key=itemgetter(0)):
+            if v is None:
+                f.write("{}\n".format(k))
+            else:
+                f.write("{}=\"{}\"\n".format(k, v))
+
+
 def main():
     parser = argparse.ArgumentParser(
         description="ARTIQ port to the ZC706 Zynq development kit")
     parser.add_argument("-r", default=None,
         help="build Rust interface into the specified file")
+    parser.add_argument("-c", default=None,
+        help="build Rust compiler configuration into the specified file")
     parser.add_argument("-g", default=None,
         help="build gateware into the specified directory")
     parser.add_argument("-V", "--variant", default="simple",
                         help="variant: "
-                             "simple/nist_clock/nist_qc2 "
+                             "[acpki_]simple/nist_clock/nist_qc2 "
                              "(default: %(default)s)")
     args = parser.parse_args()
 
+    variant = args.variant.lower()
+    acpki = variant.startswith("acpki_")
+    if acpki:
+        variant = variant[6:]
     try:
-        cls = VARIANTS[args.variant.lower()]
+        cls = VARIANTS[variant]
     except KeyError:
         raise SystemExit("Invalid variant (-V/--variant)")
 
-    soc = cls()
+    soc = cls(acpki=acpki)
     soc.finalize()
 
-    if args.g is not None:
-        soc.build(build_dir=args.g)
     if args.r is not None:
         write_csr_file(soc, args.r)
+    if args.c is not None:
+        write_rustc_cfg_file(soc, args.c)
+    if args.g is not None:
+        soc.build(build_dir=args.g)
 
 
 if __name__ == "__main__":

src/runtime/build.rs

@@ -1,6 +1,7 @@
 use std::env;
 use std::fs::File;
 use std::io::Write;
+use std::io::{BufRead, BufReader};
 use std::path::PathBuf;
 
 fn main() {
@@ -15,4 +16,13 @@ fn main() {
     // Only re-run the build script when link.x is changed,
     // instead of when any part of the source code changes.
     println!("cargo:rerun-if-changed=link.x");
+
+    // Handle rustc-cfg file
+    let cfg_path = "../../build/rustc-cfg";
+    println!("cargo:rerun-if-changed={}", cfg_path);
+
+    let f = BufReader::new(File::open(cfg_path).unwrap());
+    for line in f.lines() {
+        println!("cargo:rustc-cfg={}", line.unwrap());
+    }
 }

src/runtime/src/main.rs

@@ -31,6 +31,11 @@ mod comms;
 mod rpc;
 #[path = "../../../build/pl.rs"]
 mod pl;
+#[cfg(ki_impl = "csr")]
+#[path = "rtio_csr.rs"]
+mod rtio;
+#[cfg(ki_impl = "acp")]
+#[path = "rtio_acp.rs"]
 mod rtio;
 mod kernel;
 mod moninj;

src/runtime/src/rtio_acp.rs

@@ -0,0 +1,81 @@
+use core::ptr::{self, read_volatile, write_volatile};
+use core::ffi::VaList;
+use alloc::vec;
+use cslice::CSlice;
+use libc::{c_char, c_int, size_t};
+
+use crate::artiq_raise;
+
+use crate::pl::csr;
+
+
+#[repr(C)]
+pub struct TimestampedData {
+    timestamp: i64,
+    data: i32,
+}
+
+pub extern fn init() {
+    unsafe {
+        csr::rtio_core::reset_write(1);
+    }
+}
+
+pub extern fn get_destination_status(destination: i32) -> bool {
+    // TODO
+    destination == 0
+}
+
+pub extern fn get_counter() -> i64 {
+    unsafe {
+        csr::rtio::counter_update_write(1);
+        csr::rtio::counter_read() as i64
+    }
+}
+
+pub extern fn now_mu() -> i64 {
+    unimplemented!();
+}
+
+pub extern fn at_mu(t: i64) {
+    unimplemented!();
+}
+
+pub extern fn delay_mu(dt: i64) {
+    unimplemented!();
+}
+
+pub extern fn output(target: i32, data: i32) {
+    unimplemented!();
+}
+
+pub extern fn output_wide(target: i32, data: CSlice<i32>) {
+    unimplemented!();
+}
+
+pub extern fn input_timestamp(timeout: i64, channel: i32) -> i64 {
+   unimplemented!();
+}
+
+pub extern fn input_data(channel: i32) -> i32 {
+    unimplemented!();
+}
+
+pub extern fn input_timestamped_data(timeout: i64, channel: i32) -> TimestampedData {
+    unimplemented!();
+}
+
+extern "C" {
+    fn vsnprintf_(buffer: *mut c_char, count: size_t, format: *const c_char, va: VaList) -> c_int;
+}
+
+fn write_rtio_log(data: &[i8]) {
+    unimplemented!();
+}
+
+pub unsafe extern fn log(fmt: *const c_char, mut args: ...) {
+    let size = vsnprintf_(ptr::null_mut(), 0, fmt, args.as_va_list()) as usize;
+    let mut buf = vec![0; size + 1];
+    vsnprintf_(buf.as_mut_ptr(), size + 1, fmt, args.as_va_list());
+    write_rtio_log(buf.as_slice());
+}