artiq/artiq/gateware/rtio/core.py

469 lines
17 KiB
Python

from functools import reduce
from operator import and_
from migen import *
from migen.genlib.record import Record
from migen.genlib.cdc import *
from migen.genlib.fifo import AsyncFIFO
from migen.genlib.resetsync import AsyncResetSynchronizer
from misoc.interconnect.csr import *
from artiq.gateware.rtio import rtlink
class _GrayCodeTransfer(Module):
def __init__(self, width):
self.i = Signal(width) # in rio domain
self.o = Signal(width) # in rsys domain
# # #
# convert to Gray code
value_gray_rio = Signal(width)
self.sync.rio += value_gray_rio.eq(self.i ^ self.i[1:])
# transfer to system clock domain
value_gray_sys = Signal(width)
self.specials += [
NoRetiming(value_gray_rio),
MultiReg(value_gray_rio, value_gray_sys, "rsys")
]
# convert back to binary
value_sys = Signal(width)
self.comb += value_sys[-1].eq(value_gray_sys[-1])
for i in reversed(range(width-1)):
self.comb += value_sys[i].eq(value_sys[i+1] ^ value_gray_sys[i])
self.sync.rsys += self.o.eq(value_sys)
class _RTIOCounter(Module):
def __init__(self, width):
self.width = width
# Timestamp counter in RTIO domain
self.value_rio = Signal(width)
# Timestamp counter resynchronized to sys domain
# Lags behind value_rio, monotonic and glitch-free
self.value_sys = Signal(width)
# # #
# note: counter is in rtio domain and never affected by the reset CSRs
self.sync.rtio += self.value_rio.eq(self.value_rio + 1)
gt = _GrayCodeTransfer(width)
self.submodules += gt
self.comb += gt.i.eq(self.value_rio), self.value_sys.eq(gt.o)
# CHOOSING A GUARD TIME
#
# The buffer must be transferred to the FIFO soon enough to account for:
# * transfer of counter to sys domain: Tio + 2*Tsys + Tsys
# * FIFO latency: Tsys + 2*Tio
# * FIFO buffer latency: Tio
# Therefore we must choose:
# guard_io_cycles > (4*Tio + 4*Tsys)/Tio
#
# We are writing to the FIFO from the buffer when the guard time has been
# reached. This can fill the FIFO and deassert the writable flag. A race
# condition occurs that causes problems if the deassertion happens between
# the CPU checking the writable flag (and reading 1) and writing a new event.
#
# When the FIFO is about to be full, it contains fifo_depth-1 events of
# strictly increasing timestamps.
#
# Thus the FIFO-filling event's timestamp must satisfy:
# timestamp*Tio > (fifo_depth-1)*Tio + time
# We also have (guard time reached):
# timestamp*Tio < time + guard_io_cycles*Tio
# [NB: time > counter.value_sys*Tio]
# Thus we must have:
# guard_io_cycles > fifo_depth-1
#
# We can prevent overflows by choosing instead:
# guard_io_cycles < fifo_depth-1
class _OutputManager(Module):
def __init__(self, interface, counter, fifo_depth, guard_io_cycles):
data_width = rtlink.get_data_width(interface)
address_width = rtlink.get_address_width(interface)
fine_ts_width = rtlink.get_fine_ts_width(interface)
ev_layout = []
if data_width:
ev_layout.append(("data", data_width))
if address_width:
ev_layout.append(("address", address_width))
ev_layout.append(("timestamp", counter.width + fine_ts_width))
# ev must be valid 1 cycle before we to account for the latency in
# generating replace, sequence_error and nop
self.ev = Record(ev_layout)
self.writable = Signal()
self.we = Signal() # maximum throughput 1/2
self.underflow = Signal() # valid 1 cycle after we, pulsed
self.sequence_error = Signal()
self.collision_error = Signal()
# # #
# FIFO
fifo = ClockDomainsRenamer({"write": "rsys", "read": "rio"})(
AsyncFIFO(layout_len(ev_layout), fifo_depth))
self.submodules += fifo
fifo_in = Record(ev_layout)
fifo_out = Record(ev_layout)
self.comb += [
fifo.din.eq(fifo_in.raw_bits()),
fifo_out.raw_bits().eq(fifo.dout)
]
# Buffer
buf_pending = Signal()
buf = Record(ev_layout)
buf_just_written = Signal()
# Special cases
replace = Signal()
sequence_error = Signal()
collision_error = Signal()
any_error = Signal()
nop = Signal()
self.sync.rsys += [
# Note: replace does not perform any RTLink address checks,
# i.e. a write to a different address will be silently replaced
# as well.
replace.eq(self.ev.timestamp == buf.timestamp),
# Detect sequence errors on coarse timestamps only
# so that they are mutually exclusive with collision errors.
sequence_error.eq(self.ev.timestamp[fine_ts_width:]
< buf.timestamp[fine_ts_width:])
]
if fine_ts_width:
self.sync.rsys += collision_error.eq(
(self.ev.timestamp[fine_ts_width:] == buf.timestamp[fine_ts_width:])
& (self.ev.timestamp[:fine_ts_width] != buf.timestamp[:fine_ts_width]))
self.comb += any_error.eq(sequence_error | collision_error)
if interface.suppress_nop:
# disable NOP at reset: do not suppress a first write with all 0s
nop_en = Signal(reset=0)
addresses_equal = [getattr(self.ev, a) == getattr(buf, a)
for a in ("data", "address")
if hasattr(self.ev, a)]
if addresses_equal:
self.sync.rsys += nop.eq(
nop_en & reduce(and_, addresses_equal))
else:
self.comb.eq(nop.eq(0))
self.sync.rsys += [
# buf now contains valid data. enable NOP.
If(self.we & ~any_error, nop_en.eq(1)),
# underflows cancel the write. allow it to be retried.
If(self.underflow, nop_en.eq(0))
]
self.comb += [
self.sequence_error.eq(self.we & sequence_error),
self.collision_error.eq(self.we & collision_error)
]
# Buffer read and FIFO write
self.comb += fifo_in.eq(buf)
in_guard_time = Signal()
self.comb += in_guard_time.eq(
buf.timestamp[fine_ts_width:]
< counter.value_sys + guard_io_cycles)
self.sync.rsys += If(in_guard_time, buf_pending.eq(0))
self.comb += \
If(buf_pending,
If(in_guard_time,
If(buf_just_written,
self.underflow.eq(1)
).Else(
fifo.we.eq(1)
)
),
If(self.we & ~replace & ~nop & ~any_error,
fifo.we.eq(1)
)
)
# Buffer write
# Must come after read to handle concurrent read+write properly
self.sync.rsys += [
buf_just_written.eq(0),
If(self.we & ~nop & ~any_error,
buf_just_written.eq(1),
buf_pending.eq(1),
buf.eq(self.ev)
)
]
self.comb += self.writable.eq(fifo.writable)
# Buffer output of FIFO to improve timing
dout_stb = Signal()
dout_ack = Signal()
dout = Record(ev_layout)
self.sync.rio += \
If(fifo.re,
dout_stb.eq(1),
dout.eq(fifo_out)
).Elif(dout_ack,
dout_stb.eq(0)
)
self.comb += fifo.re.eq(fifo.readable & (~dout_stb | dout_ack))
# FIFO read through buffer
# TODO: report error on stb & busy
self.comb += [
dout_ack.eq(
dout.timestamp[fine_ts_width:] == counter.value_rio),
interface.stb.eq(dout_stb & dout_ack)
]
if data_width:
self.comb += interface.data.eq(dout.data)
if address_width:
self.comb += interface.address.eq(dout.address)
if fine_ts_width:
self.comb += interface.fine_ts.eq(dout.timestamp[:fine_ts_width])
class _InputManager(Module):
def __init__(self, interface, counter, fifo_depth):
data_width = rtlink.get_data_width(interface)
fine_ts_width = rtlink.get_fine_ts_width(interface)
ev_layout = []
if data_width:
ev_layout.append(("data", data_width))
if interface.timestamped:
ev_layout.append(("timestamp", counter.width + fine_ts_width))
self.ev = Record(ev_layout)
self.readable = Signal()
self.re = Signal()
self.overflow = Signal() # pulsed
# # #
fifo = ClockDomainsRenamer({"read": "rsys", "write": "rio"})(
AsyncFIFO(layout_len(ev_layout), fifo_depth))
self.submodules += fifo
fifo_in = Record(ev_layout)
fifo_out = Record(ev_layout)
self.comb += [
fifo.din.eq(fifo_in.raw_bits()),
fifo_out.raw_bits().eq(fifo.dout)
]
# FIFO write
if data_width:
self.comb += fifo_in.data.eq(interface.data)
if interface.timestamped:
if fine_ts_width:
full_ts = Cat(interface.fine_ts, counter.value_rio)
else:
full_ts = counter.value_rio
self.comb += fifo_in.timestamp.eq(full_ts)
self.comb += fifo.we.eq(interface.stb)
# FIFO read
self.comb += [
self.ev.eq(fifo_out),
self.readable.eq(fifo.readable),
fifo.re.eq(self.re)
]
overflow_sync = PulseSynchronizer("rio", "rsys")
overflow_ack_sync = PulseSynchronizer("rsys", "rio")
self.submodules += overflow_sync, overflow_ack_sync
overflow_blind = Signal()
self.comb += overflow_sync.i.eq(fifo.we & ~fifo.writable & ~overflow_blind)
self.sync.rio += [
If(fifo.we & ~fifo.writable, overflow_blind.eq(1)),
If(overflow_ack_sync.o, overflow_blind.eq(0))
]
self.comb += [
overflow_ack_sync.i.eq(overflow_sync.o),
self.overflow.eq(overflow_sync.o)
]
class Channel:
def __init__(self, interface, probes=[], overrides=[],
ofifo_depth=64, ififo_depth=64):
self.interface = interface
self.probes = probes
self.overrides = overrides
self.ofifo_depth = ofifo_depth
self.ififo_depth = ififo_depth
@classmethod
def from_phy(cls, phy, **kwargs):
probes = getattr(phy, "probes", [])
overrides = getattr(phy, "overrides", [])
return cls(phy.rtlink, probes, overrides, **kwargs)
class _KernelCSRs(AutoCSR):
def __init__(self, chan_sel_width,
data_width, address_width, full_ts_width):
self.reset = CSRStorage(reset=1)
self.reset_phy = CSRStorage(reset=1)
self.chan_sel = CSRStorage(chan_sel_width)
if data_width:
self.o_data = CSRStorage(data_width)
if address_width:
self.o_address = CSRStorage(address_width)
self.o_timestamp = CSRStorage(full_ts_width)
self.o_we = CSR()
self.o_status = CSRStatus(4)
self.o_underflow_reset = CSR()
self.o_sequence_error_reset = CSR()
self.o_collision_error_reset = CSR()
if data_width:
self.i_data = CSRStatus(data_width)
self.i_timestamp = CSRStatus(full_ts_width)
self.i_re = CSR()
self.i_status = CSRStatus(2)
self.i_overflow_reset = CSR()
self.counter = CSRStatus(full_ts_width)
self.counter_update = CSR()
class RTIO(Module):
def __init__(self, channels, full_ts_width=63, guard_io_cycles=20):
data_width = max(rtlink.get_data_width(c.interface)
for c in channels)
address_width = max(rtlink.get_address_width(c.interface)
for c in channels)
fine_ts_width = max(rtlink.get_fine_ts_width(c.interface)
for c in channels)
self.data_width = data_width
self.address_width = address_width
self.fine_ts_width = fine_ts_width
# CSRs
self.kcsrs = _KernelCSRs(bits_for(len(channels)-1),
data_width, address_width,
full_ts_width)
# Clocking/Reset
# Create rsys, rio and rio_phy domains based on sys and rtio
# with reset controlled by CSR.
self.clock_domains.cd_rsys = ClockDomain()
self.clock_domains.cd_rio = ClockDomain()
self.clock_domains.cd_rio_phy = ClockDomain()
self.comb += [
self.cd_rsys.clk.eq(ClockSignal()),
self.cd_rsys.rst.eq(self.kcsrs.reset.storage)
]
self.comb += self.cd_rio.clk.eq(ClockSignal("rtio"))
self.specials += AsyncResetSynchronizer(
self.cd_rio,
self.kcsrs.reset.storage | ResetSignal("rtio",
allow_reset_less=True))
self.comb += self.cd_rio_phy.clk.eq(ClockSignal("rtio"))
self.specials += AsyncResetSynchronizer(
self.cd_rio_phy,
self.kcsrs.reset_phy.storage | ResetSignal("rtio",
allow_reset_less=True))
# Managers
self.submodules.counter = _RTIOCounter(full_ts_width - fine_ts_width)
i_datas, i_timestamps = [], []
o_statuses, i_statuses = [], []
sel = self.kcsrs.chan_sel.storage
for n, channel in enumerate(channels):
selected = Signal()
self.comb += selected.eq(sel == n)
o_manager = _OutputManager(channel.interface.o, self.counter,
channel.ofifo_depth, guard_io_cycles)
self.submodules += o_manager
if hasattr(o_manager.ev, "data"):
self.comb += o_manager.ev.data.eq(
self.kcsrs.o_data.storage)
if hasattr(o_manager.ev, "address"):
self.comb += o_manager.ev.address.eq(
self.kcsrs.o_address.storage)
ts_shift = (len(self.kcsrs.o_timestamp.storage)
- len(o_manager.ev.timestamp))
self.comb += o_manager.ev.timestamp.eq(
self.kcsrs.o_timestamp.storage[ts_shift:])
self.comb += o_manager.we.eq(selected & self.kcsrs.o_we.re)
underflow = Signal()
sequence_error = Signal()
collision_error = Signal()
self.sync.rsys += [
If(selected & self.kcsrs.o_underflow_reset.re,
underflow.eq(0)),
If(selected & self.kcsrs.o_sequence_error_reset.re,
sequence_error.eq(0)),
If(selected & self.kcsrs.o_collision_error_reset.re,
collision_error.eq(0)),
If(o_manager.underflow, underflow.eq(1)),
If(o_manager.sequence_error, sequence_error.eq(1)),
If(o_manager.collision_error, collision_error.eq(1))
]
o_statuses.append(Cat(~o_manager.writable,
underflow,
sequence_error,
collision_error))
if channel.interface.i is not None:
i_manager = _InputManager(channel.interface.i, self.counter,
channel.ififo_depth)
self.submodules += i_manager
if hasattr(i_manager.ev, "data"):
i_datas.append(i_manager.ev.data)
else:
i_datas.append(0)
if channel.interface.i.timestamped:
ts_shift = (len(self.kcsrs.i_timestamp.status)
- len(i_manager.ev.timestamp))
i_timestamps.append(i_manager.ev.timestamp << ts_shift)
else:
i_timestamps.append(0)
self.comb += i_manager.re.eq(selected & self.kcsrs.i_re.re)
overflow = Signal()
self.sync.rsys += [
If(selected & self.kcsrs.i_overflow_reset.re,
overflow.eq(0)),
If(i_manager.overflow,
overflow.eq(1))
]
i_statuses.append(Cat(~i_manager.readable, overflow))
else:
i_datas.append(0)
i_timestamps.append(0)
i_statuses.append(0)
if data_width:
self.comb += self.kcsrs.i_data.status.eq(Array(i_datas)[sel])
self.comb += [
self.kcsrs.i_timestamp.status.eq(Array(i_timestamps)[sel]),
self.kcsrs.o_status.status.eq(Array(o_statuses)[sel]),
self.kcsrs.i_status.status.eq(Array(i_statuses)[sel])
]
# Counter access
self.sync += \
If(self.kcsrs.counter_update.re,
self.kcsrs.counter.status.eq(self.counter.value_sys
<< fine_ts_width)
)
def get_csrs(self):
return self.kcsrs.get_csrs()