examples/tcp: fix lifetime for smoltcp bump

Cargo.toml

@@ -13,7 +13,7 @@ license = "BSD-2-Clause"
 volatile-register = "0.2"
 aligned = "0.3"
 embedded-hal = "0.2"
-smoltcp = { version = "0.6.0", default-features = false, features = ["proto-ipv4", "proto-ipv6", "socket-icmp", "socket-udp", "socket-tcp", "log", "verbose", "ethernet"], optional = true }
+smoltcp = { version = "0.7.0", default-features = false, features = ["proto-ipv4", "proto-ipv6", "socket-icmp", "socket-udp", "socket-tcp", "log", "verbose", "ethernet"], optional = true }
 # Optional dependencies for building examples
 stm32f4xx-hal = { version = "0.8", optional = true }
 cortex-m = { version = "0.5", optional = true }
@@ -21,6 +21,9 @@ cortex-m-rt = { version = "0.6", optional = true }
 cortex-m-rtic = { version = "0.5.3", optional = true }
 panic-itm = { version = "0.4", optional = true }
 log = { version = "0.4", optional = true }
+# Optional dependencies for using NAL with somltcp and cortex_m
+heapless = { version = "0.5.6", optional = true }
+embedded-nal = { version = "0.1.0", optional = true }
 
 [features]
 smoltcp-phy = ["smoltcp"]
@@ -31,6 +34,7 @@ smoltcp-phy-all = [
 # Example-based features
 tx_stm32f407 = ["stm32f4xx-hal/stm32f407", "cortex-m", "cortex-m-rtic", "panic-itm", "log"]
 tcp_stm32f407 = ["stm32f4xx-hal/stm32f407", "cortex-m", "cortex-m-rt", "cortex-m-rtic", "smoltcp-phy-all", "smoltcp/log", "panic-itm", "log"]
+nal = [ "smoltcp-phy", "heapless", "embedded-nal", "cortex-m", "cortex-m-rtic" ]
 default = []
 
 [[example]]

examples/delay.rs

@@ -0,0 +1,33 @@
+use embedded_hal::blocking::delay::{DelayMs, DelayUs};
+
+pub struct AsmDelay {
+    frequency_us: u32,
+    frequency_ms: u32,
+}
+
+impl AsmDelay {
+    pub fn new(freq: u32) -> AsmDelay {
+        AsmDelay {
+            frequency_us: (freq / 1_000_000),
+            frequency_ms: (freq / 1_000),
+        }
+    }
+}
+
+impl<U> DelayUs<U> for AsmDelay
+where
+    U: Into<u32>,
+{
+    fn delay_us(&mut self, us: U) {
+        cortex_m::asm::delay(self.frequency_us * us.into())
+    }
+}
+
+impl<U> DelayMs<U> for AsmDelay
+where
+    U: Into<u32>,
+{
+    fn delay_ms(&mut self, ms: U) {
+        cortex_m::asm::delay(self.frequency_ms * ms.into())
+    }
+}

examples/tcp_stm32f407.rs

@@ -28,6 +28,9 @@ use smoltcp::socket::{SocketSet, TcpSocket, TcpSocketBuffer};
 use core::str;
 use core::fmt::Write;
 
+mod delay;
+use delay::AsmDelay;
+
 /// Timer
 use core::cell::RefCell;
 use cortex_m::interrupt::Mutex;
@@ -80,7 +83,9 @@ use stm32f4xx_hal::{
 };
 type BoosterSpiEth = enc424j600::SpiEth<
     Spi<SPI1, (PA5<Alternate<AF5>>, PA6<Alternate<AF5>>, PA7<Alternate<AF5>>)>,
-    PA4<Output<PushPull>>>;
+    PA4<Output<PushPull>>,
+    AsmDelay
+>;
 
 pub struct NetStorage {
     ip_addrs: [IpCidr; 1],
@@ -99,8 +104,6 @@ static mut NET_STORE: NetStorage = NetStorage {
 const APP: () = {
     struct Resources {
         eth_iface: EthernetInterface<
-            'static,
-            'static,
             'static,
             smoltcp_phy::SmoltcpDevice<BoosterSpiEth>>,
         itm: ITM
@@ -122,7 +125,8 @@ const APP: () = {
             .pclk1(42.mhz())
             .require_pll48clk()
             .freeze();
-        let mut delay = Delay::new(c.core.SYST, clocks);
+        let asm_delay = AsmDelay::new(clocks.sysclk().0);
+        let mut hal_delay = Delay::new(c.core.SYST, clocks);
 
         // Init ITM
         let mut itm = c.core.ITM;
@@ -142,7 +146,7 @@ const APP: () = {
         // Map SPISEL: see Table 1, NIC100 Manual
         let mut spisel = gpioa.pa1.into_push_pull_output();
         spisel.set_high().unwrap();
-        delay.delay_ms(1_u32);
+        hal_delay.delay_ms(1_u32);
         spisel.set_low().unwrap();
 
         // Create SPI1 for HAL
@@ -153,11 +157,11 @@ const APP: () = {
                     enc424j600::spi::interfaces::SPI_MODE,
                     Hertz(enc424j600::spi::interfaces::SPI_CLOCK_FREQ),
                     clocks);
-                enc424j600::SpiEth::new(spi_eth_port, spi1_nss)
+                enc424j600::SpiEth::new(spi_eth_port, spi1_nss, asm_delay)
             };
 
             // Init controller
-            match spi_eth.init_dev(&mut delay) {
+            match spi_eth.init_dev() {
                 Ok(_) => {
                     iprintln!(stim0, "Initializing Ethernet...")
                 }
@@ -168,7 +172,7 @@ const APP: () = {
 
             // Read MAC
             let mut eth_mac_addr: [u8; 6] = [0; 6];
-            spi_eth.read_from_mac(&mut eth_mac_addr);
+            spi_eth.read_from_mac(&mut eth_mac_addr).unwrap();
             for i in 0..6 {
                 let byte = eth_mac_addr[i];
                 match i {
@@ -180,8 +184,8 @@ const APP: () = {
             }
 
             // Init Rx/Tx buffers
-            spi_eth.init_rxbuf();
-            spi_eth.init_txbuf();
+            spi_eth.init_rxbuf().unwrap();
+            spi_eth.init_txbuf().unwrap();
             iprintln!(stim0, "Ethernet controller initialized");
 
             // Init smoltcp interface
@@ -205,7 +209,7 @@ const APP: () = {
 
         // Setup SysTick after releasing SYST from Delay
         // Reference to stm32-eth:examples/ip.rs
-        timer_setup(delay.free(), clocks);
+        timer_setup(hal_delay.free(), clocks);
         iprintln!(stim0, "Timer initialized");
 
         init::LateResources {

examples/tx_stm32f407.rs

@@ -20,6 +20,9 @@ use stm32f4xx_hal::{
 use enc424j600;
 use enc424j600::EthController;
 
+mod delay;
+use delay::AsmDelay;
+
 ///
 use stm32f4xx_hal::{
     stm32::SPI1,
@@ -30,7 +33,8 @@ use stm32f4xx_hal::{
 };
 type BoosterSpiEth = enc424j600::SpiEth<
     Spi<SPI1, (PA5<Alternate<AF5>>, PA6<Alternate<AF5>>, PA7<Alternate<AF5>>)>,
-    PA4<Output<PushPull>>>;
+    PA4<Output<PushPull>>,
+    AsmDelay>;
 
 #[rtic::app(device = stm32f4xx_hal::stm32, peripherals = true, monotonic = rtic::cyccnt::CYCCNT)]
 const APP: () = {
@@ -54,6 +58,7 @@ const APP: () = {
             //.pclk2(64.mhz())
             .require_pll48clk()
             .freeze();
+        let asm_delay = AsmDelay::new(clocks.sysclk().0);
         let mut delay = Delay::new(c.core.SYST, clocks);
 
         // Init ITM
@@ -82,11 +87,11 @@ const APP: () = {
                 enc424j600::spi::interfaces::SPI_MODE,
                 Hertz(enc424j600::spi::interfaces::SPI_CLOCK_FREQ),
                 clocks);
-            enc424j600::SpiEth::new(spi_eth_port, spi1_nss)
+            enc424j600::SpiEth::new(spi_eth_port, spi1_nss, asm_delay)
         };
 
         // Init
-        match spi_eth.init_dev(&mut delay) {
+        match spi_eth.init_dev() {
             Ok(_) => {
                 iprintln!(stim0, "Initializing Ethernet...")
             }
@@ -97,7 +102,7 @@ const APP: () = {
 
         // Read MAC
         let mut eth_mac_addr: [u8; 6] = [0; 6];
-        spi_eth.read_from_mac(&mut eth_mac_addr);
+        spi_eth.read_from_mac(&mut eth_mac_addr).unwrap();
         for i in 0..6 {
             let byte = eth_mac_addr[i];
             match i {
@@ -109,8 +114,8 @@ const APP: () = {
         }
 
         // Init Rx/Tx buffers
-        spi_eth.init_rxbuf();
-        spi_eth.init_txbuf();
+        spi_eth.init_rxbuf().unwrap();
+        spi_eth.init_txbuf().unwrap();
         iprintln!(stim0, "Ethernet controller initialized");
 
         init::LateResources {
@@ -152,7 +157,7 @@ const APP: () = {
                     _ => ()
                 };
             }
-            c.resources.spi_eth.send_raw_packet(&eth_tx_packet);
+            c.resources.spi_eth.send_raw_packet(&eth_tx_packet).unwrap();
             iprintln!(stim0, "Packet sent");
             c.resources.delay.delay_ms(100_u32);
         }

src/lib.rs

@@ -15,11 +15,14 @@ pub mod tx;
 #[cfg(feature="smoltcp")]
 pub mod smoltcp_phy;
 
+#[cfg(all(feature="smoltcp", feature="nal"))]
+pub mod nal;
+
 /// Max raw frame array size
 pub const RAW_FRAME_LENGTH_MAX: usize = 1518;
 
 pub trait EthController {
-    fn init_dev(&mut self, delay: &mut impl DelayUs<u16>) -> Result<(), EthControllerError>;
+    fn init_dev(&mut self) -> Result<(), EthControllerError>;
     fn init_rxbuf(&mut self) -> Result<(), EthControllerError>;
     fn init_txbuf(&mut self) -> Result<(), EthControllerError>;
     fn receive_next(&mut self, is_poll: bool) -> Result<rx::RxPacket, EthControllerError>;
@@ -45,17 +48,19 @@ impl From<spi::SpiPortError> for EthControllerError {
 
 /// Ethernet controller using SPI interface
 pub struct SpiEth<SPI: Transfer<u8>,
-                  NSS: OutputPin> {
-    spi_port: spi::SpiPort<SPI, NSS>,
+                  NSS: OutputPin,
+                  Delay: DelayUs<u16>> {
+    spi_port: spi::SpiPort<SPI, NSS, Delay>,
     rx_buf: rx::RxBuffer,
     tx_buf: tx::TxBuffer
 }
 
 impl <SPI: Transfer<u8>,
-      NSS: OutputPin> SpiEth<SPI, NSS> {
-    pub fn new(spi: SPI, nss: NSS) -> Self {
+      NSS: OutputPin,
+      Delay: DelayUs<u16>> SpiEth<SPI, NSS, Delay> {
+    pub fn new(spi: SPI, nss: NSS, delay: Delay) -> Self {
         SpiEth {
-            spi_port: spi::SpiPort::new(spi, nss),
+            spi_port: spi::SpiPort::new(spi, nss, delay),
             rx_buf: rx::RxBuffer::new(),
             tx_buf: tx::TxBuffer::new()
         }
@@ -63,8 +68,9 @@ impl <SPI: Transfer<u8>,
 }
 
 impl <SPI: Transfer<u8>,
-      NSS: OutputPin> EthController for SpiEth<SPI, NSS> {
-    fn init_dev(&mut self, delay: &mut impl DelayUs<u16>) -> Result<(), EthControllerError> {
+      NSS: OutputPin,
+      Delay: DelayUs<u16>> EthController for SpiEth<SPI, NSS, Delay> {
+    fn init_dev(&mut self) -> Result<(), EthControllerError> {
         // Write 0x1234 to EUDAST
         self.spi_port.write_reg_16b(spi::addrs::EUDAST, 0x1234)?;
         // Verify that EUDAST is 0x1234
@@ -81,14 +87,14 @@ impl <SPI: Transfer<u8>,
         let econ2 = self.spi_port.read_reg_8b(spi::addrs::ECON2)?;
         self.spi_port.write_reg_8b(spi::addrs::ECON2, 0x10 | (econ2 & 0b11101111))?;
         // Wait for 25us
-        delay.delay_us(25_u16);
+        self.spi_port.delay_us(25_u16);
         // Verify that EUDAST is 0x0000
         eudast = self.spi_port.read_reg_16b(spi::addrs::EUDAST)?;
         if eudast != 0x0000 {
             return Err(EthControllerError::GeneralError)
         }
         // Wait for 256us
-        delay.delay_us(256_u16);
+        self.spi_port.delay_us(256_u16);
         Ok(())
     }
 

src/nal.rs

@@ -0,0 +1,265 @@
+use core::cell::RefCell;
+use heapless::{consts, Vec};
+use embedded_nal as nal;
+use nal::nb;
+use smoltcp as net;
+use embedded_hal::{
+    blocking::spi::Transfer,
+    blocking::delay::DelayUs,
+    digital::v2::OutputPin
+};
+use rtic::cyccnt::Instant;
+
+
+#[derive(Debug)]
+pub enum NetworkError {
+    NoSocket,
+    ConnectionFailure,
+    ReadFailure,
+    WriteFailure,
+    Unsupported,
+}
+
+pub type NetworkInterface<SPI, NSS, Delay> = net::iface::EthernetInterface<
+    'static,
+    crate::smoltcp_phy::SmoltcpDevice<
+        crate::SpiEth<SPI, NSS, Delay>
+    >,
+>;
+
+pub struct NetworkStack<'a, SPI, NSS, Delay>
+where
+    SPI: 'static + Transfer<u8>,
+    NSS: 'static + OutputPin,
+    Delay: 'static + DelayUs<u16>
+{
+    network_interface: RefCell<NetworkInterface<SPI, NSS, Delay>>,
+    sockets: RefCell<net::socket::SocketSet<'a>>,
+    next_port: RefCell<u16>,
+    unused_handles: RefCell<Vec<net::socket::SocketHandle, consts::U16>>,
+    time_ms: RefCell<u32>,
+    last_update_instant: RefCell<Option<Instant>>,
+}
+
+impl<'a, SPI, NSS, Delay> NetworkStack<'a, SPI, NSS, Delay>
+where
+    SPI: Transfer<u8>,
+    NSS: OutputPin,
+    Delay: DelayUs<u16>
+{
+    pub fn new(interface: NetworkInterface<SPI, NSS, Delay>, sockets: net::socket::SocketSet<'a>) -> Self {
+        let mut unused_handles: Vec<net::socket::SocketHandle, consts::U16> = Vec::new();
+        for socket in sockets.iter() {
+            unused_handles.push(socket.handle()).unwrap();
+        }
+
+        NetworkStack {
+            network_interface: RefCell::new(interface),
+            sockets: RefCell::new(sockets),
+            next_port: RefCell::new(49152),
+            unused_handles: RefCell::new(unused_handles),
+            time_ms: RefCell::new(0),
+            last_update_instant: RefCell::new(None),
+        }
+    }
+
+    // Include auto_time_update to allow Instant::now() to not be called
+    // Instant::now() is not safe to call in `init()` context
+    pub fn update(&self, auto_time_update: bool) -> bool {
+        if auto_time_update {
+            // Check if it is the first time the stack has updated the time itself
+            let now = match *self.last_update_instant.borrow() {
+                // If it is the first time, do not advance time
+                // Simply store the current instant to initiate time updating
+                None => Instant::now(),
+
+                // If it was updated before, advance time and update last_update_instant
+                Some(instant) => {
+                    // Calculate elapsed time
+                    let now = Instant::now();
+                    let duration = now.duration_since(instant);
+                    // Adjust duration into ms (note: decimal point truncated)
+                    self.advance_time(duration.as_cycles() / 168_000);
+                    now
+                }
+            };
+
+            self.last_update_instant.replace(Some(now));
+        }
+
+        match self.network_interface.borrow_mut().poll(
+            &mut self.sockets.borrow_mut(),
+            net::time::Instant::from_millis(*self.time_ms.borrow() as i64),
+        ) {
+            Ok(changed) => changed == false,
+            Err(_e) => {
+                true
+            }
+        }
+    }
+
+    pub fn advance_time(&self, duration: u32) {
+        *self.time_ms.borrow_mut() += duration;
+    }
+
+    fn get_ephemeral_port(&self) -> u16 {
+        // Get the next ephemeral port
+        let current_port = self.next_port.borrow().clone();
+
+        let (next, wrap) = self.next_port.borrow().overflowing_add(1);
+        *self.next_port.borrow_mut() = if wrap { 49152 } else { next };
+
+        return current_port;
+    }
+}
+
+impl<'a, SPI, NSS, Delay> nal::TcpStack for NetworkStack<'a, SPI, NSS, Delay>
+where
+    SPI: Transfer<u8>,
+    NSS: OutputPin,
+    Delay: DelayUs<u16>
+{
+    type TcpSocket = net::socket::SocketHandle;
+    type Error = NetworkError;
+
+    fn open(&self, _mode: nal::Mode) -> Result<Self::TcpSocket, Self::Error> {
+        match self.unused_handles.borrow_mut().pop() {
+            Some(handle) => {
+                // Abort any active connections on the handle.
+                let mut sockets = self.sockets.borrow_mut();
+                let internal_socket: &mut net::socket::TcpSocket = &mut *sockets.get(handle);
+                internal_socket.abort();
+
+                Ok(handle)
+            }
+            None => Err(NetworkError::NoSocket),
+        }
+    }
+
+    // Ideally connect is only to be performed in `init()` of `main.rs`
+    // Calling `Instant::now()` of `rtic::cyccnt` would face correctness issue during `init()`
+    fn connect(
+        &self,
+        socket: Self::TcpSocket,
+        remote: nal::SocketAddr,
+    ) -> Result<Self::TcpSocket, Self::Error> {
+        let address = {
+            let mut sockets = self.sockets.borrow_mut();
+            let internal_socket: &mut net::socket::TcpSocket = &mut *sockets.get(socket);
+
+            // If we're already in the process of connecting, ignore the request silently.
+            if internal_socket.is_open() {
+                return Ok(socket);
+            }
+
+            match remote.ip() {
+                nal::IpAddr::V4(addr) => {
+                    let octets = addr.octets();
+                    let address =
+                        net::wire::Ipv4Address::new(octets[0], octets[1], octets[2], octets[3]);
+                    internal_socket
+                        .connect((address, remote.port()), self.get_ephemeral_port())
+                        .map_err(|_| NetworkError::ConnectionFailure)?;
+                    address
+                }
+                nal::IpAddr::V6(_) => {
+                    // Match W5500 behavior: Reject the use of IPV6
+                    return Err(NetworkError::Unsupported);
+                }
+            }
+        };
+
+        // Match W5500 behavior: Poll until connected
+        loop {
+            match self.is_connected(&socket) {
+                Ok(true) => break,
+                _ => {
+                    let mut sockets = self.sockets.borrow_mut();
+                    let internal_socket: &mut net::socket::TcpSocket = &mut *sockets.get(socket);
+                    // If the connect got ACK->RST, it will end up in Closed TCP state
+                    // Perform reconnection in this case
+                    // In all other scenario, simply wait for TCP connection to be established
+                    if internal_socket.state() == net::socket::TcpState::Closed {
+                        internal_socket.close();
+                        internal_socket
+                            .connect((address, remote.port()), self.get_ephemeral_port())
+                            .map_err(|_| NetworkError::ConnectionFailure)?;
+                    }
+                }
+            }
+
+            // Avoid using Instant::now() and Advance time manually
+            self.update(false);
+
+            // Delay for 1 ms, minimum time unit of smoltcp
+            // TODO: Allow clock configuration, if supported in main
+            cortex_m::asm::delay(168_000_000 / 1_000);
+            {
+                self.advance_time(1);
+            }
+        }
+
+        Ok(socket)
+    }
+
+    fn is_connected(&self, socket: &Self::TcpSocket) -> Result<bool, Self::Error> {
+        let mut sockets = self.sockets.borrow_mut();
+        let socket: &mut net::socket::TcpSocket = &mut *sockets.get(*socket);
+
+        Ok(socket.may_send() && socket.may_recv())
+    }
+
+    fn write(&self, socket: &mut Self::TcpSocket, buffer: &[u8]) -> nb::Result<usize, Self::Error> {
+        let mut non_queued_bytes = &buffer[..];
+        while non_queued_bytes.len() != 0 {
+            let result = {
+                let mut sockets = self.sockets.borrow_mut();
+                let socket: &mut net::socket::TcpSocket = &mut *sockets.get(*socket);
+                let result = socket.send_slice(non_queued_bytes);
+                result
+            };
+
+            match result {
+                Ok(num_bytes) => {
+                    // In case the buffer is filled up, push bytes into ethernet driver
+                    if num_bytes != non_queued_bytes.len() {
+                        self.update(true);
+                    }
+
+                    // Process the unwritten bytes again, if any
+                    non_queued_bytes = &non_queued_bytes[num_bytes..]
+                }
+                Err(_) => return Err(nb::Error::Other(NetworkError::WriteFailure)),
+            }
+        }
+
+        Ok(buffer.len())
+    }
+
+    fn read(
+        &self,
+        socket: &mut Self::TcpSocket,
+        buffer: &mut [u8],
+    ) -> nb::Result<usize, Self::Error> {
+        // Enqueue received bytes into the TCP socket buffer
+        self.update(true);
+
+        let mut sockets = self.sockets.borrow_mut();
+        let socket: &mut net::socket::TcpSocket = &mut *sockets.get(*socket);
+
+        let result = socket.recv_slice(buffer);
+        match result {
+            Ok(num_bytes) => Ok(num_bytes),
+            Err(_) => Err(nb::Error::Other(NetworkError::ReadFailure)),
+        }
+    }
+
+    fn close(&self, socket: Self::TcpSocket) -> Result<(), Self::Error> {
+        let mut sockets = self.sockets.borrow_mut();
+        let internal_socket: &mut net::socket::TcpSocket = &mut *sockets.get(socket);
+        internal_socket.close();
+
+        self.unused_handles.borrow_mut().push(socket).unwrap();
+        Ok(())
+    }
+}

src/spi.rs

@@ -1,5 +1,5 @@
 use embedded_hal::{
-    blocking::spi::Transfer,
+    blocking::{spi::Transfer, delay::DelayUs},
     digital::v2::OutputPin,
 };
 
@@ -52,9 +52,11 @@ pub mod addrs {
 /// Struct for SPI I/O interface on ENC424J600
 /// Note: stm32f4xx_hal::spi's pins include: SCK, MISO, MOSI
 pub struct SpiPort<SPI: Transfer<u8>,
-                   NSS: OutputPin> {
+                   NSS: OutputPin,
+                   Delay: DelayUs<u16>> {
     spi: SPI,
     nss: NSS,
+    delay: Delay,
 }
 
 pub enum SpiPortError {
@@ -63,14 +65,16 @@ pub enum SpiPortError {
 
 #[allow(unused_must_use)]
 impl <SPI: Transfer<u8>,
-      NSS: OutputPin> SpiPort<SPI, NSS> {
+      NSS: OutputPin,
+      Delay: DelayUs<u16>> SpiPort<SPI, NSS, Delay> {
     // TODO: return as Result()
-    pub fn new(spi: SPI, mut nss: NSS) -> Self {
+    pub fn new(spi: SPI, mut nss: NSS, delay: Delay) -> Self {
         nss.set_high();
 
         SpiPort {
             spi,
-            nss
+            nss,
+            delay
         }
     }
 
@@ -115,6 +119,10 @@ impl <SPI: Transfer<u8>,
         Ok(())
     }
 
+    pub fn delay_us(&mut self, duration: u16) {
+        self.delay.delay_us(duration)
+    }
+
     // TODO: Generalise transfer functions
     // TODO: (Make data read/write as reference to array)
     // Currently requires 1-byte addr, read/write data is only 1-byte
@@ -131,17 +139,17 @@ impl <SPI: Transfer<u8>,
         match self.spi.transfer(&mut buf) {
             Ok(_) => {
                 // Disable chip select
-                cortex_m::asm::delay(10_u32);
+                self.delay_us(1);
                 self.nss.set_high();
-                cortex_m::asm::delay(5_u32);
+                self.delay_us(1);
                 Ok(buf[2])
             },
             // TODO: Maybe too naive?
             Err(_) => {
                 // Disable chip select
-                cortex_m::asm::delay(10_u32);
+                self.delay_us(1);
                 self.nss.set_high();
-                cortex_m::asm::delay(5_u32);
+                self.delay_us(1);
                 Err(SpiPortError::TransferError)
             }
         }