README.md

@@ -1,31 +0,0 @@
-## Automatic Test Scripts
-
-#### Shell
-(Remember to **edit the settings** in `rp_ping_test` and `rp_shutdown` and the **test arguments** passed to `get_and_plot_sayma_data` according to your own setup!)
-
-* `test_mlabs`: Run an automatically-timed test using M-Labs' current setup
-* `test_creotech`: Run an automatically-timed test using Creotech's current setup
-
-## Data Collection & Analytics Tools
-
-#### Python
-(To see detailed usage, add `--help` as an argument when running the script.)
-
-* `rp_get_sayma_data.py`: Acquire and save data from RedPitayas over local network
-* `plot_sayma_data.py`: Load saved data and output phase skew measurements and data plots
-* `analyze_sayma_data.py`: Generate a data analysis report from saved data records
-
-#### Shell
-* `get_and_plot_sayma_data`: Acquire and analyse data from RedPitayas over local network
-* `get_and_plot_remote_sayma_data`: Acquire and analyse data from remote RedPitayas via SSH (often over different networks) to the local file system
-  * Requires Python testsuite scripts present on the remote host
-
-## System Utilities
-
-#### Shell
-* `mch_start`: Power on the MCH and start logging serial outputs from Metlino and Saymas
-* `mch_stop`: Power off the MCH and stop logging serial outputs from Metlino and Saymas
-* `rp_ping_test`: Loop to ping a RedPitaya until it succeeds
-  * The exit condition is where the RedPitaya is ready-to-use
-* `rp_shutdown`: Shut down the OS of a RedPitaya and loop to ping it until it fails 
-  * The exit condition is where the RedPitaya is ready for hardware power-off

analyze_sayma_data.py

@@ -1,129 +0,0 @@
-import numpy as np
-import argparse
-import os
-import datetime
-
-
-DATA_COLUMNS = np.array([
-    ["data_time_iso", "datetime64[s]"],
-    ["phase_radian", "f"],
-])
-
-
-REPORT_PARAMS_KEYS = [
-    "rpt_time",
-    "data_time_start", "data_time_end", "data_count",
-    "wave_type",    # type of DDS waveform
-    "wave_freq",    # frequency of the DDS waveform
-    "phase_ps_mean", "phase_ps_min", "phase_ps_max",
-    "phase_ps_std", # standard deviation
-    "phase_ps_meanabsdev",  # mean absolute deviation
-]
-
-
-def report(*args, **kwargs):
-    if len(args) != 1 or not isinstance(args[0], dict):
-        raise SyntaxError("Must only pass 1 dict argument, or >=1 keyword arguments.")
-    # Create and store lines of strings for the report
-    rpt = []
-    prm = args[0]
-    for k in REPORT_PARAMS_KEYS:
-        if prm.get(k) is None and kwargs.get(k) is None:
-            raise ValueError("Data missing for report item {}.".format(k))
-        # Replace or append item from kwargs onto the prm dict
-        if kwargs.get(k) is not None:
-            prm[k] = kwargs[k]
-    # Report title
-    rpt.append("Sayma DAC/TTL Data Analysis")
-    rpt.append("")
-    # Report header
-    rpt.append("Report generated at: {}".format(prm["rpt_time"]))
-    rpt.append("------")
-    rpt.append("")
-    # Report content
-    rpt.append("Data collection started at: {}".format(prm["data_time_start"]))
-    rpt.append("                ended at:   {}".format(prm["data_time_end"]))
-    rpt.append("Total number of records:    {}".format(prm["data_count"]))
-    rpt.append("")
-    rpt.append("Target wave type:      {}".format(prm["wave_type"]))
-    rpt.append("            frequency: {:.2f} MHz".format(prm["wave_freq"]/1e6))
-    rpt.append("")
-    rpt.append("Phase skew statistics:")
-    rpt.append("  Mean:                    {:>10.4f} ps".format(prm["phase_ps_mean"]))
-    rpt.append("  Minimum:                 {:>10.4f} ps".format(prm["phase_ps_min"]))
-    rpt.append("  Maximum:                 {:>10.4f} ps".format(prm["phase_ps_max"]))
-    rpt.append("  Standard Deviation:      {:>10.4f} ps".format(prm["phase_ps_std"]))
-    rpt.append("  Max Absolute Deviation:  {:>10.4f} ps".format(prm["phase_ps_maxabsdev"]))
-    rpt.append("  Mean Absolute Deviation: {:>10.4f} ps".format(prm["phase_ps_meanabsdev"]))
-    rpt.append("")
-    # TODO: Use jinja2 to produce a report
-    return '\n'.join(rpt)
-
-
-def main():
-    # Get timestamp at script start
-    now = datetime.datetime.now().replace(
-        microsecond=0,  # get rid of microseconds when printed
-    )
-    now_iso = now.isoformat(sep=' ')
-
-    parser = argparse.ArgumentParser(description="Data analysis tool for Sayma DAC/TTL")
-    parser.add_argument("log",
-        help="path of the log file containing the data records; "
-        "must be a CSV in excel-tab dialect",
-        type=str)
-    parser.add_argument("--rpt",
-        help="path of the file where the analysis will be reported (and overwrite)",
-        type=str)
-    args = parser.parse_args()
-
-    with open(args.log, 'r') as f:
-        print("Analysis of data records started at {}.".format(now_iso))
-        # Create a numpy array from the CSV data (it has no header row)
-        # Currently, log files from plot_sayma_data.py uses TAB as delimiter
-        data = np.loadtxt(f, delimiter='\t',
-            dtype={'names': DATA_COLUMNS[:, 0], 'formats': DATA_COLUMNS[:, 1]})
-        # Clean-up: remove duplicate rows
-        data_rows = len(data)       # original row count w/ potential duplicates
-        data = np.unique(data, axis=0)
-        print("{} duplicate rows detected and ignored.".format(data_rows - len(data)))
-        data_rows = len(data)       # new row count without duplicates
-        # Define dict of params for generating the report
-        rpt_params = dict()
-        rpt_params["rpt_time"] = now_iso
-        # Analyse: data collection time & count
-        rpt_params["data_time_start"] = str(data["data_time_iso"].min()).replace('T', ' ')
-        rpt_params["data_time_end"] = str(data["data_time_iso"].max()).replace('T', ' ')
-        rpt_params["data_count"] = data_rows
-        # Analyse: phase stats
-        # TODO: Check if square waves use the same calculation as sinusoids
-        WAVE_TYPES = ["Sinusoid", "Square wave"]
-        rpt_params["wave_type"] = WAVE_TYPES[int(
-            input(">> Input: target wave type (use the index)\n"
-                  "   ({}): ".format(", ".join(["{}={}".format(i,x) for i,x in enumerate(WAVE_TYPES)])))
-        )]
-        rpt_params["wave_freq"] = float(
-            input(">> Input: target wave frequency (in Hz): ")
-        )
-        # Conversion formula: phase_time = (1/wave_freq) * (phase_angle/2/pi)
-        data_phase_ps = data["phase_radian"] / rpt_params["wave_freq"] / 2 / np.pi * 1e12
-        rpt_params["phase_ps_mean"] = data_phase_ps.mean()
-        rpt_params["phase_ps_min"] = data_phase_ps.min()
-        rpt_params["phase_ps_max"] = data_phase_ps.max()
-        rpt_params["phase_ps_std"] = data_phase_ps.std()
-        rpt_params["phase_ps_maxabsdev"] = \
-            np.absolute(data_phase_ps - data_phase_ps.max()).mean()
-        rpt_params["phase_ps_meanabsdev"] = \
-            np.absolute(data_phase_ps - data_phase_ps.mean()).mean()
-        # Generate the report
-        rpt = report(rpt_params)
-        # Print the report
-        print(rpt)
-
-
-    # TODO: Implement report output feature
-    if args.rpt is not None:
-        raise NotImplementedError("Report file output feature has not been implemented.")
-
-if __name__ == "__main__":
-    main()

get_and_plot_remote_sayma_data

@@ -38,9 +38,9 @@ fi
 # Acquire data at the remote
 if [[ $1 = $3 ]]
 then
-    ssh -t -p $HOSTPORT $HOST "cd mlabs-testsuite && python3 rp_get_sayma_data.py raw $5"
+	ssh -t -p $HOSTPORT $HOST "cd mlabs-testsuite && python3 rp_get_sayma_data.py raw $5"
 else
-    ssh -t -p $HOSTPORT $HOST "cd mlabs-testsuite && python3 rp_get_sayma_data.py raw $5 $6"
+	ssh -t -p $HOSTPORT $HOST "cd mlabs-testsuite && python3 rp_get_sayma_data.py raw $5 $6"
 fi
 
 # Transfer data from remote to local

get_and_plot_sayma_data

@@ -16,7 +16,7 @@ then
     echo "Arguments: <dir name> <RP#1 name> <RP#1 channel> <RP#2 name> <RP#1 channel> <RP#1 name>:<IN1 gain>,<IN2 gain> [<RP#2 name>:<IN1 gain>,<IN2 gain>]"
     echo "(If RP#1 and RP#2 are the same, skip the bracketed arguments)"
     echo "Examples: "
-    echo "(1) mlabs-raw mlabs-1 1 mlabs-1 2 mlabs-1:LV,HV"
+    echo "(1) mlabs-raw mlabs 1 mlabs 2 mlabs:LV,HV"
     echo "(2) creotech-raw creotech-1 1 creotech-2 1 creotech-1:LV,LV creotech-2:HV,HV"
     exit
 fi
@@ -24,9 +24,9 @@ fi
 # Acquire data from RPs
 if [[ $2 = $4 ]]
 then
-    python3 rp_get_sayma_data.py $1 $6
+	python3 rp_get_sayma_data.py $1 $6
 else
-    python3 rp_get_sayma_data.py $1 $6 $7
+	python3 rp_get_sayma_data.py $1 $6 $7
 fi
 
 # Plot data from saved data

plot_sayma_data.py

@@ -3,7 +3,6 @@ import matplotlib.pyplot as plot
 from scipy import signal, constants
 import argparse
 import os
-import datetime
 
 
 def rp_raw_to_numpy(rp_raw):
@@ -15,8 +14,7 @@ def rp_raw_to_numpy(rp_raw):
 RP_IP_ADDRS = {
     "creotech-1": "192.168.1.104",
     "creotech-2": "192.168.1.105",
-    "mlabs-1": "rp-f05cc9",
+    "mlabs": "rp-f05cc9",
-    "mlabs-2": "rp-f0612e",
 }
 
 
@@ -28,12 +26,6 @@ def main():
         "CHANNEL must be 1 or 2; "
         "NAME must be any of: " + " ".join(list(RP_IP_ADDRS.keys())),
         type=str, nargs=2)
-    parser.add_argument("--log",
-        help="path of the log file where the measurement record will be appended",
-        type=str)
-    parser.add_argument("--noplot",
-        help="do not show data plot, which is blocking until the GUI is closed",
-        action="store_true")
     args = parser.parse_args()
 
     # Must only compare 2 data
@@ -45,15 +37,11 @@ def main():
         raise ValueError("Both files are the same.")
 
     with open(os.path.join(args.dir, y1_filename), 'rb') as f:
-        y1_now_iso, y1_raw = [l.decode('utf-8') for l in f.readlines()]
+        y1_raw = f.read().decode('utf-8')
     with open(os.path.join(args.dir, y2_filename), 'rb') as f:
-        y2_now_iso, y2_raw = [l.decode('utf-8') for l in f.readlines()]
+        y2_raw = f.read().decode('utf-8')
     if None in [y1_raw, y2_raw]:
         raise IOError("Raw RP string files cannot be opened.")
-    if y1_now_iso != y2_now_iso:
-        raise ValueError("Timestamps of raw RP files are not identical.")
-    now_iso = y1_now_iso.rstrip()
-    print("Reading raw RP data collected at {}.".format(now_iso))
 
     y1 = rp_raw_to_numpy(y1_raw)
     y2 = rp_raw_to_numpy(y2_raw)
@@ -71,22 +59,14 @@ def main():
     # Element-wise multiply Z[0] with the conjugate of Z[1] to get the phase difference (i.e. angle(z0) - angle(z1)), and use the mean value.
     angle = np.angle(np.mean(z[0]*z[1].conj()))
 
-    # Append the phase difference to the log file
-    log = args.log
-    if log is not None:
-        with open(log, 'a') as f:
-            f.write("{}\t{}\n".format(now_iso, angle))
-            print("Phase measurement record appended to log: {}".format(log))
-    # Print the phase difference
     print(angle)
 
-    if not args.noplot:
+    # Normalize y1 and y2 for plotting
-        # Normalize y1 and y2 for plotting
+    y1 /= abs(y1).max()
-        y1 /= abs(y1).max()
+    y2 /= abs(y2).max()
-        y2 /= abs(y2).max()
+    plot.plot(y1)
-        plot.plot(y1)
+    plot.plot(y2)
-        plot.plot(y2)
+    plot.show()
-        plot.show()
 
 if __name__ == "__main__":
     main()

rp_get_sayma_data.py

@@ -1,8 +1,8 @@
 import socket
+from time import sleep
 import asyncio
 import argparse
 import os
-import datetime
 
 
 class RPSCPI:
@@ -48,8 +48,7 @@ class RPSCPI:
 RP_IP_ADDRS = {
     "creotech-1": "192.168.1.104",
     "creotech-2": "192.168.1.105",
-    "mlabs-1": "rp-f05cc9",
+    "mlabs": "rp-f05cc9",
-    "mlabs-2": "rp-f0612e",
 }
 
 
@@ -58,12 +57,6 @@ async def gather_trigger(rp_list):
 
 
 def main():
-    # Get timestamp at script start
-    now = datetime.datetime.now().replace(
-        microsecond=0,  # get rid of microseconds when printed
-    )
-    now_iso = now.isoformat(sep=' ')
-
     parser = argparse.ArgumentParser(description="Data collection tool for Sayma DAC/TTL at RedPitaya")
     parser.add_argument("dir", help="output directory", type=str)
     parser.add_argument("rps_gains", metavar="RP_NAME:CH1_GAIN,CH2_GAIN",
@@ -102,37 +95,22 @@ def main():
         for rp in rps:
             y1_raw = rp.get_data(1)
             y2_raw = rp.get_data(2)
-            print("Data collection started at {}.".format(now_iso))
 
             with open(os.path.join(args.dir, 'rp_{}_y1_raw.bin'.format(rp.name)), 'wb') as f:
-                f.write("{}\n".format(now_iso).encode('utf-8'))
                 f.write(y1_raw.encode('utf-8'))
                 print("Succesfully written y1 raw data from RP {}.".format(rp.name))
             with open(os.path.join(args.dir, 'rp_{}_y2_raw.bin'.format(rp.name)), 'wb') as f:
-                f.write("{}\n".format(now_iso).encode('utf-8'))
                 f.write(y2_raw.encode('utf-8'))
                 print("Succesfully written y2 raw data from RP {}.".format(rp.name))
 
             if args.txt:
-                header = '''\
-                    RedPitaya Oscilloscope Reading
-
-                    RP Hostname:    {} ({})
-                    RP Channel:     Input {} ({})
-                    Date & Time:    {}
-                    ------
-                    '''
                 y1 = [float(i) for i in y1_raw.split(',')]
                 y2 = [float(i) for i in y2_raw.split(',')]
                 with open(os.path.join(args.dir, 'rp_{}_y1_raw.txt'.format(rp.name)), 'w') as f:
-                    f.write(''.join([l.lstrip(' ') for l in header.format(
-                        RP_IP_ADDRS[rp.name], rp.name, 1, ch1_gain.upper(), now_iso).splitlines(True)]))
                     for i in y1:
                         f.write(str(i) + '\n')
                     print("Succesfully written y1 human-readable data from {}.".format(rp.name))
                 with open(os.path.join(args.dir, 'rp_{}_y2_raw.txt'.format(rp.name)), 'w') as f:
-                    f.write(''.join([l.lstrip(' ') for l in header.format(
-                        RP_IP_ADDRS[rp.name], rp.name, 2, ch2_gain.upper(), now_iso).splitlines(True)]))
                     for i in y2:
                         f.write(str(i) + '\n')
                     print("Succesfully written y2 human-readable data from {}.".format(rp.name))

rp_ping_test

@@ -1,13 +0,0 @@
-#!/bin/sh
-
-# RedPitaya hostname
-export RP_HOST=rp-f0612e
-
-# Check if RedPitaya becomes connectable from network by pinging
-while true
-do
-    ping $RP_HOST -c 1 >/dev/null 2>/dev/null && break
-done
-# Print time when RedPitaya ready on network
-TIMESTAMP_PRETTY=$(date '+%Y-%m-%d %H:%M:%S')
-echo "RP becomes connectable at $TIMESTAMP_PRETTY."

rp_shutdown

@@ -1,26 +0,0 @@
-#!/bin/sh
-
-# RedPitaya hostname and user information
-# (The user should have privilege to run `shutdown` - check out `visudo`)
-# (Add local SSH key to RP's authorized keys for passwordless login)
-export RP_HOST=rp-f0612e
-export RP_USER=tester
-export RP_KEY=~/.ssh/rp
-
-# Issue shutdown on the RedPitaya
-echo "Requesting shutdown on RP at $RP_HOST."
-if [[ $RP_KEY == "" ]]
-then
-    ssh $RP_USER@$RP_HOST "sudo shutdown now"
-else
-    ssh -i $RP_KEY $RP_USER@$RP_HOST "sudo shutdown now"
-fi
-
-# Check if RedPitaya has been disconnected from network by pinging
-while ping $RP_HOST -c 1 >/dev/null 2>/dev/null
-do
-    true
-done
-# Print time when RedPitaya is down on network
-TIMESTAMP_PRETTY=$(date '+%Y-%m-%d %H:%M:%S')
-echo "RP disconnected at $TIMESTAMP_PRETTY."

test_creotech

@@ -1,30 +0,0 @@
-#!/bin/sh
-
-# Simultaneously, start testing RP's network connectivity, while
-# powering on MCH and RP using ~/power_strip/ scripts
-# Display power-on time
-TIMESTAMP_PRETTY=$(date '+%Y-%m-%d %H:%M:%S')
-echo "MCH powered on at $TIMESTAMP_PRETTY."
-# Power on MCH 
-./mch_start &
-# Wait 2 minutes before measurement
-sleep 120
-
-# Get and plot Sayma data
-# (Assumption: numpy, matplotlib and scipy have been installed on local)
-# (Assumption: directory ./creotech-raw exists)
-# NOTE: Change the RP hostname and LV/HV arguments according to
-# which two channels you are measuring!
-./get_and_plot_sayma_data creotech-raw creotech-1 1 creotech-1 2 creotech-1:LV,LV &
-# Wait 1 minute before powering off MCH and RP
-sleep 60
-
-# Wait 30 seconds
-sleep 30
-
-# Simultaneously power off MCH and RP
-# Display power-off time
-TIMESTAMP_PRETTY=$(date '+%Y-%m-%d %H:%M:%S')
-echo "MCH powered off at $TIMESTAMP_PRETTY."
-# Power off MCH
-./mch_stop

test_mlabs

@@ -1,44 +0,0 @@
-#!/bin/sh
-
-# tnetplug host
-export TNETPLUG_HOST=192.168.1.31
-export TNETPLUG_PORT=3131
-
-# Smart USB hub host
-# (uhubctl must be installed on the host)
-export HOST=rpi-3
-export HOSTPORT=22
-# Hub port where the RedPitaya is connected
-export HUBPORT=4
-
-# Simultaneously, start testing RP's network connectivity, while
-# powering on MCH using tnetplug, and RP using uhubctl
-# Display power-on time
-TIMESTAMP_PRETTY=$(date '+%Y-%m-%d %H:%M:%S')
-echo "MCH powered on at $TIMESTAMP_PRETTY."
-# Power on MCH
-echo "A" | nc -W1 $TNETPLUG_HOST $TNETPLUG_PORT &
-# Power on RedPitaya connected to the smart USB hub
-# (assume that only 1 hub is connected)
-ssh -p $HOSTPORT $HOST "uhubctl -a on -p $HUBPORT" &
-# Wait 2 minutes before measurement
-sleep 120
-
-# Get and plot Sayma data
-# NOTE: Change the RP hostname and LV/HV arguments according to
-# which two channels you are measuring!
-nix-shell -p python3Packages.matplotlib python3Packages.numpy python3Packages.scipy --run "./get_and_plot_sayma_data mlabs-raw mlabs-1 1 mlabs-1 2 mlabs-1:LV,LV" &
-# Wait 1 minute before powering off MCH and RP
-sleep 60
-
-# Wait 30 seconds
-sleep 30
-
-# Simultaneously power off MCH and RP
-# Display power-off time
-TIMESTAMP_PRETTY=$(date '+%Y-%m-%d %H:%M:%S')
-echo "MCH powered off at $TIMESTAMP_PRETTY."
-# Power off MCH
-echo "a" | nc -W1 $TNETPLUG_HOST $TNETPLUG_PORT &
-# Power off RedPitaya connected to the smart USB hub
-ssh -p $HOSTPORT $HOST "uhubctl -a off -p $HUBPORT"