From a87a0a224fc3e3146fcbcd3db215a396a7cfa9ec Mon Sep 17 00:00:00 2001 From: linuswck Date: Wed, 18 Dec 2024 11:09:38 +0800 Subject: [PATCH] Upgrade linien from v1.0.0 to v2.1.0 - Add linien GUI dev shell for fast servo platform --- fast-servo/linien-fast-servo-gateware.patch | 38 ----- ...patch => linien-gateware-fast-servo.patch} | 96 +++++++----- fast-servo/linien-pyrp3-monitor.patch | 68 ++------- fast-servo/linien-server-fast-servo.patch | 26 ++++ flake.nix | 144 +++++++++++++++--- 5 files changed, 221 insertions(+), 151 deletions(-) delete mode 100644 fast-servo/linien-fast-servo-gateware.patch rename fast-servo/{linien-fast-servo-server.patch => linien-gateware-fast-servo.patch} (90%) create mode 100644 fast-servo/linien-server-fast-servo.patch diff --git a/fast-servo/linien-fast-servo-gateware.patch b/fast-servo/linien-fast-servo-gateware.patch deleted file mode 100644 index cb294fe..0000000 --- a/fast-servo/linien-fast-servo-gateware.patch +++ /dev/null @@ -1,38 +0,0 @@ -# diff from elhep/Fast-Servo-Firmmware commit ID 7fae40c: -# https://github.com/elhep/Fast-Servo-Firmware/commit/7fae40c0f872a91218be378f8289b98b1e366729 -# Fix for migen add_source deprecation and removed xilinx bootgen command -# .bin file is being generated by bit2bin.py from Linien repository -# https://github.com/linien-org/linien/blob/master/gateware/bit2bin.py - -diff --git a/fast_servo/gateware/fast_servo_platform.py b/fast_servo/gateware/fast_servo_platform.py -index 13b4aa3..89a8103 100644 ---- a/fast_servo/gateware/fast_servo_platform.py -+++ b/fast_servo/gateware/fast_servo_platform.py -@@ -324,7 +324,12 @@ class Platform(XilinxPlatform): - self.ps7_config = ps7_config - - verilog_sources = os.listdir(verilog_dir) -- self.add_sources(verilog_dir, *verilog_sources) -+ self.add_source_dir(verilog_dir) -+ -+ def build(self, *args, **kwargs): -+ build_dir = kwargs.get('build_dir', 'build') -+ self.copy_sources(build_dir) -+ super().build(*args, **kwargs) - - def do_finalize(self, fragment): - try: -diff --git a/fast_servo/gateware/fast_servo_soc.py b/fast_servo/gateware/fast_servo_soc.py -index 02128f5..abfc583 100644 ---- a/fast_servo/gateware/fast_servo_soc.py -+++ b/fast_servo/gateware/fast_servo_soc.py -@@ -282,9 +282,3 @@ if __name__ == "__main__": - os.chdir(os.path.join(root_path, build_dir)) - with open(f"{build_name}.bif", "w") as f: - f.write(f"all:\n{{\n\t{build_name}.bit\n}}") -- -- cmd = f"bootgen -image {build_name}.bif -arch zynq -process_bitstream bin -w on".split(" ") -- subprocess.run(cmd) -- -- -- diff --git a/fast-servo/linien-fast-servo-server.patch b/fast-servo/linien-gateware-fast-servo.patch similarity index 90% rename from fast-servo/linien-fast-servo-server.patch rename to fast-servo/linien-gateware-fast-servo.patch index 018f974..1aa1086 100644 --- a/fast-servo/linien-fast-servo-server.patch +++ b/fast-servo/linien-gateware-fast-servo.patch @@ -1,3 +1,42 @@ +# diff from elhep/Fast-Servo-Firmmware commit ID 7fae40c: +# https://github.com/elhep/Fast-Servo-Firmware/commit/7fae40c0f872a91218be378f8289b98b1e366729 +# Fix for migen add_source deprecation and removed xilinx bootgen command +# .bin file is being generated by bit2bin.py from Linien repository +# https://github.com/linien-org/linien/blob/master/gateware/bit2bin.py + +diff --git a/fast_servo/gateware/fast_servo_platform.py b/fast_servo/gateware/fast_servo_platform.py +index 13b4aa3..89a8103 100644 +--- a/fast_servo/gateware/fast_servo_platform.py ++++ b/fast_servo/gateware/fast_servo_platform.py +@@ -324,7 +324,12 @@ class Platform(XilinxPlatform): + self.ps7_config = ps7_config + + verilog_sources = os.listdir(verilog_dir) +- self.add_sources(verilog_dir, *verilog_sources) ++ self.add_source_dir(verilog_dir) ++ ++ def build(self, *args, **kwargs): ++ build_dir = kwargs.get('build_dir', 'build') ++ self.copy_sources(build_dir) ++ super().build(*args, **kwargs) + + def do_finalize(self, fragment): + try: +diff --git a/fast_servo/gateware/fast_servo_soc.py b/fast_servo/gateware/fast_servo_soc.py +index 02128f5..abfc583 100644 +--- a/fast_servo/gateware/fast_servo_soc.py ++++ b/fast_servo/gateware/fast_servo_soc.py +@@ -282,9 +282,3 @@ if __name__ == "__main__": + os.chdir(os.path.join(root_path, build_dir)) + with open(f"{build_name}.bif", "w") as f: + f.write(f"all:\n{{\n\t{build_name}.bit\n}}") +- +- cmd = f"bootgen -image {build_name}.bif -arch zynq -process_bitstream bin -w on".split(" ") +- subprocess.run(cmd) +- +- +- + # diff between linen-org/linien commit ID 93f1f50: # https://github.com/linien-org/linien/commit/93f1f50ebd86fe3314cab5a549462d0fcbf6a658 # and elhep/linien commit ID b73eea0: @@ -16,41 +55,27 @@ index b3f3683..98c6e51 100644 - repo: https://github.com/pycqa/isort rev: 5.12.0 -diff --git a/gateware/build_fpga_image.sh b/gateware/build_fpga_image.sh -index f822402..be7401c 100644 ---- a/gateware/build_fpga_image.sh -+++ b/gateware/build_fpga_image.sh -@@ -16,4 +16,9 @@ export PATH=$VIVADOPATH:$PATH - - rm linien-server/linien_server/gateware.bin -f - # run with -m option to avoid errors related to relative imports without breaking pytest --python3 -m gateware.fpga_image_helper -\ No newline at end of file -+ -+if [ -z "$1" ]; then -+ python3 -m gateware.fpga_image_helper -+else -+ python3 -m gateware.fpga_image_helper -p $1 -+fi -\ No newline at end of file diff --git a/gateware/fpga_image_helper.py b/gateware/fpga_image_helper.py -index 6c34429..a0b12d0 100644 +index c3e20e7..ebead1d 100644 --- a/gateware/fpga_image_helper.py +++ b/gateware/fpga_image_helper.py -@@ -1,5 +1,6 @@ - # Copyright 2014-2015 Robert Jördens - # Copyright 2018-2022 Benjamin Wiegand -+# Copyright 2023 Jakub Matyas - # +@@ -1,6 +1,7 @@ # This file is part of Linien and based on redpid. # -@@ -23,14 +24,16 @@ from pathlib import Path - REPO_ROOT_DIR = Path(__file__).resolve().parents[1] + # Copyright (C) 2016-2024 Linien Authors (https://github.com/linien-org/linien#license) ++# Copyright 2023 Jakub Matyas + # + # Linien is free software: you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by +@@ -20,16 +21,18 @@ + from pathlib import Path from .bit2bin import bit2bin -from .hw_platform import Platform -from .linien_module import RootModule + REPO_ROOT_DIR = Path(__file__).resolve().parents[1] + def py_csrconstants(map, fil): fil.write("csr_constants = {\n") @@ -65,7 +90,7 @@ index 6c34429..a0b12d0 100644 fil.write("}\n\n") -@@ -52,26 +55,49 @@ def get_csrmap(banks): +@@ -51,26 +54,51 @@ def get_csrmap(banks): def py_csrmap(it, fil): fil.write("csr = {\n") for reg in it: @@ -84,7 +109,7 @@ index 6c34429..a0b12d0 100644 - platform = Platform() - root = RootModule(platform) + import argparse - ++ + parser = argparse.ArgumentParser() + parser.add_argument("-p", "--platform", default=None) + args = parser.parse_args() @@ -103,10 +128,11 @@ index 6c34429..a0b12d0 100644 + root = LinienFastServo(platform) + else: + raise ValueError("Unknown platform") -+ ++ + platform.add_source_dir(REPO_ROOT_DIR / "gateware" / "verilog") + build_dir = REPO_ROOT_DIR / "gateware" / "build" + platform.build(root, build_name="top", build_dir=build_dir, run=True) + with open( REPO_ROOT_DIR / "linien-server" / "linien_server" / "csrmap.py", "w" ) as fil: @@ -117,7 +143,7 @@ index 6c34429..a0b12d0 100644 py_csrmap(csr, fil) fil.write("states = {}\n".format(repr(root.linien.state_names))) fil.write("signals = {}\n".format(repr(root.linien.signal_names))) -- + - platform.add_source_dir(REPO_ROOT_DIR / "gateware" / "verilog") - build_dir = REPO_ROOT_DIR / "gateware" / "build" - platform.build(root, build_name="top", build_dir=build_dir) @@ -128,14 +154,14 @@ diff --git a/gateware/linien_module.py b/gateware/linien_module.py index 16ca186..6905ac0 100644 --- a/gateware/linien_module.py +++ b/gateware/linien_module.py -@@ -2,6 +2,7 @@ - # Copyright 2018-2022 Benjamin Wiegand - # Copyright 2021-2023 Bastian Leykauf - # Copyright 2022 Christian Freier -+# Copyright 2023 Jakub Matyas - # +@@ -1,6 +1,7 @@ # This file is part of Linien and based on redpid. # + # Copyright (C) 2016-2024 Linien Authors (https://github.com/linien-org/linien#license) ++# Copyright 2023 Jakub Matyas + # + # Linien is free software: you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by @@ -36,19 +37,13 @@ from misoc.interconnect.csr import AutoCSR, CSRStatus, CSRStorage from .logic.autolock import FPGAAutolock from .logic.chains import FastChain, SlowChain, cross_connect diff --git a/fast-servo/linien-pyrp3-monitor.patch b/fast-servo/linien-pyrp3-monitor.patch index 6830c84..abfa437 100644 --- a/fast-servo/linien-pyrp3-monitor.patch +++ b/fast-servo/linien-pyrp3-monitor.patch @@ -4,14 +4,14 @@ diff --git a/monitor/Makefile b/monitor/Makefile new file mode 100644 -index 0000000..044d88e +index 0000000..0c9bb53 --- /dev/null +++ b/monitor/Makefile @@ -0,0 +1,31 @@ +# Makefile for libmonitor + +OBJS = monitor.o -+SRCS = $(subst .o,.c, $(OBJS)) ++SRCS = $(subst .o,.c, $(OBJS))) +OSOBJS = monitor.os +TARGETLIB=libmonitor.so +CFLAGS=-g -std=gnu99 -Wall -Werror @@ -59,27 +59,16 @@ index ce1b28e..233b82a 100644 libmonitor = cdll.LoadLibrary(libmonitor_file) libmonitor.read_value.restype = c_uint32 diff --git a/setup.py b/setup.py -index 98bdaee..b0a8af4 100644 +index 9302177..2258ddc 100644 --- a/setup.py +++ b/setup.py -@@ -1,5 +1,10 @@ - import re +@@ -1,4 +1,28 @@ -from distutils.core import Extension, setup +import os -+ +from distutils.core import setup +from distutils.command.build import build -+from distutils.command.install import install + - from pathlib import Path - - # from https://stackoverflow.com/a/7071358/2750945 -@@ -11,9 +16,50 @@ if mo: - verstr = mo.group(1) - else: - raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,)) + -+# Patch from https://github.com/linien-org/pyrp3/blob/e6688acf8bd79d2dbe1d192d09c1a1baf1f6c67b/setup.py#L16-L55 +build_dir = "monitor/" + +def compile_libmonitor(): @@ -91,51 +80,16 @@ index 98bdaee..b0a8af4 100644 + finally: + os.chdir(cwd) + -+ -+def install_libmonitor(prefix=""): -+ cwd = os.getcwd() # get current directory -+ try: -+ os.chdir(build_dir) -+ os.system("make install INSTALL_DIR={prefix}".format(prefix=prefix)) -+ finally: -+ os.chdir(cwd) -+ -+ +class lib_build(build): + def run(self): + compile_libmonitor() + build.run(self) + -+ -+class lib_install(install): -+ def run(self): -+ compile_libmonitor() -+ install_libmonitor(self.prefix) -+ # install.run(self) -+ -+# Will use nix to install libmonitor -+cmdclass = { -+ "build": lib_build -+} -+ - this_directory = Path(__file__).parent - long_description = (this_directory / "README.rst").read_text() ++setup_args = dict( ++ cmdclass={ ++ "build": lib_build ++ } ++) -+ - setup( - name="pyrp3", - version=verstr, -@@ -32,6 +78,7 @@ setup( - "cached_property>=1.5.2", - "numpy>=1.11.0", - ], -+ cmdclass=cmdclass, - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Education", -@@ -45,5 +92,4 @@ setup( - "Topic :: Software Development :: Libraries :: Python Modules", - ], - keywords=["redpitaya", "FPGA", "zynq"], -- ext_modules=[Extension("monitor", ["monitor/monitor.c"])], - ) +-setup_args = dict(ext_modules=[Extension("monitor", ["monitor/monitor.c"])]) + setup(**setup_args) diff --git a/fast-servo/linien-server-fast-servo.patch b/fast-servo/linien-server-fast-servo.patch new file mode 100644 index 0000000..bdd2136 --- /dev/null +++ b/fast-servo/linien-server-fast-servo.patch @@ -0,0 +1,26 @@ +diff --git a/linien_server/cli.py b/linien_server/cli.py +index 98539b2..7781c74 100644 +--- a/linien_server/cli.py ++++ b/linien_server/cli.py +@@ -83,18 +83,9 @@ class LinienServerCLI: + else: + control = RedPitayaControlService(host=host) + +- if fake or host: +- authenticator = no_authenticator +- else: +- authenticator = username_and_password_authenticator +- +- try: +- if not (fake or host): # only available on RP +- mdio_tool.disable_ethernet_blinking() +- run_threaded_server(control, authenticator=authenticator) +- finally: +- if not (fake or host): # only available on RP +- mdio_tool.enable_ethernet_blinking() ++ authenticator = no_authenticator ++ ++ run_threaded_server(control, authenticator=authenticator) + + def enable(self) -> None: + """Enable the Linien server to start on boot.""" \ No newline at end of file diff --git a/flake.nix b/flake.nix index 87055b8..94d58f4 100644 --- a/flake.nix +++ b/flake.nix @@ -14,6 +14,14 @@ pkgs-armv7l = pkgs.pkgsCross.zynq-armv7l-linux; fsbl-support = ./fast-servo/fsbl-support; + version = "2.1.0"; + linien-src = pkgs.fetchFromGitHub { + owner = "linien-org"; + repo = "linien"; + rev = "v" + version; + sha256 = "sha256-j6oiP/usLfV5HZtKLcXQ5pHhhxRG05kP2FMwingiWm0="; + }; + patched-not-os = pkgs.applyPatches { name = "not-os-patched"; src = not-os; @@ -101,25 +109,40 @@ runScript = "vivado"; }; + cma = pkgs-armv7l.python3Packages.buildPythonPackage rec { + pname = "cma"; + version = "3.3.0"; + src = pkgs.fetchFromGitHub { + owner = "CMA-ES"; + repo = "pycma"; + rev = "refs/tags/r${version}"; + hash = "sha256-+UJI3hDVbDMfRF4bkwHED3eJCHzxS2hO4YPUzJqcoQI="; + }; + + propagatedBuildInputs = [ pkgs-armv7l.python3Packages.numpy ]; + + pythonImportsCheck = [ "cma" ]; + + checkPhase = '' + # At least one doctest fails, thus only limited amount of files is tested + python -m cma.test interfaces.py purecma.py logger.py optimization_tools.py transformations.py + ''; + }; + pyrp3 = pkgs-armv7l.python3Packages.buildPythonPackage rec { pname = "pyrp3"; - version = "1.2.0"; - pyproject = true; + version = "2.1.0"; + format = "pyproject"; src = pkgs.fetchFromGitHub { owner = "linien-org"; repo = "pyrp3"; rev = "v${version}"; - hash = "sha256-43TTlpJ5SMAjQM71bNVvrWQyciRXM3zpuA/Dw41AEgU="; + hash = "sha256-ol1QGXyCOei94iIPIocuTRHBxa5jKSH5RzjzROfZaBI="; }; patches = ./fast-servo/linien-pyrp3-monitor.patch; - nativeBuildInputs = with pkgs-armv7l.python3Packages; [ - setuptools wheel setuptools-scm - ] ++ (with pkgs-armv7l; [ gcc gnumake ]); - propagatedBuildInputs = with pkgs-armv7l.python3Packages; [ - myhdl - rpyc4 - cached-property - numpy + nativeBuildInputs = [ + pkgs-armv7l.python3Packages.setuptools + pkgs-armv7l.gcc ]; postInstall = '' cp monitor/libmonitor.so $out/lib @@ -128,13 +151,85 @@ substituteInPlace $out/${pkgs.python3.sitePackages}/pyrp3/raw_memory.py \ --replace "libmonitor.so" "$out/lib/libmonitor.so" ''; + propagatedBuildInputs = with pkgs-armv7l.python3Packages; [ + cached-property + numpy + rpyc + ]; + }; + + linien-client = pkgs.python3Packages.buildPythonPackage rec { + pname = "linien-client"; + inherit version; + src = linien-src; + + pyproject = true; + + sourceRoot = "${src.name}/linien-client"; + + preBuild = '' + export HOME=$(mktemp -d) + ''; + + nativeBuildInputs = [ pkgs.python3Packages.setuptools ]; + + doInstallCheck = false; + doCheck = false; + propagatedBuildInputs = with pkgs.python3Packages; [ + fabric + typing-extensions + linien-common + ]; + + pythonImportsCheck = [ "linien_client" ]; + + }; + + linien-gui = pkgs.python3Packages.buildPythonApplication rec { + pname = "linien-gui"; + inherit version; + src = linien-src; + pyproject = true; + + sourceRoot = "${src.name}/linien-gui"; + nativeBuildInputs = with pkgs.python3Packages; [ + setuptools + ] ++ [ + pkgs.qt5.wrapQtAppsHook + ]; + + # Makes qt-wayland appear in the qt paths injected by the wrapper - helps users + # with `QT_QPA_PLATFORM=wayland` in their environment. + buildInputs = [ + pkgs.qt5.qtwayland + ]; + + propagatedBuildInputs = with pkgs.python3Packages; [ + click + pyqtgraph + pyqt5 + requests + superqt + ] ++ [ linien-client ]; + + dontWrapQtApps = true; + + preFixup = '' + makeWrapperArgs+=("''${qtWrapperArgs[@]}") + ''; }; linien-server = pkgs-armv7l.python3Packages.buildPythonPackage rec { pname = "linien-server"; + inherit version; + src = linien-src; pyproject = true; - inherit (pkgs.python3Packages.linien-common) src version; - sourceRoot = "source/linien-server"; + + sourceRoot = "${src.name}/linien-server"; + patches = [ + ./fast-servo/linien-server-fast-servo.patch + ]; + postPatch = '' cp ${fast-servo-gateware}/csrmap.py linien_server/csrmap.py substituteInPlace linien_server/acquisition.py \ @@ -144,14 +239,13 @@ ''; nativeBuildInputs = [ pkgs-armv7l.python3Packages.setuptools ]; propagatedBuildInputs = with pkgs-armv7l.python3Packages; [ - appdirs - certifi - click - cma + fire + influxdb-client pylpsd - pyrp3 - requests linien-common + ] ++ [ + cma + pyrp3 ]; }; @@ -163,8 +257,7 @@ cp -r ${./fast-servo/linien-gateware}/. fast_servo/gateware ''; patches = [ - fast-servo/linien-fast-servo-gateware.patch - fast-servo/linien-fast-servo-server.patch + fast-servo/linien-gateware-fast-servo.patch fast-servo/autolock_pipeline.patch fast-servo/iir_pipeline.patch fast-servo/linien_module_pipeline.patch @@ -482,6 +575,15 @@ "${board}-qemu" = not-os-qemu; }; in rec { + devShell.x86_64-linux = pkgs.mkShell { + name = "nix-servo-dev_shell"; + buildInputs = with pkgs.python3Packages; [ + linien-common + linien-client + matplotlib + ] ++ [ linien-gui ]; + }; + packages.x86_64-linux = { inherit mkbootimage; inherit migen misoc vivado;