Fix Python 3.11, merge openocd into artiq

Signed-off-by: Egor Savkin <es@m-labs.hk>

flake.nix

@@ -16,13 +16,12 @@
     let
       pkgs = import artiqpkgs.inputs.nixpkgs { system = "x86_64-linux"; };
       msys2 = import ./msys2 { inherit pkgs; };
-      sipycoDependenciesFile = ./msys2/sipyco_deps.txt;
-      artiqDependenciesFile = ./msys2/artiq_deps.txt;
-      comtoolsDependenciesFile = ./msys2/comtools_deps.txt;
-      openocdDependenciesFile = ./msys2/openocd_deps.txt;
-      pythonparserDependenciesFile = ./msys2/pythonparser_deps.txt;
+      sipycoDependenciesFile = ./msys2/sipyco_dependencies.txt;
+      artiqDependenciesFile = ./msys2/artiq_dependencies.txt;
+      comtoolsDependenciesFile = ./msys2/comtools_dependencies.txt;
+      pythonparserDependenciesFile = ./msys2/pythonparser_dependencies.txt;
       # reads filename and return list of non-empty lines from it
-      readPackagesFromFile = (filename: builtins.filter (x: builtins.lessThan 0 (builtins.length (pkgs.lib.strings.stringToCharacters x)))
+      readStringListFromFile = (filename: builtins.filter (x: builtins.lessThan 0 (builtins.length (pkgs.lib.strings.stringToCharacters x)))
           (builtins.filter builtins.isString (builtins.split "\n| |\t" (builtins.readFile filename))));
 
       condaDeps = with pkgs; [ zlib xorg.libSM xorg.libICE xorg.libX11 xorg.libXau xorg.libXi xorg.libXrender libselinux libGL ];
@@ -428,19 +427,19 @@
           name = "python-pythonparser";
           version = artiqpkgs.packages.x86_64-linux.pythonparser.version;
           src = artiqpkgs.packages.x86_64-linux.pythonparser.src;
-          depends = readPackagesFromFile pythonparserDependenciesFile;
+          depends = readStringListFromFile pythonparserDependenciesFile;
         };
         msys2-sipyco = msys2.makeMsys2Package {
           name = "python-sipyco";
           src = artiqpkgs.inputs.sipyco.packages.x86_64-linux.sipyco.src;
           inherit (artiqpkgs.inputs.sipyco.packages.x86_64-linux.sipyco) version;
-          depends = readPackagesFromFile sipycoDependenciesFile;
+          depends = readStringListFromFile sipycoDependenciesFile;
         };
         msys2-artiq-comtools = msys2.makeMsys2Package {
           name = "artiq-comtools";
           src = artiqpkgs.inputs.artiq-comtools.packages.x86_64-linux.artiq-comtools.src;
           inherit (artiqpkgs.inputs.artiq-comtools.packages.x86_64-linux.artiq-comtools) version;
-          depends = (readPackagesFromFile comtoolsDependenciesFile) ++ [ "mingw-w64-x86_64-python-sipyco" ];
+          depends = (readStringListFromFile comtoolsDependenciesFile) ++ [ "mingw-w64-x86_64-python-sipyco" ];
         };
         msys2-artiq = msys2.makeMsys2Package {
           name = "artiq";
@@ -450,8 +449,7 @@
               export VERSIONEER_OVERRIDE=${artiqpkgs.packages.x86_64-linux.artiq.version}
               export VERSIONEER_REV=${artiqpkgs.sourceInfo.rev}
               '';
-          depends = (readPackagesFromFile artiqDependenciesFile)
-            ++ (readPackagesFromFile openocdDependenciesFile)
+          depends = (readStringListFromFile artiqDependenciesFile)
             ++ [ "mingw-w64-x86_64-python-sipyco" "mingw-w64-x86_64-python-pythonparser" "mingw-w64-x86_64-artiq-comtools" ];
 
         };
@@ -516,8 +514,8 @@
         ];
 
         msys2-offline-installer = pkgs.stdenv.mkDerivation {
-          name = "msys2-installer";
-          buildInputs = with pkgs; [ curl pacman fakeroot p7zip wine64 ];
+          name = "msys2-offline-installer";
+          buildInputs = with pkgs; [ p7zip wine64 ];
           src = ./.;
           msys2_qt_ifw = pkgs.stdenvNoCC.mkDerivation rec {
             name = "msys2_qt_ifw";
@@ -527,9 +525,9 @@
             installPhase = (pkgs.lib.strings.concatStringsSep "\n" (["mkdir $out"] ++ (map (p: "tar xvf ${p} -C $out") srcs)));
           };
 
-          msys2_artiq_image = pkgs.stdenvNoCC.mkDerivation rec {
-            name = "msys2_artiq_image";
-            artiq-deps-pkgs = import ./msys2/artiq_dep_packages.nix { inherit pkgs; };
+          msys2_artiq_root = pkgs.stdenvNoCC.mkDerivation rec {
+            name = "msys2_artiq_root";
+            artiq-deps-pkgs = import ./msys2/artiq_dependency_packages.nix { inherit pkgs; };
             msys2-pkgs = import ./msys2/msys2_packages.nix { inherit pkgs; };
             msys2-base = pkgs.fetchurl {
               url = "https://github.com/msys2/msys2-installer/releases/download/2024-01-13/msys2-base-x86_64-20240113.tar.xz";
@@ -568,7 +566,7 @@
              sed -i "s|@DATE@|$(date +'%Y-%m-%d')|g" qt-ifw/config/config.xml qt-ifw/packages/com.msys2.root/meta/package.xml ;
              sed -i "s|@VERSION@|$(date +'%Y%m%d')|g" qt-ifw/config/config.xml qt-ifw/packages/com.msys2.root/meta/package.xml ;
 
-             7z a -mx9 -md128m "qt-ifw/packages/com.msys2.root/data/msys64.7z" "$msys2_artiq_image/"*
+             7z a -mx9 -md128m "qt-ifw/packages/com.msys2.root/data/msys64.7z" "$msys2_artiq_root/"*
              ${pkgs.wineWowPackages.stable}/bin/wine64 $msys2_qt_ifw/mingw64/bin/binarycreator.exe \
                    -t "$msys2_qt_ifw/mingw64/bin/installerbase.exe" \
                    -p "qt-ifw/packages" \
@@ -579,7 +577,7 @@
           '';
           installPhase = ''
             mkdir -p $out
-            cp artiq-msys2-x86_64.exe $out/
+            cp artiq-msys2-x86_64.exe $out/artiq-msys2-x86_64-${artiqpkgs.packages.x86_64-linux.artiq.version}.exe
             mkdir -p $out/nix-support/
             for i in $out/*.*; do
               echo file binary-dist $i >> $out/nix-support/hydra-build-products

msys2/artiq_dependencies.txt

@@ -1,4 +1,4 @@
-mingw-w64-x86_64-python
+mingw-w64-x86_64-python3.11
 mingw-w64-x86_64-python-h5py
 mingw-w64-x86_64-python-pyqt5
 mingw-w64-x86_64-qt5-svg
@@ -15,3 +15,5 @@ mingw-w64-x86_64-python-levenshtein
 mingw-w64-x86_64-python-lmdb
 mingw-w64-x86_64-lld
 mingw-w64-x86_64-llvm
+mingw-w64-x86_64-openocd
+mingw-w64-x86_64-bscan-spi-bitstreams

msys2/artiq_dependency_packages.nix

@@ -492,48 +492,6 @@
       sha256 = "0446wv29vp1pld6drjaw1lr93lijcvn6l66jj4r5m1gyrf07jhlh";
     })
 
-    (pkgs.fetchurl {
-      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-attrs-23.2.0-1-any.pkg.tar.zst";
-      name = "mingw-w64-x86_64-python-attrs";
-      sha256 = "0jvm2jdgcrphwq61n3kj8ialy18m8lbj6xq3jgwbi8qqjlw9s1n7";
-    })
-
-    (pkgs.fetchurl {
-      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-multidict-6.0.5-1-any.pkg.tar.zst";
-      name = "mingw-w64-x86_64-python-multidict";
-      sha256 = "0hc7d031m3f8dq3qyvspgs5m2n8x7sljvr6j573ini4cn78w3y0p";
-    })
-
-    (pkgs.fetchurl {
-      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-idna-3.6-1-any.pkg.tar.zst";
-      name = "mingw-w64-x86_64-python-idna";
-      sha256 = "0fzkcd2c2c0crqg4690lrizwm9q4338bfn6w46k855gp5l83f4ww";
-    })
-
-    (pkgs.fetchurl {
-      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-yarl-1.9.4-1-any.pkg.tar.zst";
-      name = "mingw-w64-x86_64-python-yarl";
-      sha256 = "0y9790lq8kq713v08ypmbxxn48fdfbigkdv78jf1dl570vav72pw";
-    })
-
-    (pkgs.fetchurl {
-      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-frozenlist-1.4.1-1-any.pkg.tar.zst";
-      name = "mingw-w64-x86_64-python-frozenlist";
-      sha256 = "0if893h135p6qzldhl9f5a604whlj9viy3sj5p8mlkr6zyxww7p3";
-    })
-
-    (pkgs.fetchurl {
-      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-aiosignal-1.3.1-2-any.pkg.tar.zst";
-      name = "mingw-w64-x86_64-python-aiosignal";
-      sha256 = "0mpp79fx5f0sfjfrqxk0vmc5b13mkl779gwjg5ca38hjqzkycmgy";
-    })
-
-    (pkgs.fetchurl {
-      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-aiohttp-3.9.3-1-any.pkg.tar.zst";
-      name = "mingw-w64-x86_64-python-aiohttp";
-      sha256 = "17qym6rwq1w37a6ph0drvz7jqmnm8k0pc3cnbhfj135cq0z79q96";
-    })
-
     (pkgs.fetchurl {
       url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-hidapi-0.14.0-2-any.pkg.tar.zst";
       name = "mingw-w64-x86_64-hidapi";
@@ -588,6 +546,48 @@
       sha256 = "0facp60r9364fkrxx6vfrbivr00yiq99n6n24f5snqb8vqc2mrd1";
     })
 
+    (pkgs.fetchurl {
+      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-attrs-23.2.0-1-any.pkg.tar.zst";
+      name = "mingw-w64-x86_64-python-attrs";
+      sha256 = "0jvm2jdgcrphwq61n3kj8ialy18m8lbj6xq3jgwbi8qqjlw9s1n7";
+    })
+
+    (pkgs.fetchurl {
+      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-multidict-6.0.5-1-any.pkg.tar.zst";
+      name = "mingw-w64-x86_64-python-multidict";
+      sha256 = "0hc7d031m3f8dq3qyvspgs5m2n8x7sljvr6j573ini4cn78w3y0p";
+    })
+
+    (pkgs.fetchurl {
+      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-idna-3.6-1-any.pkg.tar.zst";
+      name = "mingw-w64-x86_64-python-idna";
+      sha256 = "0fzkcd2c2c0crqg4690lrizwm9q4338bfn6w46k855gp5l83f4ww";
+    })
+
+    (pkgs.fetchurl {
+      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-yarl-1.9.4-1-any.pkg.tar.zst";
+      name = "mingw-w64-x86_64-python-yarl";
+      sha256 = "0y9790lq8kq713v08ypmbxxn48fdfbigkdv78jf1dl570vav72pw";
+    })
+
+    (pkgs.fetchurl {
+      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-frozenlist-1.4.1-1-any.pkg.tar.zst";
+      name = "mingw-w64-x86_64-python-frozenlist";
+      sha256 = "0if893h135p6qzldhl9f5a604whlj9viy3sj5p8mlkr6zyxww7p3";
+    })
+
+    (pkgs.fetchurl {
+      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-aiosignal-1.3.1-2-any.pkg.tar.zst";
+      name = "mingw-w64-x86_64-python-aiosignal";
+      sha256 = "0mpp79fx5f0sfjfrqxk0vmc5b13mkl779gwjg5ca38hjqzkycmgy";
+    })
+
+    (pkgs.fetchurl {
+      url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-aiohttp-3.9.3-1-any.pkg.tar.zst";
+      name = "mingw-w64-x86_64-python-aiohttp";
+      sha256 = "17qym6rwq1w37a6ph0drvz7jqmnm8k0pc3cnbhfj135cq0z79q96";
+    })
+
     (pkgs.fetchurl {
       url = "https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-python-regex-2023.12.25-1-any.pkg.tar.zst";
       name = "mingw-w64-x86_64-python-regex";

msys2/comtools_dependencies.txt

@@ -1,3 +1,3 @@
-mingw-w64-x86_64-python
+mingw-w64-x86_64-python3.11
 mingw-w64-x86_64-python-numpy
 mingw-w64-x86_64-python-aiohttp

msys2/make_msys2_packages.sh

@@ -4,7 +4,6 @@ set -e
 
 cd $(dirname $0)
 MSYS2DIR=`pwd`/msys2
-rm -rf $MSYS2DIR
 mkdir -p $MSYS2DIR/var/lib/pacman $MSYS2DIR/msys/etc
 
 curl -L https://repo.msys2.org/msys/x86_64/pacman-mirrors-20240210-1-any.pkg.tar.zst | tar xvf - -C $MSYS2DIR --zstd
@@ -17,8 +16,8 @@ pacman --root $MSYS2DIR --config $MSYS2DIR/etc/pacman.conf \
   mingw-w64-x86_64-cc mingw-w64-x86_64-python3.11 \
   mingw-w64-x86_64-python-numpy mingw-w64-x86_64-python-setuptools > $MSYS2DIR/packages.txt
 
-cat artiq_deps.txt comtools_deps.txt openocd_deps.txt pythonparser_deps.txt sipyco_deps.txt | pacman --root $MSYS2DIR --config $MSYS2DIR/etc/pacman.conf \
-  --cachedir $MSYS2DIR/msys/cache --sync --print-format "%l %n" -  > $MSYS2DIR/artiq_dep_packages.txt
+cat artiq_dependencies.txt comtools_dependencies.txt pythonparser_dependencies.txt sipyco_dependencies.txt | pacman --root $MSYS2DIR --config $MSYS2DIR/etc/pacman.conf \
+  --cachedir $MSYS2DIR/msys/cache --sync --print-format "%l %n" -  > $MSYS2DIR/artiq_dependency_packages.txt
 
 pacman --root $MSYS2DIR --config $MSYS2DIR/etc/pacman.conf --cachedir $MSYS2DIR/msys/cache \
   --sync --print-format "%l %n" mingw-w64-x86_64-qt-installer-framework > $MSYS2DIR/qt_ifw_packages.txt
@@ -40,4 +39,4 @@ function generate_nix_packages_from_pacman() {
 
 IN=$MSYS2DIR/packages.txt OUT=msys2_packages.nix generate_nix_packages_from_pacman
 IN=$MSYS2DIR/qt_ifw_packages.txt OUT=msys2_qt_ifw_packages.nix generate_nix_packages_from_pacman
-IN=$MSYS2DIR/artiq_dep_packages.txt OUT=artiq_dep_packages.nix generate_nix_packages_from_pacman
+IN=$MSYS2DIR/artiq_dependency_packages.txt OUT=artiq_dependency_packages.nix generate_nix_packages_from_pacman

msys2/openocd_deps.txt

@@ -1,2 +0,0 @@
-mingw-w64-x86_64-openocd
-mingw-w64-x86_64-bscan-spi-bitstreams

msys2/sipyco_dependencies.txt

@@ -1,3 +1,3 @@
-mingw-w64-x86_64-python
+mingw-w64-x86_64-python3.11
 mingw-w64-x86_64-python-numpy
 mingw-w64-x86_64-python-pybase64