From a8288923d488655d97bc93ac3da8cc8438b896e1 Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Wed, 3 Mar 2021 20:35:30 +0100 Subject: [PATCH 01/12] [WIP] flakes --- artiq-board-generated/default.nix | 25 ++- artiq-full.nix | 39 +++-- flake.lock | 97 +++++++++++ flake.nix | 268 ++++++++++++++++++++++++++++++ 4 files changed, 404 insertions(+), 25 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/artiq-board-generated/default.nix b/artiq-board-generated/default.nix index 467c7b3..e63de29 100644 --- a/artiq-board-generated/default.nix +++ b/artiq-board-generated/default.nix @@ -1,9 +1,11 @@ { pkgs ? import {} , artiq-fast ? +, sinaraSystemsSrc ? +, sinaraSystemsRev ? null +, sinaraSystemsHash ? null }: let - sinaraSystemsSrc = ; generatedNix = pkgs.runCommand "generated-nix" { buildInputs = [ pkgs.nix pkgs.git ]; } '' mkdir $out @@ -11,14 +13,21 @@ let cp ${../artiq-full/artiq-targets.nix} $out/artiq-targets.nix cp -a ${artiq-fast} $out/fast - REV=`git --git-dir ${sinaraSystemsSrc}/.git rev-parse HEAD` + ${if sinaraSystemsRev != null && sinaraSystemsHash != null + then '' + REV=${sinaraSystemsRev} + HASH=${sinaraSystemsHash} + '' else '' + REV=`git --git-dir ${sinaraSystemsSrc}/.git rev-parse HEAD` + SINARA_SRC_CLEAN=`mktemp -d` + cp -a ${sinaraSystemsSrc}/. $SINARA_SRC_CLEAN + chmod -R 755 $SINARA_SRC_CLEAN/.git + chmod 755 $SINARA_SRC_CLEAN + rm -rf $SINARA_SRC_CLEAN/.git + HASH=`nix-hash --type sha256 --base32 $SINARA_SRC_CLEAN` + ''} + echo -n $REV > $out/sinara-rev.txt - SINARA_SRC_CLEAN=`mktemp -d` - cp -a ${sinaraSystemsSrc}/. $SINARA_SRC_CLEAN - chmod -R 755 $SINARA_SRC_CLEAN/.git - chmod 755 $SINARA_SRC_CLEAN - rm -rf $SINARA_SRC_CLEAN/.git - HASH=`nix-hash --type sha256 --base32 $SINARA_SRC_CLEAN` echo -n $HASH > $out/sinara-hash.txt cat > $out/default.nix << EOF diff --git a/artiq-full.nix b/artiq-full.nix index 57b44cf..033d8df 100644 --- a/artiq-full.nix +++ b/artiq-full.nix @@ -1,22 +1,25 @@ { pkgs ? import {} , a6p ? +, artiq-board-generated ? +, sinaraSystemsRev ? builtins.readFile (artiq-board-generated + "/sinara-rev.txt") +, sinaraSystemsHash ? builtins.readFile (artiq-board-generated + "/sinara-hash.txt") +, sinaraSystemsSrc ? ( + if a6p + then pkgs.fetchgit { + url = "https://git.m-labs.hk/M-Labs/sinara-systems.git"; + rev = sinaraSystemsRev; + sha256 = sinaraSystemsHash; + } + else +) +, artiq-fast ? ( + if a6p + then + else +) }: let - sinaraSystemsRev = builtins.readFile ; - sinaraSystemsHash = builtins.readFile ; - sinaraSystemsSrc = - if a6p - then pkgs.fetchgit { - url = "https://git.m-labs.hk/M-Labs/sinara-systems.git"; - rev = sinaraSystemsRev; - sha256 = sinaraSystemsHash; - } - else ; - artiq-fast = - if a6p - then - else ; artiqVersion = import (artiq-fast + "/pkgs/artiq-version.nix") { inherit (pkgs) stdenv git fetchgit; }; @@ -49,10 +52,10 @@ let ${if a6p then '' - cp -a ${} $out/board-generated + cp -a ${artiq-board-generated} $out/board-generated ln -s board-generated/fast $out/fast '' - else "cp -a ${} $out/fast"} + else "cp -a ${artiq-fast} $out/fast"} cp ${./artiq-full}/artiq-board-vivado.nix $out cp ${./artiq-full}/generate-identifier.py $out cp ${./artiq-full}/conda-artiq-board.nix $out @@ -109,7 +112,9 @@ let inherit pkgs vivado; } ''}; - conda-artiq-board = import ./conda-artiq-board.nix { inherit pkgs; }; + conda-artiq-board = import ./conda-artiq-board.nix { + inherit pkgs; + }; src = pkgs.fetchgit { url = "https://git.m-labs.hk/M-Labs/sinara-systems.git"; rev = "$REV"; diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..378ff4f --- /dev/null +++ b/flake.lock @@ -0,0 +1,97 @@ +{ + "nodes": { + "artiq-beta-src": { + "flake": false, + "locked": { + "lastModified": 1614216430, + "narHash": "sha256-iP/3vaXTjuIh7zhWpEsC2LKArB7GUttTwMZ1tg/hQI4=", + "owner": "M-Labs", + "repo": "artiq", + "rev": "e577542f6b621bebbb7a5e46b1dbab86ffbe1c11", + "type": "github" + }, + "original": { + "owner": "M-Labs", + "repo": "artiq", + "type": "github" + } + }, + "artiq-legacy-src": { + "flake": false, + "locked": { + "lastModified": 1613549523, + "narHash": "sha256-FgX/W+y8N4/6DLLTJuJVg5gUr3zk/5DxmSIxA4I6ZvE=", + "owner": "M-Labs", + "repo": "artiq", + "rev": "10c9842fc8844b524811336c6a529c648a2d2fde", + "type": "github" + }, + "original": { + "owner": "M-Labs", + "ref": "release-5", + "repo": "artiq", + "type": "github" + } + }, + "artiq-src": { + "flake": false, + "locked": { + "lastModified": 1613555154, + "narHash": "sha256-D/43YVvMNkcXV6sBBxAqxcmtEO3TagIQEQl0zsGE+ag=", + "owner": "M-Labs", + "repo": "artiq", + "rev": "c22482787ef806eb8e48d59b6ded7d93c7677c81", + "type": "github" + }, + "original": { + "owner": "M-Labs", + "ref": "release-6", + "repo": "artiq", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1614338965, + "narHash": "sha256-2HPgL9171Dhlg9lS4Ep5aeh4ejRCED+gNmU9tYzgyic=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "113092f5531ef38a45046990199873755a372646", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "release-20.09", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "artiq-beta-src": "artiq-beta-src", + "artiq-legacy-src": "artiq-legacy-src", + "artiq-src": "artiq-src", + "nixpkgs": "nixpkgs", + "sinara-systems": "sinara-systems" + } + }, + "sinara-systems": { + "flake": false, + "locked": { + "lastModified": 1614310919, + "narHash": "sha256-hHHEAUMbwATSExViuGM5foGt29Aq0Xwc0uDNF/x5Kt0=", + "ref": "master", + "rev": "52141a3fef08ee88227c48395bc8e32b5140c938", + "revCount": 200, + "type": "git", + "url": "https://git.m-labs.hk/M-Labs/sinara-systems.git" + }, + "original": { + "type": "git", + "url": "https://git.m-labs.hk/M-Labs/sinara-systems.git" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..642f35b --- /dev/null +++ b/flake.nix @@ -0,0 +1,268 @@ +# let +# system = "x86_64-linux"; +# in +{ + description = "ARTIQ"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs/release-20.09"; + artiq-beta-src = { + url = "github:M-Labs/artiq"; + flake = false; + }; + artiq-src = { + url = "github:M-Labs/artiq/release-6"; + flake = false; + }; + artiq-legacy-src = { + url = "github:M-Labs/artiq/release-5"; + flake = false; + }; + sinara-systems = { + url = "git+https://git.m-labs.hk/M-Labs/sinara-systems.git"; + flake = false; + }; + }; + + outputs = sources@{ self, nixpkgs, ... }: + let + systems = [ + "x86_64-linux" + ]; + forAllSystems = f: + builtins.foldl' (systems: system: + systems // { ${system} = f system; } + ) {} systems; + in rec { + + packages = forAllSystems (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + + artiq-fast-src = artiqInputName: + let + # locked = (builtins.fromJSON ( + # builtins.readFile ./flake.lock + # )).nodes.${artiqInputName}.locked; + inherit (sources.${artiqInputName}) lastModified rev narHash; + hash = builtins.replaceStrings ["sha256-"] [""] narHash; + + artiqSrc = sources.${artiqInputName}.outPath; + + in pkgs.runCommand "generated-nix" { buildInputs = [ pkgs.nix pkgs.git ]; } '' + cp --no-preserve=mode,ownership -R ${./artiq-fast} $out + + MAJOR_VERSION=`cat ${artiqSrc}/MAJOR_VERSION` + if [ -e ${artiqSrc}/BETA ]; then + SUFFIX=".beta" + else + SUFFIX="" + fi + + cat > $out/pkgs/artiq-src.nix << EOF + { fetchgit }: + fetchgit { + url = "git://github.com/m-labs/artiq.git"; + rev = "${rev}"; + sha256 = "${hash}"; + } + EOF + echo "{ stdenv, git, fetchgit }: \"$MAJOR_VERSION.`cut -c1-8 <<< ${rev}`$SUFFIX\"" > $out/pkgs/artiq-version.nix + echo "{ stdenv, git, fetchgit }: \"${toString lastModified}\"" > $out/pkgs/artiq-timestamp.nix + ''; + artiq-fast = artiqInputName: + import (artiq-fast-src artiqInputName) { + inherit pkgs; + }; + + artiq-board-generated = artiqInputName: + (import ./artiq-board-generated { + inherit pkgs; + sinaraSystemsSrc = null; #sources.sinara-systems.outPath; + sinaraSystemsRev = sources.sinara-systems.rev; + sinaraSystemsHash = builtins.replaceStrings ["sha256-"] [""] + sources.sinara-systems.narHash; + artiq-fast = artiq-fast-src artiqInputName; + }).generated-nix; + + artiq-full = artiqInputName: a6p: + import ./artiq-full.nix { + inherit pkgs; + inherit a6p; + artiq-board-generated = artiq-board-generated artiqInputName; + sinaraSystemsSrc = sources.sinara-systems.outPath; + artiq-fast = artiq-fast-src artiqInputName; + }; + + # artiq-full1 = artiqInputName: a6p: + # let + # pythonDeps = import ./artiq-full/python-deps.nix { inherit pkgs; }; + # ddbDeps = [ + # artiq-fast.artiq + # (pkgs.python3.withPackages (ps: [ ps.jsonschema ])) + # ]; + # artiqVersion = (artiq-fast artiqInputName).artiq.version; + + # targets = import ./artiq-full/artiq-targets.nix { + # inherit pkgs artiqVersion; + # sinaraSystemsSrc = sources.sinara-systems.outPath; + # }; + # kasliVariants = map ({ variant, ... }: variant) ( + # builtins.filter ({ target, ... }: target == "kasli") ( + # builtins.attrValues targets + # ) + # ); + # standaloneVariants = map ({ variant, ... }: variant) ( + # builtins.filter ({ target, standalone ? false, ... }: target == "kasli" && standalone) ( + # builtins.attrValues targets + # ) + # ); + + # vivado = import ./artiq-fast/vivado.nix { + # inherit pkgs; + # }; + # artiq-board = + # if a6p + # then + # import ./artiq-full/artiq-board-vivado.nix { + # inherit pkgs vivado; + # version = artiqVersion; + # board-generated = artiq-board-generated artiqInputName; + # } + # else + # import ./artiq-fast/artiq-board.nix { + # inherit pkgs vivado; + # }; + # conda-artiq-board = import ./artiq-full/conda-artiq-board.nix { + # inherit pkgs; + # }; + # artiq-targets = pkgs.lib.lists.foldr (conf: start: + # let + # inherit (conf) target variant; + # json = sources.sinara-systems.outPath + "/${variant}.json"; + # boardBinaries = artiq-board (conf // { + # src = json; + # }); + # in + # start // { + # "artiq-board-${target}-${variant}" = boardBinaries; + # "conda-artiq-board-${target}-${variant}" = conda-artiq-board { + # boardBinaries = boardBinaries; + # inherit target variant; + # }; + # } // (pkgs.lib.optionalAttrs ( + # target == "kasli" && + # builtins.elem variant standaloneVariants + # ) { + # "device-db-${target}-${variant}" = pkgs.stdenv.mkDerivation { + # name = "device-db-${target}-${variant}"; + # buildInputs = ddbDeps; + # phases = [ "buildPhase" ]; + # buildPhase = " + # mkdir \$out + # artiq_ddb_template ${json} -o \$out/device_db.py + # mkdir \$out/nix-support + # echo file device_db_template \$out/device_db.py >> \$out/nix-support/hydra-build-products + # "; + # }; + # }) + # ) {} (builtins.attrValues targets); + # drtio-systems = + # if a6p then { + # berkeley3 = { + # master = "berkeley3master"; + # satellites = { + # "1" = "berkeley3satellite"; + # }; + # }; + # bonn1 = { + # master = "bonn1master"; + # satellites = { + # "1" = "bonn1satellite"; + # }; + # }; + # hw2 = { + # master = "hw2master"; + # satellites = { + # "1" = "hw2satellite"; + # }; + # }; + # ptb3 = { + # master = "ptb3master"; + # satellites = { + # "1" = "ptb3satellite"; + # }; + # }; + # purdue = { + # master = "purduemaster"; + # satellites = { + # "1" = "purduesatellite"; + # }; + # }; + # stfc = { + # master = "stfcmaster"; + # satellites = { + # "1" = "stfcsatellite"; + # }; + # }; + # wipm7 = { + # master = "wipm7master"; + # satellites = { + # "1" = "wipm7satellite"; + # }; + # }; + # } + # else {}; + # drtio-ddbs = pkgs.lib.attrsets.mapAttrs' + # (system: crates: pkgs.lib.attrsets.nameValuePair ("device-db-" + system) + # (pkgs.stdenv.mkDerivation { + # name = "device-db-${system}"; + # buildInputs = ddbDeps; + # phases = [ "buildPhase" ]; + # buildPhase = '' + # mkdir \$out + # artiq_ddb_template \ + # ${pkgs.lib.strings.concatStringsSep " " (pkgs.lib.attrsets.mapAttrsToList (dest: desc: "-s " + dest + " " + (sources.sinara-systems.outPath + "/${desc}.json")) crates.satellites) } \ + # ${sources.sinara-systems.outPath}/${crates.master}.json -o \$out/device_db.py + # mkdir \$out/nix-support + # echo file device_db_template \$out/device_db.py >> \$out/nix-support/hydra-build-products + # ''; + # })) drtio-systems; + # extras = import ./artiq-full/extras.nix { inherit pkgs; inherit (artiq-fast artiqInputName) sipyco asyncserial artiq; }; + + # artiq-full = artiq-targets // drtio-ddbs // extras // rec { + # conda-artiq-board-kasli-tester = conda-artiq-board { + # target = "kasli"; + # variant = "tester"; + # boardBinaries = artiq-fast.artiq-board-kasli-tester; + # }; + # conda-artiq-board-kc705-nist_clock = conda-artiq-board { + # target = "kc705"; + # variant = "nist_clock"; + # boardBinaries = artiq-fast.artiq-board-kc705-nist_clock; + # }; + # }; + # sipycoManualPackages = import ./artiq-full/sipyco-manual.nix { + # inherit (pkgs) stdenv lib python3Packages texlive texinfo; + # inherit (import artiq-fast { inherit pkgs; }) sipyco; + # }; + # artiqManualPackages = import ./artiq-full/artiq-manual.nix { + # inherit (pkgs) stdenv lib fetchgit git python3Packages texlive texinfo; + # inherit (pythonDeps) sphinxcontrib-wavedrom; + # inherit artiq-fast; + # }; + # exampleUserEnv = import ./artiq-full/example-user-env.nix { inherit pkgs artiq-full; }; + # jobs = artiq-full // sipycoManualPackages // artiqManualPackages // exampleUserEnv; + # in + # jobs; + + allPackages = artiqInputName: a6p: + (artiq-fast artiqInputName) // + #(artiq-board-generated artiqInputName); + (artiq-full artiqInputName a6p); + in allPackages "artiq-beta-src" true + ); + + defaultPackage = forAllSystems (system: self.packages.${system}.artiq); + }; +} -- 2.42.0 From 3f0bc9af0c594190b4dc2b92f49581315cbbb7c9 Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Wed, 3 Mar 2021 21:29:54 +0100 Subject: [PATCH 02/12] llvm-or1k: rm NIX_BUILD_CORES --- artiq-fast/pkgs/llvm-or1k.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/artiq-fast/pkgs/llvm-or1k.nix b/artiq-fast/pkgs/llvm-or1k.nix index b305485..fc6925e 100644 --- a/artiq-fast/pkgs/llvm-or1k.nix +++ b/artiq-fast/pkgs/llvm-or1k.nix @@ -33,8 +33,6 @@ in buildInputs = [ perl groff cmake libxml2 python libffi ] ++ stdenv.lib.optional stdenv.isLinux valgrind; preBuild = '' - NIX_BUILD_CORES=4 - makeFlagsArray=(-j''$NIX_BUILD_CORES) mkdir -p $out/ ''; -- 2.42.0 From 0805a58cd8c82ab8332c3cf73d25f47f65785177 Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Thu, 4 Mar 2021 16:51:53 +0100 Subject: [PATCH 03/12] flake.nix: delint --- flake.nix | 179 +++--------------------------------------------------- 1 file changed, 10 insertions(+), 169 deletions(-) diff --git a/flake.nix b/flake.nix index 642f35b..817b113 100644 --- a/flake.nix +++ b/flake.nix @@ -23,7 +23,7 @@ flake = false; }; }; - + outputs = sources@{ self, nixpkgs, ... }: let systems = [ @@ -41,14 +41,11 @@ artiq-fast-src = artiqInputName: let - # locked = (builtins.fromJSON ( - # builtins.readFile ./flake.lock - # )).nodes.${artiqInputName}.locked; inherit (sources.${artiqInputName}) lastModified rev narHash; hash = builtins.replaceStrings ["sha256-"] [""] narHash; artiqSrc = sources.${artiqInputName}.outPath; - + in pkgs.runCommand "generated-nix" { buildInputs = [ pkgs.nix pkgs.git ]; } '' cp --no-preserve=mode,ownership -R ${./artiq-fast} $out @@ -93,173 +90,17 @@ sinaraSystemsSrc = sources.sinara-systems.outPath; artiq-fast = artiq-fast-src artiqInputName; }; - - # artiq-full1 = artiqInputName: a6p: - # let - # pythonDeps = import ./artiq-full/python-deps.nix { inherit pkgs; }; - # ddbDeps = [ - # artiq-fast.artiq - # (pkgs.python3.withPackages (ps: [ ps.jsonschema ])) - # ]; - # artiqVersion = (artiq-fast artiqInputName).artiq.version; - # targets = import ./artiq-full/artiq-targets.nix { - # inherit pkgs artiqVersion; - # sinaraSystemsSrc = sources.sinara-systems.outPath; - # }; - # kasliVariants = map ({ variant, ... }: variant) ( - # builtins.filter ({ target, ... }: target == "kasli") ( - # builtins.attrValues targets - # ) - # ); - # standaloneVariants = map ({ variant, ... }: variant) ( - # builtins.filter ({ target, standalone ? false, ... }: target == "kasli" && standalone) ( - # builtins.attrValues targets - # ) - # ); + excluded = [ + # "windows-no-hardware-tests" + # "extended-tests" + ]; - # vivado = import ./artiq-fast/vivado.nix { - # inherit pkgs; - # }; - # artiq-board = - # if a6p - # then - # import ./artiq-full/artiq-board-vivado.nix { - # inherit pkgs vivado; - # version = artiqVersion; - # board-generated = artiq-board-generated artiqInputName; - # } - # else - # import ./artiq-fast/artiq-board.nix { - # inherit pkgs vivado; - # }; - # conda-artiq-board = import ./artiq-full/conda-artiq-board.nix { - # inherit pkgs; - # }; - # artiq-targets = pkgs.lib.lists.foldr (conf: start: - # let - # inherit (conf) target variant; - # json = sources.sinara-systems.outPath + "/${variant}.json"; - # boardBinaries = artiq-board (conf // { - # src = json; - # }); - # in - # start // { - # "artiq-board-${target}-${variant}" = boardBinaries; - # "conda-artiq-board-${target}-${variant}" = conda-artiq-board { - # boardBinaries = boardBinaries; - # inherit target variant; - # }; - # } // (pkgs.lib.optionalAttrs ( - # target == "kasli" && - # builtins.elem variant standaloneVariants - # ) { - # "device-db-${target}-${variant}" = pkgs.stdenv.mkDerivation { - # name = "device-db-${target}-${variant}"; - # buildInputs = ddbDeps; - # phases = [ "buildPhase" ]; - # buildPhase = " - # mkdir \$out - # artiq_ddb_template ${json} -o \$out/device_db.py - # mkdir \$out/nix-support - # echo file device_db_template \$out/device_db.py >> \$out/nix-support/hydra-build-products - # "; - # }; - # }) - # ) {} (builtins.attrValues targets); - # drtio-systems = - # if a6p then { - # berkeley3 = { - # master = "berkeley3master"; - # satellites = { - # "1" = "berkeley3satellite"; - # }; - # }; - # bonn1 = { - # master = "bonn1master"; - # satellites = { - # "1" = "bonn1satellite"; - # }; - # }; - # hw2 = { - # master = "hw2master"; - # satellites = { - # "1" = "hw2satellite"; - # }; - # }; - # ptb3 = { - # master = "ptb3master"; - # satellites = { - # "1" = "ptb3satellite"; - # }; - # }; - # purdue = { - # master = "purduemaster"; - # satellites = { - # "1" = "purduesatellite"; - # }; - # }; - # stfc = { - # master = "stfcmaster"; - # satellites = { - # "1" = "stfcsatellite"; - # }; - # }; - # wipm7 = { - # master = "wipm7master"; - # satellites = { - # "1" = "wipm7satellite"; - # }; - # }; - # } - # else {}; - # drtio-ddbs = pkgs.lib.attrsets.mapAttrs' - # (system: crates: pkgs.lib.attrsets.nameValuePair ("device-db-" + system) - # (pkgs.stdenv.mkDerivation { - # name = "device-db-${system}"; - # buildInputs = ddbDeps; - # phases = [ "buildPhase" ]; - # buildPhase = '' - # mkdir \$out - # artiq_ddb_template \ - # ${pkgs.lib.strings.concatStringsSep " " (pkgs.lib.attrsets.mapAttrsToList (dest: desc: "-s " + dest + " " + (sources.sinara-systems.outPath + "/${desc}.json")) crates.satellites) } \ - # ${sources.sinara-systems.outPath}/${crates.master}.json -o \$out/device_db.py - # mkdir \$out/nix-support - # echo file device_db_template \$out/device_db.py >> \$out/nix-support/hydra-build-products - # ''; - # })) drtio-systems; - # extras = import ./artiq-full/extras.nix { inherit pkgs; inherit (artiq-fast artiqInputName) sipyco asyncserial artiq; }; - - # artiq-full = artiq-targets // drtio-ddbs // extras // rec { - # conda-artiq-board-kasli-tester = conda-artiq-board { - # target = "kasli"; - # variant = "tester"; - # boardBinaries = artiq-fast.artiq-board-kasli-tester; - # }; - # conda-artiq-board-kc705-nist_clock = conda-artiq-board { - # target = "kc705"; - # variant = "nist_clock"; - # boardBinaries = artiq-fast.artiq-board-kc705-nist_clock; - # }; - # }; - # sipycoManualPackages = import ./artiq-full/sipyco-manual.nix { - # inherit (pkgs) stdenv lib python3Packages texlive texinfo; - # inherit (import artiq-fast { inherit pkgs; }) sipyco; - # }; - # artiqManualPackages = import ./artiq-full/artiq-manual.nix { - # inherit (pkgs) stdenv lib fetchgit git python3Packages texlive texinfo; - # inherit (pythonDeps) sphinxcontrib-wavedrom; - # inherit artiq-fast; - # }; - # exampleUserEnv = import ./artiq-full/example-user-env.nix { inherit pkgs artiq-full; }; - # jobs = artiq-full // sipycoManualPackages // artiqManualPackages // exampleUserEnv; - # in - # jobs; - allPackages = artiqInputName: a6p: - (artiq-fast artiqInputName) // - #(artiq-board-generated artiqInputName); - (artiq-full artiqInputName a6p); + nixpkgs.lib.filterAttrs (name: _: !builtins.elem name excluded) ( + (artiq-fast artiqInputName) // + (artiq-full artiqInputName a6p) + ); in allPackages "artiq-beta-src" true ); -- 2.42.0 From fd40670b3be090b9ef746d296648b5053218cbb4 Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Fri, 5 Mar 2021 18:24:30 +0100 Subject: [PATCH 04/12] flake.lock: update artiq-beta-src --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 378ff4f..71eb6d2 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "artiq-beta-src": { "flake": false, "locked": { - "lastModified": 1614216430, - "narHash": "sha256-iP/3vaXTjuIh7zhWpEsC2LKArB7GUttTwMZ1tg/hQI4=", + "lastModified": 1614493604, + "narHash": "sha256-9P/O4SWHqHWO+OD3d8FihJXQ+Y5JRlW0ajBNYZlIZUg=", "owner": "M-Labs", "repo": "artiq", - "rev": "e577542f6b621bebbb7a5e46b1dbab86ffbe1c11", + "rev": "b8f4c6b9bb8ce4f2d7dc50e87690ad336841357c", "type": "github" }, "original": { -- 2.42.0 From 5d4a89f654a45a0d8c5a732f4126698ed9feeb9d Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Fri, 5 Mar 2021 19:08:00 +0100 Subject: [PATCH 05/12] wfvm.nix: turn wfvm-pkgs into an arg with default --- artiq-fast/conda-windows/binutils.nix | 5 ++++- artiq-fast/conda-windows/llvm-or1k.nix | 5 ++++- artiq-fast/conda-windows/llvmlite-artiq.nix | 5 ++++- artiq-fast/wfvm.nix | 14 +++++++------- artiq-fast/windows/make_conda_packages.sh | 5 ++++- artiq-fast/windows/run-test.nix | 5 ++++- 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/artiq-fast/conda-windows/binutils.nix b/artiq-fast/conda-windows/binutils.nix index 6358ebe..935c863 100644 --- a/artiq-fast/conda-windows/binutils.nix +++ b/artiq-fast/conda-windows/binutils.nix @@ -1,7 +1,10 @@ { pkgs, version, src, target }: let - wfvm = import ../wfvm.nix { inherit pkgs; }; + wfvm = import ../wfvm.nix { + inherit pkgs; + wfvm-pkgs = pkgs; + }; libiconv-filename = "libiconv-1.15-h1df5818_7.tar.bz2"; libiconv = pkgs.fetchurl { url = "https://anaconda.org/anaconda/libiconv/1.15/download/win-64/${libiconv-filename}"; diff --git a/artiq-fast/conda-windows/llvm-or1k.nix b/artiq-fast/conda-windows/llvm-or1k.nix index e23140a..7b10666 100644 --- a/artiq-fast/conda-windows/llvm-or1k.nix +++ b/artiq-fast/conda-windows/llvm-or1k.nix @@ -1,7 +1,10 @@ { pkgs, version, src }: let - wfvm = import ../wfvm.nix { inherit pkgs; }; + wfvm = import ../wfvm.nix { + inherit pkgs; + wfvm-pkgs = pkgs; + }; conda-vs2015_runtime-filename = "vs2015_runtime-14.16.27012-hf0eaf9b_2.tar.bz2"; conda-vs2015_runtime = pkgs.fetchurl { url = "https://anaconda.org/anaconda/vs2015_runtime/14.16.27012/download/win-64/${conda-vs2015_runtime-filename}"; diff --git a/artiq-fast/conda-windows/llvmlite-artiq.nix b/artiq-fast/conda-windows/llvmlite-artiq.nix index 51eaafc..b15ccd0 100644 --- a/artiq-fast/conda-windows/llvmlite-artiq.nix +++ b/artiq-fast/conda-windows/llvmlite-artiq.nix @@ -1,7 +1,10 @@ { pkgs, conda-windows-llvm-or1k, version, src }: let - wfvm = import ../wfvm.nix { inherit pkgs; }; + wfvm = import ../wfvm.nix { + inherit pkgs; + wfvm-pkgs = pkgs; + }; conda-vs2015_runtime-filename = "vs2015_runtime-14.16.27012-hf0eaf9b_2.tar.bz2"; conda-vs2015_runtime = pkgs.fetchurl { url = "https://anaconda.org/anaconda/vs2015_runtime/14.16.27012/download/win-64/${conda-vs2015_runtime-filename}"; diff --git a/artiq-fast/wfvm.nix b/artiq-fast/wfvm.nix index a520492..e9df6b3 100644 --- a/artiq-fast/wfvm.nix +++ b/artiq-fast/wfvm.nix @@ -1,16 +1,16 @@ -{ pkgs }: - -let - # Pin nixpkgs to avoid frequent resource-intensive Windows reinstallations on Hydra. - wfvm-pkgs = pkgs.fetchFromGitHub { +{ pkgs +, wfvm-pkgs ? import (pkgs.fetchFromGitHub { owner = "NixOS"; repo = "nixpkgs"; rev = "f8248ab6d9e69ea9c07950d73d48807ec595e923"; sha256 = "009i9j6mbq6i481088jllblgdnci105b2q4mscprdawg3knlyahk"; - }; + }) {} +}: + +let wfvm = pkgs.fetchgit { url = "https://git.m-labs.hk/M-Labs/wfvm.git"; rev = "4b497938ffd9fcddf84a3dbe2f01524395292adb"; sha256 = "0m3kdbbcskqc1lf8b5f7ccbll9b7vkl4r00kbyx3yjb2rs6cqvil"; }; -in import "${wfvm}/wfvm" { pkgs = (import wfvm-pkgs {}); } +in import "${wfvm}/wfvm" { pkgs = wfvm-pkgs; } diff --git a/artiq-fast/windows/make_conda_packages.sh b/artiq-fast/windows/make_conda_packages.sh index d7336d8..f734380 100755 --- a/artiq-fast/windows/make_conda_packages.sh +++ b/artiq-fast/windows/make_conda_packages.sh @@ -13,7 +13,10 @@ set -e nix-build -E " let pkgs = import {}; - wfvm = import ../wfvm.nix { inherit pkgs; }; + wfvm = import ../wfvm.nix { + inherit pkgs; + wfvm-pkgs = pkgs; + }; in wfvm.utils.wfvm-run { name = \"get-conda-packages\"; diff --git a/artiq-fast/windows/run-test.nix b/artiq-fast/windows/run-test.nix index 5fc86f4..89b35a5 100644 --- a/artiq-fast/windows/run-test.nix +++ b/artiq-fast/windows/run-test.nix @@ -10,7 +10,10 @@ let inherit port; }) tcpPorts; - wfvm = import ../wfvm.nix { inherit pkgs; }; + wfvm = import ../wfvm.nix { + inherit pkgs; + wfvm-pkgs = pkgs; + }; conda-deps = { name = "conda-deps"; script = let -- 2.42.0 From df6982376753e6881dabd15996275be1035aa47d Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Fri, 5 Mar 2021 19:08:44 +0100 Subject: [PATCH 06/12] flake.nix: remove unused `excluded` --- flake.nix | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 817b113..02c59fa 100644 --- a/flake.nix +++ b/flake.nix @@ -91,16 +91,9 @@ artiq-fast = artiq-fast-src artiqInputName; }; - excluded = [ - # "windows-no-hardware-tests" - # "extended-tests" - ]; - allPackages = artiqInputName: a6p: - nixpkgs.lib.filterAttrs (name: _: !builtins.elem name excluded) ( - (artiq-fast artiqInputName) // - (artiq-full artiqInputName a6p) - ); + (artiq-fast artiqInputName) // + (artiq-full artiqInputName a6p); in allPackages "artiq-beta-src" true ); -- 2.42.0 From 3d6ca4564e276558f0fd544082299935f209202d Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Tue, 9 Mar 2021 23:06:19 +0100 Subject: [PATCH 07/12] flake.nix: simplify forAllSystems --- flake.nix | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 02c59fa..37a8788 100644 --- a/flake.nix +++ b/flake.nix @@ -29,10 +29,7 @@ systems = [ "x86_64-linux" ]; - forAllSystems = f: - builtins.foldl' (systems: system: - systems // { ${system} = f system; } - ) {} systems; + forAllSystems = nixpkgs.lib.genAttrs systems; in rec { packages = forAllSystems (system: -- 2.42.0 From a5e807783af77fda44fc7483f326b07b49de1730 Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Tue, 9 Mar 2021 23:13:41 +0100 Subject: [PATCH 08/12] flake.nix: rm superfluous sinaraSystemsSrc --- flake.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/flake.nix b/flake.nix index 37a8788..bb056df 100644 --- a/flake.nix +++ b/flake.nix @@ -84,7 +84,6 @@ inherit pkgs; inherit a6p; artiq-board-generated = artiq-board-generated artiqInputName; - sinaraSystemsSrc = sources.sinara-systems.outPath; artiq-fast = artiq-fast-src artiqInputName; }; -- 2.42.0 From cd4a8fce8ac9cb0d132b33e965672fca378ea3d3 Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Wed, 10 Mar 2021 01:11:12 +0100 Subject: [PATCH 09/12] flake.nix: refactor for hydraJobs --- flake.nix | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/flake.nix b/flake.nix index bb056df..2f3de3f 100644 --- a/flake.nix +++ b/flake.nix @@ -30,13 +30,12 @@ "x86_64-linux" ]; forAllSystems = nixpkgs.lib.genAttrs systems; - in rec { - packages = forAllSystems (system: + artiq-jobsets = system: artiqInputName: a6p: let pkgs = nixpkgs.legacyPackages.${system}; - artiq-fast-src = artiqInputName: + artiq-fast-src = let inherit (sources.${artiqInputName}) lastModified rev narHash; hash = builtins.replaceStrings ["sha256-"] [""] narHash; @@ -64,35 +63,51 @@ echo "{ stdenv, git, fetchgit }: \"$MAJOR_VERSION.`cut -c1-8 <<< ${rev}`$SUFFIX\"" > $out/pkgs/artiq-version.nix echo "{ stdenv, git, fetchgit }: \"${toString lastModified}\"" > $out/pkgs/artiq-timestamp.nix ''; - artiq-fast = artiqInputName: - import (artiq-fast-src artiqInputName) { + artiq-fast = + import artiq-fast-src { inherit pkgs; }; - artiq-board-generated = artiqInputName: + artiq-board-generated = (import ./artiq-board-generated { inherit pkgs; sinaraSystemsSrc = null; #sources.sinara-systems.outPath; sinaraSystemsRev = sources.sinara-systems.rev; sinaraSystemsHash = builtins.replaceStrings ["sha256-"] [""] sources.sinara-systems.narHash; - artiq-fast = artiq-fast-src artiqInputName; + artiq-fast = artiq-fast-src; }).generated-nix; - artiq-full = artiqInputName: a6p: + artiq-full = import ./artiq-full.nix { - inherit pkgs; - inherit a6p; - artiq-board-generated = artiq-board-generated artiqInputName; - artiq-fast = artiq-fast-src artiqInputName; + inherit pkgs a6p artiq-board-generated; + artiq-fast = artiq-fast-src; }; - - allPackages = artiqInputName: a6p: - (artiq-fast artiqInputName) // - (artiq-full artiqInputName a6p); - in allPackages "artiq-beta-src" true + in { + inherit artiq-fast artiq-full; + }; + in { + packages = forAllSystems (system: + builtins.foldl' (a: b: a // b) {} ( + builtins.attrValues ( + artiq-jobsets system "artiq-beta-src" true + ) + ) ); defaultPackage = forAllSystems (system: self.packages.${system}.artiq); + + hydraJobs = + let + jobset = artiqInputName: a6p: name: + forAllSystems (system: (artiq-jobsets system artiqInputName a6p).${name}); + in { + fast = jobset "artiq-src" true "artiq-fast"; + fast-beta = jobset "artiq-beta-src" true "artiq-fast"; + fast-legacy = jobset "artiq-legacy-src" false "artiq-fast"; + full = jobset "artiq-src" true "artiq-full"; + full-beta = jobset "artiq-beta-src" true "artiq-full"; + full-legacy = jobset "artiq-legacy-src" false "artiq-full"; + }; }; } -- 2.42.0 From b17c7ad7b59c5e6bd4c986df1846e7a596405f67 Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Wed, 10 Mar 2021 18:10:12 +0100 Subject: [PATCH 10/12] flake.nix: include git revCount in version --- flake.lock | 44 ++++++++++++++++++++++---------------------- flake.nix | 10 +++++----- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/flake.lock b/flake.lock index 71eb6d2..2f8ef43 100644 --- a/flake.lock +++ b/flake.lock @@ -5,15 +5,15 @@ "locked": { "lastModified": 1614493604, "narHash": "sha256-9P/O4SWHqHWO+OD3d8FihJXQ+Y5JRlW0ajBNYZlIZUg=", - "owner": "M-Labs", - "repo": "artiq", + "ref": "master", "rev": "b8f4c6b9bb8ce4f2d7dc50e87690ad336841357c", - "type": "github" + "revCount": 7585, + "type": "git", + "url": "https://github.com/M-Labs/artiq.git" }, "original": { - "owner": "M-Labs", - "repo": "artiq", - "type": "github" + "type": "git", + "url": "https://github.com/M-Labs/artiq.git" } }, "artiq-legacy-src": { @@ -21,16 +21,16 @@ "locked": { "lastModified": 1613549523, "narHash": "sha256-FgX/W+y8N4/6DLLTJuJVg5gUr3zk/5DxmSIxA4I6ZvE=", - "owner": "M-Labs", - "repo": "artiq", + "ref": "release-5", "rev": "10c9842fc8844b524811336c6a529c648a2d2fde", - "type": "github" + "revCount": 7153, + "type": "git", + "url": "https://github.com/M-Labs/artiq.git" }, "original": { - "owner": "M-Labs", "ref": "release-5", - "repo": "artiq", - "type": "github" + "type": "git", + "url": "https://github.com/M-Labs/artiq.git" } }, "artiq-src": { @@ -38,16 +38,16 @@ "locked": { "lastModified": 1613555154, "narHash": "sha256-D/43YVvMNkcXV6sBBxAqxcmtEO3TagIQEQl0zsGE+ag=", - "owner": "M-Labs", - "repo": "artiq", + "ref": "release-6", "rev": "c22482787ef806eb8e48d59b6ded7d93c7677c81", - "type": "github" + "revCount": 7582, + "type": "git", + "url": "https://github.com/M-Labs/artiq.git" }, "original": { - "owner": "M-Labs", "ref": "release-6", - "repo": "artiq", - "type": "github" + "type": "git", + "url": "https://github.com/M-Labs/artiq.git" } }, "nixpkgs": { @@ -78,11 +78,11 @@ "sinara-systems": { "flake": false, "locked": { - "lastModified": 1614310919, - "narHash": "sha256-hHHEAUMbwATSExViuGM5foGt29Aq0Xwc0uDNF/x5Kt0=", + "lastModified": 1615195806, + "narHash": "sha256-UfsPJxvERsGiECOFC+sjmk4r61aWepF0nXqRgnmLR80=", "ref": "master", - "rev": "52141a3fef08ee88227c48395bc8e32b5140c938", - "revCount": 200, + "rev": "1f0605f46cc7cbfcdf427f15a93e0e4a75d0d491", + "revCount": 202, "type": "git", "url": "https://git.m-labs.hk/M-Labs/sinara-systems.git" }, diff --git a/flake.nix b/flake.nix index 2f3de3f..e3d9a92 100644 --- a/flake.nix +++ b/flake.nix @@ -7,15 +7,15 @@ inputs = { nixpkgs.url = "github:nixos/nixpkgs/release-20.09"; artiq-beta-src = { - url = "github:M-Labs/artiq"; + url = "git+https://github.com/M-Labs/artiq.git"; flake = false; }; artiq-src = { - url = "github:M-Labs/artiq/release-6"; + url = "git+https://github.com/M-Labs/artiq.git?ref=release-6"; flake = false; }; artiq-legacy-src = { - url = "github:M-Labs/artiq/release-5"; + url = "git+https://github.com/M-Labs/artiq.git?ref=release-5"; flake = false; }; sinara-systems = { @@ -37,7 +37,7 @@ artiq-fast-src = let - inherit (sources.${artiqInputName}) lastModified rev narHash; + inherit (sources.${artiqInputName}) lastModified rev revCount narHash; hash = builtins.replaceStrings ["sha256-"] [""] narHash; artiqSrc = sources.${artiqInputName}.outPath; @@ -60,7 +60,7 @@ sha256 = "${hash}"; } EOF - echo "{ stdenv, git, fetchgit }: \"$MAJOR_VERSION.`cut -c1-8 <<< ${rev}`$SUFFIX\"" > $out/pkgs/artiq-version.nix + echo "{ stdenv, git, fetchgit }: \"$MAJOR_VERSION.${toString revCount}.`cut -c1-8 <<< ${rev}`$SUFFIX\"" > $out/pkgs/artiq-version.nix echo "{ stdenv, git, fetchgit }: \"${toString lastModified}\"" > $out/pkgs/artiq-timestamp.nix ''; artiq-fast = -- 2.42.0 From d7432778e21019c0dbc0b0fc88eb644cf238a63d Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Wed, 10 Mar 2021 23:53:46 +0100 Subject: [PATCH 11/12] flake.nix: set sinaraSystemsSrc for full-legacy --- flake.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index e3d9a92..fd7388a 100644 --- a/flake.nix +++ b/flake.nix @@ -71,7 +71,7 @@ artiq-board-generated = (import ./artiq-board-generated { inherit pkgs; - sinaraSystemsSrc = null; #sources.sinara-systems.outPath; + sinaraSystemsSrc = null; sinaraSystemsRev = sources.sinara-systems.rev; sinaraSystemsHash = builtins.replaceStrings ["sha256-"] [""] sources.sinara-systems.narHash; @@ -82,6 +82,7 @@ import ./artiq-full.nix { inherit pkgs a6p artiq-board-generated; artiq-fast = artiq-fast-src; + sinaraSystemsSrc = sources.sinara-systems.outPath; }; in { inherit artiq-fast artiq-full; -- 2.42.0 From 3d825116e2339d848cdcf4e2c8367e1f9b00840d Mon Sep 17 00:00:00 2001 From: Stephan Maka Date: Thu, 11 Mar 2021 00:09:32 +0100 Subject: [PATCH 12/12] flake.nix: fix handling of sinara-systems --- artiq-full.nix | 2 +- flake.nix | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/artiq-full.nix b/artiq-full.nix index 033d8df..6110dc5 100644 --- a/artiq-full.nix +++ b/artiq-full.nix @@ -62,7 +62,7 @@ let cp ${./artiq-full}/extras.nix $out cp ${./artiq-full}/*.patch $out - ${if a6p + ${if sinaraSystemsRev != null && sinaraSystemsHash != null then '' REV=${sinaraSystemsRev} HASH=${sinaraSystemsHash} diff --git a/flake.nix b/flake.nix index fd7388a..6b75b9b 100644 --- a/flake.nix +++ b/flake.nix @@ -24,13 +24,17 @@ }; }; - outputs = sources@{ self, nixpkgs, ... }: + outputs = sources@{ self, nixpkgs, sinara-systems, ... }: let systems = [ "x86_64-linux" ]; forAllSystems = nixpkgs.lib.genAttrs systems; + sinaraSystemsSrc = sinara-systems.outPath; + sinaraSystemsRev = sinara-systems.rev; + sinaraSystemsHash = builtins.replaceStrings ["sha256-"] [""] sinara-systems.narHash; + artiq-jobsets = system: artiqInputName: a6p: let pkgs = nixpkgs.legacyPackages.${system}; @@ -71,10 +75,7 @@ artiq-board-generated = (import ./artiq-board-generated { inherit pkgs; - sinaraSystemsSrc = null; - sinaraSystemsRev = sources.sinara-systems.rev; - sinaraSystemsHash = builtins.replaceStrings ["sha256-"] [""] - sources.sinara-systems.narHash; + inherit sinaraSystemsSrc sinaraSystemsRev sinaraSystemsHash; artiq-fast = artiq-fast-src; }).generated-nix; @@ -82,7 +83,7 @@ import ./artiq-full.nix { inherit pkgs a6p artiq-board-generated; artiq-fast = artiq-fast-src; - sinaraSystemsSrc = sources.sinara-systems.outPath; + inherit sinaraSystemsSrc sinaraSystemsRev sinaraSystemsHash; }; in { inherit artiq-fast artiq-full; -- 2.42.0