From 8111b2f42be3379136a6fe9859422b236f80f56a Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 22 Jun 2020 17:45:18 +0800 Subject: [PATCH] windows: run tests using WFVM --- artiq-fast.nix | 8 +- artiq-fast/windows/README.md | 25 ------ artiq-fast/windows/install.nix | 91 -------------------- artiq-fast/windows/install.txt | 13 --- artiq-fast/windows/manual-test-run.nix | 34 ++------ artiq-fast/windows/qemu.nix | 55 ------------ artiq-fast/windows/run-test.nix | 112 +++++++++---------------- 7 files changed, 50 insertions(+), 288 deletions(-) delete mode 100644 artiq-fast/windows/README.md delete mode 100644 artiq-fast/windows/install.nix delete mode 100644 artiq-fast/windows/install.txt delete mode 100644 artiq-fast/windows/qemu.nix diff --git a/artiq-fast.nix b/artiq-fast.nix index 2a0b507..0ccd430 100644 --- a/artiq-fast.nix +++ b/artiq-fast.nix @@ -42,9 +42,7 @@ let artiqVersion = import "${generatedNix}/pkgs/artiq-version.nix" (with pkgs; { inherit stdenv fetchgit git; }); windowsRunner = overrides: import "${generatedNix}/windows/run-test.nix" ({ - inherit pkgs; - sipycoPkg = artiqpkgs.conda-sipyco; - artiqPkg = artiqpkgs.conda-artiq; + inherit pkgs artiqpkgs; } // overrides); jobs = (builtins.mapAttrs (key: value: pkgs.lib.hydraJob value) artiqpkgs); in @@ -61,7 +59,7 @@ in buildInputs = [ (windowsRunner {}) ]; phases = [ "buildPhase" ]; buildPhase = '' - ${windowsRunner {}}/bin/run.sh + ${windowsRunner {}}/bin/wfvm-run-windows-tests touch $out ''; }; @@ -119,7 +117,7 @@ in export ARTIQ_LOW_LATENCY=1 python -m unittest discover -v artiq.test.coredevice - ${windowsRunner { testCommand = "set ARTIQ_ROOT=%cd%\\anaconda\\envs\\artiq-env\\Lib\\site-packages\\artiq\\examples\\kc705_nist_clock&&python -m unittest discover -v artiq.test.coredevice"; }}/bin/run.sh + ${windowsRunner { testCommand = "set ARTIQ_ROOT=%cd%\\Anaconda3\\envs\\artiq-env\\Lib\\site-packages\\artiq\\examples\\kc705_nist_clock&& python -m unittest discover -v artiq.test.coredevice"; }}/bin/wfvm-run-windows-tests ) mkdir $out diff --git a/artiq-fast/windows/README.md b/artiq-fast/windows/README.md deleted file mode 100644 index 279e967..0000000 --- a/artiq-fast/windows/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Preparation steps - -## Install a Windows image - -```shell -nix-build install.nix -I artiqSrc=…/artiq -result/bin/windows-installer.sh -``` - -Follow the instructions. - -## Install Anaconda to the image - -```shell -result/bin/anaconda-installer.sh -``` - -Move the image `c.img` to one of Nix' `extra-sandbox-paths` (`nix.sandboxPaths` on NixOS). - - -# Running the tests manually - -```shell -nix-build --pure --arg diskImage "\"…/c.img\"" -I artiqSrc=…/artiq manual-test-run.nix -``` diff --git a/artiq-fast/windows/install.nix b/artiq-fast/windows/install.nix deleted file mode 100644 index 9ac5e7a..0000000 --- a/artiq-fast/windows/install.nix +++ /dev/null @@ -1,91 +0,0 @@ -{ pkgs ? import {}, - diskImageSize ? "22G", - qemuMem ? "4G", -}: - -with pkgs; - -let - windowsIso = fetchurl { - url = "https://software-download.microsoft.com/download/sg/17763.107.101029-1455.rs5_release_svc_refresh_CLIENT_LTSC_EVAL_x64FRE_en-us.iso"; - sha256 = "668fe1af70c2f7416328aee3a0bb066b12dc6bbd2576f40f812b95741e18bc3a"; - }; - anaconda = fetchurl { - url = "https://repo.anaconda.com/archive/Anaconda3-2019.03-Windows-x86_64.exe"; - sha256 = "1f9icm5rwab6l1f23a70dw0qixzrl62wbglimip82h4zhxlh3jfj"; - }; - - escape = builtins.replaceStrings [ "\\" ] [ "\\\\" ]; - qemu = import ./qemu.nix { - inherit pkgs qemuMem; - diskImage = "c.img"; - }; - # Double-escape because we produce a script from a shell heredoc - ssh = cmd: qemu.ssh (escape cmd); - scp = qemu.scp; - - sshCondaEnv = cmd: ssh "anaconda\\scripts\\activate && ${cmd}"; - condaEnv = "artiq-env"; - condaDepSpecs = - builtins.concatStringsSep " " - (map (s: "\"${s}\"") - (import ../conda/artiq-deps.nix)); - - instructions = - builtins.toFile "install.txt" - (builtins.readFile ./install.txt); -in -stdenv.mkDerivation { - name = "windows-installer"; - src = windowsIso; - setSourceRoot = "sourceRoot=`pwd`"; - unpackCmd = '' - ln -s $curSrc windows.iso - ''; - propagatedBuildInputs = qemu.inputs; - dontBuild = true; - installPhase = '' - mkdir -p $out/bin $out/data - ln -s $(readlink windows.iso) $out/data/windows.iso - cat > $out/bin/windows-installer.sh << EOF - #!/usr/bin/env bash - set -e -m - - ${qemu.qemu-img} create -f qcow2 c.img ${diskImageSize} - ${qemu.runQemu false [] [ - "-boot" "order=d" - "-drive" "file=c.img,index=0,media=disk,cache=unsafe" - "-drive" "file=$out/data/windows.iso,index=1,media=cdrom,cache=unsafe" - ]} & - cat ${instructions} - wait - EOF - - cat > $out/bin/anaconda-installer.sh << EOF - #!/usr/bin/env bash - set -e -m - - ${qemu.runQemu false [] [ - "-boot" "order=c" - "-drive" "file=c.img,index=0,media=disk" - ]} & - sleep 10 - ${ssh "ver"} - - ${scp anaconda "Anaconda.exe"} - ${ssh "start /wait \"\" Anaconda.exe /S /D=%cd%\\anaconda"} - - ${sshCondaEnv "conda config --add channels conda-forge"} - ${sshCondaEnv "conda config --add channels m-labs"} - ( ${sshCondaEnv "conda update -y conda"} ) || true - ${sshCondaEnv "conda update -y --all"} - ${sshCondaEnv "conda create -y -n ${condaEnv}"} - ${sshCondaEnv "conda install -y -n ${condaEnv} ${condaDepSpecs}"} - ${ssh "shutdown /p /f"} - - echo "Waiting for qemu exit" - wait - EOF - chmod a+x $out/bin/*.sh - ''; -} diff --git a/artiq-fast/windows/install.txt b/artiq-fast/windows/install.txt deleted file mode 100644 index a2576c1..0000000 --- a/artiq-fast/windows/install.txt +++ /dev/null @@ -1,13 +0,0 @@ -Add user account with expected password [user/user]. -Enable the OpenSSH server: -- "Add or remove programs" -- "Manage optional features" -- "Add a feature" -- "OpenSSH Server" -- "Install" -- Open "Services" -- Double-click the "OpenSSH SSH Server" service -- Set "Startup type" to "Automatic" -- "Start" -- "Ok" -Then press ENTER here to proceed with automatic installation diff --git a/artiq-fast/windows/manual-test-run.nix b/artiq-fast/windows/manual-test-run.nix index b04ae8d..fd4d554 100644 --- a/artiq-fast/windows/manual-test-run.nix +++ b/artiq-fast/windows/manual-test-run.nix @@ -1,30 +1,10 @@ -# This runs `run-test.nix` with `nix-build` - -{ pkgs ? import {}, - artiqpkgs ? import ../. { inherit pkgs; }, - diskImage ? "/opt/windows/c.img", - qemuMem ? "2G", - testTimeout ? 180, -}: - -with pkgs; +{ pkgs ? import {} }: let - windowsRunner = overrides: - import ./run-test.nix ({ - inherit pkgs diskImage qemuMem testTimeout; - sipycoPkg = artiqpkgs.conda-sipyco; - artiqPkg = artiqpkgs.conda-artiq; - } // overrides); + artiqpkgs = import ../. { inherit pkgs; }; + run-test = import ./run-test.nix { + inherit pkgs artiqpkgs; + testCommand = "set ARTIQ_ROOT=%cd%\\Anaconda3\\envs\\artiq-env\\Lib\\site-packages\\artiq\\examples\\kc705_nist_clock&& python -m unittest discover -v sipyco.test && python -m unittest discover -v artiq.test"; + }; in - -stdenv.mkDerivation { - name = "windows-test"; - - phases = [ "installPhase" "checkPhase" ]; - installPhase = "touch $out"; - doCheck = true; - checkPhase = '' - ${windowsRunner { testCommand = "set ARTIQ_ROOT=%cd%\\anaconda\\envs\\artiq-env\\Lib\\site-packages\\artiq\\examples\\kc705_nist_clock&&python -m unittest discover -v artiq.test"; }}/bin/run.sh - ''; -} + run-test diff --git a/artiq-fast/windows/qemu.nix b/artiq-fast/windows/qemu.nix deleted file mode 100644 index e575f0d..0000000 --- a/artiq-fast/windows/qemu.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ pkgs, - diskImage, - qemuMem, - sshUser ? "user", - sshPassword ? "user", -}: - -with pkgs; - -let - qemu-img = "${qemu_kvm}/bin/qemu-img"; - runQemu = isolateNetwork: forwardedPorts: extraArgs: - let - restrict = - if isolateNetwork - then "on" - else "off"; - # use socat instead of `tcp:…` to allow multiple connections - guestfwds = - builtins.concatStringsSep "" - (map ({ listenAddr, targetAddr, port }: - ",guestfwd=tcp:${listenAddr}:${toString port}-cmd:${socat}/bin/socat\\ -\\ tcp:${targetAddr}:${toString port}" - ) forwardedPorts); - args = [ - "-enable-kvm" - "-m" qemuMem - "-bios" "${OVMF.fd}/FV/OVMF.fd" - "-netdev" "user,id=n1,net=192.168.1.0/24,restrict=${restrict},hostfwd=tcp::2022-:22${guestfwds}" - "-device" "e1000,netdev=n1" - ]; - argStr = builtins.concatStringsSep " " (args ++ extraArgs); - in "${qemu_kvm}/bin/qemu-system-x86_64 ${argStr}"; - - # Pass empty config file to prevent ssh from failing to create ~/.ssh - sshOpts = "-F /dev/null -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=\$TMP/known_hosts"; - sshWithQuotes = quotes: cmd: '' - echo ssh windows ${quotes}${cmd}${quotes} - ${sshpass}/bin/sshpass -p${sshPassword} -- \ - ${openssh}/bin/ssh -np 2022 ${sshOpts} \ - ${sshUser}@localhost \ - ${quotes}${cmd}${quotes} - ''; - ssh = sshWithQuotes "'"; - scp = src: target: '' - echo "Copy ${src} to ${target}" - ${sshpass}/bin/sshpass -p${sshPassword} -- \ - ${openssh}/bin/scp -P 2022 ${sshOpts} \ - "${src}" "${sshUser}@localhost:${target}" - ''; - -in -{ - inherit qemu-img runQemu ssh sshWithQuotes scp; - inputs = [ qemu_kvm openssh sshpass ]; -} diff --git a/artiq-fast/windows/run-test.nix b/artiq-fast/windows/run-test.nix index 6293d2e..369af35 100644 --- a/artiq-fast/windows/run-test.nix +++ b/artiq-fast/windows/run-test.nix @@ -1,24 +1,6 @@ -{ pkgs, - sipycoPkg, - artiqPkg, - diskImage ? "/opt/windows/c.img", - qemuMem ? "2G", - testTimeout ? 600, - testCommand ? "python -m unittest discover -v sipyco.test && python -m unittest discover -v artiq.test", -}: - -with pkgs; +{ pkgs, artiqpkgs, testCommand }: let - escape = builtins.replaceStrings [ "\\" ] [ "\\\\" ]; - qemu = import ./qemu.nix { - inherit pkgs qemuMem; - diskImage = "c.img"; - }; - # Double-escape because we produce a script from a shell heredoc - ssh = cmd: qemu.ssh (escape cmd); - sshUnquoted = qemu.sshWithQuotes "\""; - scp = qemu.scp; condaEnv = "artiq-env"; tcpPorts = [ 1380 1381 1382 1383 ]; forwardedPorts = @@ -27,58 +9,44 @@ let targetAddr = "192.168.1.50"; inherit port; }) tcpPorts; + + wfvm = import ../wfvm/default.nix { inherit pkgs; }; + conda-deps = { + name = "conda-deps"; + script = let + conda-deps-noarch = import ./conda_noarch_packages.nix { inherit pkgs; }; + conda-deps-win-64 = import ./conda_win-64_packages.nix { inherit pkgs; }; + conda-packages-put = pkgs.lib.strings.concatStringsSep "\n" + ( (map (package: ''win-put ${package} 'fake-channel/noarch' '') conda-deps-noarch) + ++ (map (package: ''win-put ${package} 'fake-channel/win-64' '') conda-deps-win-64) ); + in + '' + win-exec 'mkdir fake-channel && mkdir fake-channel\noarch && mkdir fake-channel\win-64' + + ${conda-packages-put} + + win-put ${artiqpkgs.conda-windows-binutils-or1k}/win-64/*.tar.bz2 'fake-channel/win-64' + win-put ${artiqpkgs.conda-windows-llvm-or1k}/win-64/*.tar.bz2 'fake-channel/win-64' + win-put ${artiqpkgs.conda-windows-llvmlite-artiq}/win-64/*.tar.bz2 'fake-channel/win-64' + + win-put ${artiqpkgs.conda-pythonparser}/noarch/*.tar.bz2 'fake-channel/noarch' + win-put ${artiqpkgs.conda-sipyco}/noarch/*.tar.bz2 'fake-channel/noarch' + win-put ${artiqpkgs.conda-quamash}/noarch/*.tar.bz2 'fake-channel/noarch' + ''; + }; in + wfvm.utils.wfvm-run { + name = "windows-tests"; + image = wfvm.makeWindowsImage { installCommands = [ wfvm.layers.anaconda3 conda-deps ]; }; + inherit forwardedPorts; + script = + '' + ${wfvm.utils.win-put}/bin/win-put ${artiqpkgs.conda-artiq}/noarch/*.tar.bz2 'fake-channel/noarch' -stdenv.mkDerivation { - name = "windows-test-runner"; - src = ./.; - - propagatedBuildInputs = qemu.inputs; - dontBuild = true; - installPhase = '' - mkdir -p $out/bin - cat > $out/bin/run.sh << EOF - #!/usr/bin/env bash - set -e -m - - # +1 day from last modification of the disk image - CLOCK=$(date -Is -d @$(expr $(stat -c %Y ${diskImage}) + 86400)) - ${qemu.runQemu true forwardedPorts [ - "-boot" "order=c" - "-snapshot" - "-drive" "file=${diskImage},index=0,media=disk,cache=unsafe" - "-rtc" "base=\\$CLOCK" - "-display" "none" - ]} & - - echo "Wait for Windows to boot" - sleep 30 - ${ssh "ver"} - i=0 - for pkg in ${sipycoPkg}/noarch/sipyco*.tar.bz2 ${artiqPkg}/noarch/artiq*.tar.bz2 ; do - ${scp "\\$pkg" "to_install\\$i.tar.bz2"} - ${sshUnquoted "anaconda\\scripts\\activate ${condaEnv} && conda install to_install\\$i.tar.bz2"} - ((i=i+1)) - done - - # Schedule a timed shutdown against hanging test runs - ${ssh "shutdown -s -t ${toString testTimeout}"} - - FAIL=n - ( ${ssh "anaconda\\scripts\\activate ${condaEnv} && ${testCommand}"} ) || FAIL=y - - # Abort timeouted shutdown - ${ssh "shutdown -a"} - # Power off immediately - ${ssh "shutdown -p -f"} - wait - - if [ "\$FAIL" = "y" ]; then - exit 1 - else - exit 0 - fi - EOF - chmod a+x $out/bin/run.sh - ''; -} + ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate && conda index fake-channel" + ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate && conda create -n ${condaEnv} --offline" + ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate ${condaEnv} && conda install -y -c file:///C:/users/wfvm/fake-channel --offline artiq" + #${pkgs.sshpass}/bin/sshpass -p1234 -- ${pkgs.openssh}/bin/ssh -p 2022 wfvm@localhost -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null + ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate ${condaEnv} && ${testCommand}" + ''; + }