From 0f99ffc5830220e8d59237ab291980123c261877 Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Wed, 17 Jun 2020 23:40:49 +0800 Subject: [PATCH] wfvm: use sftp instead of scp scp is bogged down transferring the 13k files of the Visual Studio pile of garbage. --- artiq-fast/conda-windows/binutils.nix | 10 +++++--- artiq-fast/conda-windows/llvm-or1k.nix | 6 ++--- artiq-fast/conda-windows/llvmlite-artiq.nix | 9 ++++--- artiq-fast/wfvm/layers/default.nix | 28 +++++++++++++-------- artiq-fast/wfvm/utils.nix | 9 ++++--- 5 files changed, 37 insertions(+), 25 deletions(-) diff --git a/artiq-fast/conda-windows/binutils.nix b/artiq-fast/conda-windows/binutils.nix index e425f51..049e7c4 100644 --- a/artiq-fast/conda-windows/binutils.nix +++ b/artiq-fast/conda-windows/binutils.nix @@ -11,7 +11,8 @@ let name = "build-binutils"; image = wfvm.makeWindowsImage { installCommands = with wfvm.layers; [ anaconda3 msys2 msys2-packages ]; }; script = '' - ${wfvm.utils.win-put}/bin/win-put ${libiconv} ${libiconv-filename} + ln -s ${libiconv} ${libiconv-filename} + ${wfvm.utils.win-put}/bin/win-put ${libiconv-filename} ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate && conda create -n build ${libiconv-filename}" cat > meta.yaml << EOF @@ -54,9 +55,10 @@ let EOF ${wfvm.utils.win-exec}/bin/win-exec "mkdir binutils" - ${wfvm.utils.win-put}/bin/win-put meta.yaml ".\binutils" - ${wfvm.utils.win-put}/bin/win-put bld.bat ".\binutils" - ${wfvm.utils.win-put}/bin/win-put ${src} ".\src.tar.bz2" + ${wfvm.utils.win-put}/bin/win-put meta.yaml binutils + ${wfvm.utils.win-put}/bin/win-put bld.bat binutils + ln -s ${src} src.tar.bz2 + ${wfvm.utils.win-put}/bin/win-put src.tar.bz2 . ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate build && conda build --no-anaconda-upload --no-test binutils" diff --git a/artiq-fast/conda-windows/llvm-or1k.nix b/artiq-fast/conda-windows/llvm-or1k.nix index 217fbab..cbc882e 100644 --- a/artiq-fast/conda-windows/llvm-or1k.nix +++ b/artiq-fast/conda-windows/llvm-or1k.nix @@ -46,11 +46,11 @@ let EOF ${wfvm.utils.win-exec}/bin/win-exec "mkdir llvm-or1k" - ${wfvm.utils.win-put}/bin/win-put meta.yaml ".\llvm-or1k" - ${wfvm.utils.win-put}/bin/win-put bld.bat ".\llvm-or1k" + ${wfvm.utils.win-put}/bin/win-put meta.yaml llvm-or1k + ${wfvm.utils.win-put}/bin/win-put bld.bat llvm-or1k ln -s ${src} src tar chf src.tar src - ${wfvm.utils.win-put}/bin/win-put src.tar ".\src.tar" + ${wfvm.utils.win-put}/bin/win-put src.tar . ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate build && conda build --no-anaconda-upload --no-test llvm-or1k" diff --git a/artiq-fast/conda-windows/llvmlite-artiq.nix b/artiq-fast/conda-windows/llvmlite-artiq.nix index aa21b79..6fb8411 100644 --- a/artiq-fast/conda-windows/llvmlite-artiq.nix +++ b/artiq-fast/conda-windows/llvmlite-artiq.nix @@ -8,7 +8,8 @@ let name = "build-llvmlite-artiq"; image = wfvm.makeWindowsImage { installCommands = with wfvm.layers; [ anaconda3 msys2 msys2-packages ]; }; script = '' - ${wfvm.utils.win-put}/bin/win-put "${conda-windows-llvm-or1k}/win-64/llvm-or1k-*.tar.bz2" ".\llvm-or1k.tar.bz2" + ln -s ${conda-windows-llvm-or1k}/win-64/llvm-or1k-*.tar.bz2 llvm-or1k.tar.bz2 + ${wfvm.utils.win-put}/bin/win-put llvm-or1k.tar.bz2 . ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate && conda create -n build llvm-or1k.tar.bz2" cat > meta.yaml << EOF @@ -32,12 +33,12 @@ let EOF ${wfvm.utils.win-exec}/bin/win-exec "mkdir llvmlite-artiq" - ${wfvm.utils.win-put}/bin/win-put meta.yaml ".\llvmlite-artiq" - ${wfvm.utils.win-put}/bin/win-put bld.bat ".\llvmlite-artiq" + ${wfvm.utils.win-put}/bin/win-put meta.yaml llvmlite-artiq + ${wfvm.utils.win-put}/bin/win-put bld.bat llvmlite-artiq cp --no-preserve=mode,ownership -R ${src} src patch -d src -p1 < ${./llvmlite-msys.diff} tar chf src.tar src - ${wfvm.utils.win-put}/bin/win-put src.tar ".\src.tar" + ${wfvm.utils.win-put}/bin/win-put src.tar . ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate build && conda build --no-anaconda-upload --no-test llvmlite-artiq" diff --git a/artiq-fast/wfvm/layers/default.nix b/artiq-fast/wfvm/layers/default.nix index 415a0bd..b79929e 100644 --- a/artiq-fast/wfvm/layers/default.nix +++ b/artiq-fast/wfvm/layers/default.nix @@ -8,13 +8,14 @@ url = "https://repo.anaconda.com/archive/Anaconda3-2020.02-Windows-x86_64.exe"; sha256 = "0n31l8l89jrjrbzbifxbjnr3g320ly9i4zfyqbf3l9blf4ygbhl3"; }; - in '' + in + '' ln -s ${Anaconda3} ./Anaconda3.exe - win-put Anaconda3.exe 'C:\Users\wfvm' + win-put Anaconda3.exe . echo Running Anaconda installer... win-exec 'start /wait "" .\Anaconda3.exe /S /D=%UserProfile%\Anaconda3' echo Anaconda installer finished - ''; + ''; }; msys2 = { name = "MSYS2"; @@ -32,8 +33,8 @@ in '' ln -s ${msys2} ./msys2.exe ln -s ${msys2-auto-install} ./auto-install.js - win-put msys2.exe 'C:\Users\wfvm' - win-put auto-install.js 'C:\Users\wfvm' + win-put msys2.exe . + win-put auto-install.js . echo Running MSYS2 installer... # work around MSYS2 installer bug that prevents it from closing at the end of unattended install expect -c 'set timeout 600; spawn win-exec ".\\msys2.exe --script auto-install.js -v InstallPrefix=C:\\msys64"; expect FinishedPageCallback { close }' @@ -45,7 +46,7 @@ script = let msys-packages = import ./msys_packages.nix { inherit pkgs; }; msys-packages-put = pkgs.lib.strings.concatStringsSep "\n" - (map (package: ''win-put ${package} 'C:\Users\wfvm\msyspackages' '') msys-packages); + (map (package: ''win-put ${package} 'msyspackages' '') msys-packages); in # Windows command line is so shitty it can't even do glob expansion. Why do people use Windows? '' @@ -57,7 +58,7 @@ set PATH=%MSYS%\usr\bin;%MSYS%\mingw%ARCH%\bin;%PATH% bash -c "pacman -U --noconfirm C:/Users/wfvm/msyspackages/*" EOF - win-put installmsyspackages.bat 'C:\Users\wfvm' + win-put installmsyspackages.bat . win-exec installmsyspackages ''; }; @@ -72,7 +73,7 @@ in '' ln -s ${cmake} cmake.msi - win-put cmake.msi + win-put cmake.msi . win-exec "msiexec.exe /q /i cmake.msi ADD_CMAKE_TO_PATH=System" ''; }; @@ -99,9 +100,16 @@ buildPhase = "python $src/vsdownload.py --accept-license --dest $out"; }; in + # Yes, you need to write Windoze-side SFTP absolute paths like this or it won't work. + # Just the normal state of things on that cretinous OS. '' - win-put ${vs}/VC/Tools/MSVC 'C:\' - win-exec 'setx PATH C:\MSVC\14.26.28801\bin\Hostx64\x64;%PATH% /m' + win-exec "mkdir C:\VS" + win-exec "mkdir C:\VS\VC" + win-exec "mkdir C:\VS\VC\Tools" + win-exec "mkdir C:\VS\kits" + win-put ${vs}/VC/Tools/MSVC '/C:/VS/VC/Tools' + win-put ${vs}/kits/10 '/C:/VS/kits' + win-exec 'setx PATH "C:\VS\VC\Tools\MSVC\14.26.28801\bin\Hostx64\x64;C:\VS\kits\10\bin\10.0.18362.0\x64;%PATH%" /m' ''; }; } diff --git a/artiq-fast/wfvm/utils.nix b/artiq-fast/wfvm/utils.nix index bb6ce7b..667ad7e 100644 --- a/artiq-fast/wfvm/utils.nix +++ b/artiq-fast/wfvm/utils.nix @@ -53,14 +53,15 @@ rec { win-put = pkgs.writeShellScriptBin "win-put" '' echo win-put $1 -\> $2 ${pkgs.sshpass}/bin/sshpass -p1234 -- \ - ${pkgs.openssh}/bin/scp -r -P 2022 ${sshOpts} \ - $1 wfvm@localhost:$2 + ${pkgs.openssh}/bin/sftp -r -P 2022 ${sshOpts} \ + wfvm@localhost <<< "cd $2 + put $1" ''; win-get = pkgs.writeShellScriptBin "win-get" '' echo win-get $1 ${pkgs.sshpass}/bin/sshpass -p1234 -- \ - ${pkgs.openssh}/bin/scp -r -T -P 2022 ${sshOpts} \ - wfvm@localhost:$1 . + ${pkgs.openssh}/bin/sftp -r -P 2022 ${sshOpts} \ + wfvm@localhost:$1 ''; wfvm-run = { name, image, script, display ? false, isolateNetwork ? true, forwardedPorts ? [] }: