drewrisinger
/
nix-scripts
forked from M-Labs/nix-scripts
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

wfvm: use sftp instead of scp 

scp is bogged down transferring the 13k files of the Visual Studio pile of garbage.
This commit is contained in:
Sebastien Bourdeauducq 2020-06-17 23:40:49 +08:00
parent 465cc193ec
commit 0f99ffc583
5 changed files with 37 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
artiq-fast/conda-windows/binutils.nix
View File

@ -11,7 +11,8 @@ let
name = "build-binutils"; name = "build-binutils";
image = wfvm.makeWindowsImage { installCommands = with wfvm.layers; [ anaconda3 msys2 msys2-packages ]; }; image = wfvm.makeWindowsImage { installCommands = with wfvm.layers; [ anaconda3 msys2 msys2-packages ]; };
script = '' script = ''
${wfvm.utils.win-put}/bin/win-put ${libiconv} ${libiconv-filename} ln -s ${libiconv} ${libiconv-filename}
${wfvm.utils.win-put}/bin/win-put ${libiconv-filename}
${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate && conda create -n build ${libiconv-filename}" ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate && conda create -n build ${libiconv-filename}"
cat > meta.yaml << EOF cat > meta.yaml << EOF
@ -54,9 +55,10 @@ let
EOF EOF
${wfvm.utils.win-exec}/bin/win-exec "mkdir binutils" ${wfvm.utils.win-exec}/bin/win-exec "mkdir binutils"
${wfvm.utils.win-put}/bin/win-put meta.yaml ".\binutils" ${wfvm.utils.win-put}/bin/win-put meta.yaml binutils
${wfvm.utils.win-put}/bin/win-put bld.bat ".\binutils" ${wfvm.utils.win-put}/bin/win-put bld.bat binutils
${wfvm.utils.win-put}/bin/win-put ${src} ".\src.tar.bz2" ln -s ${src} src.tar.bz2
${wfvm.utils.win-put}/bin/win-put src.tar.bz2 .
${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate build && conda build --no-anaconda-upload --no-test binutils" ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate build && conda build --no-anaconda-upload --no-test binutils"

6
artiq-fast/conda-windows/llvm-or1k.nix
View File

@ -46,11 +46,11 @@ let
EOF EOF
${wfvm.utils.win-exec}/bin/win-exec "mkdir llvm-or1k" ${wfvm.utils.win-exec}/bin/win-exec "mkdir llvm-or1k"
${wfvm.utils.win-put}/bin/win-put meta.yaml ".\llvm-or1k" ${wfvm.utils.win-put}/bin/win-put meta.yaml llvm-or1k
${wfvm.utils.win-put}/bin/win-put bld.bat ".\llvm-or1k" ${wfvm.utils.win-put}/bin/win-put bld.bat llvm-or1k
ln -s ${src} src ln -s ${src} src
tar chf src.tar src tar chf src.tar src
${wfvm.utils.win-put}/bin/win-put src.tar ".\src.tar" ${wfvm.utils.win-put}/bin/win-put src.tar .
${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate build && conda build --no-anaconda-upload --no-test llvm-or1k" ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate build && conda build --no-anaconda-upload --no-test llvm-or1k"

9
artiq-fast/conda-windows/llvmlite-artiq.nix
View File

@ -8,7 +8,8 @@ let
name = "build-llvmlite-artiq"; name = "build-llvmlite-artiq";
image = wfvm.makeWindowsImage { installCommands = with wfvm.layers; [ anaconda3 msys2 msys2-packages ]; }; image = wfvm.makeWindowsImage { installCommands = with wfvm.layers; [ anaconda3 msys2 msys2-packages ]; };
script = '' script = ''
${wfvm.utils.win-put}/bin/win-put "${conda-windows-llvm-or1k}/win-64/llvm-or1k-*.tar.bz2" ".\llvm-or1k.tar.bz2" ln -s ${conda-windows-llvm-or1k}/win-64/llvm-or1k-*.tar.bz2 llvm-or1k.tar.bz2
${wfvm.utils.win-put}/bin/win-put llvm-or1k.tar.bz2 .
${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate && conda create -n build llvm-or1k.tar.bz2" ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate && conda create -n build llvm-or1k.tar.bz2"
cat > meta.yaml << EOF cat > meta.yaml << EOF
@ -32,12 +33,12 @@ let
EOF EOF
${wfvm.utils.win-exec}/bin/win-exec "mkdir llvmlite-artiq" ${wfvm.utils.win-exec}/bin/win-exec "mkdir llvmlite-artiq"
${wfvm.utils.win-put}/bin/win-put meta.yaml ".\llvmlite-artiq" ${wfvm.utils.win-put}/bin/win-put meta.yaml llvmlite-artiq
${wfvm.utils.win-put}/bin/win-put bld.bat ".\llvmlite-artiq" ${wfvm.utils.win-put}/bin/win-put bld.bat llvmlite-artiq
cp --no-preserve=mode,ownership -R ${src} src cp --no-preserve=mode,ownership -R ${src} src
patch -d src -p1 < ${./llvmlite-msys.diff} patch -d src -p1 < ${./llvmlite-msys.diff}
tar chf src.tar src tar chf src.tar src
${wfvm.utils.win-put}/bin/win-put src.tar ".\src.tar" ${wfvm.utils.win-put}/bin/win-put src.tar .
${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate build && conda build --no-anaconda-upload --no-test llvmlite-artiq" ${wfvm.utils.win-exec}/bin/win-exec ".\Anaconda3\scripts\activate build && conda build --no-anaconda-upload --no-test llvmlite-artiq"

26
artiq-fast/wfvm/layers/default.nix
View File

@ -8,9 +8,10 @@
url = "https://repo.anaconda.com/archive/Anaconda3-2020.02-Windows-x86_64.exe"; url = "https://repo.anaconda.com/archive/Anaconda3-2020.02-Windows-x86_64.exe";
sha256 = "0n31l8l89jrjrbzbifxbjnr3g320ly9i4zfyqbf3l9blf4ygbhl3"; sha256 = "0n31l8l89jrjrbzbifxbjnr3g320ly9i4zfyqbf3l9blf4ygbhl3";
}; };
in '' in
''
ln -s ${Anaconda3} ./Anaconda3.exe ln -s ${Anaconda3} ./Anaconda3.exe
win-put Anaconda3.exe 'C:\Users\wfvm' win-put Anaconda3.exe .
echo Running Anaconda installer... echo Running Anaconda installer...
win-exec 'start /wait "" .\Anaconda3.exe /S /D=%UserProfile%\Anaconda3' win-exec 'start /wait "" .\Anaconda3.exe /S /D=%UserProfile%\Anaconda3'
echo Anaconda installer finished echo Anaconda installer finished
@ -32,8 +33,8 @@
in '' in ''
ln -s ${msys2} ./msys2.exe ln -s ${msys2} ./msys2.exe
ln -s ${msys2-auto-install} ./auto-install.js ln -s ${msys2-auto-install} ./auto-install.js
win-put msys2.exe 'C:\Users\wfvm' win-put msys2.exe .
win-put auto-install.js 'C:\Users\wfvm' win-put auto-install.js .
echo Running MSYS2 installer... echo Running MSYS2 installer...
# work around MSYS2 installer bug that prevents it from closing at the end of unattended install # work around MSYS2 installer bug that prevents it from closing at the end of unattended install
expect -c 'set timeout 600; spawn win-exec ".\\msys2.exe --script auto-install.js -v InstallPrefix=C:\\msys64"; expect FinishedPageCallback { close }' expect -c 'set timeout 600; spawn win-exec ".\\msys2.exe --script auto-install.js -v InstallPrefix=C:\\msys64"; expect FinishedPageCallback { close }'
@ -45,7 +46,7 @@
script = let script = let
msys-packages = import ./msys_packages.nix { inherit pkgs; }; msys-packages = import ./msys_packages.nix { inherit pkgs; };
msys-packages-put = pkgs.lib.strings.concatStringsSep "\n" msys-packages-put = pkgs.lib.strings.concatStringsSep "\n"
(map (package: ''win-put ${package} 'C:\Users\wfvm\msyspackages' '') msys-packages); (map (package: ''win-put ${package} 'msyspackages' '') msys-packages);
in in
# Windows command line is so shitty it can't even do glob expansion. Why do people use Windows? # Windows command line is so shitty it can't even do glob expansion. Why do people use Windows?
'' ''
@ -57,7 +58,7 @@
set PATH=%MSYS%\usr\bin;%MSYS%\mingw%ARCH%\bin;%PATH% set PATH=%MSYS%\usr\bin;%MSYS%\mingw%ARCH%\bin;%PATH%
bash -c "pacman -U --noconfirm C:/Users/wfvm/msyspackages/*" bash -c "pacman -U --noconfirm C:/Users/wfvm/msyspackages/*"
EOF EOF
win-put installmsyspackages.bat 'C:\Users\wfvm' win-put installmsyspackages.bat .
win-exec installmsyspackages win-exec installmsyspackages
''; '';
}; };
@ -72,7 +73,7 @@
in in
'' ''
ln -s ${cmake} cmake.msi ln -s ${cmake} cmake.msi
win-put cmake.msi win-put cmake.msi .
win-exec "msiexec.exe /q /i cmake.msi ADD_CMAKE_TO_PATH=System" win-exec "msiexec.exe /q /i cmake.msi ADD_CMAKE_TO_PATH=System"
''; '';
}; };
@ -99,9 +100,16 @@
buildPhase = "python $src/vsdownload.py --accept-license --dest $out"; buildPhase = "python $src/vsdownload.py --accept-license --dest $out";
}; };
in in
# Yes, you need to write Windoze-side SFTP absolute paths like this or it won't work.
# Just the normal state of things on that cretinous OS.
'' ''
win-put ${vs}/VC/Tools/MSVC 'C:\' win-exec "mkdir C:\VS"
win-exec 'setx PATH C:\MSVC\14.26.28801\bin\Hostx64\x64;%PATH% /m' win-exec "mkdir C:\VS\VC"
win-exec "mkdir C:\VS\VC\Tools"
win-exec "mkdir C:\VS\kits"
win-put ${vs}/VC/Tools/MSVC '/C:/VS/VC/Tools'
win-put ${vs}/kits/10 '/C:/VS/kits'
win-exec 'setx PATH "C:\VS\VC\Tools\MSVC\14.26.28801\bin\Hostx64\x64;C:\VS\kits\10\bin\10.0.18362.0\x64;%PATH%" /m'
''; '';
}; };
} }

9
artiq-fast/wfvm/utils.nix
View File

@ -53,14 +53,15 @@ rec {
win-put = pkgs.writeShellScriptBin "win-put" '' win-put = pkgs.writeShellScriptBin "win-put" ''
echo win-put $1 -\> $2 echo win-put $1 -\> $2
${pkgs.sshpass}/bin/sshpass -p1234 -- \ ${pkgs.sshpass}/bin/sshpass -p1234 -- \
${pkgs.openssh}/bin/scp -r -P 2022 ${sshOpts} \ ${pkgs.openssh}/bin/sftp -r -P 2022 ${sshOpts} \
$1 wfvm@localhost:$2 wfvm@localhost <<< "cd $2
put $1"
''; '';
win-get = pkgs.writeShellScriptBin "win-get" '' win-get = pkgs.writeShellScriptBin "win-get" ''
echo win-get $1 echo win-get $1
${pkgs.sshpass}/bin/sshpass -p1234 -- \ ${pkgs.sshpass}/bin/sshpass -p1234 -- \
${pkgs.openssh}/bin/scp -r -T -P 2022 ${sshOpts} \ ${pkgs.openssh}/bin/sftp -r -P 2022 ${sshOpts} \
wfvm@localhost:$1 . wfvm@localhost:$1
''; '';
wfvm-run = { name, image, script, display ? false, isolateNetwork ? true, forwardedPorts ? [] }: wfvm-run = { name, image, script, display ? false, isolateNetwork ? true, forwardedPorts ? [] }: