windows: add script to automatically download conda dependencies and create fetchurl nix files

2020-06-22 14:45:11 +08:00 · 2020-06-22 14:45:11 +08:00 · 05cde854dd
parent bd8fa5d955
commit 05cde854dd
3 changed files with 252 additions and 0 deletions
--- a/artiq-fast/windows/conda_noarch_packages.nix
+++ b/artiq-fast/windows/conda_noarch_packages.nix
@ -0,0 +1,17 @@
+{ pkgs } : [
+
+(pkgs.fetchurl {
+  url = "https://conda.anaconda.org/conda-forge/noarch/prettytable-0.7.2-py_3.tar.bz2";
+  sha256 = "0b7s4xm6bbkcg37sf1i3mxrbac0vxhryq22m3qx4x9kh6k2c5g5q";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/noarch/six-1.15.0-py_0.conda";
+  sha256 = "057vci1j78fzkg4qnamfjhz47s0x2v6ygli565a56hvna7h11kng";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/noarch/pycparser-2.20-py_0.conda";
+  sha256 = "00rx3nqa21dpqq4lx7g7rahsds0248ky4rw8xjxv5xik5x5xrqnd";
+})
+]
--- a/artiq-fast/windows/conda_win-64_packages.nix
+++ b/artiq-fast/windows/conda_win-64_packages.nix
@ -0,0 +1,167 @@
+{ pkgs } : [
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/openssl-1.1.1g-he774522_0.conda";
+  sha256 = "1hjddbhkk8409k9x4ncxb2z76pj8gbs03bfv8nfwazxpygjhfm99";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/icu-58.2-ha925a31_3.conda";
+  sha256 = "1387x1xa1h2wivz1gw8ip2xgy0v5yb261nq1f3jfiiynn7hc8gdf";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/pyqtgraph-0.10.0-py35h28b3542_3.conda";
+  sha256 = "0n5l3p0r9fchra3dygp5jbxlb52fbwvrgnmbhq8xghnyvzyjw18c";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/sip-4.19.8-py35h6538335_0.conda";
+  sha256 = "09v80qdd6xhmmqbgvb5vm4r2hcid0zd45zzc6pyaxxnh2x9h8mss";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/certifi-2018.8.24-py35_1.conda";
+  sha256 = "06yc1a1cvc7rnqxmjk6sax8f38f41f9zksghf5h539s05cp24jhq";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/regex-2018.08.29-py35hfa6e2cd_0.conda";
+  sha256 = "0grlb566c2m2cgx4h7bcg0249fpll8n4h4idwbkjssr258sd2dbi";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/pyqt-5.9.2-py35h6538335_2.conda";
+  sha256 = "1njyr4hanppfibg075mslcbj8fq899ila9haivp8pl8f7n8zah1v";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/intel-openmp-2019.4-245.conda";
+  sha256 = "0kwvhxv219nkgcgcrlsff6gabcd2wln2c6qj7a6ahp6j99dqrz9s";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/numpy-1.15.2-py35ha559c80_0.conda";
+  sha256 = "1yqvwgw0fspxn0ckq4gcz5ag8w3wsqnrbqykz3hqspd1pvb7k7rk";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/curl-7.69.1-h2a8f88b_0.conda";
+  sha256 = "02s0wfwq8hn8l0f0j2516vabnfvywm1drlvkwrsrc4vg76n00gwx";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/numpy-base-1.15.2-py35h8128ebf_0.conda";
+  sha256 = "0z8k3z4qjgwp45p0x15w741afkamql6g9sampl3hf9sachs8mypx";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/tbb4py-2018.0.5-py35he980bc4_0.conda";
+  sha256 = "1wpw02cra3pzqw1s4x8mq447yqwd4qsv71yrsx9m2q4f5im887mp";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/setuptools-40.2.0-py35_0.conda";
+  sha256 = "19lfrbp28hng6l2pdai14x93jxd9yr0mpz585ws2kjqnz74wapn4";
+})
+
+(pkgs.fetchurl {
+  url = "https://conda.anaconda.org/conda-forge/win-64/libgit2-0.27.8-hfac1375_0.tar.bz2";
+  sha256 = "05pjzwmm5vyxwcgi6vv3i961x3dgdd9c4b1ihagq9kyxgqfy21kd";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/mkl-2018.0.3-1.conda";
+  sha256 = "0qk1nvvw37kzdxfyyzk3ibikw9s8v1y1dvzvd6ridczsyck3if9c";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/h5py-2.8.0-py35h3bdd7fb_2.conda";
+  sha256 = "1bnydwv3s6bhdw70w7vyx0vmb93002pnpwyhbi3zxibvrfdfjaqi";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/hdf5-1.10.2-hac2f561_1.conda";
+  sha256 = "12m8wnz9k6zkaja47iwfncm6srrcljgyy3l3nhkiyf0c3hwgalbg";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/vs2015_runtime-14.16.27012-hf0eaf9b_2.conda";
+  sha256 = "1k73y8r17r84m2663mffl8g0gx50va1pxspy6b2n7yp8jphhmy8d";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/mkl_random-1.0.1-py35h77b88f5_1.conda";
+  sha256 = "0p1bwjl799mxnxa0qn4bbh2w2qgg5c866zf4rgh98hlz7yx3vrzy";
+})
+
+(pkgs.fetchurl {
+  url = "https://conda.anaconda.org/conda-forge/win-64/pygit2-0.27.1-py35hfa6e2cd_0.tar.bz2";
+  sha256 = "1jypm8vxs4j4yr37ai4ki9qsslv3wz6slklmisnvjraz1a4vzaf8";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/libcurl-7.69.1-h2a8f88b_0.conda";
+  sha256 = "1j627520y8lishi60yk48291cqhjg82by2l2cnhdlc8wn1ldbi5s";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/wheel-0.31.1-py35_0.conda";
+  sha256 = "1adgbzqg5ibw8p4mdijgy13pybl273xzfvm1w8ii2k064pfazbdv";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/pip-10.0.1-py35_0.conda";
+  sha256 = "0b41585m4rc1a90xh5jrhw23x0sd4m1jdq6wn4gw1f4yazyxbn6d";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/libssh2-1.9.0-h7a1dbc1_1.conda";
+  sha256 = "1jv3nxsc1vky450v18nbz8mhk0wpw0k34ibfh53d4dfzimyii1dw";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/cffi-1.11.5-py35h74b6da3_1.conda";
+  sha256 = "1lw70385i73m3x060xdhydc5v946w5jxk13wx70z8rypvlgzqnhz";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/zlib-1.2.11-h62dcd97_4.conda";
+  sha256 = "1jcbdw7sv4mfkr70pcxa9cnvx17izif1a6wna8b5jj1barizp6g3";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/sqlite-3.32.2-h2a8f88b_0.conda";
+  sha256 = "01acaapsra2ii132vx9wykas75pgcb0yb428bs2x4nfl0v391xlp";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/wincertstore-0.2-py35hfebbdb8_0.conda";
+  sha256 = "1rg7xlygy19ici9114pyg1i1s1w0bgsd5qcb59s0g0ndj1704s08";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/python-3.5.6-he025d50_0.conda";
+  sha256 = "0hdkj7kn5dsi8gfx551n9lyan1a45iwswkh03rwxzvpxbllylm56";
+})
+
+(pkgs.fetchurl {
+  url = "https://conda.anaconda.org/conda-forge/win-64/python-levenshtein-0.12.0-py35_0.tar.bz2";
+  sha256 = "1rdpzv1y535swf17nm88chkp1m8w3wd3nwdy7jk1xfcnx1da5ss6";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/scipy-1.1.0-py35h4f6bf74_1.conda";
+  sha256 = "0f93xlmwppgs6znlc4w15bij9v41p80g25nxxpxcdqc0niv260ga";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/tk-8.6.10-he774522_0.conda";
+  sha256 = "1f1pv99y4jy7knmk3jsrhf4kj63qknr692wajzna0kp71gj0hsfm";
+})
+
+(pkgs.fetchurl {
+  url = "https://repo.anaconda.com/pkgs/main/win-64/mkl_fft-1.0.6-py35hdbbee80_0.conda";
+  sha256 = "0as5hb049gcakcqymh7k84dqisaa760xv71469b7m2658bmrni66";
+})
+]
--- a/artiq-fast/windows/make_conda_packages.sh
+++ b/artiq-fast/windows/make_conda_packages.sh
@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+set -e
+
+nix-build -E "
+let
+  pkgs = import <nixpkgs> {};
+  wfvm = import ../wfvm/default.nix { inherit pkgs; };
+in
+  wfvm.utils.wfvm-run {
+    name = \"get-conda-packages\";
+    image = wfvm.makeWindowsImage { installCommands = [ wfvm.layers.anaconda3 ]; };
+    # TODO: fix wfvm login expiry and also remove 'date' workarounds below
+    #fakeRtc = false;
+    isolateNetwork = false;
+    script = ''
+      cat > getcondapackages.bat << EOF
+      date 06-22-20
+      call conda config --prepend channels https://conda.m-labs.hk/artiq-beta
+      call conda config --append channels conda-forge
+      call conda create -n artiq -y
+      call conda install --dry-run --json -n artiq artiq > packages.json
+      date 04-20-20
+      EOF
+      \${wfvm.utils.win-put}/bin/win-put getcondapackages.bat
+      \${wfvm.utils.win-exec}/bin/win-exec '.\Anaconda3\Scripts\activate && getcondapackages'
+      \${wfvm.utils.win-get}/bin/win-get packages.json
+    '';
+  }
+"
+
+./result/bin/wfvm-run-get-conda-packages
+
+python -c "
+import json
+
+with open('packages.json') as json_file:
+    packages = json.load(json_file)
+
+with open('packages_noarch.txt', 'w') as list_noarch:
+    with open('packages_win-64.txt', 'w') as list_win64:
+        for fetch in packages['actions']['FETCH']:
+            if 'm-labs' not in fetch['channel']:
+                if fetch['subdir'] == 'noarch':
+                    list = list_noarch
+                elif fetch['subdir'] == 'win-64':
+                    list = list_win64
+                else:
+                    raise ValueError
+                print(fetch['url'], file=list)
+"
+
+for type in "noarch" "win-64"; do
+  echo Downloading $type packages
+  out=conda_$type\_packages.nix
+  echo "{ pkgs } : [" > $out
+  while read package; do
+    hash=$(nix-prefetch-url $package)
+    echo "
+(pkgs.fetchurl {
+  url = \"$package\";
+  sha256 = \"$hash\";
+})" >> $out
+  done < packages_$type.txt
+  echo "]" >> $out
+done
+
+rm result getcondapackages.bat packages.json packages_noarch.txt packages_win-64.txt