From 05cde854dd35b6624f99fdbc1580956587901ade Mon Sep 17 00:00:00 2001 From: Sebastien Bourdeauducq Date: Mon, 22 Jun 2020 14:45:11 +0800 Subject: [PATCH] windows: add script to automatically download conda dependencies and create fetchurl nix files --- artiq-fast/windows/conda_noarch_packages.nix | 17 ++ artiq-fast/windows/conda_win-64_packages.nix | 167 +++++++++++++++++++ artiq-fast/windows/make_conda_packages.sh | 68 ++++++++ 3 files changed, 252 insertions(+) create mode 100644 artiq-fast/windows/conda_noarch_packages.nix create mode 100644 artiq-fast/windows/conda_win-64_packages.nix create mode 100755 artiq-fast/windows/make_conda_packages.sh diff --git a/artiq-fast/windows/conda_noarch_packages.nix b/artiq-fast/windows/conda_noarch_packages.nix new file mode 100644 index 0000000..c677edc --- /dev/null +++ b/artiq-fast/windows/conda_noarch_packages.nix @@ -0,0 +1,17 @@ +{ pkgs } : [ + +(pkgs.fetchurl { + url = "https://conda.anaconda.org/conda-forge/noarch/prettytable-0.7.2-py_3.tar.bz2"; + sha256 = "0b7s4xm6bbkcg37sf1i3mxrbac0vxhryq22m3qx4x9kh6k2c5g5q"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/noarch/six-1.15.0-py_0.conda"; + sha256 = "057vci1j78fzkg4qnamfjhz47s0x2v6ygli565a56hvna7h11kng"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/noarch/pycparser-2.20-py_0.conda"; + sha256 = "00rx3nqa21dpqq4lx7g7rahsds0248ky4rw8xjxv5xik5x5xrqnd"; +}) +] diff --git a/artiq-fast/windows/conda_win-64_packages.nix b/artiq-fast/windows/conda_win-64_packages.nix new file mode 100644 index 0000000..9a04e9f --- /dev/null +++ b/artiq-fast/windows/conda_win-64_packages.nix @@ -0,0 +1,167 @@ +{ pkgs } : [ + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/openssl-1.1.1g-he774522_0.conda"; + sha256 = "1hjddbhkk8409k9x4ncxb2z76pj8gbs03bfv8nfwazxpygjhfm99"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/icu-58.2-ha925a31_3.conda"; + sha256 = "1387x1xa1h2wivz1gw8ip2xgy0v5yb261nq1f3jfiiynn7hc8gdf"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/pyqtgraph-0.10.0-py35h28b3542_3.conda"; + sha256 = "0n5l3p0r9fchra3dygp5jbxlb52fbwvrgnmbhq8xghnyvzyjw18c"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/sip-4.19.8-py35h6538335_0.conda"; + sha256 = "09v80qdd6xhmmqbgvb5vm4r2hcid0zd45zzc6pyaxxnh2x9h8mss"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/certifi-2018.8.24-py35_1.conda"; + sha256 = "06yc1a1cvc7rnqxmjk6sax8f38f41f9zksghf5h539s05cp24jhq"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/regex-2018.08.29-py35hfa6e2cd_0.conda"; + sha256 = "0grlb566c2m2cgx4h7bcg0249fpll8n4h4idwbkjssr258sd2dbi"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/pyqt-5.9.2-py35h6538335_2.conda"; + sha256 = "1njyr4hanppfibg075mslcbj8fq899ila9haivp8pl8f7n8zah1v"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/intel-openmp-2019.4-245.conda"; + sha256 = "0kwvhxv219nkgcgcrlsff6gabcd2wln2c6qj7a6ahp6j99dqrz9s"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/numpy-1.15.2-py35ha559c80_0.conda"; + sha256 = "1yqvwgw0fspxn0ckq4gcz5ag8w3wsqnrbqykz3hqspd1pvb7k7rk"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/curl-7.69.1-h2a8f88b_0.conda"; + sha256 = "02s0wfwq8hn8l0f0j2516vabnfvywm1drlvkwrsrc4vg76n00gwx"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/numpy-base-1.15.2-py35h8128ebf_0.conda"; + sha256 = "0z8k3z4qjgwp45p0x15w741afkamql6g9sampl3hf9sachs8mypx"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/tbb4py-2018.0.5-py35he980bc4_0.conda"; + sha256 = "1wpw02cra3pzqw1s4x8mq447yqwd4qsv71yrsx9m2q4f5im887mp"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/setuptools-40.2.0-py35_0.conda"; + sha256 = "19lfrbp28hng6l2pdai14x93jxd9yr0mpz585ws2kjqnz74wapn4"; +}) + +(pkgs.fetchurl { + url = "https://conda.anaconda.org/conda-forge/win-64/libgit2-0.27.8-hfac1375_0.tar.bz2"; + sha256 = "05pjzwmm5vyxwcgi6vv3i961x3dgdd9c4b1ihagq9kyxgqfy21kd"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/mkl-2018.0.3-1.conda"; + sha256 = "0qk1nvvw37kzdxfyyzk3ibikw9s8v1y1dvzvd6ridczsyck3if9c"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/h5py-2.8.0-py35h3bdd7fb_2.conda"; + sha256 = "1bnydwv3s6bhdw70w7vyx0vmb93002pnpwyhbi3zxibvrfdfjaqi"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/hdf5-1.10.2-hac2f561_1.conda"; + sha256 = "12m8wnz9k6zkaja47iwfncm6srrcljgyy3l3nhkiyf0c3hwgalbg"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/vs2015_runtime-14.16.27012-hf0eaf9b_2.conda"; + sha256 = "1k73y8r17r84m2663mffl8g0gx50va1pxspy6b2n7yp8jphhmy8d"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/mkl_random-1.0.1-py35h77b88f5_1.conda"; + sha256 = "0p1bwjl799mxnxa0qn4bbh2w2qgg5c866zf4rgh98hlz7yx3vrzy"; +}) + +(pkgs.fetchurl { + url = "https://conda.anaconda.org/conda-forge/win-64/pygit2-0.27.1-py35hfa6e2cd_0.tar.bz2"; + sha256 = "1jypm8vxs4j4yr37ai4ki9qsslv3wz6slklmisnvjraz1a4vzaf8"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/libcurl-7.69.1-h2a8f88b_0.conda"; + sha256 = "1j627520y8lishi60yk48291cqhjg82by2l2cnhdlc8wn1ldbi5s"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/wheel-0.31.1-py35_0.conda"; + sha256 = "1adgbzqg5ibw8p4mdijgy13pybl273xzfvm1w8ii2k064pfazbdv"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/pip-10.0.1-py35_0.conda"; + sha256 = "0b41585m4rc1a90xh5jrhw23x0sd4m1jdq6wn4gw1f4yazyxbn6d"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/libssh2-1.9.0-h7a1dbc1_1.conda"; + sha256 = "1jv3nxsc1vky450v18nbz8mhk0wpw0k34ibfh53d4dfzimyii1dw"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/cffi-1.11.5-py35h74b6da3_1.conda"; + sha256 = "1lw70385i73m3x060xdhydc5v946w5jxk13wx70z8rypvlgzqnhz"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/zlib-1.2.11-h62dcd97_4.conda"; + sha256 = "1jcbdw7sv4mfkr70pcxa9cnvx17izif1a6wna8b5jj1barizp6g3"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/sqlite-3.32.2-h2a8f88b_0.conda"; + sha256 = "01acaapsra2ii132vx9wykas75pgcb0yb428bs2x4nfl0v391xlp"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/wincertstore-0.2-py35hfebbdb8_0.conda"; + sha256 = "1rg7xlygy19ici9114pyg1i1s1w0bgsd5qcb59s0g0ndj1704s08"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/python-3.5.6-he025d50_0.conda"; + sha256 = "0hdkj7kn5dsi8gfx551n9lyan1a45iwswkh03rwxzvpxbllylm56"; +}) + +(pkgs.fetchurl { + url = "https://conda.anaconda.org/conda-forge/win-64/python-levenshtein-0.12.0-py35_0.tar.bz2"; + sha256 = "1rdpzv1y535swf17nm88chkp1m8w3wd3nwdy7jk1xfcnx1da5ss6"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/scipy-1.1.0-py35h4f6bf74_1.conda"; + sha256 = "0f93xlmwppgs6znlc4w15bij9v41p80g25nxxpxcdqc0niv260ga"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/tk-8.6.10-he774522_0.conda"; + sha256 = "1f1pv99y4jy7knmk3jsrhf4kj63qknr692wajzna0kp71gj0hsfm"; +}) + +(pkgs.fetchurl { + url = "https://repo.anaconda.com/pkgs/main/win-64/mkl_fft-1.0.6-py35hdbbee80_0.conda"; + sha256 = "0as5hb049gcakcqymh7k84dqisaa760xv71469b7m2658bmrni66"; +}) +] diff --git a/artiq-fast/windows/make_conda_packages.sh b/artiq-fast/windows/make_conda_packages.sh new file mode 100755 index 0000000..fd5624a --- /dev/null +++ b/artiq-fast/windows/make_conda_packages.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +set -e + +nix-build -E " +let + pkgs = import {}; + wfvm = import ../wfvm/default.nix { inherit pkgs; }; +in + wfvm.utils.wfvm-run { + name = \"get-conda-packages\"; + image = wfvm.makeWindowsImage { installCommands = [ wfvm.layers.anaconda3 ]; }; + # TODO: fix wfvm login expiry and also remove 'date' workarounds below + #fakeRtc = false; + isolateNetwork = false; + script = '' + cat > getcondapackages.bat << EOF + date 06-22-20 + call conda config --prepend channels https://conda.m-labs.hk/artiq-beta + call conda config --append channels conda-forge + call conda create -n artiq -y + call conda install --dry-run --json -n artiq artiq > packages.json + date 04-20-20 + EOF + \${wfvm.utils.win-put}/bin/win-put getcondapackages.bat + \${wfvm.utils.win-exec}/bin/win-exec '.\Anaconda3\Scripts\activate && getcondapackages' + \${wfvm.utils.win-get}/bin/win-get packages.json + ''; + } +" + +./result/bin/wfvm-run-get-conda-packages + +python -c " +import json + +with open('packages.json') as json_file: + packages = json.load(json_file) + +with open('packages_noarch.txt', 'w') as list_noarch: + with open('packages_win-64.txt', 'w') as list_win64: + for fetch in packages['actions']['FETCH']: + if 'm-labs' not in fetch['channel']: + if fetch['subdir'] == 'noarch': + list = list_noarch + elif fetch['subdir'] == 'win-64': + list = list_win64 + else: + raise ValueError + print(fetch['url'], file=list) +" + +for type in "noarch" "win-64"; do + echo Downloading $type packages + out=conda_$type\_packages.nix + echo "{ pkgs } : [" > $out + while read package; do + hash=$(nix-prefetch-url $package) + echo " +(pkgs.fetchurl { + url = \"$package\"; + sha256 = \"$hash\"; +})" >> $out + done < packages_$type.txt + echo "]" >> $out +done + +rm result getcondapackages.bat packages.json packages_noarch.txt packages_win-64.txt