From 43e019dfc47d248476745257f98179a2693af4a2 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Fri, 30 Sep 2016 17:21:54 -0500 Subject: [PATCH 1/4] cache Cargo artifacts notable changes in the docker-based testing infrastructure - the docker containers can now modify $CARGO_HOME, to re-use the outer Cargo registry, and the target directory to re-use build artifacts. - the docker containers are removed when their execution finishes because it's no longer necessary to re-start them to inspect them because all the interesting output is in the outer target directory --- .travis.yml | 7 ++-- .../aarch64-unknown-linux-gnu/Dockerfile | 1 - .../arm-unknown-linux-gnueabi/Dockerfile | 1 - .../arm-unknown-linux-gnueabihf/Dockerfile | 1 - .../armv7-unknown-linux-gnueabihf/Dockerfile | 1 - ci/docker/i586-unknown-linux-gnu/Dockerfile | 1 - ci/docker/i686-unknown-linux-gnu/Dockerfile | 1 - ci/docker/mips-unknown-linux-gnu/Dockerfile | 1 - ci/docker/mipsel-unknown-linux-gnu/Dockerfile | 1 - .../powerpc-unknown-linux-gnu/Dockerfile | 1 - .../powerpc64-unknown-linux-gnu/Dockerfile | 1 - .../powerpc64le-unknown-linux-gnu/Dockerfile | 2 -- ci/docker/thumbv6m-none-eabi/Dockerfile | 1 - ci/docker/thumbv7em-none-eabi/Dockerfile | 1 - ci/docker/thumbv7em-none-eabihf/Dockerfile | 1 - ci/docker/thumbv7m-none-eabi/Dockerfile | 1 - ci/docker/x86_64-unknown-linux-gnu/Dockerfile | 2 -- ci/run-docker.sh | 32 +++++++++++++------ ci/run.sh | 5 ++- 19 files changed, 30 insertions(+), 32 deletions(-) mode change 100644 => 100755 ci/run.sh diff --git a/.travis.yml b/.travis.yml index 5b84ae5..3195711 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +cache: cargo dist: trusty language: rust services: docker @@ -45,14 +46,16 @@ install: script: - cargo generate-lockfile - if [[ $TRAVIS_OS_NAME = "linux" ]]; then - sudo apt-get remove -y qemu-user-static && - sudo apt-get install -y qemu-user-static && sh ci/run-docker.sh $TARGET; else cargo test --target $TARGET && cargo test --target $TARGET --release; fi +after_success: + # Travis can't cache files that are not readable by "others" + - chmod -R a+r /home/travis/.cargo + branches: only: - auto diff --git a/ci/docker/aarch64-unknown-linux-gnu/Dockerfile b/ci/docker/aarch64-unknown-linux-gnu/Dockerfile index 6a68154..d9a243f 100644 --- a/ci/docker/aarch64-unknown-linux-gnu/Dockerfile +++ b/ci/docker/aarch64-unknown-linux-gnu/Dockerfile @@ -5,6 +5,5 @@ RUN apt-get install -y --no-install-recommends \ gcc-aarch64-linux-gnu libc6-dev-arm64-cross \ qemu-user-static ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \ - PATH=$PATH:/rust/bin \ QEMU_LD_PREFIX=/usr/aarch64-linux-gnu \ RUST_TEST_THREADS=1 diff --git a/ci/docker/arm-unknown-linux-gnueabi/Dockerfile b/ci/docker/arm-unknown-linux-gnueabi/Dockerfile index 36efcd7..cb4d021 100644 --- a/ci/docker/arm-unknown-linux-gnueabi/Dockerfile +++ b/ci/docker/arm-unknown-linux-gnueabi/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get install -y --no-install-recommends \ gcc libc6-dev ca-certificates \ gcc-arm-linux-gnueabi libc6-dev-armel-cross qemu-user-static ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABI_LINKER=arm-linux-gnueabi-gcc \ - PATH=$PATH:/rust/bin \ QEMU_LD_PREFIX=/usr/arm-linux-gnueabi \ RUST_TEST_THREADS=1 diff --git a/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile b/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile index 83e7e52..985abec 100644 --- a/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile +++ b/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile @@ -4,6 +4,5 @@ RUN apt-get install -y --no-install-recommends \ gcc libc6-dev ca-certificates \ gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user-static ENV CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc \ - PATH=$PATH:/rust/bin \ QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf \ RUST_TEST_THREADS=1 diff --git a/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile b/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile index adc15a5..94cae18 100644 --- a/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile +++ b/ci/docker/armv7-unknown-linux-gnueabihf/Dockerfile @@ -4,6 +4,5 @@ RUN apt-get install -y --no-install-recommends \ gcc libc6-dev ca-certificates \ gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user-static ENV CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc \ - PATH=$PATH:/rust/bin \ QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf \ RUST_TEST_THREADS=1 diff --git a/ci/docker/i586-unknown-linux-gnu/Dockerfile b/ci/docker/i586-unknown-linux-gnu/Dockerfile index 63450ff..3254b1a 100644 --- a/ci/docker/i586-unknown-linux-gnu/Dockerfile +++ b/ci/docker/i586-unknown-linux-gnu/Dockerfile @@ -2,4 +2,3 @@ FROM ubuntu:16.04 RUN apt-get update RUN apt-get install -y --no-install-recommends \ gcc-multilib libc6-dev ca-certificates -ENV PATH=$PATH:/rust/bin diff --git a/ci/docker/i686-unknown-linux-gnu/Dockerfile b/ci/docker/i686-unknown-linux-gnu/Dockerfile index 63450ff..3254b1a 100644 --- a/ci/docker/i686-unknown-linux-gnu/Dockerfile +++ b/ci/docker/i686-unknown-linux-gnu/Dockerfile @@ -2,4 +2,3 @@ FROM ubuntu:16.04 RUN apt-get update RUN apt-get install -y --no-install-recommends \ gcc-multilib libc6-dev ca-certificates -ENV PATH=$PATH:/rust/bin diff --git a/ci/docker/mips-unknown-linux-gnu/Dockerfile b/ci/docker/mips-unknown-linux-gnu/Dockerfile index 3281524..e332b54 100644 --- a/ci/docker/mips-unknown-linux-gnu/Dockerfile +++ b/ci/docker/mips-unknown-linux-gnu/Dockerfile @@ -7,6 +7,5 @@ RUN apt-get install -y --no-install-recommends \ binfmt-support qemu-user-static qemu-system-mips ENV CARGO_TARGET_MIPS_UNKNOWN_LINUX_GNU_LINKER=mips-linux-gnu-gcc \ - PATH=$PATH:/rust/bin \ QEMU_LD_PREFIX=/usr/mips-linux-gnu \ RUST_TEST_THREADS=1 diff --git a/ci/docker/mipsel-unknown-linux-gnu/Dockerfile b/ci/docker/mipsel-unknown-linux-gnu/Dockerfile index 088c45f..28f77be 100644 --- a/ci/docker/mipsel-unknown-linux-gnu/Dockerfile +++ b/ci/docker/mipsel-unknown-linux-gnu/Dockerfile @@ -7,6 +7,5 @@ RUN apt-get install -y --no-install-recommends \ binfmt-support qemu-user-static ENV CARGO_TARGET_MIPSEL_UNKNOWN_LINUX_GNU_LINKER=mipsel-linux-gnu-gcc \ - PATH=$PATH:/rust/bin \ QEMU_LD_PREFIX=/usr/mipsel-linux-gnu \ RUST_TEST_THREADS=1 diff --git a/ci/docker/powerpc-unknown-linux-gnu/Dockerfile b/ci/docker/powerpc-unknown-linux-gnu/Dockerfile index 542afa2..8106728 100644 --- a/ci/docker/powerpc-unknown-linux-gnu/Dockerfile +++ b/ci/docker/powerpc-unknown-linux-gnu/Dockerfile @@ -7,6 +7,5 @@ RUN apt-get install -y --no-install-recommends \ qemu-system-ppc ENV CARGO_TARGET_POWERPC_UNKNOWN_LINUX_GNU_LINKER=powerpc-linux-gnu-gcc \ - PATH=$PATH:/rust/bin \ QEMU_LD_PREFIX=/usr/powerpc-linux-gnu \ RUST_TEST_THREADS=1 diff --git a/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile b/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile index f4559c7..3259b4d 100644 --- a/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile +++ b/ci/docker/powerpc64-unknown-linux-gnu/Dockerfile @@ -8,6 +8,5 @@ RUN apt-get install -y --no-install-recommends \ ENV CARGO_TARGET_POWERPC64_UNKNOWN_LINUX_GNU_LINKER=powerpc64-linux-gnu-gcc \ CC_powerpc64_unknown_linux_gnu=powerpc64-linux-gnu-gcc \ - PATH=$PATH:/rust/bin \ QEMU_LD_PREFIX=/usr/powerpc64-linux-gnu \ RUST_TEST_THREADS=1 diff --git a/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile b/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile index 9a77788..bd0a182 100644 --- a/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile +++ b/ci/docker/powerpc64le-unknown-linux-gnu/Dockerfile @@ -7,7 +7,5 @@ RUN apt-get install -y --no-install-recommends \ qemu-system-ppc ENV CARGO_TARGET_POWERPC64LE_UNKNOWN_LINUX_GNU_LINKER=powerpc64le-linux-gnu-gcc \ - CC_powerpc64le_unknown_linux_gnu=powerpc64le-linux-gnu-gcc \ - PATH=$PATH:/rust/bin \ QEMU_LD_PREFIX=/usr/powerpc64le-linux-gnu \ RUST_TEST_THREADS=1 diff --git a/ci/docker/thumbv6m-none-eabi/Dockerfile b/ci/docker/thumbv6m-none-eabi/Dockerfile index c6f7209..6f49975 100644 --- a/ci/docker/thumbv6m-none-eabi/Dockerfile +++ b/ci/docker/thumbv6m-none-eabi/Dockerfile @@ -5,4 +5,3 @@ RUN apt-get install -y --no-install-recommends \ ENV AR_thumbv6m_none_eabi=arm-none-eabi-ar \ CARGO_TARGET_THUMBV6M_NONE_EABI_LINKER=arm-none-eabi-gcc \ CC_thumbv6m_none_eabi=arm-none-eabi-gcc \ - PATH=$PATH:/rust/bin diff --git a/ci/docker/thumbv7em-none-eabi/Dockerfile b/ci/docker/thumbv7em-none-eabi/Dockerfile index 016097b..5d26c92 100644 --- a/ci/docker/thumbv7em-none-eabi/Dockerfile +++ b/ci/docker/thumbv7em-none-eabi/Dockerfile @@ -5,4 +5,3 @@ RUN apt-get install -y --no-install-recommends \ ENV AR_thumbv7em_none_eabi=arm-none-eabi-ar \ CARGO_TARGET_THUMBV7EM_NONE_EABI_LINKER=arm-none-eabi-gcc \ CC_thumbv7em_none_eabi=arm-none-eabi-gcc \ - PATH=$PATH:/rust/bin diff --git a/ci/docker/thumbv7em-none-eabihf/Dockerfile b/ci/docker/thumbv7em-none-eabihf/Dockerfile index c7188ef..f223320 100644 --- a/ci/docker/thumbv7em-none-eabihf/Dockerfile +++ b/ci/docker/thumbv7em-none-eabihf/Dockerfile @@ -5,4 +5,3 @@ RUN apt-get install -y --no-install-recommends \ ENV AR_thumbv7em_none_eabihf=arm-none-eabi-ar \ CARGO_TARGET_THUMBV7EM_NONE_EABIHF_LINKER=arm-none-eabi-gcc \ CC_thumbv7em_none_eabihf=arm-none-eabi-gcc \ - PATH=$PATH:/rust/bin diff --git a/ci/docker/thumbv7m-none-eabi/Dockerfile b/ci/docker/thumbv7m-none-eabi/Dockerfile index 542cc92..e012103 100644 --- a/ci/docker/thumbv7m-none-eabi/Dockerfile +++ b/ci/docker/thumbv7m-none-eabi/Dockerfile @@ -5,4 +5,3 @@ RUN apt-get install -y --no-install-recommends \ ENV AR_thumbv7m_none_eabi=arm-none-eabi-ar \ CARGO_TARGET_THUMBV7M_NONE_EABI_LINKER=arm-none-eabi-gcc \ CC_thumbv7m_none_eabi=arm-none-eabi-gcc \ - PATH=$PATH:/rust/bin diff --git a/ci/docker/x86_64-unknown-linux-gnu/Dockerfile b/ci/docker/x86_64-unknown-linux-gnu/Dockerfile index a5704bb..2ca8ee9 100644 --- a/ci/docker/x86_64-unknown-linux-gnu/Dockerfile +++ b/ci/docker/x86_64-unknown-linux-gnu/Dockerfile @@ -2,5 +2,3 @@ FROM ubuntu:16.04 RUN apt-get update RUN apt-get install -y --no-install-recommends \ gcc libc6-dev ca-certificates -ENV PATH=$PATH:/rust/bin - diff --git a/ci/run-docker.sh b/ci/run-docker.sh index 6912714..82eacba 100644 --- a/ci/run-docker.sh +++ b/ci/run-docker.sh @@ -4,16 +4,30 @@ set -ex run() { - echo $1 - docker build -t $1 ci/docker/$1 + local gid=$(id -g) \ + group=$(id -g -n) \ + target=$1 \ + uid=$(id -u) \ + user=$(id -u -n) + + echo $target + docker build -t $target ci/docker/$target docker run \ - -v `rustc --print sysroot`:/rust:ro \ - -v `pwd`:/checkout:ro \ - -e CARGO_TARGET_DIR=/tmp/target \ - -w /checkout \ - --privileged \ - -it $1 \ - sh ci/run.sh $1 + --rm \ + -e CARGO_HOME=/cargo \ + -e CARGO_TARGET_DIR=/target \ + -v $HOME/.cargo:/cargo \ + -v `pwd`/target:/target \ + -v `pwd`:/checkout:ro \ + -v `rustc --print sysroot`:/rust:ro \ + -w /checkout \ + -it $target \ + sh -c " +groupadd -g $gid $group +useradd -m -g $gid -u $uid $user +chown $user /cargo /target +su -c 'PATH=\$PATH:/rust/bin ci/run.sh $target' $user +" } if [ -z "$1" ]; then diff --git a/ci/run.sh b/ci/run.sh old mode 100644 new mode 100755 index c0406e3..206b2e9 --- a/ci/run.sh +++ b/ci/run.sh @@ -44,10 +44,9 @@ esac case $TRAVIS_OS_NAME in osx) - NM=gnm - # NOTE OSx's nm doesn't accept the `--defined-only` or provide an equivalent. # Use GNU nm instead + NM=gnm brew install binutils ;; *) @@ -56,7 +55,7 @@ case $TRAVIS_OS_NAME in esac # NOTE On i586, It's normal that the get_pc_thunk symbol appears several times so ignore it -$PREFIX$NM -g --defined-only /tmp/target/${1}/debug/librustc_builtins.rlib | \ +$PREFIX$NM -g --defined-only /target/${1}/debug/librustc_builtins.rlib | \ sort | uniq -d | grep -v __x86.get_pc_thunk | grep 'T __' if test $? = 0; then From 0c9b6513891bd46289ab726eb30df37b5d33bf95 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Fri, 30 Sep 2016 18:09:52 -0500 Subject: [PATCH 2/4] do chmod jut before the script phase ends --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3195711..4ea6ddc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,8 +51,6 @@ script: cargo test --target $TARGET && cargo test --target $TARGET --release; fi - -after_success: # Travis can't cache files that are not readable by "others" - chmod -R a+r /home/travis/.cargo From d2036122f1a330965dc724e0077b8e6ca7072b6f Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Fri, 30 Sep 2016 18:25:40 -0500 Subject: [PATCH 3/4] /home/travis -> $HOME --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4ea6ddc..7b3efe4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,7 @@ script: cargo test --target $TARGET --release; fi # Travis can't cache files that are not readable by "others" - - chmod -R a+r /home/travis/.cargo + - chmod -R a+r $HOME/.cargo branches: only: From b60d2515ff62da694d2813e725e9cc35533a4f22 Mon Sep 17 00:00:00 2001 From: Jorge Aparicio Date: Fri, 30 Sep 2016 18:37:41 -0500 Subject: [PATCH 4/4] use `docker run --user` instead of manually creating a user --- ci/run-docker.sh | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/ci/run-docker.sh b/ci/run-docker.sh index 82eacba..d80e758 100644 --- a/ci/run-docker.sh +++ b/ci/run-docker.sh @@ -4,16 +4,18 @@ set -ex run() { - local gid=$(id -g) \ - group=$(id -g -n) \ - target=$1 \ - uid=$(id -u) \ - user=$(id -u -n) + local target=$1 echo $target + + # This directory needs to exist before calling docker, otherwise docker will create it but it + # will be owned by root + mkdir -p target + docker build -t $target ci/docker/$target docker run \ --rm \ + --user $(id -u):$(id -g) \ -e CARGO_HOME=/cargo \ -e CARGO_TARGET_DIR=/target \ -v $HOME/.cargo:/cargo \ @@ -22,12 +24,7 @@ run() { -v `rustc --print sysroot`:/rust:ro \ -w /checkout \ -it $target \ - sh -c " -groupadd -g $gid $group -useradd -m -g $gid -u $uid $user -chown $user /cargo /target -su -c 'PATH=\$PATH:/rust/bin ci/run.sh $target' $user -" + sh -c "PATH=\$PATH:/rust/bin ci/run.sh $target" } if [ -z "$1" ]; then