diff --git a/artiq/windows/run-test.nix b/artiq/windows/run-test.nix
index 8afa2fa..f775d95 100644
--- a/artiq/windows/run-test.nix
+++ b/artiq/windows/run-test.nix
@@ -60,7 +60,8 @@ stdenv.mkDerivation {
     # Allow tests to run for 2 minutes
     ${ssh "shutdown -s -t ${toString testTimeout}"}
 
-    ${ssh "anaconda\\scripts\\activate ${condaEnv} && ${testCommand}"}
+    FAIL=n
+    ${ssh "anaconda\\scripts\\activate ${condaEnv} && ${testCommand}"} || FAIL=y
 
     # Abort timeouted shutdown
     ${ssh "shutdown -a"}
@@ -68,6 +69,8 @@ stdenv.mkDerivation {
     ${ssh "shutdown -p -f"}
     wait
 
+    [ "\\$FAIL" = "y" ] && exit 1
+
     EOF
     chmod a+x $out/bin/run.sh
   '';