web fixes and improvements

hydra/web.json

@@ -0,0 +1,16 @@
+{
+    "enabled": 1,
+    "hidden": true,
+    "description": "js",
+    "nixexprinput": "nixScripts",
+    "nixexprpath": "hydra/web.nix",
+    "checkinterval": 300,
+    "schedulingshares": 10,
+    "enableemail": false,
+    "emailoverride": "",
+    "keepnr": 10,
+    "inputs": {
+        "nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs-channels nixos-unstable", "emailresponsible": false },
+        "nixScripts": { "type": "git", "value": "https://git.m-labs.hk/M-Labs/nix-scripts.git", "emailresponsible": false }
+    }
+}

hydra/web.nix

@@ -0,0 +1,27 @@
+{ pkgs ? import <nixpkgs> {}}:
+{
+  jobsets = pkgs.runCommand "spec.json" {}
+    ''
+    cat > $out << EOF
+    {
+        "web": {
+            "enabled": 1,
+            "hidden": false,
+            "description": "M-Labs website",
+            "nixexprinput": "nixScripts",
+            "nixexprpath": "web.nix",
+            "checkinterval": 300,
+            "schedulingshares": 10,
+            "enableemail": false,
+            "emailoverride": "",
+            "keepnr": 10,
+            "inputs": {
+                "nixpkgs": { "type": "git", "value": "git://github.com/NixOS/nixpkgs-channels nixos-unstable", "emailresponsible": false },
+                "nixScripts": { "type": "git", "value": "https://git.m-labs.hk/M-Labs/nix-scripts.git", "emailresponsible": false },
+                "webSrc": { "type": "git", "value": "https://git.m-labs.hk/M-Labs/web2019.git", "emailresponsible": false }
+            }
+        }
+    }
+    EOF
+    '';
+}

nixbld-etc-nixos/configuration.nix

@@ -141,17 +141,21 @@ ACTION=="add", SUBSYSTEM=="tty", \
       binary_cache_secret_key_file = /etc/nixos/secret/nixbld.m-labs.hk-1
       max_output_size = 10000000000
 
+      <runcommand>
+        job = web:web:web
+        command = ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/web
+      </runcommand>
       <runcommand>
         job = artiq:full:artiq-manual-html
-        command = ln -sf $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-html-beta
+        command = ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-html-beta
       </runcommand>
       <runcommand>
         job = artiq:full:artiq-manual-latexpdf
-        command = ln -sf $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-latexpdf-beta
+        command = ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-latexpdf-beta
       </runcommand>
       <runcommand>
         job = artiq:full:conda-channel
-        command = ln -sf $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/conda-channel
+        command = ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/conda-channel
       </runcommand>
       '';
   };
@@ -211,6 +215,18 @@ ACTION=="add", SUBSYSTEM=="tty", \
   };
  
   nixpkgs.config.packageOverrides = super: let self = super.pkgs; in {
+    # nginx etag patch merged in Nixpkgs 19.09, remove after upgrading
+    # https://github.com/NixOS/nixpkgs/pull/60578
+    nginx = super.nginx.overrideAttrs(oa: {
+      patches = oa.patches ++ [
+        (super.substituteAll {
+          src = ./nix-etag-1.15.4.patch;
+          preInstall = ''
+            export nixStoreDir="$NIX_STORE" nixStoreDirLen="''${#NIX_STORE}"
+          '';
+        })
+      ];
+    });
     hydra = super.hydra.overrideAttrs(oa: {
       patches = oa.patches ++ [ ./hydra-conda.patch ./hydra-retry.patch ];
       hydraPath = oa.hydraPath + ":" + super.lib.makeBinPath [ super.jq ];
@@ -247,8 +263,8 @@ ACTION=="add", SUBSYSTEM=="tty", \
   services.nginx = {
     enable = true;
     recommendedProxySettings = true;
-    virtualHosts = {
+    virtualHosts = let
-      "m-labs.hk" = {
+      mainWebsite = {
         addSSL = true;
         useACMEHost = "nixbld.m-labs.hk";
         root = "/var/www/m-labs.hk";
@@ -263,45 +279,21 @@ ACTION=="add", SUBSYSTEM=="tty", \
         '';
         locations."/artiq/manual-beta" = {
           alias = "${hydraWwwOutputs}/artiq-manual-html-beta/share/doc/artiq-manual/html";
-          extraConfig = ''
-            etag off;
-            if_modified_since off;
-            add_header last-modified "";
-          '';
         };
         locations."/artiq/manual-beta.pdf" = {
           alias = "${hydraWwwOutputs}/artiq-manual-latexpdf-beta/share/doc/artiq-manual/ARTIQ.pdf";
-          extraConfig = ''
-            etag off;
-            if_modified_since off;
-            add_header last-modified "";
-          '';
         };
         locations."/artiq/conda" = {
           alias = "${hydraWwwOutputs}/conda-channel";
           extraConfig = ''
-            etag off;
-            if_modified_since off;
-            add_header last-modified "";
             autoindex on;
             index bogus_index_file;
           '';
         };
       };
-      "www.m-labs.hk" = {
+    in {
-        addSSL = true;
+      "m-labs.hk" = mainWebsite;
-        useACMEHost = "nixbld.m-labs.hk";
+      "www.m-labs.hk" = mainWebsite;
-        root = "/var/www/m-labs.hk";
-        locations."/gateware.html".extraConfig = ''
-          return 301 /gateware/migen/;
-        '';
-        locations."/migen".extraConfig = ''
-          return 301 /gateware/migen/;
-        '';
-        locations."/artiq".extraConfig = ''
-          return 301 /experiment-control/artiq/;
-        '';
-      };
       "lab.m-labs.hk" = {
         addSSL = true;
         useACMEHost = "nixbld.m-labs.hk";
@@ -396,7 +388,6 @@ ACTION=="add", SUBSYSTEM=="tty", \
         useACMEHost = "nixbld.m-labs.hk";
         locations."/".proxyPass = "http://192.168.1.204:3007";
       };
-
     };
   };
   services.uwsgi = {

nixbld-etc-nixos/nix-etag-1.15.4.patch

@@ -0,0 +1,92 @@
+From f6a978f024d01202f954483423af1b2d5d5159a6 Mon Sep 17 00:00:00 2001
+From: Yegor Timoshenko <yegortimoshenko@riseup.net>
+Date: Fri, 28 Sep 2018 03:27:04 +0000
+Subject: [PATCH] If root is in Nix store, set ETag to its path hash
+
+---
+ src/http/ngx_http_core_module.c | 56 +++++++++++++++++++++++++++++----
+ 1 file changed, 50 insertions(+), 6 deletions(-)
+
+diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c
+index c57ec00c..b7992de2 100644
+--- a/src/http/ngx_http_core_module.c
++++ b/src/http/ngx_http_core_module.c
+@@ -1583,6 +1583,7 @@ ngx_http_set_etag(ngx_http_request_t *r)
+ {
+     ngx_table_elt_t           *etag;
+     ngx_http_core_loc_conf_t  *clcf;
++    u_char                    *real, *ptr1, *ptr2;
+ 
+     clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
+ 
+@@ -1598,16 +1599,61 @@ ngx_http_set_etag(ngx_http_request_t *r)
+     etag->hash = 1;
+     ngx_str_set(&etag->key, "ETag");
+ 
+-    etag->value.data = ngx_pnalloc(r->pool, NGX_OFF_T_LEN + NGX_TIME_T_LEN + 3);
+-    if (etag->value.data == NULL) {
++    real = ngx_realpath(clcf->root.data, NULL);
++
++    if (real == NULL) {
+         etag->hash = 0;
+         return NGX_ERROR;
+     }
+ 
+-    etag->value.len = ngx_sprintf(etag->value.data, "\"%xT-%xO\"",
+-                                  r->headers_out.last_modified_time,
+-                                  r->headers_out.content_length_n)
+-                      - etag->value.data;
++    #define NIX_STORE_DIR "@nixStoreDir@"
++    #define NIX_STORE_LEN @nixStoreDirLen@
++
++    if (r->headers_out.last_modified_time == 1
++        && !ngx_strncmp(real, NIX_STORE_DIR, NIX_STORE_LEN)
++        && real[NIX_STORE_LEN] == '/'
++        && real[NIX_STORE_LEN + 1] != '\0')
++    {
++        ptr1 = real + NIX_STORE_LEN;
++        *ptr1 = '"';
++
++        ptr2 = (u_char *) ngx_strchr(ptr1, '-');
++
++        if (ptr2 == NULL) {
++            ngx_free(real);
++            etag->hash = 0;
++            return NGX_ERROR;
++        }
++
++        *ptr2++ = '"';
++        *ptr2 = '\0';
++
++        etag->value.len = ngx_strlen(ptr1);
++        etag->value.data = ngx_pnalloc(r->pool, etag->value.len);
++
++        if (etag->value.data == NULL) {
++            ngx_free(real);
++            etag->hash = 0;
++            return NGX_ERROR;
++        }
++
++        ngx_memcpy(etag->value.data, ptr1, etag->value.len);
++    } else {
++        etag->value.data = ngx_pnalloc(r->pool, NGX_OFF_T_LEN + NGX_TIME_T_LEN + 3);
++
++        if (etag->value.data == NULL) {
++            ngx_free(real);
++            etag->hash = 0;
++            return NGX_ERROR;
++        }
++
++        etag->value.len = ngx_sprintf(etag->value.data, "\"%xT-%xO\"",
++                                      r->headers_out.last_modified_time,
++                                      r->headers_out.content_length_n)
++                          - etag->value.data;
++    }
++
++    ngx_free(real);
+ 
+     r->headers_out.etag = etag;
+ 
+-- 
+2.19.0
+

web.nix

@@ -0,0 +1,7 @@
+let
+   pkgs = import <nixpkgs> {};
+   src = <webSrc>;
+in
+  {
+    web = pkgs.runCommand "web" {} "cd ${src}; ${pkgs.zola}/bin/zola build -o $out";
+  }