Revert "Break cycle dependency of tunnel netdev services on network setup"

Does not solve the problem. This reverts commit b1779b57cc.
afws: enable file logging with afws group permissions
2024-11-25 12:11:36 +08:00 · 2024-11-22 15:34:23 +08:00 · 2024-11-16 18:49:26 +08:00 · 2024-11-06 15:25:21 +08:00 · 2024-11-05 18:45:40 +08:00 · 2024-10-30 17:23:52 +08:00
46 changed files with 11000 additions and 216 deletions
--- a/m-labs-intl/60-tunnels.yaml
+++ b/m-labs-intl/60-tunnels.yaml
@ -0,0 +1,18 @@
+network:
+   version: 2
+   renderer: networkd
+   ethernets:
+     eth0:
+       addresses:
+       - 5.78.86.156/32
+       - 2a01:4ff:1f0:83de::2/64
+       - 2a01:4ff:1f0:83de::3/64
+       - 2a01:4ff:1f0:83de::4/64
+   tunnels:
+     gre1:
+       mode: gre
+       local: 5.78.86.156
+       remote: 94.190.212.123
+       addresses:
+        - 10.47.3.0/31
+
--- a/m-labs-intl/gretun.service
+++ b/m-labs-intl/gretun.service
@ -0,0 +1,14 @@
+[Unit]
+Description=GRE tunnel to the main host
+After=network.target
+
+[Service]
+Type=simple
+User=root
+ExecStart=/root/gretun.sh
+ExecStop=/root/gretun_down.sh
+Restart=on-failure
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
--- a/m-labs-intl/gretun.sh
+++ b/m-labs-intl/gretun.sh
@ -0,0 +1,10 @@
+#!/bin/bash
+
+/usr/sbin/iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 25 -j DNAT --to-destination 10.47.3.1:25
+/usr/sbin/iptables -A FORWARD -p tcp -d 10.47.3.1/31 --dport 25 -j ACCEPT
+
+/usr/sbin/iptables -t nat -A PREROUTING -p tcp -i eth0 --dport 587 -j DNAT --to-destination 10.47.3.1:587
+/usr/sbin/iptables -A FORWARD -p tcp -d 10.47.3.1/31 --dport 587 -j ACCEPT
+
+/usr/sbin/iptables -A FORWARD -m state --state ESTABLISHED,RELATED -j ACCEPT
+/usr/sbin/iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
--- a/m-labs-intl/gretun_down.sh
+++ b/m-labs-intl/gretun_down.sh
@ -0,0 +1,10 @@
+#!/bin/bash
+
+/usr/sbin/iptables -t nat -D PREROUTING -p tcp -i eth0 --dport 25 -j DNAT --to-destination 10.47.3.1:25
+/usr/sbin/iptables -D FORWARD -p tcp -d 10.47.3.1/31 --dport 25 -j ACCEPT
+
+/usr/sbin/iptables -t nat -D PREROUTING -p tcp -i eth0 --dport 587 -j DNAT --to-destination 10.47.3.1:587
+/usr/sbin/iptables -D FORWARD -p tcp -d 10.47.3.1/31 --dport 587 -j ACCEPT
+
+/usr/sbin/iptables -D FORWARD -m state --state ESTABLISHED,RELATED -j ACCEPT
+/usr/sbin/iptables -t nat -D POSTROUTING -o eth0 -j MASQUERADE
--- a/m-labs-intl/m-labs-intl.com
+++ b/m-labs-intl/m-labs-intl.com
@ -0,0 +1,81 @@
+upstream rfq_server {
+    server 127.0.0.1:5000;
+}
+
+server {
+    limit_conn addr 5;
+
+    root /var/www/m-labs-intl.com/html;
+    index index.html index.htm index.nginx-debian.html;
+
+    server_name m-labs-intl.com;
+
+    location / {
+        try_files $uri $uri/ =404;
+    }
+
+    listen [::]:443 ssl ipv6only=on; # managed by Certbot
+    listen 443 ssl; # managed by Certbot
+    ssl_certificate /etc/letsencrypt/live/m-labs-intl.com/fullchain.pem; # managed by Certbot
+    ssl_certificate_key /etc/letsencrypt/live/m-labs-intl.com/privkey.pem; # managed by Certbot
+    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+}
+
+server {
+    server_name www.m-labs-intl.com;
+    return 301 https://m-labs-intl.com$request_uri;
+
+    listen [::]:443 ssl; # managed by Certbot
+    listen 443 ssl; # managed by Certbot
+    ssl_certificate /etc/letsencrypt/live/m-labs-intl.com/fullchain.pem; # managed by Certbot
+    ssl_certificate_key /etc/letsencrypt/live/m-labs-intl.com/privkey.pem; # managed by Certbot
+    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+}
+
+server {
+    server_name hooks.m-labs-intl.com;
+    limit_conn addr 5;
+
+    location /rfq {
+        proxy_pass http://rfq_server/rfq;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_read_timeout 30;
+        proxy_connect_timeout 30;
+        proxy_send_timeout 30;
+    }
+
+    location / {
+        return 418;
+    }
+
+    listen [::]:443 ssl; # managed by Certbot
+    listen 443 ssl; # managed by Certbot
+    ssl_certificate /etc/letsencrypt/live/m-labs-intl.com/fullchain.pem; # managed by Certbot
+    ssl_certificate_key /etc/letsencrypt/live/m-labs-intl.com/privkey.pem; # managed by Certbot
+    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
+    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
+}
+
+server {
+    limit_conn addr 5;
+    if ($host = m-labs-intl.com) {
+        return 301 https://$host$request_uri;
+    } # managed by Certbot
+
+
+    if ($host = www.m-labs-intl.com) {
+        return 301 https://m-labs-intl.com$request_uri;
+    } # managed by Certbot
+
+
+    listen 80;
+    listen [::]:80;
+
+    server_name m-labs-intl.com www.m-labs-intl.com hooks.m-labs-intl.com;
+    return 301 https://$host$request_uri;
+}
--- a/m-labs-intl/m-labs.hk.conf
+++ b/m-labs-intl/m-labs.hk.conf
@ -0,0 +1,34 @@
+
+
+connections {
+  m_labs {
+    version = 2
+    encap = no
+    mobike = no
+    send_certreq = no
+    proposals = aes128gcm128-sha256-prfsha256-curve25519,aes128gcm128-sha256-prfsha256-ecp256
+    local_addrs = 5.78.86.156
+    remote_addrs = 94.190.212.123
+    local {
+      auth = pubkey
+      id = fqdn:m-labs-intl.com
+      pubkeys = m-labs-intl.com
+    }
+    remote {
+      auth = pubkey
+      id = fqdn:m-labs.hk
+      pubkeys = m-labs.hk
+    }
+    children {
+      con1 {
+        mode = transport
+        ah_proposals = sha256-curve25519,sha256-ecp256
+        esp_proposals =
+        local_ts = 5.78.86.156[gre]
+        remote_ts = 94.190.212.123[gre]
+        start_action = start
+        close_action = none
+      }
+    }
+  } 
+}
--- a/m-labs-intl/mail.secret
+++ b/m-labs-intl/mail.secret
--- a/m-labs-intl/nginx.conf
+++ b/m-labs-intl/nginx.conf
@ -0,0 +1,65 @@
+user www-data;
+worker_processes auto;
+pid /run/nginx.pid;
+error_log /var/log/nginx/error.log;
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+	worker_connections 768;
+	# multi_accept on;
+}
+
+http {
+
+	##
+	# Basic Settings
+	##
+
+	sendfile on;
+	tcp_nopush on;
+	types_hash_max_size 2048;
+	# server_tokens off;
+
+	server_names_hash_bucket_size 64;
+	# server_name_in_redirect off;
+
+	include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+
+	##
+	# SSL Settings
+	##
+
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
+	ssl_prefer_server_ciphers on;
+
+	# Rate limiting
+	limit_conn_zone $binary_remote_addr zone=addr:10m;
+
+	##
+	# Logging Settings
+	##
+
+	access_log /var/log/nginx/access.log;
+
+	##
+	# Gzip Settings
+	##
+
+	gzip on;
+
+	# gzip_vary on;
+	# gzip_proxied any;
+	# gzip_comp_level 6;
+	# gzip_buffers 16 8k;
+	# gzip_http_version 1.1;
+	gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+
+	##
+	# Virtual Host Configs
+	##
+
+	include /etc/nginx/conf.d/*.conf;
+	include /etc/nginx/sites-enabled/*;
+}
+
--- a/m-labs-intl/rfq.service
+++ b/m-labs-intl/rfq.service
@ -0,0 +1,12 @@
+[Unit]
+Description=RFQ service
+After=network.target
+
+[Service]
+Type=simple
+User=rfqserver
+ExecStart=/home/rfqserver/runrfq.sh
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
--- a/m-labs-intl/runrfq.sh
+++ b/m-labs-intl/runrfq.sh
@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+export FLASK_DEBUG=0
+export FLASK_MAIL_SERVER=mail.m-labs.hk
+export FLASK_MAIL_PORT=465
+export FLASK_MAIL_USE_SSL=True
+export FLASK_MAIL_USERNAME=sysop-intl@m-labs-intl.com
+export FLASK_MAIL_PASSWORD_FILE=/home/rfqserver/mail.secret
+export FLASK_MAIL_RECIPIENT=sales@m-labs.hk
+export FLASK_MAIL_SENDER=sysop-intl@m-labs-intl.com
+
+cd /home/rfqserver/web2019/server
+source venv/bin/activate
+python3 -m flask --app rfq run --port=5000
--- a/m-labs-intl/setup.md
+++ b/m-labs-intl/setup.md
@ -0,0 +1,99 @@
+# Setup m-labs-intl.com server
+
+```shell
+# Install required packages
+apt install git nginx-full python3 python3.12-venv python3-pip iptables ufw \
+  strongswan strongswan-swanctl strongswan-pki strongswan-libcharon
+snap install --classic certbot
+ln -s /snap/bin/certbot /usr/bin/certbot
+
+# Set up networks (includes GRE)
+cp 60-tunnels.yaml /etc/netplan/
+netplan apply
+
+# set up IPsec-AH connection
+cp m-labs.hk.conf /etc/swanctl/conf.d/
+echo 'net.ipv4.ip_forward=1' >> /etc/sysctl.conf
+sysctl -p
+cp m-labs.hk /etc/swanctl/pubkey/m-labs.hk # get pubkey from nixbld
+pki --gen --type rsa --size 4096 --outform pem > /etc/swanctl/private/m-labs-intl.com
+pki --pub --in /etc/swanctl/private/m-labs-intl.com --outform pem > /etc/swanctl/pubkey/m-labs-intl.com
+cp /etc/swanctl/pubkey/m-labs-intl.com m-labs-intl.com # add it to the nixbld
+systemctl enable strongswan --now
+systemctl restart strongswan
+
+# Set up website
+cp m-labs-intl.com /etc/nginx/sites-available/
+cp nginx.conf /etc/nginx/
+ln -s /etc/nginx/sites-available/m-labs-intl.com /etc/nginx/sites-enabled/
+systemctl enable nginx --now
+service nginx restart
+
+# Issue SSL certificate - website only, the mail is on the HK side
+certbot --nginx
+service nginx restart
+
+# Create a user for automatic website deployment from nixbld
+useradd -m zolaupd
+mkdir -p /var/www/m-labs-intl.com/html
+chown -R zolaupd /var/www/m-labs-intl.com/
+sudo -u zolaupd sh -c '
+  cd /home/zolaupd;
+  mkdir /home/zolaupd/.ssh;
+  echo -n "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP1OJJM8g/1ffxDjN31XKEfGmrYaW03lwpyTa1UGWqVx
+  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF6R6XK0IiuAKxVKvSABm4m9bfOlvfJcMvTpjenuXUPv" > /home/zolaupd/.ssh/authorized_keys
+  chmod 700 .ssh/
+  chmod 600 .ssh/authorized_keys
+  '
+
+# Create a user for RFQ hooks service
+useradd -m rfqserver
+cp runrfq.sh /home/rfqserver/
+cp mail.secret /home/rfqserver/
+chown rfqserver /home/rfqserver/runrfq.sh
+chmod +x /home/rfqserver/runrfq.sh
+chown rfqserver /home/rfqserver/mail.secret
+
+sudo -u rfqserver sh -c '
+  cd /home/rfqserver;
+  git clone https://git.m-labs.hk/M-Labs/web2019.git;
+  cd web2019;
+  python3 -m venv ./venv;
+  source venv/bin/activate;
+  pip install -r requirements.txt;
+'
+cp rfq.service /etc/systemd/system/
+
+# Automate port forwarding rules creation
+cp gretun.sh /root/gretun.sh
+cp gretun_down.sh /root/gretun_down.sh
+chmod u+x /root/gretun.sh
+chmod u+x /root/gretun_down.sh
+cp gretun.service /etc/systemd/system/
+
+# Enable custom services
+systemctl daemon-reload
+systemctl enable rfq.service --now
+systemctl enable gretun.service --now
+
+# Setup basic firewall rules  
+ufw default deny
+ufw default allow outgoing
+
+ufw allow from 94.190.212.123
+ufw allow from 2001:470:f891:1::/64
+ufw allow from 202.77.7.238
+ufw allow from 2001:470:18:390::2
+ufw allow "Nginx HTTP"
+ufw allow "Nginx HTTPS"
+ufw limit OpenSSH
+ufw allow 25/tcp
+ufw allow 587/tcp
+ufw limit 500,4500/udp
+
+ufw route allow in on gre1 out on eth0
+ufw allow from 10.47.3.0/31
+
+ufw show added
+ufw enable
+```
--- a/nixbld-etc-nixos/afws-module.nix
+++ b/nixbld-etc-nixos/afws-module.nix
@ -10,16 +10,34 @@ in
      default = false;
      description = "Enable AFWS server";
    };
+    logFile = mkOption {
+      type = types.str;
+      default = "/var/lib/afws/logs/afws.log";
+      description = "Path to the log file";
+    };
+    logBackupCount = mkOption {
+      type = types.int;
+      default = 30;
+      description = "Number of daily log files to keep";
+    };
  };

  config = mkIf config.services.afws.enable {
    systemd.services.afws = {
      description = "AFWS server";
      wantedBy = [ "multi-user.target" ];
+      preStart = ''
+        mkdir -p "$(dirname ${config.services.afws.logFile})"
+        chown afws:afws "$(dirname ${config.services.afws.logFile})"
+      '';
      serviceConfig = {
        User = "afws";
        Group = "afws";
-        ExecStart = "${afws}/bin/afws_server";
+        ExecStart = ''
+          ${afws}/bin/afws_server \
+            --log-file ${config.services.afws.logFile} \
+            --log-backup-count ${toString config.services.afws.logBackupCount}
+        '';
        ExecReload = "${pkgs.coreutils}/bin/kill -USR1 $MAINPID";
      };
      path = [ pkgs.nix pkgs.git ];
--- a/nixbld-etc-nixos/backup-module.nix
+++ b/nixbld-etc-nixos/backup-module.nix
@ -26,9 +26,10 @@ let
    ${config.services.mysql.package}/bin/mysqldump --user=root --single-transaction flarum > flarum.sql
    ${config.services.postgresql.package}/bin/pg_dump mattermost > mattermost.sql
    ${config.services.postgresql.package}/bin/pg_dump rt5 > rt.sql
+    ${config.services.postgresql.package}/bin/pg_dump gitea > gitea.sql

    exec 6< /etc/nixos/secret/backup-passphrase
-    ${pkgs.gnutar}/bin/tar cf - ${lib.concatMapStringsSep " " (p: "--exclude \"${p}\"") excludePaths} /etc/nixos /var/vmail /var/lib/hedgedoc /var/lib/gitea /var/lib/afws /var/lib/mattermost/data /var/www/193thz flarum.sql mattermost.sql rt.sql | \
+    ${pkgs.gnutar}/bin/tar cf - ${lib.concatMapStringsSep " " (p: "--exclude \"${p}\"") excludePaths} /etc/nixos /var/vmail /var/lib/hedgedoc /var/lib/gitea /var/lib/afws /var/lib/mattermost/data /var/www/193thz flarum.sql mattermost.sql rt.sql gitea.sql | \
        ${pkgs.bzip2}/bin/bzip2 | \
        ${pkgs.gnupg}/bin/gpg --symmetric --batch --passphrase-fd 6
  '';
--- a/nixbld-etc-nixos/configuration.nix
+++ b/nixbld-etc-nixos/configuration.nix
@ -1,14 +1,14 @@
-# Edit this configuration file to define what should be installed on
-# your system.  Help is available in the configuration.nix(5) man page
-# and in the NixOS manual (accessible by running ‘nixos-help’).
-
 { config, pkgs, ... }:

 let
  netifWan = "enp4s0";
+  netifWanBackup = "enp11s0";
  netifLan = "enp5s0f1";
  netifWifi = "wlp6s0";
  netifSit = "henet0";
+  netifUSA = "trump0";
+  netifAlt = "alt0";
+  netifAltVlan = "vlan0";
  hydraWwwOutputs = "/var/www/hydra-outputs";
 in
 {
@ -20,8 +20,8 @@ in
      ./afws-module.nix
      ./rt.nix
      (builtins.fetchTarball {
-        url = "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/archive/008d78cc21959e33d0d31f375b88353a7d7121ae/nixos-mailserver-nixos.tar.gz";
-        sha256 = "sha256:0pnfyg4icsvrw390a227m8b1j5w8awicx5aza3d0fiyyzpnrpn5a";
+        url = "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/archive/29916981e7b3b5782dc5085ad18490113f8ff63b/nixos-mailserver-nixos.tar.gz";
+        sha256 = "sha256:0clvw4622mqzk1aqw1qn6shl9pai097q62mq1ibzscnjayhp278b";
      })
    ];

@ -31,7 +31,7 @@ in
  boot.loader.grub.efiSupport = true;
  boot.loader.efi.canTouchEfiVariables = true;
  hardware.cpu.amd.updateMicrocode = true;
-  boot.supportedFilesystems = ["zfs"];
+  boot.supportedFilesystems.zfs = true;
  boot.kernelParams = ["zfs.l2arc_write_max=536870912"];
  boot.binfmt.emulatedSystems = [ "armv7l-linux" "aarch64-linux" ];

@ -90,11 +90,35 @@ in
    hostName = "nixbld";
    hostId = "e423f012";
    firewall = {
-      allowedTCPPorts = [ 53 80 443 7402 ];
+      allowedTCPPorts = [ 53 80 443 2222 7402 ];
      allowedUDPPorts = [ 53 67 500 4500 ];
      trustedInterfaces = [ netifLan ];
+      logRefusedConnections = false;
+      extraCommands = ''
+        iptables -A INPUT -s 5.78.86.156 -p gre -j ACCEPT
+        iptables -A INPUT -s 5.78.86.156 -p ah -j ACCEPT
+      '';
+      extraStopCommands = ''
+        iptables -D INPUT -s 5.78.86.156 -p gre -j ACCEPT
+        iptables -D INPUT -s 5.78.86.156 -p ah -j ACCEPT
+      '';
+    };
+    useDHCP = false;
+    interfaces."${netifWan}".useDHCP = true;  # PCCW - always wants active DHCP lease or cuts you off
+    interfaces."${netifWanBackup}" = {        # HKBN - no DHCP with static IP service
+      ipv4.addresses = [{
+        address = "202.77.7.238";
+        prefixLength = 30;
+      }];
+      ipv4.routes = [
+        {
+          address = "0.0.0.0";
+          prefixLength = 0;
+          via = "202.77.7.237";
+          options.table = "2";
+        }
+      ];
    };
-    interfaces."${netifWan}".useDHCP = true;
    interfaces."${netifLan}" = {
      ipv4.addresses = [{
        address = "192.168.1.1";
@ -112,6 +136,11 @@ in
          prefixLength = 24;
          options.table = "1";
        }
+        {
+          address = "192.168.1.0";
+          prefixLength = 24;
+          options.table = "2";
+        }
      ];
    };
    interfaces."${netifWifi}" = {
@ -123,6 +152,19 @@ in
        address = "2001:470:f891:2::";
        prefixLength = 64;
      }];
+      # same hack as above
+      ipv4.routes = [
+        {
+          address = "192.168.12.0";
+          prefixLength = 24;
+          options.table = "1";
+        }
+        {
+          address = "192.168.12.0";
+          prefixLength = 24;
+          options.table = "2";
+        }
+      ];
    };
    nat = {
      enable = true;
@ -135,11 +177,6 @@ in
        { sourcePort = 2204; destination = "192.168.1.204:22"; proto = "tcp"; }
      ];
      extraCommands = ''
-        iptables -w -N block-lan-from-wifi
-        iptables -w -A block-lan-from-wifi -i ${netifLan} -o ${netifWifi} -j DROP
-        iptables -w -A block-lan-from-wifi -i ${netifWifi} -o ${netifLan} -j DROP
-        iptables -w -A FORWARD -j block-lan-from-wifi
-
        iptables -w -N block-insecure-devices
        iptables -w -A block-insecure-devices -m mac --mac-source 00:20:0c:6c:ee:ba -j DROP  # keysight SA
        iptables -w -A block-insecure-devices -m mac --mac-source 74:5b:c5:20:c1:5f -j DROP  # siglent scope
@ -151,15 +188,21 @@ in
        iptables -w -A block-insecure-devices -m mac --mac-source d8:9c:67:ab:83:e7 -j DROP  # HP printer, wifi
        iptables -w -A block-insecure-devices -m mac --mac-source f4:39:09:f7:3c:d7 -j DROP  # HP printer, ethernet
        iptables -w -A FORWARD -j block-insecure-devices
+
+        iptables -w -N pccw-sucks
+        iptables -A pccw-sucks -o ${netifSit} -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1440
+        iptables -A pccw-sucks -o ${netifAlt} -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1380
+        iptables -A pccw-sucks -o ${netifUSA} -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1380
+        iptables -w -A FORWARD -j pccw-sucks
      '';
      extraStopCommands = ''
-        iptables -w -D FORWARD -j block-lan-from-wifi 2>/dev/null|| true
-        iptables -w -F block-lan-from-wifi 2>/dev/null|| true
-        iptables -w -X block-lan-from-wifi 2>/dev/null|| true
-
        iptables -w -D FORWARD -j block-insecure-devices 2>/dev/null|| true
        iptables -w -F block-insecure-devices 2>/dev/null|| true
        iptables -w -X block-insecure-devices 2>/dev/null|| true
+
+        iptables -w -D FORWARD -j pccw-sucks 2>/dev/null|| true
+        iptables -w -F pccw-sucks 2>/dev/null|| true
+        iptables -w -X pccw-sucks 2>/dev/null|| true
      '';
    };
    sits."${netifSit}" = {
@ -172,14 +215,37 @@ in
      addresses = [{ address = "2001:470:18:390::2"; prefixLength = 64; }];
      routes = [{ address = "::"; prefixLength = 0; }];
    };
-    greTunnels.alt0 = {
+    greTunnels."${netifUSA}" = {
+      dev = netifWan;
+      remote = "5.78.86.156";
+      local = "94.190.212.123";
+      ttl = 255;
+      type = "tun";
+    };
+    greTunnels."${netifAlt}" = {
      dev = netifWan;
      remote = "103.206.98.1";
      local = "94.190.212.123";
      ttl = 255;
      type = "tun";
    };
-    interfaces.alt0 = {
+    interfaces."${netifUSA}" = {
+      ipv4.addresses = [
+        {
+          address = "10.47.3.1";
+          prefixLength = 31;
+        }
+      ];
+      ipv4.routes = [
+        {
+          address = "0.0.0.0";
+          prefixLength = 0;
+          via = "10.47.3.0";
+          options.table = "3";
+        }
+      ];
+    };
+    interfaces."${netifAlt}" = {
      ipv4.addresses = [
        {
          address = "103.206.98.227";
@ -196,12 +262,12 @@ in
      ];
    };
    vlans = {
-      vlan0 = {
+      "${netifAltVlan}" = {
        id = 2;
        interface = netifLan;
      };
    };
-    interfaces.vlan0 = {
+    interfaces."${netifAltVlan}" = {
      ipv4.addresses = [{
        address = "103.206.98.200";
        prefixLength = 29;
@ -234,7 +300,7 @@ in
      id = "fqdn:igw0.hkg.as150788.net";
      pubkeys = [ "/etc/swanctl/pubkey/igw0.hkg.as150788.net" ];
    };
-    children.alt0 = {
+    children."${netifAlt}" = {
      mode = "transport";
      ah_proposals = [ "sha256-curve25519" ];
      remote_ts = [ "103.206.98.1[gre]" ];
@ -242,13 +308,52 @@ in
      start_action = "start";
    };
  };
+  services.strongswan-swanctl.swanctl.connections.usa = {
+    local_addrs = [ "94.190.212.123" ];
+    remote_addrs = [ "5.78.86.156" ];
+    local.main = {
+      auth = "pubkey";
+      id = "fqdn:m-labs.hk";
+      pubkeys = [ "/etc/swanctl/pubkey/m-labs.hk" ];
+    };
+    remote.main = {
+      auth = "pubkey";
+      id = "fqdn:m-labs-intl.com";
+      pubkeys = [ "/etc/swanctl/pubkey/m-labs-intl.com" ];
+    };
+    children."${netifUSA}" = {
+      mode = "transport";
+      ah_proposals = [ "sha256-curve25519" ];
+      remote_ts = [ "5.78.86.156[gre]" ];
+      local_ts = [ "94.190.212.123[gre]" ];
+      start_action = "start";
+    };
+  };

-  systemd.services.custom-network-setup = {
+  systemd.services.network-custom-route-backup = {
    wantedBy = [ "network.target" ];
    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
-      ExecStart = "${pkgs.iproute2}/bin/ip rule add from 103.206.98.0/24 table 1";
+      ExecStart = "${pkgs.iproute2}/bin/ip rule add from 202.77.7.238/30 table 2";
+      ExecStop = "${pkgs.iproute2}/bin/ip rule del table 2";
+    };
+  };
+  systemd.services.network-custom-route-usa = {
+    wantedBy = [ "network.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      ExecStart = "${pkgs.iproute2}/bin/ip rule add from 10.47.3.0/31 table 3";
+      ExecStop = "${pkgs.iproute2}/bin/ip rule del table 3";
+    };
+  };
+  systemd.services.network-custom-route-alt = {
+    wantedBy = [ "network.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      ExecStart = "${pkgs.iproute2}/bin/ip rule add from 103.206.98.200/29 table 1";
      ExecStop = "${pkgs.iproute2}/bin/ip rule del table 1";
    };
  };
@ -276,11 +381,13 @@ in
           also-notify {
             213.239.220.50;  # ns1.qnetp.net
             216.218.130.2;   # ns1.he.net
+             88.198.32.245;   # new qnetp
           };
           '';
        slaves = [
          "213.239.220.50" "2a01:4f8:a0:7041::1"  # ns1.qnetp.net
          "216.218.133.2" "2001:470:600::2"       # slave.dns.he.net
+          "88.198.32.245"                         # new qnetp
        ];
      };
      "m-labs.ph" = {
@ -332,6 +439,27 @@ in
          "216.218.133.2" "2001:470:600::2"       # slave.dns.he.net
        ];
      };
+      "m-labs-intl.com" = {
+        name = "m-labs-intl.com";
+        master = true;
+        file = "/etc/nixos/named/m-labs-intl.com";
+        extraConfig =
+           ''
+           dnssec-policy "default";
+           inline-signing yes;
+           notify explicit;
+           also-notify {
+             216.218.130.2;   # ns1.he.net
+             213.239.220.50;  # ns1.qnetp.net
+             88.198.32.245;   # new qnetp
+           };
+           '';
+        slaves = [
+          "216.218.133.2" "2001:470:600::2"       # slave.dns.he.net
+          "213.239.220.50" "2a01:4f8:a0:7041::1"  # ns1.qnetp.net
+          "88.198.32.245"                         # new qnetp
+        ];
+      };
      "200-29.98.206.103.in-addr.arpa" = {
        name = "200-29.98.206.103.in-addr.arpa";
        master = true;
@ -412,11 +540,6 @@ in
        "/kasli/192.168.1.70"
        "/kasli-customer/192.168.1.75"
        "/stabilizer-customer/192.168.1.76"
-
-        # Google can't do DNS geolocation correctly and slows down websites of everyone using
-        # their shitty font cloud hosting. In HK, you sometimes get IPs behind the GFW that you
-        # cannot reach.
-        "/fonts.googleapis.com/142.250.207.74"
      ];

      dhcp-match = "set:ipxe,175"; # https://forum.ipxe.org/showthread.php?tid=6077
@ -442,10 +565,23 @@ in
  # List packages installed in system profile. To search, run:
  # $ nix search wget
  environment.systemPackages = with pkgs; [
-    wget vim git file lm_sensors acpi pciutils psmisc nixopsUnstable
-    irssi tmux usbutils imagemagick jq zip unzip
+    lm_sensors
+    acpi
+    usbutils
+    pciutils
    iw
    nvme-cli
+    smartmontools
+    psmisc
+
+    wget
+    vim
+    git
+    file
+    imagemagick
+    jq
+
+    nixops_unstable_minimal
    borgbackup
    bind
    waypipe
@ -472,7 +608,10 @@ in
  services.openssh.enable = true;
  services.openssh.settings.PasswordAuthentication = false;
  services.openssh.settings.GatewayPorts = "clientspecified";
+  services.openssh.settings.X11Forwarding = true;
+  services.openssh.authorizedKeysInHomedir = false;
  programs.mosh.enable = true;
+  programs.tmux.enable = true;

  programs.fish.enable = true;
  programs.zsh.enable = true;
@ -499,6 +638,20 @@ in
    SUBSYSTEM=="usb", ATTRS{idVendor}=="07cf", ATTRS{idProduct}=="4204", MODE="0660", GROUP="lp"
    '';

+  sound.enable = true;
+  services.mpd.enable = true;
+  services.mpd.musicDirectory = "/tank/sb-public/FLAC";
+  services.mpd.network.listenAddress = "192.168.1.1";
+  services.mpd.extraConfig =
+    ''
+    audio_output_format "192000:24:2"
+    audio_output {
+      type "alsa"
+      name "alsa"
+      device "hw:1,1"
+    }
+    '';
+
  users.extraUsers.root = {
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBNdIiLvP2hmDUFyyE0oLOIXrjrMdWWpBV9/gPR5m4AiARx4JkufIDZzmptdYQ5FhJORJ4lluPqp7dAmahoSwg4lv9Di0iNQpHMJvNGZLHYKM1H1FWCCFIEDJ8bD4SVfrDg=="
@ -509,66 +662,73 @@ in
  # https://github.com/NixOS/nixpkgs/issues/155357
  security.sudo.enable = true;

+  # M-Labs HK
  users.extraUsers.sb = {
    isNormalUser = true;
-    extraGroups = ["lp" "scanner" "afws"];
+    extraGroups = ["lp" "scanner" "afws" "audio"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBF/YybP+fQ0J+bNqM5Vgx5vDmVqVWsgUdF1moUxghv7d73GZAFaM6IFBdrXTAa33AwnWwDPMrTgP1V6SXBkb3ciJo/lD1urJGbydbSI5Ksq9d59wvOeANvyWYrQw6+eqTQ=="
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCMALVC8RDTHec+PC8y1s3tcpUAODgq6DEzQdHDf/cyvDMfmCaPiMxfIdmkns5lMa03hymIfSmLUF0jFFDc7biRp7uf9AAXNsrTmplHii0l0McuOOZGlSdZM4eL817P7UwJqFMxJyFXDjkubhQiX6kp25Kfuj/zLnupRCaiDvE7ho/xay6Jrv0XLz935TPDwkc7W1asLIvsZLheB+sRz9SMOb9gtrvk5WXZl5JTOFOLu+JaRwQLHL/xdcHJTOod7tqHYfpoC5JHrEwKzbhTOwxZBQBfTQjQktKENQtBxXHTe71rUEWfEZQGg60/BC4BrRmh4qJjlJu3v4VIhC7SSHn1"
    ];
    shell = pkgs.fish;
  };
-  users.extraUsers.rj = {
-    isNormalUser = true;
-    extraGroups = ["afws"];
-  };
-  users.extraUsers.nkrackow = {
-    isNormalUser = true;
-    extraGroups = ["afws"];
-    openssh.authorizedKeys.keys = [
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDBNsAtZdp0BMvw0rRpMDgJ0V9hqB/BVSyhZ3m8LEx0im939ya6Urmlz3x7+RilD1LMl/p4B1Yxt+z/w7J5NB7unTYlKigJr/s9aH/0IKCFRvO5Omw88k3tWCCRA9KbbXAh0OE/Kli09rrgRuB6++c+XBZ4IvFvohfML0eAjdofn6ePnLWt+R/RNvNjmSb5y7rSIbJ9t+B7O1QOr7u+1GgZEexhG79o52I4rsrgyhUJOK4FbDGPnIkFYFeB2alijzbM1bAu9GR6BD4HBoqeW+DF7tUZs8GYtJsBX8rMnzuR3t8pM7RcGjY5IHQM9MM5WpHokJCFNSSzrvFgbK7CBFklOtipo1H1fwOuDuT3sCE3/ZTK5UgfKGdsb+vsvZub7KBNXfgru2webpl/rLcDJpn3eSDX/ZMGXVV8zskteQHtakra52bc2IeFaPiE1V+WeUB/LpIvRWG+Eh1VEgbUcjoVkaIBu6tQflW7US3uCGYan9Hw80MkwxAmqY1pogAJgzxsYbqdcNb8Xrra6LYFeMD8HXKdW9sXh7mzxDwwkzqjXCKPavWPT7ujicTRlJC6TfmZTdZUPh2mjvzUZI9ZPr50hkV0EAdERn57HwPGMlHiOCntPI/Jw3XmZXIOxChkyss5YFF5mWIzYOp5YxWBlWusNpnMeZCk2ncJmdXcAd6GzQ=="
-    ];
-  };
-  users.extraUsers.occheung = {
-    isNormalUser = true;
-    openssh.authorizedKeys.keys = [
-      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBPEvmWmxpFpMgp5fpjKud8ev0cyf/+X5fEpQt/YD/+u4mbvZYPE300DLqQ0h/qjgvaGMz1ndf4idYnRdy+plJEC/+hmlRW5NlcpAr3S/LYAisacgKToFVl+MlBo+emS9Ig=="
-    ];
-  };
  users.extraUsers.spaqin = {
    isNormalUser = true;
-    extraGroups = ["lp" "scanner" "afws"];
+    extraGroups = ["lp" "afws"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBOtmlQmIK/cEUkcwA/y9jC4AohjoEmikerpxzPhZZtOcENidN/vFum58jIcSxBvjHnILOzhfCTeLvbvGbQOFE53a7FOyEHmIzXRKS86Mg5bPHUBJxRSq9MjulGZXES3HOQ=="
    ];
    shell = pkgs.zsh;
  };
-  users.extraUsers.esavkin = {
+  users.extraUsers.therobs12 = {
    isNormalUser = true;
    extraGroups = ["lp" "afws"];
    openssh.authorizedKeys.keys = [
-      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBLDJI4GFLBmScbeR8Jh4Gi8A/2nuGlYtFTJVT+Es/bzdiPRk8DLG62T0hyRR+8LfHjbrCsDuYFNztT8hHGXd7h3xp3y2X7ArkJo8xUK5QxGd5D2Zn4ANfZTTVkoGlEHbFA=="
+      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBK1tUg7TtceARRnGI80Ai5kNFolFfZ++LH9v1UoRCiJdxeQWPdNYO0Gj7+ejJvgZXwvN4yHGgcZHraEml4Mj/dKrEMFygfuYLDRmXtPFwX6TNMrWlxMhPzuNY+yCaxlqYg=="
    ];
  };
  users.extraUsers.morgan = {
    isNormalUser = true;
-    extraGroups = ["lp"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBDXMbJEPn0mM2Bgt6eMAd+c0J5oPSvquZG+BxKdUf0qbeQldRaoB26NHMZnLte/fS00U/cqStLWDiwtEvH5WlbbawsMBymm65zbWMByebXhBDjdr6a1kkOFcKJvAL9qVBQ=="
    ];
  };
+
+  # M-Labs PH
+  users.extraUsers.flo = {
+    isNormalUser = true;
+    extraGroups = ["afws"];
+    openssh.authorizedKeys.keys = [
+      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBF4ZYNBYqJPQCKBYjMatFj5eGMyzh/X2TSraJEG6XBdg3jnJ3WcsOd7sm+vx+o9Y1EJ2kvwW/Vy9c3OYVU2U45njox//sKtt8Eyzszws3EYJqHQ6KAwXtW9ao4aamRtK3Q=="
+    ];
+  };
+
+  # QUARTIQ
+  users.extraUsers.rj = {
+    isNormalUser = true;
+    extraGroups = ["afws"];
+    openssh.authorizedKeys.keys = [
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC27krR8G8Pb59YuYm7+X2mmNnVdk/t9myYgO8LH0zfb2MeeXX5+90nW9kMjKflJss/oLl8dkD85jbJ0fRbRkfJd20pGCqCUuYAbYKkowigFVEkbrbWSLkmf+clRjzJOuBuUA0uq0XKS17uMC3qhu+dDdBOAIKb3L83NfVE8p8Pjb4BPktQrdxefM43/x4jTMuc7tgxVmTOEge3+rmVPK2GnLkUBgBn8b6S+9ElPd63HXI5J5f61v21l5N9V0mhTu1pv6PiDRdFIlFDK9dLVZcZ2qlzpKmCnFrOoreBEgre44SpfFe5/MMItxvWiVsj/rij/rHZZiol1k7JiQCnEHeCCbjjvcBBka5HxZgcb3vBZVceTOawrmjbdbA2dq35sUptz/bEgdZ1UVCmVpWsdROAlEDBmSSbcVwxzcvhoKnkpbuP4Q0V3tVKSLW053ADFNB4frtwY5nAZfsVErFLLphjwb8nlyJoDRNapQrn5syEiW0ligX2AAskZTYIl2A5AYyWPrmX6HJOPqZGatMU3qQiRMxs+hFqhyyCmBgl0kcsgW09MBKtJWk1Fbii98MHqgRUN9R7AUiYy5p78Pnv9DC8DT8Ubl9zoP0g5d40P9NGK2LAhMxLXvtckJ4ERqbSEcNZJw+q4jBrOHnMTz+NLdAUiEtru+6T2OdhaHv+eiNlFQ== robert-jordens-rsa4096"
+     "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCUdbne3NtIG+iy/jer76/OY+IksuS3BDLSXPnWrGejWnig9h+L6sUV0lEVI6dqp+W/b8jWqPB8nh5S0NZsCd3Ta3Go82k/SPPkh9lB2PpfquhCjLnmC/RNc3TgC4FuiS+NZHqXaTggYHubNwEK+8gynMqkMQXjOGU02U0CtUfsYdAm75AW60DySZCRNwOcU0Ndpn1UCpha7fL1k179Dd/OtArkYsIL24ohlfxFeOB3jGYQK6ATmzbvCRjwIKXcyECuajWwfnDg9FtDWrqHNzu5dJlvmxoWm8zCDgMj53uiA7TjujQN81MYrIJNeEwSr5jXQMqzA3mzlk4k3Z0qs3TP robert-jordens-64FEFBAF-4D0749B2-rsa2048"
+     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMUaB2G1jexxfkdlly3fdWslH54/s/bOuvk9AxqpjtAY robert-jordens-ed25519"
+    ];
+  };
+  users.extraUsers.eduardotenholder = {
+    isNormalUser = true;
+    extraGroups = ["afws"];
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIu6yhjCoZ62eamYrAXtFefDhplTRUIdD4tncwlkyAEH"
+    ];
+  };
+
+  # HKUST
  users.extraUsers.derppening = {
    isNormalUser = true;
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBOKwN4ui94QfouYYlkI1lc3WgtjURVYLTdAizJIBnY3dNRNblAiuvTD4pQ+LEI+eOTg4SnQz1NeqH4YOQhbT5+/nZojvGTb3UVN13ZYND+Gci3DdqB2mwIYop7kMXwHgLQ=="
    ];
  };
-  users.extraUsers.dpn = {
-    isNormalUser = true;
-    openssh.authorizedKeys.keys = [
-      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBGChLocYJi8XcSJkIjT2Olm3jPGjtRq5aORa5G9F3OqmjCfvav9Q5+2Mc64XqHtNTffnJuDe4gv+lVJatC0URvPs2HyxXmxRK0jgkkLSUsV2SYLlgMqHW3jsrdh6wKBmkg=="
-    ];
-  };
+
  users.extraUsers.nix = {
    isNormalUser = true;
  };
@ -577,7 +737,6 @@ in

  nix.settings.max-jobs = 10;
  nix.nrBuildUsers = 64;
-  nix.settings.trusted-users = ["sb"];
  services.hydra = {
    enable = true;
    useSubstitutes = true;
@ -594,6 +753,10 @@ in
        job = web:web:web
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/web
      </runcommand>
+      <runcommand>
+        job = web:web:web-intl
+        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ${pkgs.rsync}/bin/rsync -r -c $(jq -r '.outputs[0].path' < $HYDRA_JSON)/ zolaupd@10.47.3.0:/var/www/m-labs-intl.com/html/
+      </runcommand>
      <runcommand>
        job = web:web:nmigen-docs
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/nmigen-docs
@ -620,6 +783,10 @@ in
        job = artiq:extra-beta:conda-channel
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel-beta
      </runcommand>
+      <runcommand>
+        job = artiq:extra-beta:msys2-repos
+        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-msys2-repos-beta
+      </runcommand>

      <runcommand>
        job = artiq:main:artiq-manual-html
@ -633,17 +800,21 @@ in
        job = artiq:extra:conda-channel
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel
      </runcommand>
+      <runcommand>
+        job = artiq:extra:msys2-repos
+        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-msys2-repos
+      </runcommand>

      <runcommand>
-        job = artiq:full-legacy:artiq-manual-html
+        job = artiq:main-legacy:artiq-manual-html
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-html-legacy
      </runcommand>
      <runcommand>
-        job = artiq:full-legacy:artiq-manual-latexpdf
-        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-latexpdf-legacy
+        job = artiq:main-legacy:artiq-manual-pdf
+        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-pdf-legacy
      </runcommand>
      <runcommand>
-        job = artiq:full-legacy:conda-channel
+        job = artiq:extra-legacy:conda-channel
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel-legacy
      </runcommand>

@ -652,11 +823,6 @@ in
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel-archives/$(jq -r '.build' < $HYDRA_JSON)
      </runcommand>

-      <runcommand>
-        job = artiq:extra-beta:msys2-repos
-        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-msys2-repos-beta
-      </runcommand>
-
      <runcommand>
        job = artiq:main-nac3:msys2-repos
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-msys2-repos-nac3
@ -693,6 +859,7 @@ in
    secret-key-files = /etc/nixos/secret/nixbld.m-labs.hk-1
    experimental-features = nix-command flakes
  '';
+  nix.settings.allowed-uris = "github: gitlab: git+https://";  # https://github.com/NixOS/nix/issues/5039
  nix.settings.extra-sandbox-paths = ["/opt"];

  services.mlabs-backup.enable = true;
@ -701,11 +868,19 @@ in
  services.gitea = {
    enable = true;
    appName = "M-Labs Git";
+    database = {
+      type = "postgres";
+      socket = "/run/postgresql";
+    };
    mailerPasswordFile = "/etc/nixos/secret/mailerpassword";
    settings = {
      server = {
        ROOT_URL = "https://git.m-labs.hk/";
        HTTP_PORT = 3001;
+        DISABLE_SSH = false;
+        SSH_CREATE_AUTHORIZED_KEYS_FILE = false;
+        START_SSH_SERVER = true;
+        SSH_PORT = 2222;
      };

      indexer = {
@ -714,7 +889,8 @@ in

      mailer = {
        ENABLED = true;
-        HOST = "mail.m-labs.hk:587";
+        SMTP_ADDR = "mail.m-labs.hk";
+        SMTP_PORT = "587";
        FROM = "sysop@m-labs.hk";
        USER = "sysop@m-labs.hk";
      };
@ -743,12 +919,20 @@ in
    siteUrl = "https://chat.m-labs.hk/";
    mutableConfig = true;
  };
-  services.postgresql.package = pkgs.postgresql_12;
  services.matterbridge = {
    enable = true;
    configPath = "/etc/nixos/secret/matterbridge.toml";
  };
- 
+
+  services.postgresql = {
+    package = pkgs.postgresql_15;
+    settings.listen_addresses = pkgs.lib.mkForce "";
+    identMap =
+      ''
+      rt rt rt_user
+      '';
+  };
+
  nixpkgs.config.packageOverrides = super: let self = super.pkgs; in {
    nix = super.nix.overrideAttrs(oa: {
      patches = oa.patches or [] ++ [ ./nix-networked-derivations.patch ];
@ -758,7 +942,6 @@ in
        ./hydra-conda.patch
        ./hydra-msys2.patch
        ./hydra-restrictdist.patch
-        ./hydra-hack-allowed-uris.patch  # work around https://github.com/NixOS/nix/issues/5039
      ];
      hydraPath = oa.hydraPath + ":" + super.lib.makeBinPath [ super.jq ];
      doCheck = false;  # FIXME: ldap tests fail on hydra rebuild, seems unrelated to patches above.
@ -799,7 +982,7 @@ in
            expires 60d;
          '';
        };
-        locations."/nuc-netboot/".alias = "${import ./defenestrate}/";
+        locations."/nuc-netboot/".alias = "${import ./defenestrate { prioNixbld = true; } }/";

        # legacy URLs, redirect to avoid breaking people's bookmarks
        locations."/gateware.html".extraConfig = ''
@ -835,10 +1018,10 @@ in
          alias = "${hydraWwwOutputs}/artiq-manual-pdf/ARTIQ.pdf";
        };
        locations."/artiq/manual-legacy/" = {
-          alias = "${hydraWwwOutputs}/artiq-manual-html-legacy/share/doc/artiq-manual/html/";
+          alias = "${hydraWwwOutputs}/artiq-manual-html-legacy/";
        };
        locations."=/artiq/manual-legacy.pdf" = {
-          alias = "${hydraWwwOutputs}/artiq-manual-latexpdf-legacy/share/doc/artiq-manual/ARTIQ.pdf";
+          alias = "${hydraWwwOutputs}/artiq-manual-pdf-legacy/ARTIQ.pdf";
        };

        # legacy content
@ -874,6 +1057,12 @@ in
            autoindex on;
          '';
        };
+        locations."/artiq/" = {
+          alias = "${hydraWwwOutputs}/artiq-msys2-repos/";
+          extraConfig = ''
+            autoindex on;
+          '';
+        };
        locations."/artiq-nac3/" = {
          alias = "${hydraWwwOutputs}/artiq-msys2-repos-nac3/";
          extraConfig = ''
@ -954,20 +1143,14 @@ in
      "forum.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
-        root = "/var/www/flarum/public";
-        locations."~ \.php$".extraConfig = ''
-          fastcgi_pass unix:${config.services.phpfpm.pools.flarum.socket};
-          fastcgi_index index.php;
-        '';
-        extraConfig = ''
-          index index.php;
-          include /var/www/flarum/.nginx.conf;
-        '';
      };
      "perso.m-labs.hk" = {
        addSSL = true;
        enableACME = true;
        root = "/var/www/perso";
+        extraConfig = ''
+          autoindex on;
+        '';
      };
      "rt.m-labs.hk" = {
        forceSSL = true;
@ -1031,23 +1214,17 @@ in
      };
    };
  };
+
  services.mysql = {
    enable = true;
-    package = pkgs.mariadb;
+    package = pkgs.lib.mkForce pkgs.mariadb;
+    ensureDatabases = pkgs.lib.mkForce [];
+    ensureUsers = pkgs.lib.mkForce [];
  };
-  services.phpfpm.pools.flarum = {
-    user = "nobody";
-    settings = {
-      "listen.owner" = "nginx";
-      "listen.group" = "nginx";
-      "listen.mode" = "0600";
-      "pm" = "dynamic";
-      "pm.max_children" = 5;
-      "pm.start_servers" = 2;
-      "pm.min_spare_servers" = 1;
-      "pm.max_spare_servers" = 3;
-      "pm.max_requests" = 500;
-    };
+  services.flarum = {
+    enable = true;
+    package = pkgs.callPackage ./flarum {};
+    domain = "forum.m-labs.hk";
  };

  services.rt = {
@ -1072,7 +1249,18 @@ in
      Restart = "on-failure";
      User = "rt";
      Group = "rt";
-      ExecStart = "${pkgs.bash}/bin/bash -c 'PATH=${pkgs.rt}/bin HOME=/tmp ${pkgs.fetchmail}/bin/fetchmail -f /etc/nixos/secret/rt_fetchmailrc'";
+      ExecStart = "${pkgs.bash}/bin/bash -c 'PATH=${pkgs.rt}/bin HOME=/tmp ${pkgs.fetchmail}/bin/fetchmail --pidfile /tmp/.fetchmail.pid -f /etc/nixos/secret/rt_fetchmailrc'";
+    };
+  };
+  systemd.services.rt-fetchmail-intl = {
+    description = "Fetchmail for RT (intl)";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "dovecot2.service" ];
+    serviceConfig = {
+      Restart = "on-failure";
+      User = "rt";
+      Group = "rt";
+      ExecStart = "${pkgs.bash}/bin/bash -c 'PATH=${pkgs.rt}/bin HOME=/tmp ${pkgs.fetchmail}/bin/fetchmail --pidfile /tmp/.fetchmail-intl.pid -f /etc/nixos/secret/rt_fetchmailrc_intl'";
    };
  };

@ -1080,11 +1268,28 @@ in
    enable = true;
    localDnsResolver = false;  # conflicts with dnsmasq
    fqdn = "mail.m-labs.hk";
-    domains = [ "m-labs.hk" "m-labs.ph" "193thz.com" "malloctech.fr" ];
+    domains = [ "m-labs.hk" "m-labs.ph" "m-labs-intl.com" "193thz.com" "malloctech.fr" ];
    enablePop3 = true;
    enablePop3Ssl = true;
    certificateScheme = "acme-nginx";
  } // (import /etc/nixos/secret/email_settings.nix);
+  services.postfix = {
+    mapFiles."sender_transport" = builtins.toFile "sender_transport" ''
+        @m-labs-intl.com intltunnel:
+    '';
+    config = {
+      sender_dependent_default_transport_maps = "hash:/var/lib/postfix/conf/sender_transport";
+    };
+    masterConfig."intltunnel" = {
+      type = "unix";
+      command = "smtp";
+      args = [
+        "-o" "inet_interfaces=10.47.3.1"
+        "-o" "smtp_helo_name=mail.m-labs-intl.com"
+        "-o" "inet_protocols=ipv4"
+       ];
+    };
+  };
  services.roundcube = {
    enable = true;
    hostName = "mail.m-labs.hk";
@ -1097,12 +1302,15 @@ in

  services.nextcloud = {
    enable = true;
-    package = pkgs.nextcloud27;
+    package = pkgs.nextcloud30;
+    extraApps = { inherit (config.services.nextcloud.package.packages.apps) forms; };
    hostName = "files.m-labs.hk";
    https = true;
    maxUploadSize = "2G";
    config.adminpassFile = "/etc/nixos/secret/nextcloud_pass.txt";
-    config.defaultPhoneRegion = "HK";
+    settings.default_phone_region = "HK";
+    settings.log_type = "file";
+    phpOptions."opcache.interned_strings_buffer" = "12";
  };

  services.hedgedoc = {
--- a/nixbld-etc-nixos/flarum/composer.lock
+++ b/nixbld-etc-nixos/flarum/composer.lock
--- a/nixbld-etc-nixos/flarum/default.nix
+++ b/nixbld-etc-nixos/flarum/default.nix
@ -0,0 +1,39 @@
+{
+  lib,
+  php,
+  fetchFromGitHub,
+  fetchpatch,
+}:
+
+php.buildComposerProject (finalAttrs: {
+  pname = "flarum";
+  version = "1.8.1";
+
+  src = fetchFromGitHub {
+    owner = "flarum";
+    repo = "flarum";
+    rev = "v${finalAttrs.version}";
+    hash = "sha256-kigUZpiHTM24XSz33VQYdeulG1YI5s/M02V7xue72VM=";
+  };
+
+  patches = [
+    # Add useful extensions from https://github.com/FriendsOfFlarum
+    # Extensions included: fof/upload, fof/polls, fof/subscribed
+    ./fof-extensions.patch
+  ];
+
+  composerLock = ./composer.lock;
+  composerStrictValidation = false;
+  vendorHash = "sha256-z3KVGmILw8MZ4aaSf6IP/0l16LI/Y2yMzY2KMHf4qSg=";
+
+  meta = with lib; {
+    changelog = "https://github.com/flarum/framework/blob/main/CHANGELOG.md";
+    description = "Flarum is a delightfully simple discussion platform for your website";
+    homepage = "https://github.com/flarum/flarum";
+    license = lib.licenses.mit;
+    maintainers = with maintainers; [
+      fsagbuya
+      jasonodoom
+    ];
+  };
+})
--- a/nixbld-etc-nixos/flarum/fof-extensions.patch
+++ b/nixbld-etc-nixos/flarum/fof-extensions.patch
@ -0,0 +1,16 @@
+diff --git a/composer.json b/composer.json
+index c63b5f8..5ad1186 100644
+--- a/composer.json
+++ b/composer.json
+@@ -37,7 +37,10 @@
+         "flarum/sticky": "*",
+         "flarum/subscriptions": "*",
+         "flarum/suspend": "*",
+-        "flarum/tags": "*"
+        "flarum/tags": "*",
+        "fof/polls": "*",
+        "fof/subscribed": "*",
+        "fof/upload": "*"
+     },
+     "config": {
+         "preferred-install": "dist",
--- a/nixbld-etc-nixos/hydra-hack-allowed-uris.patch
+++ b/nixbld-etc-nixos/hydra-hack-allowed-uris.patch
@ -1,13 +0,0 @@
-diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc
-index 934bf42e..48f2d248 100644
--- a/src/hydra-eval-jobs/hydra-eval-jobs.cc
-+++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc
-@@ -281,6 +281,8 @@ int main(int argc, char * * argv)
-            to the environment. */
-         evalSettings.restrictEval = true;
- 
-+        evalSettings.allowedUris = {"https://github.com/m-labs/", "https://git.m-labs.hk/m-labs/", "https://gitlab.com/duke-artiq/"};
-+
-         /* When building a flake, use pure evaluation (no access to
-            'getEnv', 'currentSystem' etc. */
-         evalSettings.pureEval = myArgs.flake;
--- a/nixbld-etc-nixos/named/193thz.com
+++ b/nixbld-etc-nixos/named/193thz.com
@ -1,7 +1,7 @@
 $TTL 7200

@						SOA	ns.193thz.com. sb.m-labs.hk. (
-							2023121301
+							2024060201
 							7200
 							3600
 							86400
@ -12,11 +12,12 @@ $TTL 7200
 						NS		ns1.he.net.

 						A		94.190.212.123
+						A		202.77.7.238
 						AAAA	2001:470:18:390::2
 						MX		10 mail.m-labs.hk.
 						TXT		"v=spf1 mx a:router.alt.m-labs.hk -all"
 						TXT		"google-site-verification=5eIjLyhM_siRg5Fc2Z3AMSbheH0JFOn5iR3TCEXakqU"
-						CAA		0 issue "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/496268470"
+						CAA		0 issue "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1758987512"


 ns						A		94.190.212.123
--- a/nixbld-etc-nixos/named/200-29.98.206.103.in-addr.arpa
+++ b/nixbld-etc-nixos/named/200-29.98.206.103.in-addr.arpa
@ -1,7 +1,7 @@
 $TTL 7200

@						SOA	NS.XN--WBTZ5WPQAJ35CFXC.XN--J6W193G. sb.m-labs.hk. (
-							2024010901
+							2024060201
 							7200
 							3600
 							86400
@ -10,7 +10,7 @@ $TTL 7200

 						NS		NS.XN--WBTZ5WPQAJ35CFXC.XN--J6W193G.
 						NS		ns1.he.net.
-						CAA		0 issue "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/496268470"
+						CAA		0 issue "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1758987512"

 200	PTR	router.alt.m-labs.hk.
 201	PTR	stewardship1.alt.m-labs.hk.
--- a/nixbld-etc-nixos/named/m-labs-intl.com
+++ b/nixbld-etc-nixos/named/m-labs-intl.com
@ -0,0 +1,30 @@
+$TTL 7200
+
+@						SOA	ns.m-labs-intl.com. sb.m-labs.hk. (
+							2024101401
+							7200
+							3600
+							86400
+							600)
+
+
+						NS		ns.m-labs-intl.com.
+						NS		ns1.he.net.
+						NS		ns1.qnetp.net.
+
+						A		5.78.86.156
+						AAAA	2a01:4ff:1f0:83de::1
+						MX		10 mail.m-labs-intl.com.
+						TXT		"v=spf1 mx -all"
+						TXT		"google-site-verification=BlQd5_5wWW7calKC7bZA0GdoxR8-zj4gwJEg9sGJ3l8"
+						CAA		0 issue "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1768317117"
+
+ns						A		94.190.212.123
+ns						AAAA	2001:470:18:390::2
+
+mail					A		5.78.86.156
+mail._domainkey			IN	TXT	"v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDJVPuhSGXghO7ib8Em/Se3jfCCIJK5g4zn5pGZ3/e0I0f+zGHMuvwpjkAKf6eSmo/AAXEaco28pDi3qE5xfV512AJsORCfPoPFyNhLsj/qtri6hc5KVSWW0Ja3MSFBINDCaX78c7PXPY+3jJJGpwSBDLjdxj9AQwtfiCVlH4qE/QIDAQAB"
+_dmarc					TXT		"v=DMARC1; p=none"
+
+www						CNAME	@
+hooks					CNAME	@
--- a/nixbld-etc-nixos/named/m-labs.hk
+++ b/nixbld-etc-nixos/named/m-labs.hk
@ -1,7 +1,7 @@
 $TTL 7200

@						SOA	NS.XN--WBTZ5WPQAJ35CFXC.XN--J6W193G. sb.m-labs.hk. (
-							2024010901
+							2024080501
 							7200
 							3600
 							86400
@ -13,14 +13,16 @@ $TTL 7200
 						NS		ns1.he.net.

 						A		94.190.212.123
+						A		202.77.7.238
 						AAAA	2001:470:18:390::2
 						MX		10 mail.m-labs.hk.
 						TXT		"v=spf1 mx a:router.alt.m-labs.hk -all"
 						TXT		"google-site-verification=Tf_TEGZLG7-2BE70hMjLnzjDZ1qUeUZ6vxzbl1sagT8"
-						CAA		0 issue "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/496268470"
+						CAA		0 issue "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1758987512"


 mail					A		94.190.212.123
+mail					A		202.77.7.238
 mail					AAAA	2001:470:18:390::2
 mail._domainkey			TXT		"v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCl38A/Z0IInVU157qzrWgMfYm2iDHoWZsTyiiOoZdT7kHMzS/M2OMXMt7r5g1/7pCPClsGUDJvKGqVMmjJuPleMyKHwpGeT92qDNEFpt6ahneap/oYx5eBYM/vGcgmleNxyIoBHsptaZvqD4vCEFaC22f8UL5QAgQD3wCH3FwlpQIDAQAB"
 _dmarc					TXT		"v=DMARC1; p=none"
@ -41,17 +43,7 @@ files					CNAME	@
 docs					CNAME	@

 rpi-1					AAAA	2001:470:f891:1:dea6:32ff:fe8a:6a93
-rpi-2					AAAA	2001:470:f891:1:ba27:ebff:fef0:e9e6
 rpi-4					AAAA	2001:470:f891:1:dea6:32ff:fe14:fce9
-chiron					AAAA	2001:470:f891:1:7f02:9ebf:bee9:3dc7
-old-nixbld				AAAA	2001:470:f891:1:a07b:f49a:a4ef:aad9
-zeus					AAAA	2001:470:f891:1:4fd7:e70a:68bf:e9c1
-franz					AAAA	2001:470:f891:1:1b65:a743:2335:f5c6
-hera					AAAA	2001:470:f891:1:8b5e:404d:ef4e:9d92
-hestia					AAAA	2001:470:f891:1:881c:f409:a090:8401
-vulcan					AAAA	2001:470:f891:1:105d:3f15:bd53:c5ac
-
-aux						A		42.200.147.171

 router.alt				A		103.206.98.200
 stewardship1.alt		A		103.206.98.201
--- a/nixbld-etc-nixos/named/m-labs.ph
+++ b/nixbld-etc-nixos/named/m-labs.ph
@ -1,7 +1,7 @@
 $TTL 7200

@						SOA	ns1.m-labs.ph. sb.m-labs.hk. (
-							2024010901
+							2024060201
 							7200
 							3600
 							86400
@ -12,11 +12,12 @@ $TTL 7200
 						NS		ns1.he.net.

 						A		94.190.212.123
+						A		202.77.7.238
 						AAAA	2001:470:18:390::2
 						MX		10 mail.m-labs.hk.
 						TXT		"v=spf1 mx a:router.alt.m-labs.hk -all"
 						TXT		"google-site-verification=g2k8M1fhbYOPs4C37SeGCfNlD6paWcexamji1DXrp0o"
-						CAA		0 issue "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/496268470"
+						CAA		0 issue "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1758987512"

 ns1						A		94.190.212.123
 ns1						AAAA	2001:470:18:390::2
--- a/nixbld-etc-nixos/named/malloctech.fr
+++ b/nixbld-etc-nixos/named/malloctech.fr
@ -1,7 +1,7 @@
 $TTL 7200

@						SOA	ns.malloctech.fr. sb.m-labs.hk. (
-							2024010901
+							2024060201
 							7200
 							3600
 							86400
@ -14,7 +14,7 @@ $TTL 7200
 						MX		10 mail.m-labs.hk.
 						TXT		"v=spf1 mx a:router.alt.m-labs.hk -all"
 						TXT		"google-site-verification=LALF-fafTnmkL-18m3CzwFjSwEV1C7NeKexiNfMYsOw"
-						CAA		0 issue "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/496268470"
+						CAA		0 issue "letsencrypt.org; accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1758987512"

 ns						A		94.190.212.123
 ns						AAAA	2001:470:18:390::2
--- a/nixbld-etc-nixos/nix-networked-derivations.patch
+++ b/nixbld-etc-nixos/nix-networked-derivations.patch
@ -1,8 +1,8 @@
 diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
-index 64b55ca6a..9b4e52b8e 100644
+index 763045a80..d7c5cc82e 100644
 --- a/src/libstore/build/local-derivation-goal.cc
 +++ b/src/libstore/build/local-derivation-goal.cc
-@@ -180,6 +180,8 @@ void LocalDerivationGoal::tryLocalBuild()
+@@ -190,6 +190,8 @@ void LocalDerivationGoal::tryLocalBuild()
 
     assert(derivationType);
 
@ -11,7 +11,7 @@ index 64b55ca6a..9b4e52b8e 100644
     /* Are we doing a chroot build? */
     {
         auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-@@ -197,7 +199,7 @@ void LocalDerivationGoal::tryLocalBuild()
+@@ -207,7 +209,7 @@ void LocalDerivationGoal::tryLocalBuild()
         else if (settings.sandboxMode == smDisabled)
             useChroot = false;
         else if (settings.sandboxMode == smRelaxed)
@ -20,7 +20,7 @@ index 64b55ca6a..9b4e52b8e 100644
     }
 
     auto & localStore = getLocalStore();
-@@ -691,7 +693,7 @@ void LocalDerivationGoal::startBuilder()
+@@ -717,7 +719,7 @@ void LocalDerivationGoal::startBuilder()
                 "nogroup:x:65534:\n", sandboxGid()));
 
         /* Create /etc/hosts with localhost entry. */
@ -29,7 +29,7 @@ index 64b55ca6a..9b4e52b8e 100644
             writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
 
         /* Make the closure of the inputs available in the chroot,
-@@ -895,7 +897,7 @@ void LocalDerivationGoal::startBuilder()
+@@ -921,7 +923,7 @@ void LocalDerivationGoal::startBuilder()
            us.
         */
 
@ -38,7 +38,7 @@ index 64b55ca6a..9b4e52b8e 100644
             privateNetwork = true;
 
         userNamespaceSync.create();
-@@ -1134,7 +1136,7 @@ void LocalDerivationGoal::initEnv()
+@@ -1160,7 +1162,7 @@ void LocalDerivationGoal::initEnv()
        to the builder is generally impure, but the output of
        fixed-output derivations is by definition pure (since we
        already know the cryptographic hash of the output). */
@ -47,7 +47,7 @@ index 64b55ca6a..9b4e52b8e 100644
         for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
             env[i] = getEnv(i).value_or("");
     }
-@@ -1799,7 +1801,7 @@ void LocalDerivationGoal::runChild()
+@@ -1829,7 +1831,7 @@ void LocalDerivationGoal::runChild()
             /* Fixed-output derivations typically need to access the
                network, so give them access to /etc/resolv.conf and so
                on. */
@ -56,21 +56,21 @@ index 64b55ca6a..9b4e52b8e 100644
                 // Only use nss functions to resolve hosts and
                 // services. Don’t use it for anything else that may
                 // be configured for this system. This limits the
-@@ -2050,7 +2052,7 @@ void LocalDerivationGoal::runChild()
-                     #include "sandbox-defaults.sb"
-                     ;
+@@ -2071,7 +2073,7 @@ void LocalDerivationGoal::runChild()
+                 #include "sandbox-defaults.sb"
+                 ;
 
-                if (!derivationType->isSandboxed())
-+                if (networked || !derivationType->isSandboxed())
-                     sandboxProfile +=
-                         #include "sandbox-network.sb"
-                         ;
+-            if (!derivationType->isSandboxed())
+            if (networked || !derivationType->isSandboxed())
+                 sandboxProfile +=
+                     #include "sandbox-network.sb"
+                     ;
 diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
-index 0a05081c7..4c251718c 100644
+index 86b86c01e..95b03aae8 100644
 --- a/src/libstore/build/local-derivation-goal.hh
 +++ b/src/libstore/build/local-derivation-goal.hh
-@@ -66,6 +66,8 @@ struct LocalDerivationGoal : public DerivationGoal
- 
+@@ -82,6 +82,8 @@ struct LocalDerivationGoal : public DerivationGoal
+      */
     Path chrootRootDir;
 
 +    bool networked;
--- a/nixbld-etc-nixos/rt.nix
+++ b/nixbld-etc-nixos/rt.nix
@ -19,14 +19,9 @@ let
      Set($Timezone, '${cfg.timeZone}');

      Set($DatabaseType, 'Pg');
-      Set($DatabaseHost, 'localhost');
-      Set($DatabaseUser, 'rt_user');
+      Set($DatabaseHost, '/run/postgresql');
+      Set($DatabaseUser, 'rt');
      Set($DatabaseName, 'rt5');
-      # Read database password from file
-      open my $fh, '<', '${cfg.dbPasswordFile}' or die 'Can\'t open file $!';
-      my $dbpw = do { local $/; <$fh> };
-      $dbpw =~ s/^\s+|\s+$//g;
-      Set($DatabasePassword, $dbpw);

      # System (Logging)
      Set($LogToSTDERR, undef); # Don't log twice
@ -35,7 +30,7 @@ let
      Set($OwnerEmail, '${cfg.ownerEmail}');
      Set($MaxAttachmentSize, 15360000);
      Set($CheckMoreMSMailHeaders, 1);
-      Set($RTAddressRegexp, '^(helpdesk|sales)\@(m-labs.hk)$');
+      Set($RTAddressRegexp, '^(helpdesk)\@(m-labs.hk|m-labs-intl.com)$');
      Set($LoopsToRTOwner, 0);

      # System (Outgoing mail)
@ -154,13 +149,6 @@ in {
      type = str;
    };

-    dbPasswordFile = mkOption {
-      description = "File containing the database password";
-      type = str;
-      default = "/etc/nixos/secret/rtpasswd";
-      internal = true;
-    };
-
    domain = mkOption {
      description = "Which domain RT is running on";
      type = str;
@ -245,8 +233,6 @@ in {

          PrivateNetwork = false;
          MemoryDenyWriteExecute = false;
-
-          ReadOnlyPaths = [ cfg.dbPasswordFile ];
        };

        environment = {
--- a/nixops/athena-hardware-configuration.nix
+++ b/nixops/athena-hardware-configuration.nix
@ -0,0 +1,45 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.kernelPackages = pkgs.linuxPackages_latest;
+  boot.initrd.availableKernelModules = [ "xhci_pci" "thunderbolt" "ahci" "nvme" "usbhid" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/89463254-b38d-45db-92b6-0f7d92a44f47";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/F84B-ACC5";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp86s0.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+
+  hardware.cpu.intel.updateMicrocode = true;
+
+  system.stateVersion = "23.11";
+}
--- a/nixops/chiron-hardware-configuration.nix
+++ b/nixops/chiron-hardware-configuration.nix
@ -18,6 +18,7 @@
  fileSystems."/boot" =
    { device = "/dev/disk/by-uuid/060C-8772";
      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
    };

  swapDevices = [ ];
--- a/nixops/common-users.nix
+++ b/nixops/common-users.nix
@ -57,7 +57,7 @@
  };
  esavkin = {
    isNormalUser = true;
-    extraGroups = ["plugdev" "dialout" "libvirtd"];
+    extraGroups = ["plugdev" "dialout" "libvirtd" "wireshark"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBLDJI4GFLBmScbeR8Jh4Gi8A/2nuGlYtFTJVT+Es/bzdiPRk8DLG62T0hyRR+8LfHjbrCsDuYFNztT8hHGXd7h3xp3y2X7ArkJo8xUK5QxGd5D2Zn4ANfZTTVkoGlEHbFA=="
    ];
@ -78,14 +78,14 @@
  };
  linuswck = {
    isNormalUser = true;
-    extraGroups = ["plugdev" "dialout"];
+    extraGroups = ["plugdev" "dialout" "wireshark" "libvirtd"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBAFYwmik6/xY1vb9aKBOpKklKOwSJJ0PEgNwWNULghZGJ0g4CTk04LXLSMYBm1SW74df8YMgaE/eoidq6smN6hKIgo8s3qPQGZAi4UXffMs2ciqXNa/zZcCu3PyZvyksxA=="
    ];
  };
  morgan = {
    isNormalUser = true;
-    extraGroups = ["plugdev" "dialout"];
+    extraGroups = ["plugdev" "dialout" "wireshark" "libvirtd"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBDXMbJEPn0mM2Bgt6eMAd+c0J5oPSvquZG+BxKdUf0qbeQldRaoB26NHMZnLte/fS00U/cqStLWDiwtEvH5WlbbawsMBymm65zbWMByebXhBDjdr6a1kkOFcKJvAL9qVBQ=="
    ];
@ -104,6 +104,13 @@
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBOKwN4ui94QfouYYlkI1lc3WgtjURVYLTdAizJIBnY3dNRNblAiuvTD4pQ+LEI+eOTg4SnQz1NeqH4YOQhbT5+/nZojvGTb3UVN13ZYND+Gci3DdqB2mwIYop7kMXwHgLQ=="
    ];
  };
+  therobs12 = {
+    isNormalUser = true;
+    extraGroups = ["plugdev" "dialout"];
+    openssh.authorizedKeys.keys = [
+      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBK1tUg7TtceARRnGI80Ai5kNFolFfZ++LH9v1UoRCiJdxeQWPdNYO0Gj7+ejJvgZXwvN4yHGgcZHraEml4Mj/dKrEMFygfuYLDRmXtPFwX6TNMrWlxMhPzuNY+yCaxlqYg=="
+    ];
+  };

  dpn = {
    isNormalUser = true;
--- a/nixops/demeter-hardware-configuration.nix
+++ b/nixops/demeter-hardware-configuration.nix
@ -21,6 +21,7 @@
  fileSystems."/boot" =
    { device = "/dev/disk/by-uuid/76A2-F01F";
      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
    };

  swapDevices = [ ];
--- a/nixops/desktop.nix
+++ b/nixops/desktop.nix
@ -10,6 +10,9 @@ in
  nix.nixPath = [ "nixpkgs=${pkgs.path}" ];
  programs.command-not-found.dbPath = "${pkgs.path}/programs.sqlite";

+  boot.loader.systemd-boot.memtest86.enable = true;
+  boot.loader.grub.memtest86.enable = true;
+
  imports =
    [
      (./. + "/${host}-hardware-configuration.nix")
@ -61,7 +64,8 @@ in
    xournal
    xsane
    gtkwave unzip zip gnupg
-    gnome3.gnome-tweaks
+    gnome-tweaks
+    ghex
    jq sublime3 rink qemu_kvm
    tmux screen gdb minicom picocom
    artiq.packages.x86_64-linux.openocd-bscanspi
@ -89,6 +93,7 @@ in
  services.avscan.enable = true;

  services.openssh.enable = true;
+  services.openssh.authorizedKeysInHomedir = false;
  services.openssh.settings.PasswordAuthentication = false;
  services.openssh.extraConfig =
    ''
@ -121,29 +126,17 @@ in
  };
  services.avahi = {
    enable = true;
-    nssmdns = true;
+    nssmdns4 = true;
  };

-  # Enable sound.
-  sound.enable = true;
-  hardware.pulseaudio = {
-    enable = true;
-    package = pkgs.pulseaudioFull;
-  };
+  hardware.graphics.enable32Bit = true;

-  hardware.opengl.driSupport32Bit = true;
-  hardware.pulseaudio.support32Bit = true;
-
-  i18n.inputMethod = {
-    enabled = "fcitx5";
-    fcitx5.addons = [ pkgs.fcitx5-table-extra pkgs.fcitx5-m17n ];
-  };
-  fonts.packages = [ pkgs.noto-fonts pkgs.noto-fonts-cjk pkgs.noto-fonts-emoji pkgs.noto-fonts-extra pkgs.emacs-all-the-icons-fonts ];
+  fonts.packages = [ pkgs.noto-fonts pkgs.noto-fonts-cjk-sans pkgs.noto-fonts-emoji pkgs.noto-fonts-extra pkgs.emacs-all-the-icons-fonts ];

  # Enable the X11 windowing system.
  services.xserver.enable = true;
-  services.xserver.layout = "us";
-  services.xserver.xkbOptions = "eurosign:e";
+  services.xserver.xkb.layout = "us";
+  services.xserver.xkb.options = "eurosign:e";

  services.xserver.displayManager.gdm.enable = true;
  services.xserver.desktopManager.gnome.enable = true;
--- a/nixops/experimental-users.nix
+++ b/nixops/experimental-users.nix
@ -1,4 +0,0 @@
-{ pkgs, ... }:
-
-{
-}
--- a/nixops/extra-udev.nix
+++ b/nixops/extra-udev.nix
@ -21,4 +21,7 @@ SUBSYSTEM=="usb", ATTRS{idVendor}=="2109", ATTRS{idProduct}=="2812", MODE="0660"
 # LibreVNA
 SUBSYSTEM=="usb", ATTRS{idVendor}=="0483", ATTRS{idProduct}=="4121", MODE="0660", GROUP="plugdev"
 SUBSYSTEM=="usb", ATTRS{idVendor}=="1209", ATTRS{idProduct}=="4121", MODE="0660", GROUP="plugdev"
+# DSLogic
+SUBSYSTEM=="usb", ATTRS{idVendor}=="2a0e", ATTRS{idProduct}=="0020", MODE="0660", GROUP="plugdev"
+SUBSYSTEM=="usb", ATTRS{idVendor}=="2a0e", ATTRS{idProduct}=="0034", MODE="0660", GROUP="plugdev"
 ''
--- a/nixops/franz-hardware-configuration.nix
+++ b/nixops/franz-hardware-configuration.nix
@ -10,7 +10,6 @@

  boot.initrd.availableKernelModules = [ "xhci_pci" "thunderbolt" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" "rtsx_pci_sdmmc" ];
  boot.initrd.kernelModules = [ ];
-  boot.kernelPackages = pkgs.linuxPackages_5_15;
  boot.kernelModules = [ "kvm-intel" ];
  boot.blacklistedKernelModules = [ "iwlwifi" ];
  boot.extraModulePackages = [ ];
@ -24,6 +23,7 @@
  fileSystems."/boot" =
    { device = "/dev/disk/by-uuid/A33B-F001";
      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
    };

  swapDevices = [ ];
--- a/nixops/hera-hardware-configuration.nix
+++ b/nixops/hera-hardware-configuration.nix
@ -18,6 +18,7 @@
  fileSystems."/boot" =
    { device = "/dev/disk/by-uuid/8C30-F6DC";
      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
    };

  swapDevices = [ ];
--- a/nixops/hestia-hardware-configuration.nix
+++ b/nixops/hestia-hardware-configuration.nix
@ -18,6 +18,7 @@
  fileSystems."/boot" =
    { device = "/dev/disk/by-uuid/E085-5F21";
      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
    };

  swapDevices = [ ];
--- a/nixops/juno-hardware-configuration.nix
+++ b/nixops/juno-hardware-configuration.nix
@ -8,9 +8,9 @@
    [ (modulesPath + "/installer/scan/not-detected.nix")
    ];

-  boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" "sr_mod" ];
+  boot.initrd.availableKernelModules = [ "ehci_pci" "ata_piix" "xhci_pci" "ahci" "usbhid" "usb_storage" "sd_mod" ];
  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
  boot.extraModulePackages = [ ];

  fileSystems."/" =
@ -18,11 +18,6 @@
      fsType = "ext4";
    };

-  fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/4E51-B390";
-      fsType = "vfat";
-    };
-
  swapDevices = [ ];

  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
@ -30,13 +25,13 @@
  # still possible to use this option, but it's recommended to use it in conjunction
  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
  networking.useDHCP = lib.mkDefault true;
-  # networking.interfaces.enp3s0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp5s0.useDHCP = lib.mkDefault true;

  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
  hardware.cpu.intel.updateMicrocode = true;

-  boot.loader.systemd-boot.enable = true;
-  boot.loader.efi.canTouchEfiVariables = true;
+  boot.loader.grub.enable = true;
+  boot.loader.grub.device = "/dev/sda";

  nixpkgs.config.nvidia.acceptLicense = true;
  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.legacy_470;
--- a/nixops/jupiter-hardware-configuration.nix
+++ b/nixops/jupiter-hardware-configuration.nix
@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "thunderbolt" "ahci" "nvme" "usbhid" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/315af039-6799-43ac-8999-7da69a6fbd1e";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/45B7-790E";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp86s0.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode = true;
+  
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+  
+  system.stateVersion = "24.05";
+}
--- a/nixops/nixops.nix
+++ b/nixops/nixops.nix
@ -6,8 +6,6 @@
  network.enableRollback = true;

  rpi-1 = import ./rpi.nix { host = "rpi-1"; rpi4 = true; };
-  rpi-2 = import ./rpi.nix { host = "rpi-2"; rpi4 = false; experimental-users = true; };
-  rpi-3 = import ./rpi.nix { host = "rpi-3"; rpi4 = true; };
  rpi-4 = import ./rpi.nix { host = "rpi-4"; rpi4 = true; };
  zeus = import ./desktop.nix { host = "zeus"; };
  hera = import ./desktop.nix { host = "hera"; };
@ -17,4 +15,9 @@
  franz = import ./desktop.nix { host = "franz"; };
  juno = import ./desktop.nix { host = "juno"; };
  demeter = import ./desktop.nix { host = "demeter"; };
+  vulcan = import ./desktop.nix { host = "vulcan"; };
+  rc = import ./desktop.nix { host = "rc"; };
+  athena = import ./desktop.nix { host = "athena"; };
+  jupiter = import ./desktop.nix { host = "jupiter"; };
+  saturn = import ./desktop.nix { host = "saturn"; };
 }
--- a/nixops/old-nixbld-hardware-configuration.nix
+++ b/nixops/old-nixbld-hardware-configuration.nix
@ -21,6 +21,7 @@
  fileSystems."/boot" =
    { device = "/dev/disk/by-uuid/D0A3-DDAE";
      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
    };

  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
--- a/nixops/rc-hardware-configuration.nix
+++ b/nixops/rc-hardware-configuration.nix
@ -0,0 +1,50 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "usbhid" "sd_mod" "rtsx_usb_sdmmc" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/348c924c-1d86-44ff-84af-2594f414e7d0";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/1BDC-44BB";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+
+  fileSystems."/opt" =
+    { device = "/dev/disk/by-uuid/cf0f51b6-7b95-4c74-9390-37dc4c86f32b";
+      fsType = "ext4";
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp1s0.useDHCP = lib.mkDefault true;
+  # networking.interfaces.wlp2s0.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+
+  hardware.cpu.intel.updateMicrocode = true;
+
+  system.stateVersion = "23.11";
+}
--- a/nixops/rpi.nix
+++ b/nixops/rpi.nix
@ -24,6 +24,7 @@ in
  };

  services.openssh.enable = true;
+  services.openssh.authorizedKeysInHomedir = false;
  services.openssh.settings.PasswordAuthentication = false;
  services.openssh.settings.GatewayPorts = "clientspecified";
  services.openssh.extraConfig =
@ -34,15 +35,12 @@ in

  networking.hostName = host;

-  networking.firewall.allowedTCPPorts = if host == "rpi-2" then [ 6000 ] else [];
-
  time.timeZone = "Asia/Hong_Kong";

  users.extraGroups.plugdev = { };
  users.mutableUsers = false;
  users.defaultUserShell = pkgs.fish;
-  users.extraUsers = (import ./common-users.nix { inherit pkgs; }) //
-    (pkgs.lib.optionalAttrs experimental-users (import ./experimental-users.nix { inherit pkgs; })) // {
+  users.extraUsers = (import ./common-users.nix { inherit pkgs; }) // {
    nixbld = {
      isNormalUser = true;
      extraGroups = ["plugdev" "dialout"];
--- a/nixops/saturn-hardware-configuration.nix
+++ b/nixops/saturn-hardware-configuration.nix
@ -0,0 +1,43 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "thunderbolt" "ahci" "nvme" "usbhid" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/51d521ec-4807-4b71-8a89-116b89f72d2e";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/877D-AF6A";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.enp86s0.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  hardware.cpu.intel.updateMicrocode = true;
+  
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+  
+  system.stateVersion = "24.05";
+}
--- a/nixops/vulcan-hardware-configuration.nix
+++ b/nixops/vulcan-hardware-configuration.nix
@ -0,0 +1,41 @@
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" =
+    { device = "/dev/disk/by-uuid/67168ae0-6448-4b40-b278-406290224b4f";
+      fsType = "ext4";
+    };
+
+  fileSystems."/boot" =
+    { device = "/dev/disk/by-uuid/8F4B-AD84";
+      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
+    };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+
+  hardware.cpu.intel.updateMicrocode = true;
+  
+  system.stateVersion = "23.05";
+}
--- a/nixops/zeus-hardware-configuration.nix
+++ b/nixops/zeus-hardware-configuration.nix
@ -18,6 +18,7 @@
  fileSystems."/boot" =
    { device = "/dev/disk/by-uuid/91B4-E546";
      fsType = "vfat";
+      options = [ "fmask=0022" "dmask=0022" ];
    };

  swapDevices = [ ];
--- a/remote-ipsec.txt
+++ b/remote-ipsec.txt
@ -0,0 +1,49 @@
+connections {
+  bypass-ipsec {
+    remote_addrs = 127.0.0.1
+    children {
+      bypass-isakmp-v4 {
+        local_ts = 0.0.0.0/0[udp/isakmp]
+        remote_ts = 0.0.0.0/0[udp/isakmp]
+        mode = pass
+        start_action = trap
+      }
+      bypass-isakmp-v6 {
+        local_ts = ::/0[udp/isakmp]
+        remote_ts = ::/0[udp/isakmp]
+        mode = pass
+        start_action = trap
+      }
+    }
+  }
+  m_labs {
+    version = 2
+    encap = no
+    mobike = no
+    send_certreq = no
+    proposals = aes128gcm128-sha256-prfsha256-curve25519,aes128gcm128-sha256-prfsha256-ecp256
+    local_addrs = 103.206.98.1
+    remote_addrs = 94.190.212.123
+    local {
+      auth = pubkey
+      id = fqdn:igw0.hkg.as150788.net
+      pubkeys = igw0.hkg.as150788.net
+    }
+    remote {
+      auth = pubkey
+      id = fqdn:m-labs.hk
+      pubkeys = m-labs.hk
+    }
+    children {
+      con1 {
+        mode = transport
+        ah_proposals = sha256-curve25519,sha256-ecp256
+        esp_proposals =
+        local_ts = 103.206.98.1[gre]
+        remote_ts = 94.190.212.123[gre]
+        start_action = none
+        close_action = none
+      }
+    }
+  }
+}
Author	SHA1	Message	Date
Sébastien Bourdeauducq	c7ea537622	Revert "Break cycle dependency of tunnel netdev services on network setup" Does not solve the problem. This reverts commit `b1779b57cc`.	2024-11-25 12:11:36 +08:00
Florian Agbuya	d1236d548d	afws: enable file logging with afws group permissions	2024-11-22 15:34:23 +08:00
Sebastien Bourdeauducq	98c1ecd325	nixops: nixpkgs 24.11 compatibility	2024-11-16 18:49:26 +08:00
Sébastien Bourdeauducq	45e718d65a	nixops: add esavkin to wireshark group	2024-11-06 15:25:21 +08:00
Sébastien Bourdeauducq	243deb96be	nixbld: update Nix patch	2024-11-05 18:45:40 +08:00
Egor Savkin	b1779b57cc	Break cycle dependency of tunnel netdev services on network setup This changes the following chain after nixos-rebuild switch with modified tunnel interfaces: stop network-setup -> stop TUN-netdev -> stop network-addresses-TUN -> start network-addresses-TUN (fails since it depends on TUN-netdev which is off). Chain after this change: stop TUN-netdev -> stop network-setup -> stop network-addresses-TUN -> start TUN-netdev -> start network-addresses-TUN -> start network-setup Signed-off-by: Egor Savkin <es@m-labs.hk>	2024-10-30 17:23:52 +08:00
Sébastien Bourdeauducq	4f8d84e3ef	nixbld: enable prioNixbld for new defenestrate	2024-10-30 14:53:56 +08:00
Egor Savkin	eabd92d2e8	Use tunnel for uploading web-intl Signed-off-by: Egor Savkin <es@m-labs.hk>	2024-10-24 17:35:34 +08:00
Sébastien Bourdeauducq	04a64c3710	nixbld: set up RT for m-labs-intl.com	2024-10-24 15:49:41 +08:00
Egor Savkin	d27ee750a2	m-labs-intl.com VPS setup information Co-authored-by: Egor Savkin <es@m-labs.hk> Co-committed-by: Egor Savkin <es@m-labs.hk>	2024-10-21 15:48:17 +08:00
Sébastien Bourdeauducq	14e9d63ab7	nixbld: apply TCP MSS clamping to USA tunnel	2024-10-17 15:08:27 +08:00
Sébastien Bourdeauducq	19aee9b59f	nixbld: send mail from m-labs-intl.com through trump0	2024-10-17 15:04:50 +08:00
Sébastien Bourdeauducq	f8a3d54b54	nixbld: update simple-nixos-mailserver	2024-10-17 15:04:14 +08:00
Sébastien Bourdeauducq	c499a7ce86	nixbld: keep checking SPF for email from tunnel GRE preserves source IP information.	2024-10-17 14:48:04 +08:00
Sébastien Bourdeauducq	476f5d1d6c	nixbld: update to nextcloud 30	2024-10-16 11:33:07 +08:00
Sebastien Bourdeauducq	ecf40fb2db	nixbld: fix firewall issue with incoming USA tunnel connections	2024-10-15 21:27:43 +08:00
Sébastien Bourdeauducq	34102e66ad	nixbld: install nextcloud forms app	2024-10-15 16:22:33 +08:00
Sébastien Bourdeauducq	93ae830468	nixbld: disable IPv6 MX for m-labs-intl.com	2024-10-14 14:23:15 +08:00
Sébastien Bourdeauducq	8af66556b9	nixbld: remove google fonts workaround	2024-10-11 17:27:10 +08:00
Sébastien Bourdeauducq	94cff9bb09	nixbld: revert `233998b8` (did not fix the problem)	2024-10-08 16:11:12 +08:00
Sébastien Bourdeauducq	2bf7bb0638	nixbld: connect to USA VPN	2024-10-08 16:09:56 +08:00
Sébastien Bourdeauducq	3419fe6013	nixbld: remove nkrackow user	2024-10-05 10:15:13 +08:00
Sébastien Bourdeauducq	ec53c0cbdd	nixbld: add eduardotenholder user	2024-10-02 18:41:45 +08:00
Sébastien Bourdeauducq	0258f5cff4	nixbld: reorganize users (NFC)	2024-10-02 18:40:48 +08:00
Sébastien Bourdeauducq	b723b7f8c0	nixbld: clean up/update systemPackages	2024-09-30 15:12:01 +08:00
Sébastien Bourdeauducq	0c336f3dd7	nixbld: do not log refused connections Happen all the time and spam the kernel log.	2024-09-30 14:40:09 +08:00
Sebastien Bourdeauducq	11181f0397	nixbld: flarum createDatabaseLocally no longer needed https://github.com/NixOS/nixpkgs/pull/341340	2024-09-23 10:52:08 +08:00
Sebastien Bourdeauducq	aaf70f36df	nixops: remove user accounts	2024-09-13 13:23:15 +08:00
Sébastien Bourdeauducq	4a288abe2b	nixbld: keep automatic flarum DB migrations	2024-09-10 17:12:44 +08:00
Sébastien Bourdeauducq	246a375dfb	add remote IPsec settings	2024-09-05 14:36:37 +08:00
Sébastien Bourdeauducq	635f90f0c7	nixbld/flarum: use nix	2024-08-31 17:27:16 +08:00
Sébastien Bourdeauducq	8a187ba5b9	nixbld: SIT can take larger packets	2024-08-29 18:55:52 +08:00
Sébastien Bourdeauducq	9383227c5b	nixbld: consistent netif variables	2024-08-29 18:53:33 +08:00
Sébastien Bourdeauducq	233998b8f3	nixbld: work around tunnel bring-up race condition	2024-08-29 18:40:17 +08:00
Sébastien Bourdeauducq	90a6b84c09	nixbld: work around tunnel TCPMSS issues	2024-08-29 18:39:52 +08:00
Sébastien Bourdeauducq	23e1fa029a	nixbld: upgrade postgresql	2024-08-25 11:06:19 +08:00
Egor Savkin	75035b387e	Skip SPF for mails originating from intl Signed-off-by: Egor Savkin <es@m-labs.hk>	2024-08-20 10:59:27 +08:00
Sébastien Bourdeauducq	4f48ea611a	nixops: remove wanglm user	2024-08-19 11:18:06 +08:00
Sébastien Bourdeauducq	6dc8214102	nixbld/backup: include gitea DB dump	2024-08-17 18:26:46 +08:00
Sébastien Bourdeauducq	a6b216bb87	nixbld/gitea: move to postgresql	2024-08-17 18:18:56 +08:00
Sébastien Bourdeauducq	6e21a95ba8	nixbld/named: add qnetp slave DNS for m-labs-intl.com	2024-08-15 19:52:42 +08:00
Sébastien Bourdeauducq	d08186a27a	nixbld/named: enable CAA for m-labs-intl.com	2024-08-14 11:52:25 +08:00
Sébastien Bourdeauducq	5d132565e6	nixbld/named: add hooks.m-labs-intl.com	2024-08-14 11:42:38 +08:00
Sébastien Bourdeauducq	97ca7ea3ce	nixbld: mail setup for m-labs-intl.com WIP	2024-08-14 11:38:19 +08:00
Sébastien Bourdeauducq	e24c167f8b	Revert "nixbld: block SAP spam" Option seems to have no effect. This reverts commit `b769b47075`.	2024-08-14 10:58:49 +08:00
Egor Savkin	18194be5c3	nixbld: deploy web2019 to the intl domain Co-authored-by: Egor Savkin <es@m-labs.hk> Co-committed-by: Egor Savkin <es@m-labs.hk>	2024-08-14 10:54:52 +08:00
Sébastien Bourdeauducq	7781d6236e	nixbld/rt: disable TCP	2024-08-11 12:19:15 +08:00
Sébastien Bourdeauducq	93e19c74e9	nixbld/rt: use psql peer authentication	2024-08-11 12:12:28 +08:00
Sébastien Bourdeauducq	4ccab3cf2b	nixbld: remove outdated DNS records	2024-08-05 19:13:34 +08:00
Sebastien Bourdeauducq	69fe8c9866	nixbld: add flo user	2024-08-01 07:32:11 +08:00
Sebastien Bourdeauducq	b769b47075	nixbld: block SAP spam	2024-07-02 09:56:02 +02:00
Sébastien Bourdeauducq	f0668fa5b7	juno: mobo swap	2024-06-27 14:20:30 +08:00
Sébastien Bourdeauducq	8422d16978	nixops: add new DSLogic USB ID	2024-06-26 13:29:20 +08:00
Sébastien Bourdeauducq	872dcaa6bc	nixbld: serve m-labs-intl.com domain	2024-06-06 17:29:07 +08:00
Sébastien Bourdeauducq	ca895df9f3	nixbld: switch to gitea built-in SSH server	2024-06-06 16:27:39 +08:00
Sébastien Bourdeauducq	4e6686dbe9	nixbld: fix gitea emails	2024-06-06 13:52:35 +08:00
Sébastien Bourdeauducq	f973d2969a	nixbld: fix gitea emails	2024-06-05 11:23:24 +08:00
Sebastien Bourdeauducq	18a41e1c88	nixbld: work around for hydra input issues in restricted mode	2024-06-03 22:39:00 +08:00
Sébastien Bourdeauducq	f49a0f825e	nixops: typo	2024-06-02 20:29:37 +08:00
Sébastien Bourdeauducq	6c3a89df02	nixops: update wanglm key	2024-06-02 20:24:25 +08:00
Sébastien Bourdeauducq	bbc4d663a9	nixops: add new machines	2024-06-02 17:55:40 +08:00
Sébastien Bourdeauducq	adad8e9894	nixops: add new users	2024-06-02 17:55:19 +08:00
Sébastien Bourdeauducq	f07b292d3b	nixbld: disallow user SSH keys	2024-06-02 14:10:10 +08:00
Sébastien Bourdeauducq	d91ff8300d	nixops: disallow user SSH keys	2024-06-02 14:04:02 +08:00
Sébastien Bourdeauducq	bd6c61094f	nixbld: update letsencrypt CAA URI https://github.com/NixOS/nixpkgs/issues/316608	2024-06-02 13:50:48 +08:00
Sébastien Bourdeauducq	cc0bf224df	nixbld: install mpd	2024-06-02 13:50:24 +08:00
Sébastien Bourdeauducq	41aeae7b2d	nixbld: update simple-nixos-mailserver	2024-06-02 12:59:47 +08:00
Sébastien Bourdeauducq	1eac9d249d	nixbld: nixos 24.05	2024-06-02 12:52:17 +08:00
Sébastien Bourdeauducq	c3d9b9a7a1	nixbld: small cleanup	2024-06-02 12:52:01 +08:00
Sébastien Bourdeauducq	b6263c7dd9	nixops: fix /boot mount options	2024-05-30 18:43:54 +08:00
Sébastien Bourdeauducq	2446d0c946	nixops: mount /opt on rc	2024-05-30 18:41:43 +08:00
Sébastien Bourdeauducq	8879147c8d	nixops: add DSLogic to udev	2024-05-30 12:21:04 +08:00
Sebastien Bourdeauducq	ca37637771	desktop: install memtest86	2024-05-29 17:59:37 +08:00
Sebastien Bourdeauducq	6e7b4aa4a9	nixops: add athena machine	2024-05-28 15:27:21 +08:00
Sebastien Bourdeauducq	a22e270ac8	nixbld: replace deprecated gitea mailer setting	2024-05-28 11:44:55 +08:00
Sébastien Bourdeauducq	bc20cf499b	add architeuthis user	2024-05-26 20:24:01 +08:00
Sebastien Bourdeauducq	9285123f8e	desktop: install ghex	2024-05-24 21:46:12 +08:00
Sébastien Bourdeauducq	9845ea7832	nixops: nixos 24.05 renamed options	2024-05-24 11:09:02 +08:00
Sébastien Bourdeauducq	a202452e5c	nixops: add missing stateVersion	2024-05-24 10:40:37 +08:00
Sebastien Bourdeauducq	55cfda91e0	nixbld: fix nextcloud logging	2024-05-24 09:58:19 +08:00
Sebastien Bourdeauducq	543e9468cc	nixbld: fix nextcloud opcache warning	2024-05-24 09:49:17 +08:00
Sebastien Bourdeauducq	6487eab3c7	nixbld: update nextcloud	2024-05-23 22:58:39 +08:00
Sebastien Bourdeauducq	96f7264258	nixbld: enable audio	2024-05-23 22:21:10 +08:00
Sebastien Bourdeauducq	6018eca294	nixbld: enable SSH X11 forwarding	2024-05-23 22:20:56 +08:00
Sebastien Bourdeauducq	584c9e560c	nixops: add rc machine	2024-05-23 22:07:42 +08:00
Sébastien Bourdeauducq	37f24f5898	nixops: update permissions	2024-05-16 13:40:57 +08:00
Sébastien Bourdeauducq	4c7f35bc75	nixbld: allow routing between wifi and LAN	2024-05-06 10:57:33 +08:00
Sébastien Bourdeauducq	2cd3ae1337	nixbld: fix routing policy for wifi	2024-05-06 10:57:11 +08:00
Sébastien Bourdeauducq	7f1972fc9d	nixbld: add backup IP to DNS	2024-05-06 10:44:54 +08:00
Sébastien Bourdeauducq	5729c4998a	nixbld: add backup internet connection	2024-05-06 10:32:10 +08:00
Sébastien Bourdeauducq	60aacb6a1b	nixbld: cleanup routing policy	2024-05-06 10:31:45 +08:00
Sébastien Bourdeauducq	bfd5e0289a	nixbld: enable indexing on perso.m-labs.hk	2024-05-06 09:48:16 +08:00
Sébastien Bourdeauducq	2c9be41ce4	Revert "nixbld: disable substituters" NixOS bug festival This reverts commit `5576b82d15`.	2024-04-13 16:47:30 +08:00
Sébastien Bourdeauducq	8ceaad6e16	nixops: franz no longer need special kernel	2024-04-12 10:41:02 +08:00
Sébastien Bourdeauducq	b1fc3914bf	nixbld: update lp group	2024-04-12 09:40:40 +08:00
Sébastien Bourdeauducq	16fbcef5bf	nixbld: add IP for new qnetp DNS slave	2024-04-12 09:40:17 +08:00
Sébastien Bourdeauducq	0f54fbc893	nixops: add vulcan machine	2024-04-10 12:01:18 +08:00
Sébastien Bourdeauducq	bae360f4ca	nixbld: update users	2024-04-10 11:40:16 +08:00
Sébastien Bourdeauducq	325fe232b3	nixops: add back therobs12	2024-04-10 11:21:58 +08:00
Sébastien Bourdeauducq	5576b82d15	nixbld: disable substituters	2024-04-05 12:51:33 +08:00
Sébastien Bourdeauducq	576087913a	nixbld: remove sb from trusted users	2024-04-05 12:51:20 +08:00
Sebastien Bourdeauducq	38dbad4488	nixops: remove old rpi	2024-02-16 18:10:05 +08:00
Sébastien Bourdeauducq	219268fcfd	desktop: remove fcitx	2024-02-14 16:51:04 +08:00
Sébastien Bourdeauducq	a2b4e61e1d	nixops: add linus to wireshark	2024-02-02 16:32:45 +08:00