it-infra/nixbld-etc-nixos/configuration.nix

{ config, pkgs, ... }:

let
  netifWan = "enp4s0";
  netifWanBackup = "enp11s0";
  netifLan = "enp5s0f1";
  netifWifi = "wlp6s0";
  netifSit = "henet0";
  netifUSA = "trump0";
  netifAlt = "alt0";
  netifAltVlan = "vlan0";
  hydraWwwOutputs = "/var/www/hydra-outputs";
in
{
  imports =
    [
      ./hardware-configuration.nix
      ./backup-module.nix
      ./github-backup-module.nix
      ./afws-module.nix
      ./rt.nix
      (builtins.fetchTarball {
        url = "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/archive/41059fc548088e49e3ddb3a2b4faeb5de018e60f/nixos-mailserver-nixos.tar.gz";
        sha256 = "sha256:0xvch92yi4mc1acj08461wrgrva63770aiis02vpvaa7a1xqaibv";
      })
    ];

  boot.loader.grub.enable = true;
  boot.loader.grub.copyKernels = true;
  boot.loader.grub.device = "nodev";
  boot.loader.grub.efiSupport = true;
  boot.loader.efi.canTouchEfiVariables = true;
  hardware.cpu.amd.updateMicrocode = true;
  boot.supportedFilesystems.zfs = true;
  boot.kernelParams = ["zfs.l2arc_write_max=536870912"];
  boot.binfmt.emulatedSystems = [ "armv7l-linux" "aarch64-linux" ];

  services.zfs.autoScrub.enable = true;
  services.zfs.autoScrub.interval = "monthly";
  services.zfs.autoSnapshot.enable = true;

  systemd.suppressedSystemUnits = [
      "hibernate.target"
      "suspend.target"
      "suspend-then-hibernate.target"
      "sleep.target"
      "hybrid-sleep.target"
      "systemd-hibernate.service"
      "systemd-hybrid-sleep.service"
      "systemd-suspend.service"
      "systemd-suspend-then-hibernate.service"
  ];

  services.fail2ban.enable = true;
  services.fail2ban.ignoreIP = [ "94.190.212.123" "2001:470:18:390::2" ];
  services.fail2ban.maxretry = 9;
  services.fail2ban.bantime-increment.enable = true;
  services.fail2ban.jails.sshd = {
    settings = {
      filter = "sshd";
      action = "iptables-allports";
    };
  };
  services.fail2ban.jails.nginx-botsearch = {
    settings = {
      filter = "nginx-botsearch";
      action = "iptables-allports";
    };
  };
  services.fail2ban.jails.nginx-limit-req = {
    settings = {
      filter = "nginx-limit-req";
      action = "iptables-allports";
    };
  };
  services.fail2ban.jails.postfix = {
    settings = {
      filter = "postfix";
      action = "iptables-allports";
    };
  };
  services.fail2ban.jails.dovecot = {
    settings = {
      filter = "dovecot";
      action = "iptables-allports";
    };
  };

  networking = {
    hostName = "nixbld";
    hostId = "e423f012";
    firewall = {
      allowedTCPPorts = [ 53 80 443 2222 7402 ];
      allowedUDPPorts = [ 53 67 500 4500 ];
      trustedInterfaces = [ netifLan ];
      logRefusedConnections = false;
    };
    useDHCP = false;
    interfaces."${netifWan}".useDHCP = true;  # PCCW - always wants active DHCP lease or cuts you off
    interfaces."${netifWanBackup}" = {        # HKBN - no DHCP with static IP service
      ipv4.addresses = [{
        address = "202.77.7.238";
        prefixLength = 30;
      }];
      ipv4.routes = [
        {
          address = "0.0.0.0";
          prefixLength = 0;
          via = "202.77.7.237";
          options.table = "2";
        }
      ];
    };
    interfaces."${netifLan}" = {
      ipv4.addresses = [{
        address = "192.168.1.1";
        prefixLength = 24;
      }];
      ipv6.addresses = [{
        address = "2001:470:f891:1::";
        prefixLength = 64;
      }];
      # https://unix.stackexchange.com/questions/423502/iproute2-inherit-or-copy-table
      # we just copy what matters here. Ugly but easier.
      ipv4.routes = [
        {
          address = "192.168.1.0";
          prefixLength = 24;
          options.table = "1";
        }
        {
          address = "192.168.1.0";
          prefixLength = 24;
          options.table = "2";
        }
      ];
    };
    interfaces."${netifWifi}" = {
      ipv4.addresses = [{
        address = "192.168.12.1";
        prefixLength = 24;
      }];
      ipv6.addresses = [{
        address = "2001:470:f891:2::";
        prefixLength = 64;
      }];
      # same hack as above
      ipv4.routes = [
        {
          address = "192.168.12.0";
          prefixLength = 24;
          options.table = "1";
        }
        {
          address = "192.168.12.0";
          prefixLength = 24;
          options.table = "2";
        }
      ];
    };
    nat = {
      enable = true;
      externalInterface = netifWan;
      internalInterfaces = [ netifLan netifWifi ];
      forwardPorts = [
        { sourcePort = 2201; destination = "192.168.1.201:22"; proto = "tcp"; }
        { sourcePort = 2202; destination = "192.168.1.202:22"; proto = "tcp"; }
        { sourcePort = 2203; destination = "192.168.1.203:22"; proto = "tcp"; }
        { sourcePort = 2204; destination = "192.168.1.204:22"; proto = "tcp"; }
      ];
      extraCommands = ''
        iptables -w -N block-insecure-devices
        iptables -w -A block-insecure-devices -m mac --mac-source 00:20:0c:6c:ee:ba -j DROP  # keysight SA
        iptables -w -A block-insecure-devices -m mac --mac-source 74:5b:c5:20:c1:5f -j DROP  # siglent scope
        iptables -w -A block-insecure-devices -m mac --mac-source 00:0a:35:00:01:23 -j DROP  # function generator
        iptables -w -A block-insecure-devices -m mac --mac-source 74:5b:c5:21:f1:ee -j DROP  # siglent scope #2
        iptables -w -A block-insecure-devices -m mac --mac-source 00:19:af:5b:dd:58 -j DROP  # power supply
        iptables -w -A block-insecure-devices -m mac --mac-source 28:58:be:dc:66:1f -j DROP  # hikvision low-cost 780nm laser viewer
        iptables -w -A block-insecure-devices -m mac --mac-source bc:99:11:a4:d2:ac -j DROP  # zyxel cloud switch
        iptables -w -A block-insecure-devices -m mac --mac-source d8:9c:67:ab:83:e7 -j DROP  # HP printer, wifi
        iptables -w -A block-insecure-devices -m mac --mac-source f4:39:09:f7:3c:d7 -j DROP  # HP printer, ethernet
        iptables -w -A FORWARD -j block-insecure-devices

        iptables -w -N pccw-sucks
        iptables -A pccw-sucks -o ${netifSit} -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1440
        iptables -A pccw-sucks -o ${netifAlt} -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --set-mss 1380
        iptables -w -A FORWARD -j pccw-sucks
      '';
      extraStopCommands = ''
        iptables -w -D FORWARD -j block-insecure-devices 2>/dev/null|| true
        iptables -w -F block-insecure-devices 2>/dev/null|| true
        iptables -w -X block-insecure-devices 2>/dev/null|| true

        iptables -w -D FORWARD -j pccw-sucks 2>/dev/null|| true
        iptables -w -F pccw-sucks 2>/dev/null|| true
        iptables -w -X pccw-sucks 2>/dev/null|| true
      '';
    };
    sits."${netifSit}" = {
      dev = netifWan;
      remote = "216.218.221.6";
      local = "94.190.212.123";
      ttl = 255;
    };
    interfaces."${netifSit}".ipv6 = {
      addresses = [{ address = "2001:470:18:390::2"; prefixLength = 64; }];
      routes = [{ address = "::"; prefixLength = 0; }];
    };
    greTunnels."${netifUSA}" = {
      dev = netifWan;
      remote = "5.78.86.156";
      local = "94.190.212.123";
      ttl = 255;
      type = "tun";
    };
    greTunnels."${netifAlt}" = {
      dev = netifWan;
      remote = "103.206.98.1";
      local = "94.190.212.123";
      ttl = 255;
      type = "tun";
    };
    interfaces."${netifUSA}" = {
      ipv4.addresses = [
        {
          address = "10.47.3.1";
          prefixLength = 31;
        }
      ];
      ipv4.routes = [
        {
          address = "0.0.0.0";
          prefixLength = 0;
          via = "10.47.3.0";
          options.table = "3";
        }
      ];
    };
    interfaces."${netifAlt}" = {
      ipv4.addresses = [
        {
          address = "103.206.98.227";
          prefixLength = 31;
        }
      ];
      ipv4.routes = [
        {
          address = "0.0.0.0";
          prefixLength = 0;
          via = "103.206.98.226";
          options.table = "1";
        }
      ];
    };
    greTunnels.intl0 = {
      dev = netifWan;
      remote = "5.78.16.0";
      local = "94.190.212.123";
      ttl = 255;
      type = "tun";
    };
    interfaces.intl0 = {
      ipv4.addresses = [
        {
          address = "10.42.0.2";
          prefixLength = 30;
        }
      ];
      ipv4.routes = [
        {
          address = "0.0.0.0";
          prefixLength = 0;
          via = "10.42.0.1";
          options.table = "3";
        }
      ];
    };
    vlans = {
      "${netifAltVlan}" = {
        id = 2;
        interface = netifLan;
      };
    };
    interfaces."${netifAltVlan}" = {
      ipv4.addresses = [{
        address = "103.206.98.200";
        prefixLength = 29;
      }];
      ipv4.routes = [
        {
          address = "103.206.98.200";
          prefixLength = 29;
          options.table = "1";
        }
      ];
    };
  };
  boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = "1";
  boot.kernel.sysctl."net.ipv6.conf.default.forwarding" = "1";
  boot.kernel.sysctl."net.ipv6.conf.${netifLan}.accept_dad" = "0";
  boot.kernel.sysctl."net.ipv6.conf.${netifWifi}.accept_dad" = "0";

  services.strongswan-swanctl.enable = true;
  services.strongswan-swanctl.swanctl.connections.altnet = {
    local_addrs = [ "94.190.212.123" ];
    remote_addrs = [ "103.206.98.1" ];
    local.main = {
      auth = "pubkey";
      id = "fqdn:m-labs.hk";
      pubkeys = [ "/etc/swanctl/pubkey/m-labs.hk" ];
    };
    remote.main = {
      auth = "pubkey";
      id = "fqdn:igw0.hkg.as150788.net";
      pubkeys = [ "/etc/swanctl/pubkey/igw0.hkg.as150788.net" ];
    };
    children."${netifAlt}" = {
      mode = "transport";
      ah_proposals = [ "sha256-curve25519" ];
      remote_ts = [ "103.206.98.1[gre]" ];
      local_ts = [ "94.190.212.123[gre]" ];
      start_action = "start";
    };
  };
  services.strongswan-swanctl.swanctl.connections.usa = {
    local_addrs = [ "94.190.212.123" ];
    remote_addrs = [ "5.78.86.156" ];
    local.main = {
      auth = "pubkey";
      id = "fqdn:m-labs.hk";
      pubkeys = [ "/etc/swanctl/pubkey/m-labs.hk" ];
    };
    remote.main = {
      auth = "pubkey";
      id = "fqdn:m-labs-intl.com";
      pubkeys = [ "/etc/swanctl/pubkey/m-labs-intl.com" ];
    };
    children."${netifUSA}" = {
      mode = "transport";
      ah_proposals = [ "sha256-curve25519" ];
      remote_ts = [ "5.78.86.156[gre]" ];
      local_ts = [ "94.190.212.123[gre]" ];
      start_action = "start";
    };
  };
  services.strongswan-swanctl.swanctl.connections.intl = {
    local_addrs = [ "94.190.212.123" ];
    remote_addrs = [ "5.78.16.0" ];
    local.main = {
      auth = "pubkey";
      id = "fqdn:m-labs.hk";
      pubkeys = [ "/etc/swanctl/pubkey/m-labs.hk" ];
    };
    remote.main = {
      auth = "pubkey";
      id = "fqdn:m-labs-intl.com";
      pubkeys = [ "/etc/swanctl/pubkey/m-labs-intl.com" ];
    };
    children.intl0 = {
      mode = "transport";
      ah_proposals = [ "sha256-curve25519" ];
      remote_ts = [ "5.78.16.0[gre]" ];
      local_ts = [ "94.190.212.123[gre]" ];
      start_action = "start";
    };
  };

  systemd.services.network-custom-route-backup = {
    wantedBy = [ "network.target" ];
    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
      ExecStart = "${pkgs.iproute2}/bin/ip rule add from 202.77.7.238/30 table 2";
      ExecStop = "${pkgs.iproute2}/bin/ip rule del table 2";
    };
  };
  systemd.services.network-custom-route-usa = {
    wantedBy = [ "network.target" ];
    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
      ExecStart = "${pkgs.iproute2}/bin/ip rule add from 10.47.3.0/31 table 3";
      ExecStop = "${pkgs.iproute2}/bin/ip rule del table 3";
    };
  };
  systemd.services.network-custom-route-alt = {
    wantedBy = [ "network.target" ];
    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
      ExecStart = "${pkgs.iproute2}/bin/ip rule add from 103.206.98.200/29 table 1";
      ExecStop = "${pkgs.iproute2}/bin/ip rule del table 1";
    };
  };
  systemd.services.network-custom-route-intl = {
    wantedBy = [ "network.target" ];
    serviceConfig = {
      Type = "oneshot";
      RemainAfterExit = true;
      ExecStart = "${pkgs.iproute2}/bin/ip rule add from 5.78.16.0/32 table 3";
      ExecStop = "${pkgs.iproute2}/bin/ip rule del table 3";
    };
  };

  # https://kb.isc.org/docs/dnssec-key-and-signing-policy
  # chown named.named /etc/nixos/named
  services.bind = {
    enable = true;
    listenOn = [ "94.190.212.123" ];
    listenOnIpv6 = [ "2001:470:18:390::2" ];
    forwarders = [];
    extraOptions = "listen-on-v6 port 5354 { ::1; };";
    cacheNetworks = [ "::1/128" ];
    directory = "/etc/nixos/named";
    zones = {
      "m-labs.hk" = {
        name = "m-labs.hk";
        master = true;
        file = "/etc/nixos/named/m-labs.hk";
        extraConfig =
           ''
           dnssec-policy "default";
           inline-signing yes;
           notify explicit;
           also-notify {
             213.239.220.50;  # ns1.qnetp.net
             216.218.130.2;   # ns1.he.net
             88.198.32.245;   # new qnetp
           };
           '';
        slaves = [
          "213.239.220.50" "2a01:4f8:a0:7041::1"  # ns1.qnetp.net
          "216.218.133.2" "2001:470:600::2"       # slave.dns.he.net
          "88.198.32.245"                         # new qnetp
        ];
      };
      "m-labs.ph" = {
        name = "m-labs.ph";
        master = true;
        file = "/etc/nixos/named/m-labs.ph";
        extraConfig =
           ''
           notify explicit;
           also-notify {
             216.218.130.2;   # ns1.he.net
           };
           '';
        slaves = [
          "216.218.133.2" "2001:470:600::2"       # slave.dns.he.net
        ];
      };
      "193thz.com" = {
        name = "193thz.com";
        master = true;
        file = "/etc/nixos/named/193thz.com";
        extraConfig =
           ''
           dnssec-policy "default";
           inline-signing yes;
           notify explicit;
           also-notify {
             216.218.130.2;   # ns1.he.net
           };
           '';
        slaves = [
          "216.218.133.2" "2001:470:600::2"       # slave.dns.he.net
        ];
      };
      "malloctech.fr" = {
        name = "malloctech.fr";
        master = true;
        file = "/etc/nixos/named/malloctech.fr";
        extraConfig =
           ''
           dnssec-policy "default";
           inline-signing yes;
           notify explicit;
           also-notify {
             216.218.130.2;   # ns1.he.net
           };
           '';
        slaves = [
          "216.218.133.2" "2001:470:600::2"       # slave.dns.he.net
        ];
      };
      "m-labs-intl.com" = {
        name = "m-labs-intl.com";
        master = true;
        file = "/etc/nixos/named/m-labs-intl.com";
        extraConfig =
           ''
           dnssec-policy "default";
           inline-signing yes;
           notify explicit;
           also-notify {
             216.218.130.2;   # ns1.he.net
             213.239.220.50;  # ns1.qnetp.net
             88.198.32.245;   # new qnetp
           };
           '';
        slaves = [
          "216.218.133.2" "2001:470:600::2"       # slave.dns.he.net
          "213.239.220.50" "2a01:4f8:a0:7041::1"  # ns1.qnetp.net
          "88.198.32.245"                         # new qnetp
        ];
      };
      "200-29.98.206.103.in-addr.arpa" = {
        name = "200-29.98.206.103.in-addr.arpa";
        master = true;
        file = "/etc/nixos/named/200-29.98.206.103.in-addr.arpa";
        extraConfig =
           ''
           dnssec-policy "default";
           inline-signing yes;
           notify explicit;
           also-notify {
             216.218.130.2;   # ns1.he.net
           };
           '';
        slaves = [
          "216.218.133.2" "2001:470:600::2"       # slave.dns.he.net
        ];
      };
    };
    extraConfig = ''
      zone "mil." IN {
        type forward;
        forward only;
        forwarders { 74.82.42.42; };
      };
    '';
  };

  services.hostapd = {
    enable = true;
    radios.${netifWifi} = {
      band = "2g";
      countryCode = "HK";
      networks.${netifWifi} = {
        ssid = "M-Labs";
        authentication.saePasswords = [
          { password = (import /etc/nixos/secret/wifi_password.nix); }
        ];
      };
    };
  };
  services.dnsmasq = {
    enable = true;
    settings = {
      server = ["::1#5354"];
      interface = [ netifLan netifWifi ];
      bind-interfaces = true;
      dhcp-range = [
        "interface:${netifLan},192.168.1.81,192.168.1.254,24h"
        "interface:${netifWifi},192.168.12.10,192.168.12.254,24h"
        "interface:${netifLan},::,constructor:${netifLan},ra-names"
        "interface:${netifWifi},::,constructor:${netifWifi},ra-only"
      ];
      enable-ra = true;

      no-resolv = true;

      dhcp-host = [
        # Static IPv4s to make Red Pitayas with factory firmware less annoying
        "rp-f05cc9,192.168.1.190"
        "rp-f0612e,192.168.1.191"
        # Static IPv4s to make port redirections work
        "rpi-1,192.168.1.201"
        "rpi-2,192.168.1.202"
        "rpi-3,192.168.1.203"
        "rpi-4,192.168.1.204"
      ];

      address = [
        # Static IP addresses for non-DHCP boards
        "/thermostat/192.168.1.26"
        "/powercycler/192.168.1.31"
        "/kc705/192.168.1.50"
        "/zynq-experiments/192.168.1.51"
        "/zc706/192.168.1.52"
        "/zc706-2/192.168.1.53"
        "/cora-z7/192.168.1.54"
        "/rust-pitaya/192.168.1.55"
        "/kasli/192.168.1.70"
        "/kasli-customer/192.168.1.75"
        "/stabilizer-customer/192.168.1.76"

        # Google can't do DNS geolocation correctly and slows down websites of everyone using
        # their shitty font cloud hosting. In HK, you sometimes get IPs behind the GFW that you
        # cannot reach.
        "/fonts.googleapis.com/142.250.207.74"
      ];

      dhcp-match = "set:ipxe,175"; # https://forum.ipxe.org/showthread.php?tid=6077
      dhcp-boot = [
        "tag:!ipxe,ipxe.efi"
        "tag:ipxe,http://m-labs.hk/nuc-netboot/netboot.ipxe"
      ];
      enable-tftp = netifLan;
      tftp-root = "${pkgs.ipxe}";
    };
  };

  # Select internationalisation properties.
  i18n.defaultLocale = "en_US.UTF-8";
  console = {
    font = "Lat2-Terminus16";
    keyMap = "de";
  };

  # Set your time zone.
  time.timeZone = "Asia/Hong_Kong";

  # List packages installed in system profile. To search, run:
  # $ nix search wget
  environment.systemPackages = with pkgs; [
    lm_sensors
    acpi
    usbutils
    pciutils
    iw
    nvme-cli
    smartmontools
    psmisc

    wget
    vim
    git
    file
    imagemagick
    jq

    nixops_unstable_minimal
    borgbackup
    bind
    waypipe
    (callPackage ./afws { inherit pkgs; })
    (callPackage ./labelprinter { inherit pkgs; })
  ];

  # Some programs need SUID wrappers, can be configured further or are
  # started in user sessions.
  # programs.mtr.enable = true;
  # programs.gnupg.agent = { enable = true; enableSSHSupport = true; };

  # List services that you want to enable:

  services.apcupsd.enable = true;
  services.apcupsd.configText = ''
    UPSTYPE usb
    NISIP 127.0.0.1
    BATTERYLEVEL 10
    MINUTES 5
  '';

  # Enable the OpenSSH daemon.
  services.openssh.enable = true;
  services.openssh.settings.PasswordAuthentication = false;
  services.openssh.settings.GatewayPorts = "clientspecified";
  services.openssh.settings.X11Forwarding = true;
  services.openssh.authorizedKeysInHomedir = false;
  programs.mosh.enable = true;
  programs.tmux.enable = true;

  programs.fish.enable = true;
  programs.zsh.enable = true;

  # Enable CUPS to print documents.
  services.avahi.enable = true;
  services.avahi.allowInterfaces = [ netifLan ];
  services.avahi.publish.enable = true;
  services.avahi.publish.userServices = true;
  nixpkgs.config.allowUnfree = true;
  services.printing.enable = true;
  services.printing.drivers = [ pkgs.hplipWithPlugin ];
  services.printing.browsing = true;
  services.printing.listenAddresses = [ "*:631" ];
  services.printing.defaultShared = true;
  hardware.sane.enable = true;
  hardware.sane.extraBackends = [ pkgs.hplipWithPlugin ];
  services.saned.enable = true;
  services.saned.extraConfig = "192.168.1.0/24";

  services.udev.extraRules =
    ''
    # label printer
    SUBSYSTEM=="usb", ATTRS{idVendor}=="07cf", ATTRS{idProduct}=="4204", MODE="0660", GROUP="lp"
    '';

  sound.enable = true;
  services.mpd.enable = true;
  services.mpd.musicDirectory = "/tank/sb-public/FLAC";
  services.mpd.network.listenAddress = "192.168.1.1";
  services.mpd.extraConfig =
    ''
    audio_output_format "192000:24:2"
    audio_output {
      type "alsa"
      name "alsa"
      device "hw:1,1"
    }
    '';

  users.extraUsers.root = {
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBNdIiLvP2hmDUFyyE0oLOIXrjrMdWWpBV9/gPR5m4AiARx4JkufIDZzmptdYQ5FhJORJ4lluPqp7dAmahoSwg4lv9Di0iNQpHMJvNGZLHYKM1H1FWCCFIEDJ8bD4SVfrDg=="
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCMALVC8RDTHec+PC8y1s3tcpUAODgq6DEzQdHDf/cyvDMfmCaPiMxfIdmkns5lMa03hymIfSmLUF0jFFDc7biRp7uf9AAXNsrTmplHii0l0McuOOZGlSdZM4eL817P7UwJqFMxJyFXDjkubhQiX6kp25Kfuj/zLnupRCaiDvE7ho/xay6Jrv0XLz935TPDwkc7W1asLIvsZLheB+sRz9SMOb9gtrvk5WXZl5JTOFOLu+JaRwQLHL/xdcHJTOod7tqHYfpoC5JHrEwKzbhTOwxZBQBfTQjQktKENQtBxXHTe71rUEWfEZQGg60/BC4BrRmh4qJjlJu3v4VIhC7SSHn1"
    ];
    shell = pkgs.fish;
  };
  # https://github.com/NixOS/nixpkgs/issues/155357
  security.sudo.enable = true;

  # M-Labs HK
  users.extraUsers.sb = {
    isNormalUser = true;
    extraGroups = ["lp" "scanner" "afws" "audio"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBF/YybP+fQ0J+bNqM5Vgx5vDmVqVWsgUdF1moUxghv7d73GZAFaM6IFBdrXTAa33AwnWwDPMrTgP1V6SXBkb3ciJo/lD1urJGbydbSI5Ksq9d59wvOeANvyWYrQw6+eqTQ=="
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCMALVC8RDTHec+PC8y1s3tcpUAODgq6DEzQdHDf/cyvDMfmCaPiMxfIdmkns5lMa03hymIfSmLUF0jFFDc7biRp7uf9AAXNsrTmplHii0l0McuOOZGlSdZM4eL817P7UwJqFMxJyFXDjkubhQiX6kp25Kfuj/zLnupRCaiDvE7ho/xay6Jrv0XLz935TPDwkc7W1asLIvsZLheB+sRz9SMOb9gtrvk5WXZl5JTOFOLu+JaRwQLHL/xdcHJTOod7tqHYfpoC5JHrEwKzbhTOwxZBQBfTQjQktKENQtBxXHTe71rUEWfEZQGg60/BC4BrRmh4qJjlJu3v4VIhC7SSHn1"
    ];
    shell = pkgs.fish;
  };
  users.extraUsers.spaqin = {
    isNormalUser = true;
    extraGroups = ["lp" "afws"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBOtmlQmIK/cEUkcwA/y9jC4AohjoEmikerpxzPhZZtOcENidN/vFum58jIcSxBvjHnILOzhfCTeLvbvGbQOFE53a7FOyEHmIzXRKS86Mg5bPHUBJxRSq9MjulGZXES3HOQ=="
    ];
    shell = pkgs.zsh;
  };
  users.extraUsers.therobs12 = {
    isNormalUser = true;
    extraGroups = ["lp" "afws"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBK1tUg7TtceARRnGI80Ai5kNFolFfZ++LH9v1UoRCiJdxeQWPdNYO0Gj7+ejJvgZXwvN4yHGgcZHraEml4Mj/dKrEMFygfuYLDRmXtPFwX6TNMrWlxMhPzuNY+yCaxlqYg=="
    ];
  };
  users.extraUsers.morgan = {
    isNormalUser = true;
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBDXMbJEPn0mM2Bgt6eMAd+c0J5oPSvquZG+BxKdUf0qbeQldRaoB26NHMZnLte/fS00U/cqStLWDiwtEvH5WlbbawsMBymm65zbWMByebXhBDjdr6a1kkOFcKJvAL9qVBQ=="
    ];
  };

  # M-Labs PH
  users.extraUsers.flo = {
    isNormalUser = true;
    extraGroups = ["afws"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBF4ZYNBYqJPQCKBYjMatFj5eGMyzh/X2TSraJEG6XBdg3jnJ3WcsOd7sm+vx+o9Y1EJ2kvwW/Vy9c3OYVU2U45njox//sKtt8Eyzszws3EYJqHQ6KAwXtW9ao4aamRtK3Q=="
    ];
  };

  # QUARTIQ
  users.extraUsers.rj = {
    isNormalUser = true;
    extraGroups = ["afws"];
    openssh.authorizedKeys.keys = [
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC27krR8G8Pb59YuYm7+X2mmNnVdk/t9myYgO8LH0zfb2MeeXX5+90nW9kMjKflJss/oLl8dkD85jbJ0fRbRkfJd20pGCqCUuYAbYKkowigFVEkbrbWSLkmf+clRjzJOuBuUA0uq0XKS17uMC3qhu+dDdBOAIKb3L83NfVE8p8Pjb4BPktQrdxefM43/x4jTMuc7tgxVmTOEge3+rmVPK2GnLkUBgBn8b6S+9ElPd63HXI5J5f61v21l5N9V0mhTu1pv6PiDRdFIlFDK9dLVZcZ2qlzpKmCnFrOoreBEgre44SpfFe5/MMItxvWiVsj/rij/rHZZiol1k7JiQCnEHeCCbjjvcBBka5HxZgcb3vBZVceTOawrmjbdbA2dq35sUptz/bEgdZ1UVCmVpWsdROAlEDBmSSbcVwxzcvhoKnkpbuP4Q0V3tVKSLW053ADFNB4frtwY5nAZfsVErFLLphjwb8nlyJoDRNapQrn5syEiW0ligX2AAskZTYIl2A5AYyWPrmX6HJOPqZGatMU3qQiRMxs+hFqhyyCmBgl0kcsgW09MBKtJWk1Fbii98MHqgRUN9R7AUiYy5p78Pnv9DC8DT8Ubl9zoP0g5d40P9NGK2LAhMxLXvtckJ4ERqbSEcNZJw+q4jBrOHnMTz+NLdAUiEtru+6T2OdhaHv+eiNlFQ== robert-jordens-rsa4096"
     "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCUdbne3NtIG+iy/jer76/OY+IksuS3BDLSXPnWrGejWnig9h+L6sUV0lEVI6dqp+W/b8jWqPB8nh5S0NZsCd3Ta3Go82k/SPPkh9lB2PpfquhCjLnmC/RNc3TgC4FuiS+NZHqXaTggYHubNwEK+8gynMqkMQXjOGU02U0CtUfsYdAm75AW60DySZCRNwOcU0Ndpn1UCpha7fL1k179Dd/OtArkYsIL24ohlfxFeOB3jGYQK6ATmzbvCRjwIKXcyECuajWwfnDg9FtDWrqHNzu5dJlvmxoWm8zCDgMj53uiA7TjujQN81MYrIJNeEwSr5jXQMqzA3mzlk4k3Z0qs3TP robert-jordens-64FEFBAF-4D0749B2-rsa2048"
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMUaB2G1jexxfkdlly3fdWslH54/s/bOuvk9AxqpjtAY robert-jordens-ed25519"
    ];
  };
  users.extraUsers.eduardotenholder = {
    isNormalUser = true;
    extraGroups = ["afws"];
    openssh.authorizedKeys.keys = [
      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIu6yhjCoZ62eamYrAXtFefDhplTRUIdD4tncwlkyAEH"
    ];
  };

  # HKUST
  users.extraUsers.derppening = {
    isNormalUser = true;
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBOKwN4ui94QfouYYlkI1lc3WgtjURVYLTdAizJIBnY3dNRNblAiuvTD4pQ+LEI+eOTg4SnQz1NeqH4YOQhbT5+/nZojvGTb3UVN13ZYND+Gci3DdqB2mwIYop7kMXwHgLQ=="
    ];
  };

  users.extraUsers.nix = {
    isNormalUser = true;
  };
  boot.kernel.sysctl."kernel.dmesg_restrict" = true;
  services.udev.packages = [ pkgs.sane-backends ];

  nix.settings.max-jobs = 10;
  nix.nrBuildUsers = 64;
  services.hydra = {
    enable = true;
    useSubstitutes = true;
    hydraURL = "https://nixbld.m-labs.hk";
    notificationSender = "hydra@m-labs.hk";
    minimumDiskFree = 15;  # in GB
    minimumDiskFreeEvaluator = 1;
    extraConfig =
      ''
      binary_cache_secret_key_file = /etc/nixos/secret/nixbld.m-labs.hk-1
      max_output_size = 10000000000

      <runcommand>
        job = web:web:web
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/web
      </runcommand>
      <runcommand>
        job = web:web:web-intl
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ${pkgs.rsync}/bin/rsync -r -c $(jq -r '.outputs[0].path' < $HYDRA_JSON)/ zolaupd@5.78.86.156:/var/www/m-labs-intl.com/html/
      </runcommand>
      <runcommand>
        job = web:web:nmigen-docs
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/nmigen-docs
      </runcommand>

      <runcommand>
        job = artiq:sipyco:sipyco-manual-html
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/sipyco-manual-html
      </runcommand>
      <runcommand>
        job = artiq:sipyco:sipyco-manual-pdf
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/sipyco-manual-pdf
      </runcommand>

      <runcommand>
        job = artiq:main-beta:artiq-manual-html
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-html-beta
      </runcommand>
      <runcommand>
        job = artiq:main-beta:artiq-manual-pdf
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-pdf-beta
      </runcommand>
      <runcommand>
        job = artiq:extra-beta:conda-channel
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel-beta
      </runcommand>
      <runcommand>
        job = artiq:extra-beta:msys2-repos
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-msys2-repos-beta
      </runcommand>

      <runcommand>
        job = artiq:main:artiq-manual-html
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-html
      </runcommand>
      <runcommand>
        job = artiq:main:artiq-manual-pdf
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-pdf
      </runcommand>
      <runcommand>
        job = artiq:extra:conda-channel
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel
      </runcommand>
      <runcommand>
        job = artiq:extra:msys2-repos
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-msys2-repos
      </runcommand>

      <runcommand>
        job = artiq:main-legacy:artiq-manual-html
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-html-legacy
      </runcommand>
      <runcommand>
        job = artiq:main-legacy:artiq-manual-pdf
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-pdf-legacy
      </runcommand>
      <runcommand>
        job = artiq:extra-legacy:conda-channel
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel-legacy
      </runcommand>

      <runcommand>
        job = artiq:*:conda-channel
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel-archives/$(jq -r '.build' < $HYDRA_JSON)
      </runcommand>

      <runcommand>
        job = artiq:main-nac3:msys2-repos
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-msys2-repos-nac3
      </runcommand>


      <githubstatus>
        jobs = artiq:fast.*:extended-tests
        inputs = artiqSrc
        useShortContext = 1
        authorization = token ${(import /etc/nixos/secret/github_tokens.nix).artiq}
      </githubstatus>
      <gitea_authorization>
        sb=${(import /etc/nixos/secret/gitea_tokens.nix).artiq-zynq}
      </gitea_authorization>
      '';
  };
  systemd.services.hydra-www-outputs-init = {
    description = "Set up a hydra-owned directory for build outputs";
    wantedBy = [ "multi-user.target" ];
    requiredBy = [ "hydra-queue-runner.service" ];
    before = [ "hydra-queue-runner.service" ];
    serviceConfig = {
      Type = "oneshot";
      ExecStart = [
        "${pkgs.coreutils}/bin/mkdir -p ${hydraWwwOutputs} ${hydraWwwOutputs}/artiq-conda-channel-archives"
        "${pkgs.coreutils}/bin/chown hydra-queue-runner:hydra ${hydraWwwOutputs} ${hydraWwwOutputs}/artiq-conda-channel-archives"
      ];
    };
  };
  services.afws.enable = true;

  nix.extraOptions = ''
    secret-key-files = /etc/nixos/secret/nixbld.m-labs.hk-1
    experimental-features = nix-command flakes
  '';
  nix.settings.allowed-uris = "github: gitlab: git+https://";  # https://github.com/NixOS/nix/issues/5039
  nix.settings.extra-sandbox-paths = ["/opt"];

  services.mlabs-backup.enable = true;
  services.ghbackup.enable = true;

  services.gitea = {
    enable = true;
    appName = "M-Labs Git";
    database = {
      type = "postgres";
      socket = "/run/postgresql";
    };
    mailerPasswordFile = "/etc/nixos/secret/mailerpassword";
    settings = {
      server = {
        ROOT_URL = "https://git.m-labs.hk/";
        HTTP_PORT = 3001;
        DISABLE_SSH = false;
        SSH_CREATE_AUTHORIZED_KEYS_FILE = false;
        START_SSH_SERVER = true;
        SSH_PORT = 2222;
      };

      indexer = {
        REPO_INDEXER_ENABLED = true;
      };

      mailer = {
        ENABLED = true;
        SMTP_ADDR = "mail.m-labs.hk";
        SMTP_PORT = "587";
        FROM = "sysop@m-labs.hk";
        USER = "sysop@m-labs.hk";
      };

      service = {
        ENABLE_NOTIFY_MAIL = true;
        DISABLE_REGISTRATION = true;
      };

      attachment = {
        ALLOWED_TYPES = "*/*";
      };

      log.LEVEL = "Warn";

      session.COOKIE_SECURE = true;
    };
  };
  systemd.tmpfiles.rules = [
    "L+ '${config.services.gitea.stateDir}/custom/templates/home.tmpl' - - - - ${./gitea-home.tmpl}"
    "L+ '${config.services.gitea.stateDir}/custom/templates/user/auth/signin.tmpl' - - - - ${./gitea-signin.tmpl}"
  ];

  services.mattermost = {
    enable = true;
    siteUrl = "https://chat.m-labs.hk/";
    mutableConfig = true;
  };
  services.matterbridge = {
    enable = true;
    configPath = "/etc/nixos/secret/matterbridge.toml";
  };

  services.postgresql = {
    package = pkgs.postgresql_15;
    settings.listen_addresses = pkgs.lib.mkForce "";
    identMap =
      ''
      rt rt rt_user
      '';
  };

  nixpkgs.config.packageOverrides = super: let self = super.pkgs; in {
    nix = super.nix.overrideAttrs(oa: {
      patches = oa.patches or [] ++ [ ./nix-networked-derivations.patch ];
    });
    hydra_unstable = super.hydra_unstable.overrideAttrs(oa: {
      patches = oa.patches or [] ++ [
        ./hydra-conda.patch
        ./hydra-msys2.patch
        ./hydra-restrictdist.patch
      ];
      hydraPath = oa.hydraPath + ":" + super.lib.makeBinPath [ super.jq ];
      doCheck = false;  # FIXME: ldap tests fail on hydra rebuild, seems unrelated to patches above.
    });
    matterbridge = super.matterbridge.overrideAttrs(oa: {
      patches = oa.patches or [] ++ [ ./matterbridge-disable-github.patch ];
    });
  };

  security.acme.acceptTerms = true;
  security.acme.defaults.email = "sb" + "@m-labs.hk";

  # https://github.com/NixOS/nixpkgs/issues/106862
  systemd.services."acme-fixperms".wants = [ "bind.service" "dnsmasq.service" ];
  systemd.services."acme-fixperms".after = [ "bind.service" "dnsmasq.service" ];
  services.nginx = {
    enable = true;
    recommendedProxySettings = true;
    recommendedGzipSettings = true;
    recommendedTlsSettings = true;
    virtualHosts = let
      mainWebsite = {
        addSSL = true;
        enableACME = true;
        root = "${hydraWwwOutputs}/web";
        extraConfig = ''
          error_page 404 /404.html;
        '';
        locations."^~ /fonts/".extraConfig = ''
          expires 60d;
        '';
        locations."^~ /js/".extraConfig = ''
          expires 60d;
        '';
        locations."/MathJax/" = {
          alias = "/var/www/MathJax/";
          extraConfig = ''
            expires 60d;
          '';
        };
        locations."/nuc-netboot/".alias = "${import ./defenestrate}/";

        # legacy URLs, redirect to avoid breaking people's bookmarks
        locations."/gateware.html".extraConfig = ''
          return 301 /gateware/migen/;
        '';
        locations."/migen".extraConfig = ''
          return 301 /gateware/migen/;
        '';
        locations."/artiq".extraConfig = ''
          return 301 /experiment-control/artiq/;
        '';
        locations."/artiq/resources.html".extraConfig = ''
          return 301 /experiment-control/resources/;
        '';

        # autogenerated manuals
        locations."/artiq/sipyco-manual/" = {
          alias = "${hydraWwwOutputs}/sipyco-manual-html/";
        };
        locations."=/artiq/sipyco-manual.pdf" = {
          alias = "${hydraWwwOutputs}/sipyco-manual-pdf/SiPyCo.pdf";
        };
        locations."/artiq/manual-beta/" = {
          alias = "${hydraWwwOutputs}/artiq-manual-html-beta/";
        };
        locations."=/artiq/manual-beta.pdf" = {
          alias = "${hydraWwwOutputs}/artiq-manual-pdf-beta/ARTIQ.pdf";
        };
        locations."/artiq/manual/" = {
          alias = "${hydraWwwOutputs}/artiq-manual-html/";
        };
        locations."=/artiq/manual.pdf" = {
          alias = "${hydraWwwOutputs}/artiq-manual-pdf/ARTIQ.pdf";
        };
        locations."/artiq/manual-legacy/" = {
          alias = "${hydraWwwOutputs}/artiq-manual-html-legacy/";
        };
        locations."=/artiq/manual-legacy.pdf" = {
          alias = "${hydraWwwOutputs}/artiq-manual-pdf-legacy/ARTIQ.pdf";
        };

        # legacy content
        locations."/migen/manual/" = {
          alias = "/var/www/m-labs.hk.old/migen/manual/";
        };
        locations."/artiq/manual-release-4/" = {
          alias = "/var/www/m-labs.hk.old/artiq/manual-release-4/";
        };
        locations."/artiq/manual-release-3/" = {
          alias = "/var/www/m-labs.hk.old/artiq/manual-release-3/";
        };
        locations."/artiq/manual-release-2/" = {
          alias = "/var/www/m-labs.hk.old/artiq/manual-release-2/";
        };
      };
    in {
      "m-labs.hk" = mainWebsite;
      "www.m-labs.hk" = mainWebsite;
      "m-labs.ph" = mainWebsite;
      "www.m-labs.ph" = mainWebsite;
      "nixbld.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/".proxyPass = "http://127.0.0.1:3000";
      };
      "msys2.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/artiq-beta/" = {
          alias = "${hydraWwwOutputs}/artiq-msys2-repos-beta/";
          extraConfig = ''
            autoindex on;
          '';
        };
        locations."/artiq/" = {
          alias = "${hydraWwwOutputs}/artiq-msys2-repos/";
          extraConfig = ''
            autoindex on;
          '';
        };
        locations."/artiq-nac3/" = {
          alias = "${hydraWwwOutputs}/artiq-msys2-repos-nac3/";
          extraConfig = ''
            autoindex on;
          '';
        };
      };
      "conda.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/artiq-beta/" = {
          alias = "${hydraWwwOutputs}/artiq-conda-channel-beta/";
          extraConfig = ''
            autoindex on;
            index bogus_index_file;
          '';
        };
        locations."/artiq/" = {
          alias = "${hydraWwwOutputs}/artiq-conda-channel/";
          extraConfig = ''
            autoindex on;
            index bogus_index_file;
          '';
        };
        locations."/artiq-legacy/" = {
          alias = "${hydraWwwOutputs}/artiq-conda-channel-legacy/";
          extraConfig = ''
            autoindex on;
            index bogus_index_file;
          '';
        };
        locations."/artiq-archives/" = {
          alias = "${hydraWwwOutputs}/artiq-conda-channel-archives/";
          extraConfig = ''
            autoindex on;
            index bogus_index_file;
          '';
        };
      };
      "afws.m-labs.hk" = {
        enableACME = true;
        locations."/" = {
          proxyPass = "http://localhost:3771";
          proxyWebsockets = true;
          extraConfig =
            ''
            proxy_read_timeout 2h;
            '';
        };
      };
      "git.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/".proxyPass = "http://127.0.0.1:3001";
        extraConfig = ''
          client_max_body_size 300M;
        '';
      };
      "chat.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/".proxyPass = "http://127.0.0.1:8065";
        locations."~ /api/v[0-9]+/(users/)?websocket$".proxyPass = "http://127.0.0.1:8065";
        locations."~ /api/v[0-9]+/(users/)?websocket$".proxyWebsockets = true;
      };
      "hooks.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/mattermost-github".extraConfig = ''
          include ${pkgs.nginx}/conf/uwsgi_params;
          uwsgi_pass unix:${config.services.uwsgi.runDir}/uwsgi-mgi.sock;
        '';
        locations."/rfq".extraConfig = ''
          include ${pkgs.nginx}/conf/uwsgi_params;
          uwsgi_pass unix:${config.services.uwsgi.runDir}/uwsgi-rfq.sock;
        '';
      };
      "forum.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
      };
      "perso.m-labs.hk" = {
        addSSL = true;
        enableACME = true;
        root = "/var/www/perso";
        extraConfig = ''
          autoindex on;
        '';
      };
      "rt.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/" = {
          proxyPass = "http://127.0.0.1:4201";
          extraConfig = ''
            client_max_body_size 100M;
          '';
        };
        locations."/REST/1.0/NoAuth" = {
          proxyPass = "http://127.0.0.1:4201";
          extraConfig = ''
            client_max_body_size 100M;
            allow 127.0.0.1;
            deny all;
          '';
        };
      };
      "files.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
      };
      "docs.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/".proxyPass = "http://localhost:9825";
        locations."/socket.io/".proxyPass = "http://localhost:9825";
        locations."/socket.io/".proxyWebsockets = true;
      };
      "193thz.com" = {
        addSSL = true;
        enableACME = true;
        root = "/var/www/193thz";
      };
      "www.193thz.com" = {
        addSSL = true;
        enableACME = true;
        root = "/var/www/193thz";
      };
      "nmigen.net" = {
        addSSL = true;
        enableACME = true;
        root = "${hydraWwwOutputs}/nmigen-docs";
      };
      "www.nmigen.net" = {
        addSSL = true;
        enableACME = true;
        root = "${hydraWwwOutputs}/nmigen-docs";
      };
    };
  };
  services.uwsgi = {
    enable = true;
    plugins = [ "python3" ];
    instance = {
      type = "emperor";
      vassals = {
        mattermostgithub = import ./mattermost-github-integration/uwsgi-config.nix { inherit config pkgs; };
        rfq = import ./rfq/uwsgi-config.nix { inherit config pkgs; };
      };
    };
  };

  services.mysql = {
    enable = true;
    package = pkgs.lib.mkForce pkgs.mariadb;
    ensureDatabases = pkgs.lib.mkForce [];
    ensureUsers = pkgs.lib.mkForce [];
  };
  services.flarum = {
    enable = true;
    package = pkgs.callPackage ./flarum {};
    domain = "forum.m-labs.hk";
  };

  services.rt = {
    enable = true;
    package = pkgs.rt.overrideAttrs(oa: {
      patches = oa.patches or [] ++ [ ./rt-session.patch ];
    });
    organization = "M-Labs";
    domain = "rt.m-labs.hk";
    rtName = "Helpdesk";
    ownerEmail = "sb" + "@m-labs.hk";
    commentAddress = "helpdesk" + "@m-labs.hk";
    correspondAddress = "helpdesk" + "@m-labs.hk";
    sendmailPath = "${pkgs.msmtp}/bin/msmtp";
    sendmailArguments = ["-t" "-C" "/etc/nixos/secret/rt_msmtp.conf"];
  };
  systemd.services.rt-fetchmail = {
    description = "Fetchmail for RT";
    wantedBy = [ "multi-user.target" ];
    after = [ "dovecot2.service" ];
    serviceConfig = {
      Restart = "on-failure";
      User = "rt";
      Group = "rt";
      ExecStart = "${pkgs.bash}/bin/bash -c 'PATH=${pkgs.rt}/bin HOME=/tmp ${pkgs.fetchmail}/bin/fetchmail -f /etc/nixos/secret/rt_fetchmailrc'";
    };
  };

  environment.etc."postfix/sender_relay".text = ''
      m-labs-intl.com smtptun:
      * :
  '';
  systemd.services.postfix-rebuild-sender-relay = {
      description = "Postfix Rebuild Sender Dependent Relayhost Maps";
      serviceConfig = {
        ExecStart = "${pkgs.postfix}/sbin/postmap /etc/postfix/sender_relay";
      };
      wantedBy = [ "multi-user.target" ];
  };

  mailserver = {
    enable = true;
    localDnsResolver = false;  # conflicts with dnsmasq
    fqdn = "mail.m-labs.hk";
    domains = [ "m-labs.hk" "m-labs.ph" "m-labs-intl.com" "193thz.com" "malloctech.fr" ];
    enablePop3 = true;
    enablePop3Ssl = true;
    certificateScheme = "acme-nginx";
  } // (import /etc/nixos/secret/email_settings.nix);
  services.postfix = {
    config = {
      sender_dependent_relayhost_maps = "hash:/etc/postfix/sender_relay";
      masterConfig.smtptun = {
        type = "smtp";
        command = "smtp";
        args = "-o smtp_bind_address=10.42.0.2";
      };
    };
  };
  services.roundcube = {
    enable = true;
    hostName = "mail.m-labs.hk";
    extraConfig = ''
      $config['smtp_server'] = "tls://${config.mailserver.fqdn}";
      $config['smtp_user'] = "%u";
      $config['smtp_pass'] = "%p";
    '';
  };

  services.nextcloud = {
    enable = true;
    package = pkgs.nextcloud29;
    hostName = "files.m-labs.hk";
    https = true;
    maxUploadSize = "2G";
    config.adminpassFile = "/etc/nixos/secret/nextcloud_pass.txt";
    settings.default_phone_region = "HK";
    settings.log_type = "file";
    phpOptions."opcache.interned_strings_buffer" = "12";
  };

  services.hedgedoc = {
    enable = true;
    settings = {
      port = 9825;
      domain = "docs.m-labs.hk";
      protocolUseSSL = true;
      allowEmailRegister = false;
      allowAnonymous = false;
      db = {
        dialect = "sqlite";
        storage = "/var/lib/hedgedoc/db.hedgedoc.sqlite";
      };
    };
  };

  system.stateVersion = "21.05";
}