it-infra/nixbld-etc-nixos/configuration.nix

# Edit this configuration file to define what should be installed on
# your system.  Help is available in the configuration.nix(5) man page
# and in the NixOS manual (accessible by running ‘nixos-help’).

{ config, pkgs, ... }:

let
  netifWan = "enp4s0";
  netifLan = "enp5s0f1";
  netifWifi = "wlp6s0";
  netifSit = "henet0";
  hydraWwwOutputs = "/var/www/hydra-outputs";
in
{
  imports =
    [
      ./hardware-configuration.nix
      ./backup-module.nix
      ./github-backup-module.nix
      ./afws-module.nix
      ./rt.nix
      (builtins.fetchTarball {
        url = "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/archive/70a970f5a02b7febec1c3065e10c4155b99ecf86/nixos-mailserver-nixos.tar.gz";
        sha256 = "sha256:0samfdxzsg14bwjl50p5x92v9scpa6l8nxjlshrwhib03dai47n1";
      })
    ];

  boot.loader.grub.enable = true;
  boot.loader.grub.copyKernels = true;
  boot.loader.grub.device = "nodev";
  boot.loader.grub.efiSupport = true;
  boot.loader.efi.canTouchEfiVariables = true;
  boot.supportedFilesystems = ["zfs"];
  boot.kernelParams = ["zfs.l2arc_write_max=536870912"];
  boot.binfmt.emulatedSystems = [ "aarch64-linux" ];

  services.zfs.autoScrub.enable = true;
  services.zfs.autoScrub.interval = "monthly";
  services.zfs.autoSnapshot.enable = true;

  systemd.suppressedSystemUnits = [
      "hibernate.target"
      "suspend.target"
      "suspend-then-hibernate.target"
      "sleep.target"
      "hybrid-sleep.target"
      "systemd-hibernate.service"
      "systemd-hybrid-sleep.service"
      "systemd-suspend.service"
      "systemd-suspend-then-hibernate.service"
  ];

  security.apparmor.enable = true;
  services.fail2ban.enable = true;
  services.fail2ban.maxretry = 9;
  services.fail2ban.bantime-increment.enable = true;
  services.fail2ban.jails.sshd =
    ''
    enabled = true
    filter = sshd
    action = iptables-allports
    '';
  services.fail2ban.jails.nginx-botsearch =
    ''
    enabled = true
    filter = nginx-botsearch
    action = iptables-allports
    '';
  services.fail2ban.jails.nginx-limit-req =
    ''
    enabled = true
    filter = nginx-limit-req
    action = iptables-allports
    '';
  services.fail2ban.jails.postfix =
    ''
    enabled = true
    filter = postfix
    action = iptables-allports
    '';
  services.fail2ban.jails.dovecot =
    ''
    enabled = true
    filter = dovecot
    action = iptables-allports
    '';

  networking = {
    hostName = "nixbld";
    hostId = "e423f012";
    firewall = {
      allowedTCPPorts = [ 53 80 443 7402 3939 3940 ];
      allowedUDPPorts = [ 53 67 ];
      trustedInterfaces = [ netifLan ];
    };
    interfaces."${netifWan}".useDHCP = true;
    interfaces."${netifLan}" = {
      ipv4.addresses = [{
        address = "192.168.1.1";
        prefixLength = 24;
      }];
      ipv6.addresses = [{
        address = "2001:470:f821:1::";
        prefixLength = 64;
      }];
      ipv4.routes = [{
        address = "192.168.13.0";
        prefixLength = 24;
        via = "192.168.1.30";
      }];
      ipv6.routes = [{
        address = "2001:470:f821:3::";
        prefixLength = 64;
        via = "2001:470:f821:1:dea6:32ff:fe95:2fcf";
      }];
    };
    interfaces."${netifWifi}" = {
      ipv4.addresses = [{
        address = "192.168.12.1";
        prefixLength = 24;
      }];
      ipv6.addresses = [{
        address = "2001:470:f821:2::";
        prefixLength = 64;
      }];
    };
    nat = {
      enable = true;
      externalInterface = netifWan;
      internalInterfaces = [ netifLan netifWifi ];
      forwardPorts = [
        { sourcePort = 2201; destination = "192.168.1.201:22"; proto = "tcp"; }
        { sourcePort = 2202; destination = "192.168.1.202:22"; proto = "tcp"; }
        { sourcePort = 2203; destination = "192.168.1.203:22"; proto = "tcp"; }
        { sourcePort = 2204; destination = "192.168.1.204:22"; proto = "tcp"; }
      ];
      extraCommands = ''
        iptables -w -N block-lan-from-wifi
        iptables -w -A block-lan-from-wifi -i ${netifLan} -o ${netifWifi} -j DROP
        iptables -w -A block-lan-from-wifi -i ${netifWifi} -o ${netifLan} -j DROP
        iptables -w -A FORWARD -j block-lan-from-wifi

        iptables -w -N block-insecure-devices
        iptables -w -A block-insecure-devices -m mac --mac-source 00:20:0c:6c:ee:ba -j DROP  # keysight SA
        iptables -w -A block-insecure-devices -m mac --mac-source 74:5b:c5:20:c1:5f -j DROP  # siglent scope
        iptables -w -A FORWARD -j block-insecure-devices
      '';
      extraStopCommands = ''
        iptables -w -D FORWARD -j block-lan-from-wifi 2>/dev/null|| true
        iptables -w -F block-lan-from-wifi 2>/dev/null|| true
        iptables -w -X block-lan-from-wifi 2>/dev/null|| true

        iptables -w -D FORWARD -j block-insecure-devices 2>/dev/null|| true
        iptables -w -F block-insecure-devices 2>/dev/null|| true
        iptables -w -X block-insecure-devices 2>/dev/null|| true
      '';
    };
    sits."${netifSit}" = {
      dev = netifWan;
      remote = "216.218.221.6";
      local = "42.200.147.171";
      ttl = 255;
    };
    interfaces."${netifSit}".ipv6 = {
      addresses = [{ address = "2001:470:18:629::2"; prefixLength = 64; }];
      routes = [{ address = "::"; prefixLength = 0; }];
    };
  };
  boot.kernel.sysctl."net.ipv6.conf.all.forwarding" = "1";
  boot.kernel.sysctl."net.ipv6.conf.default.forwarding" = "1";
  boot.kernel.sysctl."net.ipv6.conf.${netifLan}.accept_dad" = "0";
  boot.kernel.sysctl."net.ipv6.conf.${netifWifi}.accept_dad" = "0";

  # https://kb.isc.org/docs/dnssec-key-and-signing-policy
  # chown named.named /etc/nixos/named
  services.bind = {
    enable = true;
    listenOn = [ "42.200.147.171" ];
    listenOnIpv6 = [ "2001:470:18:629::2" ];
    forwarders = [];
    extraOptions = "listen-on-v6 port 5354 { ::1; };";
    cacheNetworks = [ "::1/128" ];
    zones = {
      "m-labs.hk" = {
        name = "m-labs.hk";
        master = true;
        file = "/etc/nixos/named/m-labs.hk";
        extraConfig =
           ''
           dnssec-policy "default";
           inline-signing yes;
           '';
        slaves = [
          "213.239.220.50" "2a01:4f8:a0:7041::1"  # qnetp
        ];
      };
    };
  };

  services.hostapd = {
    enable        = true;
    interface     = netifWifi;
    hwMode        = "g";
    ssid          = "M-Labs";
    wpaPassphrase = (import /etc/nixos/secret/wifi_password.nix);
    extraConfig   = ''
      ieee80211d=1
      country_code=HK
      ieee80211n=1
      wmm_enabled=1
      auth_algs=1
      wpa_key_mgmt=WPA-PSK
      rsn_pairwise=CCMP
    '';
  };
  services.dnsmasq = {
    enable = true;
    servers = ["::1#5354"];
    extraConfig = ''
      interface=${netifLan}
      interface=${netifWifi}
      bind-interfaces
      dhcp-range=interface:${netifLan},192.168.1.81,192.168.1.254,24h
      dhcp-range=interface:${netifWifi},192.168.12.10,192.168.12.254,24h
      enable-ra
      dhcp-range=interface:${netifLan},::,constructor:${netifLan},ra-names
      dhcp-range=interface:${netifWifi},::,constructor:${netifWifi},ra-only

      no-resolv

      # Static IPv4s to make Red Pitayas with factory firmware less annoying
      dhcp-host=rp-f05cc9,192.168.1.190
      dhcp-host=rp-f0612e,192.168.1.191
      # Static IPv4s to make port redirections work
      dhcp-host=rpi-1,192.168.1.201
      dhcp-host=rpi-2,192.168.1.202
      dhcp-host=rpi-3,192.168.1.203
      dhcp-host=rpi-4,192.168.1.204

      # Static IP addresses for non-DHCP boards
      address=/rpi-ext/192.168.1.30
      address=/rpi-ext/2001:470:f821:1:dea6:32ff:fe95:2fcf

      address=/thermostat/192.168.1.26
      address=/powercycler/192.168.1.31
      address=/kc705/192.168.1.50
      address=/zynq-experiments/192.168.1.51
      address=/zc706/192.168.1.52
      address=/zc706-2/192.168.1.53
      address=/cora-z7/192.168.1.54
      address=/rust-pitaya/192.168.1.55
      address=/sayma/192.168.1.60
      address=/metlino/192.168.1.65
      address=/kasli/192.168.1.70
      address=/kasli-customer/192.168.1.75
      address=/stabilizer-customer/192.168.1.76

      # uTCA MCH from NAT
      address=/tschernobyl/192.168.1.80

      # Google can't do DNS geolocation correctly and slows down websites of everyone using
      # their shitty font cloud hosting. In HK, you sometimes get IPs behind the GFW that you
      # cannot reach.
      address=/fonts.googleapis.com/142.250.207.74
    '';
  };

  # Select internationalisation properties.
  i18n.defaultLocale = "en_US.UTF-8";
  console = {
    font = "Lat2-Terminus16";
    keyMap = "de";
  };

  # Set your time zone.
  time.timeZone = "Asia/Hong_Kong";

  # List packages installed in system profile. To search, run:
  # $ nix search wget
  environment.systemPackages = with pkgs; [
    wget vim git file lm_sensors acpi pciutils psmisc nixopsUnstable
    irssi tmux usbutils imagemagick jq zip unzip
    iw
    nvme-cli
    borgbackup
    bind
    (callPackage ./afws { inherit pkgs; })
  ];

  # Some programs need SUID wrappers, can be configured further or are
  # started in user sessions.
  # programs.mtr.enable = true;
  # programs.gnupg.agent = { enable = true; enableSSHSupport = true; };

  # List services that you want to enable:

  services.apcupsd.enable = true;
  services.apcupsd.configText = ''
    UPSTYPE usb
    NISIP 127.0.0.1
    BATTERYLEVEL 10
    MINUTES 5
  '';

  # Enable the OpenSSH daemon.
  services.openssh.enable = true;
  services.openssh.forwardX11 = true;
  services.openssh.passwordAuthentication = false;
  services.openssh.gatewayPorts = "clientspecified";
  programs.mosh.enable = true;

  programs.fish.enable = true;

  # Enable CUPS to print documents.
  services.avahi.enable = true;
  services.avahi.interfaces = [ netifLan ];
  services.avahi.publish.enable = true;
  services.avahi.publish.userServices = true;
  nixpkgs.config.allowUnfree = true;
  services.printing.enable = true;
  services.printing.drivers = [ pkgs.hplipWithPlugin ];
  services.printing.browsing = true;
  services.printing.listenAddresses = [ "*:631" ];
  services.printing.defaultShared = true;
  hardware.sane.enable = true;
  hardware.sane.extraBackends = [ pkgs.hplipWithPlugin ];

  users.extraUsers.root = {
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBNdIiLvP2hmDUFyyE0oLOIXrjrMdWWpBV9/gPR5m4AiARx4JkufIDZzmptdYQ5FhJORJ4lluPqp7dAmahoSwg4lv9Di0iNQpHMJvNGZLHYKM1H1FWCCFIEDJ8bD4SVfrDg=="
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCMALVC8RDTHec+PC8y1s3tcpUAODgq6DEzQdHDf/cyvDMfmCaPiMxfIdmkns5lMa03hymIfSmLUF0jFFDc7biRp7uf9AAXNsrTmplHii0l0McuOOZGlSdZM4eL817P7UwJqFMxJyFXDjkubhQiX6kp25Kfuj/zLnupRCaiDvE7ho/xay6Jrv0XLz935TPDwkc7W1asLIvsZLheB+sRz9SMOb9gtrvk5WXZl5JTOFOLu+JaRwQLHL/xdcHJTOod7tqHYfpoC5JHrEwKzbhTOwxZBQBfTQjQktKENQtBxXHTe71rUEWfEZQGg60/BC4BrRmh4qJjlJu3v4VIhC7SSHn1"
    ];
    shell = pkgs.fish;
  };
  # https://github.com/NixOS/nixpkgs/issues/155357
  security.sudo.enable = true;

  users.extraUsers.sb = {
    isNormalUser = true;
    extraGroups = ["lp" "scanner" "afws"];
    openssh.authorizedKeys.keys = [
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCyPk5WyFoWSvF4ozehxcVBoZ+UHgrI7VW/OoQfFFwIQe0qvetUZBMZwR2FwkLPAMZV8zz1v4EfncudEkVghy4P+/YVLlDjqDq9zwZnh8Nd/ifu84wmcNWHT2UcqnhjniCdshL8a44memzABnxfLLv+sXhP2x32cJAamo5y6fukr2qLp2jbXzR+3sv3klE0ruUXis/BR1lLqNJEYP8jB6fLn2sLKinnZPfn6DwVOk10mGeQsdME/eGl3phpjhODH9JW5V2V5nJBbC0rBnq+78dyArKVqjPSmIcSy72DEIpTctnMEN1W34BGrnsDd5Xd/DKxKxHKTMCHtZRwLC2X0NWN"
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCMALVC8RDTHec+PC8y1s3tcpUAODgq6DEzQdHDf/cyvDMfmCaPiMxfIdmkns5lMa03hymIfSmLUF0jFFDc7biRp7uf9AAXNsrTmplHii0l0McuOOZGlSdZM4eL817P7UwJqFMxJyFXDjkubhQiX6kp25Kfuj/zLnupRCaiDvE7ho/xay6Jrv0XLz935TPDwkc7W1asLIvsZLheB+sRz9SMOb9gtrvk5WXZl5JTOFOLu+JaRwQLHL/xdcHJTOod7tqHYfpoC5JHrEwKzbhTOwxZBQBfTQjQktKENQtBxXHTe71rUEWfEZQGg60/BC4BrRmh4qJjlJu3v4VIhC7SSHn1"
    ];
    shell = pkgs.fish;
  };
  users.extraUsers.rj = {
    isNormalUser = true;
    extraGroups = ["afws"];
  };
  users.extraUsers.nkrackow = {
    isNormalUser = true;
    extraGroups = ["afws"];
    openssh.authorizedKeys.keys = [
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDBNsAtZdp0BMvw0rRpMDgJ0V9hqB/BVSyhZ3m8LEx0im939ya6Urmlz3x7+RilD1LMl/p4B1Yxt+z/w7J5NB7unTYlKigJr/s9aH/0IKCFRvO5Omw88k3tWCCRA9KbbXAh0OE/Kli09rrgRuB6++c+XBZ4IvFvohfML0eAjdofn6ePnLWt+R/RNvNjmSb5y7rSIbJ9t+B7O1QOr7u+1GgZEexhG79o52I4rsrgyhUJOK4FbDGPnIkFYFeB2alijzbM1bAu9GR6BD4HBoqeW+DF7tUZs8GYtJsBX8rMnzuR3t8pM7RcGjY5IHQM9MM5WpHokJCFNSSzrvFgbK7CBFklOtipo1H1fwOuDuT3sCE3/ZTK5UgfKGdsb+vsvZub7KBNXfgru2webpl/rLcDJpn3eSDX/ZMGXVV8zskteQHtakra52bc2IeFaPiE1V+WeUB/LpIvRWG+Eh1VEgbUcjoVkaIBu6tQflW7US3uCGYan9Hw80MkwxAmqY1pogAJgzxsYbqdcNb8Xrra6LYFeMD8HXKdW9sXh7mzxDwwkzqjXCKPavWPT7ujicTRlJC6TfmZTdZUPh2mjvzUZI9ZPr50hkV0EAdERn57HwPGMlHiOCntPI/Jw3XmZXIOxChkyss5YFF5mWIzYOp5YxWBlWusNpnMeZCk2ncJmdXcAd6GzQ=="
    ];
  };
  users.extraUsers.backupdl = {
    isNormalUser = true;
    openssh.authorizedKeys.keys = [
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCyPk5WyFoWSvF4ozehxcVBoZ+UHgrI7VW/OoQfFFwIQe0qvetUZBMZwR2FwkLPAMZV8zz1v4EfncudEkVghy4P+/YVLlDjqDq9zwZnh8Nd/ifu84wmcNWHT2UcqnhjniCdshL8a44memzABnxfLLv+sXhP2x32cJAamo5y6fukr2qLp2jbXzR+3sv3klE0ruUXis/BR1lLqNJEYP8jB6fLn2sLKinnZPfn6DwVOk10mGeQsdME/eGl3phpjhODH9JW5V2V5nJBbC0rBnq+78dyArKVqjPSmIcSy72DEIpTctnMEN1W34BGrnsDd5Xd/DKxKxHKTMCHtZRwLC2X0NWN"
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCMALVC8RDTHec+PC8y1s3tcpUAODgq6DEzQdHDf/cyvDMfmCaPiMxfIdmkns5lMa03hymIfSmLUF0jFFDc7biRp7uf9AAXNsrTmplHii0l0McuOOZGlSdZM4eL817P7UwJqFMxJyFXDjkubhQiX6kp25Kfuj/zLnupRCaiDvE7ho/xay6Jrv0XLz935TPDwkc7W1asLIvsZLheB+sRz9SMOb9gtrvk5WXZl5JTOFOLu+JaRwQLHL/xdcHJTOod7tqHYfpoC5JHrEwKzbhTOwxZBQBfTQjQktKENQtBxXHTe71rUEWfEZQGg60/BC4BrRmh4qJjlJu3v4VIhC7SSHn1"
      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQCrDyB76mzasnoVlkq0lsX8etaJycwOPFbMIG7vlcTqAoJzDbr3QO6X0SEVwQGAqNhyn7ZxI6Xnw0Ib2FncgoXc5heMYXC7GlI+9lUitUnn/oPIh1vFRoMf1oo7pAlqbO2IdAYyap77A+/Ftqwn2LFy22d7z+DyXOJn6BVVIOYH55mpdadZKXPQSS4wi4THHXzO2uIjTBDGJYUNUVGKG8ggn1/fbsTnerz8vQ3kQ8F+xofxo4W6nMFSohmlk24J+/mcCUWBdNChQwZKPuEG7dBVvAIUkDnmwAaTQGxKveZlOx1CTcHNZAjh53zZdixO8wDJ+qNNQyWEhXJt58iAo2BWx91g5f0ObMYfGk7ytP30EHJN6UKU0+aNz3hno9igLX2hhKoB9S6BAyKbdpuC6kWido3WMde1tcudrtrl4M7VipqS6N3rWU3d/KMPGK1T9LfLJXqiy8q1g66q+uB1ZUZ9Wbip7KFWEVPq5muXobYmF3sgzJ8BMoUQPQfbrejR0= backupdl@aux"
    ];
  };
  users.extraUsers.occheung = {
    isNormalUser = true;
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBPEvmWmxpFpMgp5fpjKud8ev0cyf/+X5fEpQt/YD/+u4mbvZYPE300DLqQ0h/qjgvaGMz1ndf4idYnRdy+plJEC/+hmlRW5NlcpAr3S/LYAisacgKToFVl+MlBo+emS9Ig=="
    ];
  };
  users.extraUsers.spaqin = {
    isNormalUser = true;
    extraGroups = ["lp" "scanner" "afws"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBOtmlQmIK/cEUkcwA/y9jC4AohjoEmikerpxzPhZZtOcENidN/vFum58jIcSxBvjHnILOzhfCTeLvbvGbQOFE53a7FOyEHmIzXRKS86Mg5bPHUBJxRSq9MjulGZXES3HOQ=="
    ];
    shell = pkgs.zsh;
  };
  users.extraUsers.topquark12 = {
    isNormalUser = true;
    extraGroups = ["afws"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBGtaB32Ri1O7zRVG+1ADXmhwbK0r8R+c5W1RJd7cab5O9jp10wI+0JArFgK8u4m9oaR1rrVy4ym8wWMNu8vGKVTgyK+6UK87qTD96pnzwTyvzKCMOIHVBAPsnVfdOSKXLQ=="
    ];
  };
  users.extraUsers.therobs12 = {
    isNormalUser = true;
    extraGroups = ["afws"];
    openssh.authorizedKeys.keys = [
      "ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBGKbLMWCWrcUIVDKc7wrkn1Oo1PCH3K0HWIXnPjgASqKdzrNkgrHcg8UsXS0tLKtoYWbWisVUU+V05p6RKMOP7TP6hc2xoVRov6fBMuKHnVJbR9BAkn3O2EaJ7Rlo0y8aQ=="
    ];
    shell = pkgs.fish;
  };
  users.extraUsers.nix = {
    isNormalUser = true;
  };
  boot.kernel.sysctl."kernel.dmesg_restrict" = true;
  services.udev.packages = [ pkgs.sane-backends ];

  nix.settings.max-jobs = 10;
  nix.nrBuildUsers = 64;
  nix.settings.trusted-users = ["sb"];
  services.hydra = {
    enable = true;
    useSubstitutes = true;
    hydraURL = "https://nixbld.m-labs.hk";
    notificationSender = "hydra@m-labs.hk";
    minimumDiskFree = 15;  # in GB
    minimumDiskFreeEvaluator = 1;
    extraConfig =
      ''
      binary_cache_secret_key_file = /etc/nixos/secret/nixbld.m-labs.hk-1
      max_output_size = 10000000000

      <runcommand>
        job = web:web:web
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/web
      </runcommand>
      <runcommand>
        job = web:web:nmigen-docs
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/nmigen-docs
      </runcommand>

      <runcommand>
        job = artiq:sipyco:sipyco-manual-html
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/sipyco-manual-html
      </runcommand>
      <runcommand>
        job = artiq:sipyco:sipyco-manual-pdf
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/sipyco-manual-pdf
      </runcommand>

      <runcommand>
        job = artiq:main-beta:artiq-manual-html
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-html-beta
      </runcommand>
      <runcommand>
        job = artiq:main-beta:artiq-manual-pdf
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-pdf-beta
      </runcommand>
      <runcommand>
        job = artiq:extra-beta:conda-channel
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel-beta
      </runcommand>

      <runcommand>
        job = artiq:main:artiq-manual-html
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-html
      </runcommand>
      <runcommand>
        job = artiq:main:artiq-manual-pdf
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-pdf
      </runcommand>
      <runcommand>
        job = artiq:extra:conda-channel
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel
      </runcommand>

      <runcommand>
        job = artiq:full-legacy:artiq-manual-html
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-html-legacy
      </runcommand>
      <runcommand>
        job = artiq:full-legacy:artiq-manual-latexpdf
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-manual-latexpdf-legacy
      </runcommand>
      <runcommand>
        job = artiq:full-legacy:conda-channel
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel-legacy
      </runcommand>

      <runcommand>
        job = artiq:*:conda-channel
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-conda-channel-archives/$(jq -r '.build' < $HYDRA_JSON)
      </runcommand>

      <runcommand>
        job = artiq:extra-beta:msys2-repos
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-msys2-repos-beta
      </runcommand>

      <runcommand>
        job = artiq:artiq-nac3:msys2-repos
        command = [ $(jq '.buildStatus' < $HYDRA_JSON) = 0 ] && ln -sfn $(jq -r '.outputs[0].path' < $HYDRA_JSON) ${hydraWwwOutputs}/artiq-msys2-repos-nac3
      </runcommand>


      <githubstatus>
        jobs = artiq:fast.*:extended-tests
        inputs = artiqSrc
        useShortContext = 1
        authorization = token ${(import /etc/nixos/secret/github_tokens.nix).artiq}
      </githubstatus>
      <gitea_authorization>
        sb=${(import /etc/nixos/secret/gitea_tokens.nix).artiq-zynq}
      </gitea_authorization>
      '';
  };
  systemd.services.hydra-www-outputs-init = {
    description = "Set up a hydra-owned directory for build outputs";
    wantedBy = [ "multi-user.target" ];
    requiredBy = [ "hydra-queue-runner.service" ];
    before = [ "hydra-queue-runner.service" ];
    serviceConfig = {
      Type = "oneshot";
      ExecStart = [
        "${pkgs.coreutils}/bin/mkdir -p ${hydraWwwOutputs} ${hydraWwwOutputs}/artiq-conda-channel-archives"
        "${pkgs.coreutils}/bin/chown hydra-queue-runner:hydra ${hydraWwwOutputs} ${hydraWwwOutputs}/artiq-conda-channel-archives"
      ];
    };
  };
  services.afws.enable = true;

  nix.extraOptions = ''
    secret-key-files = /etc/nixos/secret/nixbld.m-labs.hk-1
    experimental-features = nix-command flakes
  '';
  nix.settings.extra-sandbox-paths = ["/opt"];

  services.munin-node.enable = true;
  services.munin-cron = {
    enable = true;
     hosts = ''
       [${config.networking.hostName}]
       address localhost
     '';
  };
  services.mlabs-backup.enable = true;
  services.ghbackup.enable = true;

  services.gitea = {
    enable = true;
    httpPort = 3001;
    rootUrl = "https://git.m-labs.hk/";
    appName = "M-Labs Git";
    mailerPasswordFile = "/etc/nixos/secret/mailerpassword";
    settings = {
      indexer = {
        REPO_INDEXER_ENABLED = true;
      };

      mailer = {
        ENABLED = true;
        HOST = "mail.m-labs.hk:587";
        FROM = "sysop@m-labs.hk";
        USER = "sysop@m-labs.hk";
      };

      service = {
        ENABLE_NOTIFY_MAIL = true;
        DISABLE_REGISTRATION = true;
      };

      attachment = {
        ALLOWED_TYPES = "*/*";
      };

      log.LEVEL = "Warn";

      session.COOKIE_SECURE = true;
    };
  };
  systemd.tmpfiles.rules = [
    "L+ '${config.services.gitea.stateDir}/custom/templates/home.tmpl' - - - - ${./gitea-home.tmpl}"
    "L+ '${config.services.gitea.stateDir}/custom/templates/user/auth/signin.tmpl' - - - - ${./gitea-signin.tmpl}"
  ];

  services.mattermost = {
    enable = true;
    siteUrl = "https://chat.m-labs.hk/";
    mutableConfig = true;
  };
  services.postgresql.package = pkgs.postgresql_11;
  services.matterbridge = {
    enable = true;
    configPath = "/etc/nixos/secret/matterbridge.toml";
  };
 
  nixpkgs.config.packageOverrides = super: let self = super.pkgs; in {
    nix = super.nix.overrideAttrs(oa: {
      patches = oa.patches or [] ++ [ ./nix-28-networked-derivations.patch ];
    });
    hydra_unstable = super.hydra_unstable.overrideAttrs(oa: {
      patches = oa.patches or [] ++ [
        ./hydra-conda.patch
        ./hydra-msys2.patch
        ./hydra-restrictdist.patch
        ./hydra-hack-allowed-uris.patch  # work around https://github.com/NixOS/nix/issues/5039
      ];
      hydraPath = oa.hydraPath + ":" + super.lib.makeBinPath [ super.jq ];
      doCheck = false;  # FIXME: ldap tests fail on hydra rebuild, seems unrelated to patches above.
    });
    matterbridge = super.matterbridge.overrideAttrs(oa: {
      patches = oa.patches or [] ++ [ ./matterbridge-disable-github.patch ];
    });
  };

  security.acme.acceptTerms = true;
  security.acme.defaults.email = "sb" + "@m-labs.hk";

  # https://github.com/NixOS/nixpkgs/issues/106862
  systemd.services."acme-fixperms".wants = [ "bind.service" "dnsmasq.service" ];
  systemd.services."acme-fixperms".after = [ "bind.service" "dnsmasq.service" ];
  services.nginx = {
    enable = true;
    recommendedProxySettings = true;
    recommendedGzipSettings = true;
    virtualHosts = let
      mainWebsite = {
        addSSL = true;
        enableACME = true;
        root = "${hydraWwwOutputs}/web";
        extraConfig = ''
          error_page 404 /404.html;
        '';
        locations."^~ /fonts/".extraConfig = ''
          expires 60d;
        '';
        locations."^~ /js/".extraConfig = ''
          expires 60d;
        '';
        locations."/MathJax/" = {
          alias = "/var/www/MathJax/";
          extraConfig = ''
            expires 60d;
          '';
        };

        # legacy URLs, redirect to avoid breaking people's bookmarks
        locations."/gateware.html".extraConfig = ''
          return 301 /gateware/migen/;
        '';
        locations."/migen".extraConfig = ''
          return 301 /gateware/migen/;
        '';
        locations."/artiq".extraConfig = ''
          return 301 /experiment-control/artiq/;
        '';
        locations."/artiq/resources.html".extraConfig = ''
          return 301 /experiment-control/resources/;
        '';

        # autogenerated manuals
        locations."/artiq/sipyco-manual/" = {
          alias = "${hydraWwwOutputs}/sipyco-manual-html/";
        };
        locations."=/artiq/sipyco-manual.pdf" = {
          alias = "${hydraWwwOutputs}/sipyco-manual-pdf/SiPyCo.pdf";
        };
        locations."/artiq/manual-beta/" = {
          alias = "${hydraWwwOutputs}/artiq-manual-html-beta/";
        };
        locations."=/artiq/manual-beta.pdf" = {
          alias = "${hydraWwwOutputs}/artiq-manual-pdf-beta/ARTIQ.pdf";
        };
        locations."/artiq/manual/" = {
          alias = "${hydraWwwOutputs}/artiq-manual-html/";
        };
        locations."=/artiq/manual.pdf" = {
          alias = "${hydraWwwOutputs}/artiq-manual-pdf/ARTIQ.pdf";
        };
        locations."/artiq/manual-legacy/" = {
          alias = "${hydraWwwOutputs}/artiq-manual-html-legacy/share/doc/artiq-manual/html/";
        };
        locations."=/artiq/manual-legacy.pdf" = {
          alias = "${hydraWwwOutputs}/artiq-manual-latexpdf-legacy/share/doc/artiq-manual/ARTIQ.pdf";
        };

        # legacy content
        locations."/migen/manual/" = {
          alias = "/var/www/m-labs.hk.old/migen/manual/";
        };
        locations."/artiq/manual-release-4/" = {
          alias = "/var/www/m-labs.hk.old/artiq/manual-release-4/";
        };
        locations."/artiq/manual-release-3/" = {
          alias = "/var/www/m-labs.hk.old/artiq/manual-release-3/";
        };
        locations."/artiq/manual-release-2/" = {
          alias = "/var/www/m-labs.hk.old/artiq/manual-release-2/";
        };
      };
    in {
      "m-labs.hk" = mainWebsite;
      "www.m-labs.hk" = mainWebsite;
      "m-labs.science" = mainWebsite;
      "www.m-labs.science" = mainWebsite;
      "lab.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/munin/".alias = "/var/www/munin/";
        locations."/munin/".extraConfig = ''
          auth_basic "Munin";
          auth_basic_user_file /etc/nixos/secret/muninpasswd;
        '';
      };
      "nixbld.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/".proxyPass = "http://127.0.0.1:3000";
      };
      "msys2.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/artiq-beta/" = {
          alias = "${hydraWwwOutputs}/artiq-msys2-repos-beta/";
          extraConfig = ''
            autoindex on;
          '';
        };
        locations."/artiq-nac3/" = {
          alias = "${hydraWwwOutputs}/artiq-msys2-repos-nac3/";
          extraConfig = ''
            autoindex on;
          '';
        };
      };
      "conda.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/artiq-beta/" = {
          alias = "${hydraWwwOutputs}/artiq-conda-channel-beta/";
          extraConfig = ''
            autoindex on;
            index bogus_index_file;
          '';
        };
        locations."/artiq/" = {
          alias = "${hydraWwwOutputs}/artiq-conda-channel/";
          extraConfig = ''
            autoindex on;
            index bogus_index_file;
          '';
        };
        locations."/artiq-legacy/" = {
          alias = "${hydraWwwOutputs}/artiq-conda-channel-legacy/";
          extraConfig = ''
            autoindex on;
            index bogus_index_file;
          '';
        };
        locations."/artiq-archives/" = {
          alias = "${hydraWwwOutputs}/artiq-conda-channel-archives/";
          extraConfig = ''
            autoindex on;
            index bogus_index_file;
          '';
        };
      };
      "git.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/".proxyPass = "http://127.0.0.1:3001";
        extraConfig = ''
          client_max_body_size 300M;
        '';
      };
      "chat.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/".proxyPass = "http://127.0.0.1:8065";
        locations."~ /api/v[0-9]+/(users/)?websocket$".proxyPass = "http://127.0.0.1:8065";
        locations."~ /api/v[0-9]+/(users/)?websocket$".proxyWebsockets = true;
      };
      "hooks.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/mattermost-github".extraConfig = ''
          include ${pkgs.nginx}/conf/uwsgi_params;
          uwsgi_pass unix:${config.services.uwsgi.runDir}/uwsgi-mgi.sock;
        '';
        locations."/rfq".extraConfig = ''
          include ${pkgs.nginx}/conf/uwsgi_params;
          uwsgi_pass unix:${config.services.uwsgi.runDir}/uwsgi-rfq.sock;
        '';
      };
      "forum.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        root = "/var/www/flarum/public";
        locations."~ \.php$".extraConfig = ''
          fastcgi_pass unix:${config.services.phpfpm.pools.flarum.socket};
          fastcgi_index index.php;
        '';
        extraConfig = ''
          index index.php;
          include /var/www/flarum/.nginx.conf;
        '';
      };
      "perso.m-labs.hk" = {
        addSSL = true;
        enableACME = true;
        root = "/var/www/perso";
      };
      "rt.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/" = {
          proxyPass = "http://127.0.0.1:4201";
          extraConfig = ''
            client_max_body_size 100M;
          '';
        };
        locations."/REST/1.0/NoAuth" = {
          proxyPass = "http://127.0.0.1:4201";
          extraConfig = ''
            client_max_body_size 100M;
            allow 127.0.0.1;
            deny all;
          '';
        };
      };
      "files.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
      };
      "docs.m-labs.hk" = {
        forceSSL = true;
        enableACME = true;
        locations."/".proxyPass = "http://localhost:9825";
        locations."/socket.io/".proxyPass = "http://localhost:9825";
        locations."/socket.io/".proxyWebsockets = true;
      };
      "nmigen.net" = {
        addSSL = true;
        enableACME = true;
        root = "${hydraWwwOutputs}/nmigen-docs";
      };
      "www.nmigen.net" = {
        addSSL = true;
        enableACME = true;
        root = "${hydraWwwOutputs}/nmigen-docs";
      };
    };
  };
  services.uwsgi = {
    enable = true;
    plugins = [ "python3" ];
    instance = {
      type = "emperor";
      vassals = {
        mattermostgithub = import ./mattermost-github-integration/uwsgi-config.nix { inherit config pkgs; };
        rfq = import ./rfq/uwsgi-config.nix { inherit config pkgs; };
      };
    };
  };
  services.mysql = {
    enable = true;
    package = pkgs.mariadb;
  };
  services.phpfpm.pools.flarum = {
    user = "nobody";
    settings = {
      "listen.owner" = "nginx";
      "listen.group" = "nginx";
      "listen.mode" = "0600";
      "pm" = "dynamic";
      "pm.max_children" = 5;
      "pm.start_servers" = 2;
      "pm.min_spare_servers" = 1;
      "pm.max_spare_servers" = 3;
      "pm.max_requests" = 500;
    };
  };

  services.rt = {
    enable = true;
    package = pkgs.rt.overrideAttrs(oa: {
      patches = oa.patches or [] ++ [ ./rt-session.patch ];
    });
    organization = "M-Labs";
    domain = "rt.m-labs.hk";
    rtName = "Helpdesk";
    ownerEmail = "sb" + "@m-labs.hk";
    commentAddress = "helpdesk" + "@m-labs.hk";
    correspondAddress = "helpdesk" + "@m-labs.hk";
    sendmailPath = "${pkgs.msmtp}/bin/msmtp";
    sendmailArguments = ["-t" "-C" "/etc/nixos/secret/rt_msmtp.conf"];
  };
  systemd.services.rt-fetchmail = {
    description = "Fetchmail for RT";
    wantedBy = [ "multi-user.target" ];
    after = [ "network.target" ];
    serviceConfig = {
      Restart = "on-failure";
      User = "rt";
      Group = "rt";
      ExecStart = "${pkgs.bash}/bin/bash -c 'PATH=${pkgs.rt}/bin HOME=/tmp ${pkgs.fetchmail}/bin/fetchmail -f /etc/nixos/secret/rt_fetchmailrc'";
    };
  };

  mailserver = {
    enable = true;
    localDnsResolver = false;  # conflicts with dnsmasq
    fqdn = "mail.m-labs.hk";
    domains = [ "m-labs.hk" ];
    certificateScheme = 3;
  } // (import /etc/nixos/secret/email_settings.nix);
  services.roundcube = {
    enable = true;
    hostName = "mail.m-labs.hk";
    extraConfig = ''
      $config['smtp_server'] = "tls://${config.mailserver.fqdn}";
      $config['smtp_user'] = "%u";
      $config['smtp_pass'] = "%p";
    '';
  };

  services.nextcloud = {
    enable = true;
    package = pkgs.nextcloud25;
    hostName = "files.m-labs.hk";
    https = true;
    enableBrokenCiphersForSSE = false;
    config.adminpassFile = "/etc/nixos/secret/nextcloud_pass.txt";
  };

  services.hedgedoc = {
    enable = true;
    settings = {
      port = 9825;
      domain = "docs.m-labs.hk";
      protocolUseSSL = true;
      allowEmailRegister = false;
      allowAnonymous = false;
      db = {
        dialect = "sqlite";
        storage = "/var/lib/hedgedoc/db.hedgedoc.sqlite";
      };
    };
  };

  system.stateVersion = "21.05";
}