Add iron

hosts/default.nix

@@ -17,4 +17,8 @@
     system = "x86_64-linux";
     targetHost = "91.107.235.15";
   };
+  iron = {
+    system = "x86_64-linux";
+    targetHost = "192.168.42.1";
+  };
 }

hosts/iron/ZFS.md

@@ -0,0 +1,149 @@
+# ZFS setup
+I want to set up a 5-disk RAIDZ array on an EFI system. To be considerate of my paranoia in doing so, I don't want to rely on ZFS buitin encryption as it is prone to metadata leaks. Instead, I want to use LUKS to encrypt on the block-device level.
+
+Two pools will be used in the setup.
+ - one mirrored pool named `bpool` for booting (kernel+initrd stuff)
+ - the main RAIDZ pool named `rpool`
+
+## Partition table
+To simplify the setup, I'm using the same partition table for each of the disks. A `gpt` Disklabel type is used in order to use EFI boot. My disks still use `512` bytes sector size. When using disks with `4K` sectors, the setup might need to be changed to correctly align the partitions to the sector size.
+
+| Partition number | Start    | End      | Sectors | Size  | Type             |
+| ---------------- | -------- | -------- | ------- | ----- | ---------------- |
+|                1 |     2048 |  2099199 | 2097152 |    1G | EFI System       |
+|                2 |  2099200 | 10487807 | 8388608 |    4G | Solaris boot     |
+|                3 | 10487808 |      ... |     ... |   ... | Linux filesystem |
+|                4 |       48 |     2047 |    2000 | 1000K | BIOS boot        |
+
+The third partition takes the remaining size of the disk.
+
+First, TRIM all SSDs. All data will be lost! Then create the partition table.
+
+```bash
+for disk in /dev/sd{a..e}; do
+    blkdiscard -f $disk
+    parted --script --align=optimal $disk -- \
+        mklabel gpt \
+        mkpart EFI 2MiB 1GiB \
+        mkpart bpool 1GiB 5GiB \
+        mkpart rpool 5GiB 100% \
+        mkpart BIOS 1MiB 2MiB \
+        set 1 esp on \
+        set 4 bios_grub on \
+        set 4 legacy_boot on
+    partprobe $disk
+    udevadm settle
+done
+```
+
+## LUKS
+
+```bash
+cryptsetup luksFormat --hash sha512 --use-random --pbkdf argon2id --iter-time 12000 --pbkdf-memory $((4*1024*1024)) --pbkdf-parallel 4 $disk
+cryptsetup open --allow-discards $disk $name
+```
+
+### Host-Id
+The host id `networking.hostId` should be unique among our machines. Let's generate a random 32bit ID.
+```bash
+head -c4 /dev/urandom | od -A none -t x4
+```
+
+## Create ZFS pools
+
+`ashift` needs to be set depending on the sector size. For example if ashift=9, 2^9=512.
+
+If the disk uses a sector size of 512 Bytes, use `ashift=9`. For 4K sector size, use `ashift=12`.
+
+Create root pool
+```bash
+zpool create \
+    -o compatibility=grub2 \
+    -o ashift=9 \
+    -o autotrim=on \
+    -O acltype=posixacl \
+    -O canmount=off \
+    -O compression=lz4 \
+    -O devices=off \
+    -O normalization=formD \
+    -O relatime=on \
+    -O xattr=sa \
+    -O mountpoint=/boot \
+    -R /mnt \
+    bpool \
+    mirror \
+    /dev/disk/by-id/ata-Samsung_SSD*-part2
+```
+
+Create boot pool
+```bash
+zpool create \
+    -o ashift=9 \
+    -o autotrim=on \
+    -R "/mnt" \
+    -O acltype=posixacl \
+    -O canmount=off \
+    -O compression=zstd \
+    -O dnodesize=auto \
+    -O normalization=formD \
+    -O relatime=on \
+    -O xattr=sa \
+    -O mountpoint=/ \
+    rpool \
+    raidz \
+    /dev/mapper/LUKS-*
+```
+
+Create root system container
+```bash
+zfs create \
+    -o canmount=off \
+    -o mountpoint=none \
+    rpool/nixos
+```
+
+Create system datasets, manage mountpoints with `mountpoint=legacy`
+```bash
+zfs create -o mountpoint=legacy rpool/nixos/root
+mount -t zfs rpool/nixos/root /mnt/
+zfs create -o mountpoint=legacy rpool/nixos/home
+mkdir /mnt/home
+mount -t zfs rpool/nixos/home /mnt/home
+zfs create -o mountpoint=legacy  rpool/nixos/var
+zfs create -o mountpoint=legacy rpool/nixos/var/lib
+zfs create -o mountpoint=legacy rpool/nixos/var/log
+zfs create -o mountpoint=none bpool/nixos
+zfs create -o mountpoint=legacy bpool/nixos/root
+mkdir /mnt/boot
+mount -t zfs bpool/nixos/root /mnt/boot
+mkdir -p /mnt/var/log
+mkdir -p /mnt/var/lib
+mount -t zfs rpool/nixos/var/lib /mnt/var/lib
+mount -t zfs rpool/nixos/var/log /mnt/var/log
+zfs create -o mountpoint=legacy rpool/nixos/empty
+zfs snapshot rpool/nixos/empty@start
+```
+
+Format and mount ESP
+```bash
+for disk in \
+    /dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R103837K \
+    /dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R103838A \
+    /dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R104926N \
+    /dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R104934H \
+    /dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNJ0W206517Y
+do
+    mkfs.vfat -n EFI "${disk}"-part1
+    mkdir -p "/mnt/boot/efis/${disk##*/}"-part1
+    mount -t vfat -o iocharset=iso8859-1 "${disk}"-part1 "/mnt/boot/efis/${disk##*/}"-part1
+done
+```
+
+```bash
+umount -Rl /mnt
+```
+
+```bash
+zpool export -a
+```
+

hosts/iron/configuration.nix

@@ -0,0 +1,143 @@
+{ inputs, config, pkgs, lib, ... }:
+let
+  zfsKernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages;
+  disks = [
+    "ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R103837K"
+    "ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R103838A"
+    "ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R104926N"
+    "ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R104934H"
+    "ata-Samsung_SSD_870_QVO_8TB_S5SSNJ0W206517Y"
+  ];
+  removableEfi = true;
+  devNodes = "/dev/disk/by-id/";
+  datasets = {
+    "bpool/nixos/root" = "/boot";
+    "rpool/nixos/root" = "/";
+    "rpool/nixos/home" = "/home";
+    "rpool/nixos/var/lib" = "/var/lib";
+    "rpool/nixos/var/log" = "/var/log";
+  };
+  partitionScheme = {
+    efiBoot = "-part1";
+    bootPool = "-part2";
+    luksDev = "-part3";
+    biosBoot = "-part4";
+  };
+  efiSystemPartitions = (map (diskName: diskName + partitionScheme.efiBoot) disks);
+in
+with lib; {
+  imports = [
+    ../../home-manager/users/jalr.nix
+    ./services
+  ];
+  config = {
+
+    system.stateVersion = "22.11";
+
+    security.sudo.wheelNeedsPassword = false;
+
+    networking = {
+      hostName = "iron";
+      hostId = "b141e72f";
+      useDHCP = false;
+      networkmanager.enable = false;
+
+      interfaces = {
+        enp3s4.ipv4.addresses = [{
+          address = "192.168.42.1";
+          prefixLength = 24;
+        }];
+        enp4s5 = {
+          useDHCP = true;
+        };
+      };
+
+      nat = {
+        enable = true;
+        externalInterface = "enp4s5";
+        internalInterfaces = [
+          "enp3s4"
+        ];
+      };
+
+      firewall = {
+        extraCommands = ''
+          iptables -A FORWARD -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
+        '';
+      };
+    };
+
+    boot = {
+      kernelPackages = zfsKernelPackages;
+      initrd = {
+        availableKernelModules = [ "ahci" ];
+        luks.devices = lib.listToAttrs (
+          map
+            (dev: {
+              name = "LUKS-${dev}${partitionScheme.luksDev}";
+              value = {
+                device = "${devNodes}${dev}${partitionScheme.luksDev}";
+                allowDiscards = true;
+              };
+            })
+            disks
+        );
+      };
+      supportedFilesystems = [ "zfs" ];
+      zfs = {
+        devNodes = devNodes;
+        forceImportRoot = false;
+      };
+      loader = {
+        efi = {
+          canTouchEfiVariables = (if removableEfi then false else true);
+          efiSysMountPoint = ("/boot/efis/" + (head disks)
+            + partitionScheme.efiBoot);
+        };
+        generationsDir.copyKernels = true;
+        grub = {
+          enable = true;
+          devices = (map (diskName: devNodes + diskName) disks);
+          efiInstallAsRemovable = removableEfi;
+          version = 2;
+          copyKernels = true;
+          efiSupport = true;
+          zfsSupport = true;
+          extraInstallCommands = (toString (map
+            (diskName: ''
+              ${pkgs.coreutils-full}/bin/cp -r ${config.boot.loader.efi.efiSysMountPoint}/EFI /boot/efis/${diskName}${partitionScheme.efiBoot}
+            '')
+            (tail disks)));
+        };
+      };
+    };
+
+    fileSystems = mkMerge (mapAttrsToList
+      (dataset: mountpoint: {
+        "${mountpoint}" = {
+          device = "${dataset}";
+          fsType = "zfs";
+          options = [ "X-mount.mkdir" "noatime" ];
+          neededForBoot = true;
+        };
+      })
+      datasets ++ map
+      (esp: {
+        "/boot/efis/${esp}" = {
+          device = "${devNodes}/${esp}";
+          fsType = "vfat";
+          options = [
+            "x-systemd.idle-timeout=1min"
+            "x-systemd.automount"
+            "noauto"
+            "nofail"
+            "noatime"
+            "X-mount.mkdir"
+          ];
+        };
+      })
+      efiSystemPartitions);
+
+    hardware.enableRedistributableFirmware = true;
+  };
+}

hosts/iron/services/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./dnsmasq.nix
+    ./unifi-controller.nix
+  ];
+}

hosts/iron/services/dnsmasq.nix

@@ -0,0 +1,38 @@
+{ pkgs, ... }:
+
+let
+  stateDir = "/var/lib/dnsmasq";
+in
+{
+  services.dnsmasq = {
+    enable = true;
+
+    extraConfig = ''
+      listen-address=192.168.42.1
+      interface=lo
+
+      expand-hosts
+      domain=lan.bw.jalr.de
+      dhcp-range=192.168.42.20,192.168.42.254,4h
+
+      cache-size=10000
+      dns-forward-max=1000
+
+      no-hosts
+      addn-hosts=${pkgs.writeText "hosts.dnsmasq" ''
+        192.168.42.1 aluminium unifi
+      ''}
+    '';
+
+    servers = [
+      "142.250.185.78" # dns.as250.net
+      "2001:470:20::2" # ordns.he.net
+      "74.82.42.42" # ordns.he.net
+    ];
+  };
+
+  networking.firewall = {
+    allowedUDPPorts = [ 53 67 ];
+    allowedTCPPorts = [ 53 ];
+  };
+}

hosts/iron/services/unifi-controller.nix

@@ -0,0 +1,9 @@
+{ pkgs, ... }:
+{
+  services.unifi = {
+    enable = true;
+    openFirewall = true;
+    unifiPackage = pkgs.unifi;
+  };
+  networking.firewall.allowedTCPPorts = [ 8443 ];
+}