From a77cb59e96355a1cf502984b91b17248743b7174 Mon Sep 17 00:00:00 2001 From: Jakob Lechner Date: Mon, 1 May 2023 16:11:44 +0000 Subject: [PATCH] Add iron --- hosts/default.nix | 4 + hosts/iron/ZFS.md | 149 +++++++++++++++++++++++ hosts/iron/configuration.nix | 143 ++++++++++++++++++++++ hosts/iron/services/default.nix | 6 + hosts/iron/services/dnsmasq.nix | 38 ++++++ hosts/iron/services/unifi-controller.nix | 9 ++ 6 files changed, 349 insertions(+) create mode 100644 hosts/iron/ZFS.md create mode 100644 hosts/iron/configuration.nix create mode 100644 hosts/iron/services/default.nix create mode 100644 hosts/iron/services/dnsmasq.nix create mode 100644 hosts/iron/services/unifi-controller.nix diff --git a/hosts/default.nix b/hosts/default.nix index f3b7436..2511c21 100644 --- a/hosts/default.nix +++ b/hosts/default.nix @@ -17,4 +17,8 @@ system = "x86_64-linux"; targetHost = "91.107.235.15"; }; + iron = { + system = "x86_64-linux"; + targetHost = "192.168.42.1"; + }; } diff --git a/hosts/iron/ZFS.md b/hosts/iron/ZFS.md new file mode 100644 index 0000000..f7cdf23 --- /dev/null +++ b/hosts/iron/ZFS.md @@ -0,0 +1,149 @@ +# ZFS setup +I want to set up a 5-disk RAIDZ array on an EFI system. To be considerate of my paranoia in doing so, I don't want to rely on ZFS buitin encryption as it is prone to metadata leaks. Instead, I want to use LUKS to encrypt on the block-device level. + +Two pools will be used in the setup. + - one mirrored pool named `bpool` for booting (kernel+initrd stuff) + - the main RAIDZ pool named `rpool` + +## Partition table +To simplify the setup, I'm using the same partition table for each of the disks. A `gpt` Disklabel type is used in order to use EFI boot. My disks still use `512` bytes sector size. When using disks with `4K` sectors, the setup might need to be changed to correctly align the partitions to the sector size. + +| Partition number | Start | End | Sectors | Size | Type | +| ---------------- | -------- | -------- | ------- | ----- | ---------------- | +| 1 | 2048 | 2099199 | 2097152 | 1G | EFI System | +| 2 | 2099200 | 10487807 | 8388608 | 4G | Solaris boot | +| 3 | 10487808 | ... | ... | ... | Linux filesystem | +| 4 | 48 | 2047 | 2000 | 1000K | BIOS boot | + +The third partition takes the remaining size of the disk. + +First, TRIM all SSDs. All data will be lost! Then create the partition table. + +```bash +for disk in /dev/sd{a..e}; do + blkdiscard -f $disk + parted --script --align=optimal $disk -- \ + mklabel gpt \ + mkpart EFI 2MiB 1GiB \ + mkpart bpool 1GiB 5GiB \ + mkpart rpool 5GiB 100% \ + mkpart BIOS 1MiB 2MiB \ + set 1 esp on \ + set 4 bios_grub on \ + set 4 legacy_boot on + partprobe $disk + udevadm settle +done +``` + +## LUKS + +```bash +cryptsetup luksFormat --hash sha512 --use-random --pbkdf argon2id --iter-time 12000 --pbkdf-memory $((4*1024*1024)) --pbkdf-parallel 4 $disk +cryptsetup open --allow-discards $disk $name +``` + +### Host-Id +The host id `networking.hostId` should be unique among our machines. Let's generate a random 32bit ID. +```bash +head -c4 /dev/urandom | od -A none -t x4 +``` + +## Create ZFS pools + +`ashift` needs to be set depending on the sector size. For example if ashift=9, 2^9=512. + +If the disk uses a sector size of 512 Bytes, use `ashift=9`. For 4K sector size, use `ashift=12`. + +Create root pool +```bash +zpool create \ + -o compatibility=grub2 \ + -o ashift=9 \ + -o autotrim=on \ + -O acltype=posixacl \ + -O canmount=off \ + -O compression=lz4 \ + -O devices=off \ + -O normalization=formD \ + -O relatime=on \ + -O xattr=sa \ + -O mountpoint=/boot \ + -R /mnt \ + bpool \ + mirror \ + /dev/disk/by-id/ata-Samsung_SSD*-part2 +``` + +Create boot pool +```bash +zpool create \ + -o ashift=9 \ + -o autotrim=on \ + -R "/mnt" \ + -O acltype=posixacl \ + -O canmount=off \ + -O compression=zstd \ + -O dnodesize=auto \ + -O normalization=formD \ + -O relatime=on \ + -O xattr=sa \ + -O mountpoint=/ \ + rpool \ + raidz \ + /dev/mapper/LUKS-* +``` + +Create root system container +```bash +zfs create \ + -o canmount=off \ + -o mountpoint=none \ + rpool/nixos +``` + +Create system datasets, manage mountpoints with `mountpoint=legacy` +```bash +zfs create -o mountpoint=legacy rpool/nixos/root +mount -t zfs rpool/nixos/root /mnt/ +zfs create -o mountpoint=legacy rpool/nixos/home +mkdir /mnt/home +mount -t zfs rpool/nixos/home /mnt/home +zfs create -o mountpoint=legacy rpool/nixos/var +zfs create -o mountpoint=legacy rpool/nixos/var/lib +zfs create -o mountpoint=legacy rpool/nixos/var/log +zfs create -o mountpoint=none bpool/nixos +zfs create -o mountpoint=legacy bpool/nixos/root +mkdir /mnt/boot +mount -t zfs bpool/nixos/root /mnt/boot +mkdir -p /mnt/var/log +mkdir -p /mnt/var/lib +mount -t zfs rpool/nixos/var/lib /mnt/var/lib +mount -t zfs rpool/nixos/var/log /mnt/var/log +zfs create -o mountpoint=legacy rpool/nixos/empty +zfs snapshot rpool/nixos/empty@start +``` + +Format and mount ESP +```bash +for disk in \ + /dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R103837K \ + /dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R103838A \ + /dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R104926N \ + /dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R104934H \ + /dev/disk/by-id/ata-Samsung_SSD_870_QVO_8TB_S5SSNJ0W206517Y +do + mkfs.vfat -n EFI "${disk}"-part1 + mkdir -p "/mnt/boot/efis/${disk##*/}"-part1 + mount -t vfat -o iocharset=iso8859-1 "${disk}"-part1 "/mnt/boot/efis/${disk##*/}"-part1 +done +``` + +```bash +umount -Rl /mnt +``` + +```bash +zpool export -a +``` + diff --git a/hosts/iron/configuration.nix b/hosts/iron/configuration.nix new file mode 100644 index 0000000..3b427e6 --- /dev/null +++ b/hosts/iron/configuration.nix @@ -0,0 +1,143 @@ +{ inputs, config, pkgs, lib, ... }: +let + zfsKernelPackages = config.boot.zfs.package.latestCompatibleLinuxPackages; + disks = [ + "ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R103837K" + "ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R103838A" + "ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R104926N" + "ata-Samsung_SSD_870_QVO_8TB_S5SSNG0R104934H" + "ata-Samsung_SSD_870_QVO_8TB_S5SSNJ0W206517Y" + ]; + removableEfi = true; + devNodes = "/dev/disk/by-id/"; + datasets = { + "bpool/nixos/root" = "/boot"; + "rpool/nixos/root" = "/"; + "rpool/nixos/home" = "/home"; + "rpool/nixos/var/lib" = "/var/lib"; + "rpool/nixos/var/log" = "/var/log"; + }; + partitionScheme = { + efiBoot = "-part1"; + bootPool = "-part2"; + luksDev = "-part3"; + biosBoot = "-part4"; + }; + efiSystemPartitions = (map (diskName: diskName + partitionScheme.efiBoot) disks); +in +with lib; { + imports = [ + ../../home-manager/users/jalr.nix + ./services + ]; + config = { + + system.stateVersion = "22.11"; + + security.sudo.wheelNeedsPassword = false; + + networking = { + hostName = "iron"; + hostId = "b141e72f"; + useDHCP = false; + networkmanager.enable = false; + + interfaces = { + enp3s4.ipv4.addresses = [{ + address = "192.168.42.1"; + prefixLength = 24; + }]; + enp4s5 = { + useDHCP = true; + }; + }; + + nat = { + enable = true; + externalInterface = "enp4s5"; + internalInterfaces = [ + "enp3s4" + ]; + }; + + firewall = { + extraCommands = '' + iptables -A FORWARD -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu + ''; + }; + }; + + boot = { + kernelPackages = zfsKernelPackages; + initrd = { + availableKernelModules = [ "ahci" ]; + luks.devices = lib.listToAttrs ( + map + (dev: { + name = "LUKS-${dev}${partitionScheme.luksDev}"; + value = { + device = "${devNodes}${dev}${partitionScheme.luksDev}"; + allowDiscards = true; + }; + }) + disks + ); + }; + supportedFilesystems = [ "zfs" ]; + zfs = { + devNodes = devNodes; + forceImportRoot = false; + }; + loader = { + efi = { + canTouchEfiVariables = (if removableEfi then false else true); + efiSysMountPoint = ("/boot/efis/" + (head disks) + + partitionScheme.efiBoot); + }; + generationsDir.copyKernels = true; + grub = { + enable = true; + devices = (map (diskName: devNodes + diskName) disks); + efiInstallAsRemovable = removableEfi; + version = 2; + copyKernels = true; + efiSupport = true; + zfsSupport = true; + extraInstallCommands = (toString (map + (diskName: '' + ${pkgs.coreutils-full}/bin/cp -r ${config.boot.loader.efi.efiSysMountPoint}/EFI /boot/efis/${diskName}${partitionScheme.efiBoot} + '') + (tail disks))); + }; + }; + }; + + fileSystems = mkMerge (mapAttrsToList + (dataset: mountpoint: { + "${mountpoint}" = { + device = "${dataset}"; + fsType = "zfs"; + options = [ "X-mount.mkdir" "noatime" ]; + neededForBoot = true; + }; + }) + datasets ++ map + (esp: { + "/boot/efis/${esp}" = { + device = "${devNodes}/${esp}"; + fsType = "vfat"; + options = [ + "x-systemd.idle-timeout=1min" + "x-systemd.automount" + "noauto" + "nofail" + "noatime" + "X-mount.mkdir" + ]; + }; + }) + efiSystemPartitions); + + hardware.enableRedistributableFirmware = true; + }; +} diff --git a/hosts/iron/services/default.nix b/hosts/iron/services/default.nix new file mode 100644 index 0000000..7fdd64b --- /dev/null +++ b/hosts/iron/services/default.nix @@ -0,0 +1,6 @@ +{ + imports = [ + ./dnsmasq.nix + ./unifi-controller.nix + ]; +} diff --git a/hosts/iron/services/dnsmasq.nix b/hosts/iron/services/dnsmasq.nix new file mode 100644 index 0000000..05e15f8 --- /dev/null +++ b/hosts/iron/services/dnsmasq.nix @@ -0,0 +1,38 @@ +{ pkgs, ... }: + +let + stateDir = "/var/lib/dnsmasq"; +in +{ + services.dnsmasq = { + enable = true; + + extraConfig = '' + listen-address=192.168.42.1 + interface=lo + + expand-hosts + domain=lan.bw.jalr.de + dhcp-range=192.168.42.20,192.168.42.254,4h + + cache-size=10000 + dns-forward-max=1000 + + no-hosts + addn-hosts=${pkgs.writeText "hosts.dnsmasq" '' + 192.168.42.1 aluminium unifi + ''} + ''; + + servers = [ + "142.250.185.78" # dns.as250.net + "2001:470:20::2" # ordns.he.net + "74.82.42.42" # ordns.he.net + ]; + }; + + networking.firewall = { + allowedUDPPorts = [ 53 67 ]; + allowedTCPPorts = [ 53 ]; + }; +} diff --git a/hosts/iron/services/unifi-controller.nix b/hosts/iron/services/unifi-controller.nix new file mode 100644 index 0000000..f34d12d --- /dev/null +++ b/hosts/iron/services/unifi-controller.nix @@ -0,0 +1,9 @@ +{ pkgs, ... }: +{ + services.unifi = { + enable = true; + openFirewall = true; + unifiPackage = pkgs.unifi; + }; + networking.firewall.allowedTCPPorts = [ 8443 ]; +}