Files
nixfiles/nixos/boxes/home/palace/default.nix
T
jackos1998andClaude Opus 5.5 a6491c7535
CI / Check, build and cache nixfiles (push) Successful in 45m39s
Update docs / update (push) Successful in 1m0s
nixos/slop-cannon: Move to a VM on palace
Give agents their own kernel, Nix daemon and disk for workloads that
need root. Put the VM on the `untrusted` VLAN with MAC spoof
checking on VF 5.

Export a dedicated partition from `cellar` and connect it on
`palace`. Give the VM its own SSH host key and age secrets for Paseo
and Gitea. Keep the agent home and Paseo identity on persistent
storage and provide `slop-pair` for pairing clients.

Retire the container, restore `sfh` to 8 cores and 32 GiB, and rekey
the shared password secret. Document the VM and its operating
procedures on the relevant box pages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 22:02:14 +00:00

271 lines
8.0 KiB
Nix

{ lib, ... }:
let
inherit (lib.my) net mkVLAN;
inherit (lib.my.c) pubDomain;
inherit (lib.my.c.home) domain vlans prefixes vips vlanDns hiMTU;
in
{
imports = [ ./vms ];
nixos.systems.palace = {
system = "x86_64-linux";
nixpkgs = "mine-stable";
home-manager = "mine-stable";
assignments = {
hi = {
inherit domain;
mtu = hiMTU;
ipv4 = {
address = net.cidr.host 22 prefixes.hi.v4;
mask = 22;
gateway = vips.hi.v4;
};
ipv6 = {
iid = "::2:1";
address = net.cidr.host (65536*2+1) prefixes.hi.v6;
};
};
core = {
inherit domain;
name = "palace-core";
mtu = 1500;
ipv4 = {
address = net.cidr.host 20 prefixes.core.v4;
gateway = null;
};
};
};
configuration = { lib, pkgs, modulesPath, config, systems, assignments, allAssignments, ... }:
let
inherit (lib) mkForce mkMerge;
inherit (lib.my) networkdAssignment;
inherit (lib.my.c) networkd;
in
{
boot = {
kernelPackages = (lib.my.c.kernel.lts pkgs).extend (self: super: {
kernel = super.kernel.override {
structuredExtraConfig = with lib.kernel; {
ACPI_APEI_PCIEAER = yes;
PCIEAER = yes;
};
};
});
kernelModules = [ "dm-raid" "kvm-amd" "nvme-rdma" ];
kernelParams = [ "amd_iommu=on" ];
initrd = {
availableKernelModules = [ "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
# LVM autoactivation can discover the RAID LVs before stage-two module loading.
kernelModules = [ "dm-raid" ];
};
};
hardware = {
enableRedistributableFirmware = true;
cpu = {
amd.updateMicrocode = true;
};
};
fileSystems = {
"/boot" = {
device = "/dev/disk/by-partuuid/c06a8d24-2af9-4416-bf5e-cfe6defdbd47";
fsType = "vfat";
};
"/nix" = {
device = "/dev/disk/by-uuid/450e1f72-238a-4160-98b8-b5e6d0d6fdf6";
fsType = "ext4";
};
"/persist" = {
device = "/dev/disk/by-uuid/9d6d53a8-dff8-49e0-9bc3-fb5f7c6760d0";
fsType = "ext4";
neededForBoot = true;
};
};
# NVMe-oF disks connected here belong to guests (`palace` has no NVMe of its own; its
# drives are passed to `cellar`); never activate their volume groups
environment.etc."lvm/lvm.conf".text = ''
devices/global_filter = [ "r|^/dev/nvme|" "a|.*|" ]
'';
services = {
lvm = {
boot.thin.enable = true;
dmeventd.enable = true;
};
smartd = {
enable = true;
autodetect = true;
extraOptions = [ "-A /var/log/smartd/" "--interval=600" ];
};
udev.extraRules = ''
ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="6"
'';
};
environment.systemPackages = with pkgs; [
pciutils
usbutils
partclone
lm_sensors
linuxPackages.cpupower
smartmontools
mstflint
ethtool
hwloc
];
networking = { inherit domain; };
systemd = {
tmpfiles.rules = [
"d /var/log/smartd 0755 root root"
];
network = {
links = {
"10-river-jim-phy" = {
matchConfig = {
PermanentMACAddress = "e0:d5:5e:68:0c:6e";
Driver = "igb";
};
linkConfig.Name = "river-jim-phy";
};
"10-host-jim" = {
matchConfig.PermanentMACAddress = "e0:d5:5e:68:0c:70";
linkConfig.Name = "host-jim";
};
"10-uplink-dave" = {
matchConfig = {
PermanentMACAddress = "24:8a:07:ac:59:c0";
Driver = "mlx5_core";
};
linkConfig = {
Name = "uplink-dave";
MTUBytes = toString hiMTU;
};
};
};
netdevs = mkMerge [
(mkVLAN "lan-hi" vlans.hi)
(mkVLAN "lan-lo-phy" vlans.lo)
{
"25-lan-core".netdevConfig = {
Name = "lan-core";
Kind = "bridge";
};
}
{
"25-lan-lo".netdevConfig = {
Name = "lan-lo";
Kind = "bridge";
};
}
];
networks = {
"50-lan-core" = mkMerge [
(networkdAssignment "lan-core" assignments.core)
{
matchConfig.Name = "lan-core";
vlan = [ "lan-lo-phy" ];
networkConfig.IPv6AcceptRA = mkForce false;
}
];
"50-uplink-dave" = {
matchConfig.Name = "uplink-dave";
vlan = [ "lan-hi" ];
networkConfig = networkd.noL3;
linkConfig.RequiredForOnline = "no";
extraConfig = ''
# cellar
[SR-IOV]
VirtualFunction=0
VLANId=${toString vlans.hi}
LinkState=yes
MACAddress=52:54:00:cc:3e:70
# river RDMA
[SR-IOV]
VirtualFunction=1
VLANId=${toString vlans.hi}
LinkState=yes
MACAddress=52:54:00:8a:8a:f2
# sfh
[SR-IOV]
VirtualFunction=2
VLANId=${toString vlans.hi}
LinkState=yes
MACAddress=52:54:00:ac:15:a9
# sfh containers
[SR-IOV]
VirtualFunction=3
VLANId=${toString vlans.hi}
LinkState=yes
MACAddress=52:54:00:90:34:95
# river LAN trunk
[SR-IOV]
VirtualFunction=4
LinkState=auto
MACAddress=52:54:00:8a:8a:f4
Trust=yes
MACSpoofCheck=no
# slop-cannon (untrusted)
[SR-IOV]
VirtualFunction=5
VLANId=${toString vlans.untrusted}
LinkState=auto
MACAddress=52:54:00:71:22:ea
MACSpoofCheck=yes
'';
};
"60-lan-hi" = mkMerge [
(networkdAssignment "lan-hi" assignments.hi)
{ networkConfig = vlanDns "hi"; }
];
"50-host-jim" = {
matchConfig.Name = "host-jim";
networkConfig = {
Bridge = "lan-core";
} // networkd.noL3;
};
"50-lan-lo-phy" = {
matchConfig.Name = "lan-lo-phy";
networkConfig = {
Bridge = "lan-lo";
} // networkd.noL3;
};
"60-lan-lo" = {
matchConfig.Name = "lan-lo";
linkConfig.RequiredForOnline = "no";
networkConfig = networkd.noL3;
};
};
};
};
my = {
secrets = {
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHzVJpoDz/AAYLJGzU8t6DgZ2sY3oehRqrlSO7C+GWiK";
};
server.enable = true;
deploy.node.hostname = "192.168.68.22";
# Host NQN for guest disks on `cellar`; nothing on `palace` itself boots from it
nvme.uuid = "a5b573fe-59e0-4760-802d-4b6d2df6dcf2";
};
};
};
}