nixos/slop-cannon: Move to a VM on palace
CI / Check, build and cache nixfiles (push) Successful in 45m39s
Update docs / update (push) Successful in 1m0s

Give agents their own kernel, Nix daemon and disk for workloads that
need root. Put the VM on the `untrusted` VLAN with MAC spoof
checking on VF 5.

Export a dedicated partition from `cellar` and connect it on
`palace`. Give the VM its own SSH host key and age secrets for Paseo
and Gitea. Keep the agent home and Paseo identity on persistent
storage and provide `slop-pair` for pairing clients.

Retire the container, restore `sfh` to 8 cores and 32 GiB, and rekey
the shared password secret. Document the VM and its operating
procedures on the relevant box pages.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-18 22:02:14 +00:00
co-authored by Claude Opus 5.5
parent 5058dec3f1
commit a6491c7535
20 changed files with 588 additions and 353 deletions
@@ -128,6 +128,13 @@ in
hostnqn =
"nqn.2014-08.org.nvmexpress:uuid:85d7df36-0de0-431b-b06e-51f7c0a455b4";
serial = "SPDK00000000000003";
}) ++ (nvmfBdev {
# `slop-cannon`'s disk; `palace` connects it and hands it to the VM
bdev = "NVMeRaidp4";
nqn = "nqn.2016-06.io.spdk:slop-cannon";
hostnqn =
"nqn.2014-08.org.nvmexpress:uuid:a5b573fe-59e0-4760-802d-4b6d2df6dcf2";
serial = "SPDK00000000000004";
});
};
};
+65 -2
View File
@@ -3,11 +3,13 @@
./cellar
./river.nix
./sfh
./slop-cannon.nix
];
nixos.systems.palace.configuration = { lib, pkgs, config, systems, allAssignments, ... }:
let
inherit (lib) mkMerge;
inherit (config.my.nvme) uuid;
inherit (lib.my) vm;
inherit (lib.my.c) networkd;
inherit (lib.my.c.home) hiMTU;
@@ -25,6 +27,13 @@
bootindex = 1;
};
};
# `slop-cannon`'s disk: a partition of `cellar`'s RAID, connected here so the guest never
# speaks NVMe-oF itself
slopDisk = {
nqn = "nqn.2016-06.io.spdk:slop-cannon";
device = "/dev/disk/by-id/nvme-SPDK_bdev_Controller_SPDK00000000000004";
};
in
{
systemd.network = {
@@ -99,6 +108,38 @@
}
];
"vm@sfh" = (awaitVM "river");
# Only `vm@slop-cannon` needs this; stops before `cellar` does, and gives up on a dead
# target rather than hanging I/O on `palace`
slop-cannon-disk = mkMerge [
(awaitVM "cellar")
{
description = "Connect slop-cannon's NVMe-oF disk";
path = with pkgs; [ nvme-cli systemd ];
script = ''
until nvme connect -t rdma -a ${allAssignments.cellar.hi.ipv4.address} -s 4420 \
-n ${slopDisk.nqn} -q nqn.2014-08.org.nvmexpress:uuid:${uuid} --ctrl-loss-tmo 60; do
sleep 5
done
udevadm wait --timeout 30 ${slopDisk.device}
'';
preStop = ''
nvme disconnect -n ${slopDisk.nqn}
'';
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
TimeoutStartSec = "5min";
};
}
];
"vm@slop-cannon" = {
requires = [ "slop-cannon-disk.service" ];
bindsTo = [ "slop-cannon-disk.service" ];
after = [ "slop-cannon-disk.service" ];
# Coding agents; keep storage and the router responsive
serviceConfig.CPUWeight = 20;
};
};
my = {
@@ -181,10 +222,10 @@
uuid = "82ec149d-577c-421a-93e2-a9307c756cd8";
cpu = "host,topoext";
smp = {
cpus = 16;
cpus = 8;
threads = 2;
};
memory = 98304;
memory = 32768;
cleanShutdown.timeout = 120;
networks = {
netboot = {
@@ -222,6 +263,28 @@
"device usb-host,hostbus=1,hostport=3"
];
};
slop-cannon = {
uuid = "9de7dc84-7506-445f-8e37-37c4cc6d8ed5";
cpu = "host,topoext";
# Overcommitted so agents can burst; `vm@slop-cannon` has a low CPU weight
smp = {
cpus = 16;
threads = 2;
};
memory = 49152;
cleanShutdown.timeout = 120;
drives = [
installerDisk
(mkMerge [ (vm.blockDisk "disk" slopDisk.device) { frontendOpts.bootindex = 0; } ])
];
hostDevices = {
slop-cannon-ut = {
index = 0;
hostBDF = "44:00.6";
};
};
};
};
};
};
@@ -2,6 +2,5 @@
imports = [
./unifi.nix
./hass.nix
./slop-cannon.nix
];
}
@@ -168,10 +168,6 @@ in
MACVLAN = mkForce "ctrs-hi:host0 ctrs-core:lan-core";
};
};
# nspawn's default `/tmp` is 10% of RAM; agents need more scratch space
slop-cannon = {
filesConfig.TemporaryFileSystem = [ "/tmp:mode=1777,size=32G" ];
};
};
systemd.services = {
@@ -184,11 +180,6 @@ in
RuntimeDirectory = "hass-webcam";
BindPaths = [ "-/dev/video0:/run/hass-webcam/video0" ];
};
# Coding agents; keep the other containers responsive
"systemd-nspawn@slop-cannon".serviceConfig = {
CPUWeight = 20;
MemoryMax = "64G";
};
};
my = {
@@ -212,7 +203,6 @@ in
let
instances = {
unifi = {};
slop-cannon = {};
hass = {
bindMounts = {
"/mnt/frigate" = {
@@ -1,33 +1,30 @@
{ lib, ... }:
let
inherit (lib.my) net;
inherit (lib.my.c.home) domain prefixes vips vlanDns hiMTU;
inherit (lib.my.c.home) domain prefixes vips;
in
{
nixos.systems.slop-cannon = { config, ... }: {
nixos.systems.slop-cannon = {
system = "x86_64-linux";
nixpkgs = "mine";
home-manager = "mine";
rendered = config.configuration.config.my.asContainer;
assignments = {
hi = {
name = "slop-cannon-ctr";
inherit domain;
mtu = hiMTU;
ipv4 = {
address = net.cidr.host 104 prefixes.hi.v4;
mask = 22;
gateway = vips.hi.v4;
};
ipv6 = {
iid = "::5:4";
address = net.cidr.host (65536*5+4) prefixes.hi.v6;
};
# The routers limit traffic from the untrusted VLAN to the Internet
assignments.untrusted = {
inherit domain;
mtu = prefixes.untrusted.mtu;
ipv4 = {
address = net.cidr.host 5 prefixes.untrusted.v4;
mask = 24;
gateway = vips.untrusted.v4;
};
ipv6 = {
iid = "::5";
address = net.cidr.host 5 prefixes.untrusted.v6;
};
};
configuration = { lib, config, pkgs, inputs, assignments, ... }:
configuration = { lib, modulesPath, config, pkgs, assignments, ... }:
let
inherit (builtins) toJSON;
inherit (lib) mkMerge;
@@ -35,15 +32,8 @@ in
agentUser = "slop-cannon";
agentHome = "/home/${agentUser}";
# TODO: Drop once Codex recognises bubblewrap's newer `/proc` error wording.
# Restore the diagnostic its procfs preflight expects so it uses its container fallback.
codexBubblewrap = pkgs.bubblewrap.overrideAttrs (old: {
postPatch = (old.postPatch or "") + ''
substituteInPlace bubblewrap.c \
--replace-fail "Can't mount proc on %s" "Can't mount proc on /newroot%s"
'';
});
giteaKey = config.age.secrets."slop-cannon/gitea-key".path;
paseoPasswordFile = config.age.secrets."slop-cannon/paseo.env".path;
# Steer agents to the approval broker; these match command text, so they aren't a boundary
claudeDeny = [
@@ -71,23 +61,107 @@ in
if ! jj git clone --colocate "git@git.nul.ie:$repo.git" "$dest"; then
echo >&2
echo "Clone failed. If Gitea refused the key, add it as a write-enabled deploy key on $repo:" >&2
cat "$HOME/.ssh/id_ed25519.pub" >&2
ssh-keygen -y -f ${giteaKey} >&2
exit 1
fi
paseo workspace create --isolation local --path "$dest"
'';
};
slopPair = pkgs.writeShellApplication {
name = "slop-pair";
runtimeInputs = [ pkgs.paseo ];
text = ''
if [ "$#" -ne 0 ]; then
echo "usage: slop-pair" >&2
exit 2
fi
paseo daemon pair
printf '\n'
exec doas ${pkgs.coreutils}/bin/cat ${paseoPasswordFile}
'';
};
in
{
imports = [
"${modulesPath}/profiles/qemu-guest.nix"
];
config = {
boot = {
# Load the passed-through ConnectX VF even when PCI probing doesn't autoload it.
kernelModules = [ "kvm-amd" "mlx5_core" ];
kernelParams = [ "console=ttyS0,115200n8" ];
initrd = {
availableKernelModules = [ "virtio_pci" "ahci" "sr_mod" "virtio_blk" ];
kernelModules = [ "dm-snapshot" ];
};
# On disk rather than in the tmpfs root, so agents' scratch files don't use RAM
tmp.cleanOnBoot = true;
};
hardware = {
enableRedistributableFirmware = true;
};
fileSystems = {
"/boot" = {
device = "/dev/disk/by-partlabel/esp";
fsType = "vfat";
};
"/nix" = {
device = "/dev/main/slop-cannon-nix";
fsType = "ext4";
};
"/persist" = {
device = "/dev/main/slop-cannon-persist";
fsType = "ext4";
neededForBoot = true;
};
"/tmp" = {
device = "/dev/main/slop-cannon-tmp";
fsType = "ext4";
# Return deleted scratch space to the thin pool straight away
options = [ "discard" ];
};
};
networking = { inherit domain; };
services = {
# The LVs are thin, so space moves between them as needed
lvm = {
boot.thin.enable = true;
dmeventd.enable = true;
};
fstrim.enable = true;
};
# Grow the pool into the VG's spare space before it fills
environment.etc."lvm/lvm.conf".text = ''
activation/thin_pool_autoextend_threshold = 80
activation/thin_pool_autoextend_percent = 10
'';
my = {
deploy.enable = false;
server.enable = true;
# Root inside this VM belongs to the agents, so it gets no shared secrets: in particular
# not `user-passwd.txt`
user = {
passwordSecret = null;
config = {
password = null;
hashedPassword = "!";
};
};
secrets = {
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWhNyGER3E0cgpsl6eIiG1mJFcS6hckU74t9vzMY4Le";
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBCkgBZh/YejY+gDnGa9r5SPo1+aHWPAAAIfENE7t0Om";
files = {
"slop-cannon/gitea-key".owner = agentUser;
# `PASEO_PASSWORD=...`
"slop-cannon/paseo.env" = { };
};
@@ -109,6 +183,8 @@ in
isNormalUser = true;
uid = uids.slop-cannon;
group = agentUser;
# Agents get root (passwordless `doas`) and can run nested VMs
extraGroups = [ "wheel" "kvm" ];
hashedPassword = "!";
shell = pkgs.fish;
openssh.authorizedKeys.keyFiles = [ lib.my.c.sshKeyFiles.me ];
@@ -118,7 +194,7 @@ in
home-manager.users.${agentUser} = {
my.agents.enable = true;
home.packages = [ slopClone codexBubblewrap ];
home.packages = [ slopClone slopPair ];
programs = {
claude-code.settings.permissions = {
@@ -129,12 +205,12 @@ in
# The shared config pins `IdentityFile` for every host
ssh.settings."git.nul.ie" = {
IdentityFile = [ "~/.ssh/id_ed25519" ];
IdentityFile = [ giteaKey ];
};
};
};
# Managed settings can't be edited by agents (unlike `~/.claude`)
# Not editable by the agent user without `doas`; a convenience, not a boundary
environment.etc."claude-code/managed-settings.json".text = toJSON {
permissions = {
disableBypassPermissionsMode = "disable";
@@ -143,34 +219,37 @@ in
};
systemd = {
network.networks."80-container-host0" = mkMerge [
(networkdAssignment "host0" assignments.hi)
{ networkConfig = vlanDns "hi"; }
];
services = {
# Box-wide key for Gitea deploy keys
slop-cannon-ssh-key = {
description = "Generate ${agentUser}'s SSH key";
wantedBy = [ "multi-user.target" ];
unitConfig.RequiresMountsFor = agentHome;
path = [ pkgs.openssh ];
script = ''
if [ ! -e ~/.ssh/id_ed25519 ]; then
install -d -m 700 ~/.ssh
ssh-keygen -q -t ed25519 -N "" -C "${agentUser}@${config.networking.hostName}" -f ~/.ssh/id_ed25519
fi
'';
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
User = agentUser;
network = {
links = {
"10-lan-untrusted" = {
matchConfig = {
Driver = "mlx5_core";
PermanentMACAddress = "52:54:00:71:22:ea";
};
linkConfig = {
Name = "lan-untrusted";
MTUBytes = toString prefixes.untrusted.mtu;
};
};
};
networks = {
"80-lan-untrusted" = mkMerge [
(networkdAssignment "lan-untrusted" assignments.untrusted)
{
# As `untrusted`'s DHCP and RAs hand out; the routers' resolvers aren't reachable
networkConfig.DNS = [
"1.1.1.1" "1.0.0.1"
"2606:4700:4700::1111" "2606:4700:4700::1001"
];
}
];
};
};
services = {
paseo = {
unitConfig.RequiresMountsFor = agentHome;
serviceConfig.EnvironmentFile = config.age.secrets."slop-cannon/paseo.env".path;
serviceConfig.EnvironmentFile = paseoPasswordFile;
# Don't interrupt running agents on deploy; restart by hand when convenient
restartIfChanged = false;
};