dev/nixfiles
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Working "shill From Home" full network boot

Browse Source
This commit is contained in:
Jack O'Sullivan
2024-06-29 23:12:21 +01:00
parent 84ca556c47
commit f9870abc9e
75 changed files with 1550 additions and 1088 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
nixos/boxes/home/palace/vms/cellar/default.nix
View File

@@ -66,6 +66,7 @@ in
];
services = {
fstrim.enable = true;
netdata.enable = true;
};

6
nixos/boxes/home/palace/vms/cellar/spdk.nix
View File

@@ -129,6 +129,12 @@ in
hostnqn =
"nqn.2014-08.org.nvmexpress:uuid:2230b066-a674-4f45-a1dc-f7727b3a9e7b";
serial = "SPDK00000000000002";
}) ++ (nvmfBdev {
bdev = "NVMeRaidp3";
nqn = "nqn.2016-06.io.spdk:sfh";
hostnqn =
"nqn.2014-08.org.nvmexpress:uuid:85d7df36-0de0-431b-b06e-51f7c0a455b4";
serial = "SPDK00000000000003";
});
};
};

13
nixos/boxes/home/palace/vms/default.nix
View File

@@ -2,6 +2,7 @@
imports = [
./cellar
./river.nix
./sfh
];
nixos.systems.palace.configuration = { lib, pkgs, config, systems, allAssignments, ... }:
@@ -57,11 +58,11 @@
systemd.services =
let
awaitCellar = {
after = [ "vm@cellar.service" ];
bindsTo = [ "vm@cellar.service" ];
awaitVM = system: {
after = [ "vm@${system}.service" ];
bindsTo = [ "vm@${system}.service" ];
preStart = ''
until ${pkgs.netcat}/bin/nc -w1 -z ${allAssignments.cellar.hi.ipv4.address} 22; do
until ${pkgs.netcat}/bin/nc -w1 -z ${allAssignments.${system}.hi.ipv4.address} 22; do
sleep 1
done
'';
@@ -81,13 +82,13 @@
vtapUnit = "sys-subsystem-net-devices-vm\\x2det1g0.device";
in
mkMerge [
awaitCellar
(awaitVM "cellar")
{
requires = [ vtapUnit ];
after = [ vtapUnit ];
}
];
"vm@sfh" = awaitCellar;
"vm@sfh" = (awaitVM "river");
};
my = {

57
nixos/boxes/home/palace/vms/river.nix
View File

@@ -10,18 +10,7 @@
let
inherit (lib.my) networkdAssignment mkVLAN;
inherit (lib.my.c) networkd;
inherit (lib.my.c.home) vlans;
lanLink = {
matchConfig = {
Driver = "mlx5_core";
PermanentMACAddress = "52:54:00:8a:8a:f2";
};
linkConfig = {
Name = "lan";
MTUBytes = toString lib.my.c.home.hiMTU;
};
};
inherit (lib.my.c.home) vlans domain prefixes roceBootModules;
in
{
imports = [
@@ -30,29 +19,16 @@
config = {
boot = {
kernelModules = [ "kvm-intel" ];
kernelModules = [ "kvm-amd" ];
kernelParams = [ "console=ttyS0,115200n8" ];
initrd = {
availableKernelModules = [
"virtio_pci" "ahci" "sr_mod" "virtio_blk"
"ib_core" "ib_uverbs" "mlx5_core" "mlx5_ib" "8021q"
"rdma_cm" "iw_cm" "ib_cm" "nvme_core" "nvme_rdma"
];
kernelModules = [ "dm-snapshot" "nvme-fabrics" ];
] ++ roceBootModules;
kernelModules = [ "dm-snapshot" ];
systemd = {
extraBin = with pkgs; {
dmesg = "${util-linux}/bin/dmesg";
ip = "${iproute2}/bin/ip";
};
extraConfig = ''
DefaultTimeoutStartSec=50
DefaultDeviceTimeoutSec=50
'';
network = {
enable = true;
wait-online.enable = true;
links."10-lan" = lanLink;
# Don't need to put the link config here, they're copied from main config
netdevs = mkVLAN "lan-hi" vlans.hi;
networks = {
"20-lan" = {
@@ -70,9 +46,6 @@
hardware = {
enableRedistributableFirmware = true;
cpu = {
intel.updateMicrocode = true;
};
};
fileSystems = {
@@ -96,6 +69,7 @@
boot.thin.enable = true;
dmeventd.enable = true;
};
fstrim.enable = true;
};
systemd.network = {
@@ -114,7 +88,16 @@
};
};
"10-lan" = lanLink;
"10-lan" = {
matchConfig = {
Driver = "mlx5_core";
PermanentMACAddress = "52:54:00:8a:8a:f2";
};
linkConfig = {
Name = "lan";
MTUBytes = toString lib.my.c.home.hiMTU;
};
};
};
# So we don't drop the IP we use to connect to NVMe-oF!
@@ -134,6 +117,14 @@
};
};
netboot.server = {
enable = true;
ip = assignments.lo.ipv4.address;
host = "boot.${domain}";
allowedPrefixes = with prefixes; [ hi.v4 hi.v6 lo.v4 lo.v6 ];
instances = [ "sfh" ];
};
deploy.node.hostname = "192.168.68.1";
};
};

122
nixos/boxes/home/palace/vms/sfh/default.nix Normal file
View File

@@ -0,0 +1,122 @@
{ lib, ... }:
let
inherit (lib.my) net;
inherit (lib.my.c) pubDomain;
inherit (lib.my.c.home) domain prefixes vips hiMTU roceBootModules;
in
{
config.nixos.systems.sfh = {
system = "x86_64-linux";
nixpkgs = "mine";
home-manager = "mine";
assignments = {
hi = {
inherit domain;
mtu = hiMTU;
ipv4 = {
address = net.cidr.host 81 prefixes.hi.v4;
mask = 22;
gateway = vips.hi.v4;
};
ipv6 = {
iid = "::4:2";
address = net.cidr.host (65536*4+2) prefixes.hi.v6;
};
};
};
configuration = { lib, modulesPath, pkgs, config, assignments, allAssignments, ... }:
let
inherit (lib) mkMerge;
inherit (lib.my) networkdAssignment;
inherit (lib.my.c.home) domain;
in
{
imports = [
"${modulesPath}/profiles/qemu-guest.nix"
];
config = {
boot = {
kernelModules = [ "kvm-amd" ];
kernelParams = [ "console=ttyS0,115200n8" ];
initrd = {
availableKernelModules = [
"virtio_pci" "ahci" "sr_mod" "virtio_blk"
] ++ roceBootModules;
kernelModules = [ "dm-snapshot" ];
systemd = {
network = {
networks = {
"20-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
};
};
};
};
};
hardware = {
enableRedistributableFirmware = true;
};
fileSystems = {
"/nix" = {
device = "/dev/main/nix";
fsType = "ext4";
};
"/persist" = {
device = "/dev/main/persist";
fsType = "ext4";
neededForBoot = true;
};
};
services = {
lvm = {
boot.thin.enable = true;
dmeventd.enable = true;
};
};
systemd.network = {
links = {
"10-lan-hi" = {
matchConfig = {
Driver = "mlx5_core";
PermanentMACAddress = "52:54:00:ac:15:a9";
};
linkConfig = {
Name = "lan-hi";
MTUBytes = toString lib.my.c.home.hiMTU;
};
};
};
networks."30-lan-hi" = mkMerge [
(networkdAssignment "lan-hi" assignments.hi)
# So we don't drop the IP we use to connect to NVMe-oF!
{ networkConfig.KeepConfiguration = "static"; }
];
};
my = {
secrets = {
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAaav5Se1E/AbqEXmADryVszYfNDscyP6jrWioN57R7";
};
server.enable = true;
netboot.client = {
enable = true;
};
nvme = {
uuid = "85d7df36-0de0-431b-b06e-51f7c0a455b4";
boot = {
nqn = "nqn.2016-06.io.spdk:sfh";
address = "192.168.68.80";
};
};
};
};
};
};
}