dev/nixfiles
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

nixos/river: Initial NVMe-oF booting river :)
All checks were successful
CI / Check, build and cache Nix flake (push) Successful in 17m32s
Details

Browse Source
This commit is contained in:
Jack O'Sullivan
2023-12-11 01:55:02 +00:00
parent 33eded0626
commit 5e5f70501c
55 changed files with 1116 additions and 862 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
nixos/boxes/home/palace/default.nix
View File

@@ -35,6 +35,7 @@ in
let
inherit (lib) mkForce mkMerge;
inherit (lib.my) networkdAssignment;
inherit (lib.my.c) networkd;
in
{
boot = {
@@ -87,7 +88,7 @@ in
extraOptions = [ "-A /var/log/smartd/" "--interval=600" ];
};
udev.extraRules = ''
ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="3"
ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="2"
'';
};
@@ -113,11 +114,14 @@ in
network = {
links = {
"10-et1g0" = {
matchConfig.MACAddress = "e0:d5:5e:68:0c:6e";
matchConfig = {
PermanentMACAddress = "e0:d5:5e:68:0c:6e";
Driver = "igb";
};
linkConfig.Name = "et1g0";
};
"10-lan-core" = {
matchConfig.MACAddress = "e0:d5:5e:68:0c:70";
matchConfig.PermanentMACAddress = "e0:d5:5e:68:0c:70";
linkConfig.Name = "lan-core";
};
"10-et100g" = {
@@ -148,7 +152,8 @@ in
"50-et100g" = {
matchConfig.Name = "et100g";
vlan = [ "lan-hi" ];
networkConfig.IPv6AcceptRA = false;
networkConfig = networkd.noL3;
linkConfig.RequiredForOnline = "no";
extraConfig = ''
# cellar
[SR-IOV]
@@ -156,6 +161,12 @@ in
VLANId=${toString vlans.hi}
LinkState=yes
MACAddress=52:54:00:cc:3e:70
# river
[SR-IOV]
VirtualFunction=1
LinkState=yes
MACAddress=52:54:00:8a:8a:f2
'';
};
"60-lan-hi" = mkMerge [

54
nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json
View File

@@ -242,7 +242,7 @@
{
"method": "nvmf_create_subsystem",
"params": {
"nqn": "nqn.2016-06.io.spdk:cellar",
"nqn": "nqn.2016-06.io.spdk:river",
"allow_any_host": false,
"serial_number": "SPDK00000000000001",
"model_number": "SPDK bdev Controller",
@@ -255,26 +255,64 @@
{
"method": "nvmf_subsystem_add_host",
"params": {
"nqn": "nqn.2016-06.io.spdk:cellar",
"host": "nqn.2014-08.org.nvmexpress:uuid:de712327-7fc8-6e54-15af-c87f546e170f"
"nqn": "nqn.2016-06.io.spdk:river",
"host": "nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9"
}
},
{
"method": "nvmf_subsystem_add_ns",
"params": {
"nqn": "nqn.2016-06.io.spdk:cellar",
"nqn": "nqn.2016-06.io.spdk:river",
"namespace": {
"nsid": 1,
"bdev_name": "Raid0",
"nguid": "FA9D43CFD8D54885BB73F038BDC31B25",
"uuid": "fa9d43cf-d8d5-4885-bb73-f038bdc31b25"
"bdev_name": "9c545b0b-5b9c-4a32-b1eb-483adf5369fc",
"nguid": "9C545B0B5B9C4A32B1EB483ADF5369FC",
"uuid": "9c545b0b-5b9c-4a32-b1eb-483adf5369fc"
}
}
},
{
"method": "nvmf_subsystem_add_listener",
"params": {
"nqn": "nqn.2016-06.io.spdk:cellar",
"nqn": "nqn.2016-06.io.spdk:river",
"listen_address": {
"trtype": "RDMA",
"adrfam": "IPv4",
"traddr": "192.168.68.80",
"trsvcid": "4420"
},
"secure_channel": false
}
},
{
"method": "nvmf_create_subsystem",
"params": {
"nqn": "nqn.2016-06.io.spdk:sfh",
"allow_any_host": false,
"serial_number": "SPDK00000000000002",
"model_number": "SPDK bdev Controller",
"max_namespaces": 32,
"min_cntlid": 1,
"max_cntlid": 65519,
"ana_reporting": false
}
},
{
"method": "nvmf_subsystem_add_ns",
"params": {
"nqn": "nqn.2016-06.io.spdk:sfh",
"namespace": {
"nsid": 1,
"bdev_name": "780ddeb9-646d-4331-a0d5-3b0aecd3bf3e",
"nguid": "780DDEB9646D4331A0D53B0AECD3BF3E",
"uuid": "780ddeb9-646d-4331-a0d5-3b0aecd3bf3e"
}
}
},
{
"method": "nvmf_subsystem_add_listener",
"params": {
"nqn": "nqn.2016-06.io.spdk:sfh",
"listen_address": {
"trtype": "RDMA",
"adrfam": "IPv4",

70
nixos/boxes/home/palace/vms/default.nix
View File

@@ -1,12 +1,14 @@
{
imports = [
./cellar
./river.nix
];
nixos.systems.palace.configuration = { lib, pkgs, config, systems, ... }:
let
inherit (lib) mkMerge;
inherit (lib.my) vm;
inherit (lib.my.c) networkd;
installerDisk = {
name = "installer";
@@ -23,6 +25,36 @@
};
in
{
systemd.network = {
netdevs = {
"25-vm-et1g0" = {
netdevConfig = {
Name = "vm-et1g0";
Kind = "macvtap";
};
# TODO: Upstream this missing section
extraConfig = ''
[MACVTAP]
Mode=passthru
'';
};
};
networks = {
"75-et1g0" = {
matchConfig.Name = "et1g0";
linkConfig.RequiredForOnline = "no";
networkConfig = {
MACVTAP = "vm-et1g0";
} // networkd.noL3;
};
"75-vm-et1g0" = {
matchConfig.Name = "vm-et1g0";
linkConfig.RequiredForOnline = "no";
networkConfig = networkd.noL3;
};
};
};
systemd.services = {
"vm@cellar" = {
serviceConfig = {
@@ -31,6 +63,14 @@
NUMAMask = "1";
};
};
"vm@river" =
let
vtapUnit = "sys-subsystem-net-devices-vm\\x2det1g0.device";
in
{
requires = [ vtapUnit ];
after = [ vtapUnit ];
};
};
my = {
@@ -68,6 +108,36 @@
};
};
};
river = {
uuid = "12b52d80-ccb6-418d-9b2e-2be34bff3cd9";
cpu = "host,topoext";
smp = {
cpus = 3;
threads = 2;
};
memory = 4096;
networks = {
et1g0 = {
ifname = "vm-et1g0";
bridge = null;
tapFD = 100;
# Real hardware MAC
mac = "e0:d5:5e:68:0c:6e";
waitOnline = false;
};
};
drives = [
installerDisk
(mkMerge [ (vm.disk "river" "esp") { frontendOpts.bootindex = 0; } ])
];
hostDevices = {
et100g0vf1 = {
index = 0;
hostBDF = "44:00.2";
};
};
};
};
};
};

154
nixos/boxes/home/palace/vms/river.nix Normal file
View File

@@ -0,0 +1,154 @@
{
imports = [ (import ../../routing-common 0) ];
config.nixos.systems.river = {
system = "x86_64-linux";
nixpkgs = "mine";
home-manager = "mine";
configuration = { lib, modulesPath, pkgs, config, assignments, allAssignments, ... }:
let
inherit (lib.my) networkdAssignment mkVLAN;
inherit (lib.my.c) networkd;
inherit (lib.my.c.home) vlans;
lanLink = {
matchConfig = {
Driver = "mlx5_core";
PermanentMACAddress = "52:54:00:8a:8a:f2";
};
linkConfig = {
Name = "lan";
MTUBytes = toString lib.my.c.home.hiMTU;
};
};
in
{
imports = [
"${modulesPath}/profiles/qemu-guest.nix"
];
config = {
boot = {
kernelModules = [ "kvm-intel" ];
kernelParams = [ "console=ttyS0,115200n8" ];
initrd = {
availableKernelModules = [
"virtio_pci" "ahci" "sr_mod" "virtio_blk"
"ib_core" "ib_uverbs" "mlx5_core" "mlx5_ib" "8021q"
"rdma_cm" "iw_cm" "ib_cm" "nvme_core" "nvme_rdma"
];
kernelModules = [ "dm-snapshot" "nvme-fabrics" ];
systemd = {
extraBin = with pkgs; {
dmesg = "${util-linux}/bin/dmesg";
ip = "${iproute2}/bin/ip";
nvme = "${nvme-cli}/bin/nvme";
};
extraConfig = ''
DefaultTimeoutStartSec=50
DefaultDeviceTimeoutSec=50
'';
network = {
enable = true;
wait-online.enable = true;
links."10-lan" = lanLink;
netdevs = mkVLAN "lan-hi" vlans.hi;
networks = {
"20-lan" = {
matchConfig.Name = "lan";
vlan = [ "lan-hi" ];
linkConfig.RequiredForOnline = "no";
networkConfig = networkd.noL3;
};
"30-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
};
};
services.connect-nvme = {
description = "Connect NVMe-oF";
before = [ "initrd-root-device.target" ];
after = [ "systemd-networkd-wait-online.service" ];
requires = [ "systemd-networkd-wait-online.service" ];
serviceConfig = {
Type = "oneshot";
Restart = "on-failure";
RestartSec = 10;
};
script = ''
${pkgs.nvme-cli}/bin/nvme connect -t rdma -a 192.168.68.80 \
-n nqn.2016-06.io.spdk:river -q nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9
'';
wantedBy = [ "initrd-root-device.target" ];
};
};
};
};
hardware = {
enableRedistributableFirmware = true;
cpu = {
intel.updateMicrocode = true;
};
};
fileSystems = {
"/boot" = {
device = "/dev/disk/by-partuuid/3ec6c49e-b485-40cb-8eff-315581ac6fe9";
fsType = "vfat";
};
"/nix" = {
device = "/dev/main/nix";
fsType = "ext4";
};
"/persist" = {
device = "/dev/main/persist";
fsType = "ext4";
neededForBoot = true;
};
};
services = {
lvm = {
boot.thin.enable = true;
dmeventd.enable = true;
};
};
systemd.network = {
links = {
"10-wan" = {
matchConfig = {
# Matching against MAC address seems to break VLAN interfaces
# (since they share the same MAC address)
Driver = "virtio_net";
PermanentMACAddress = "e0:d5:5e:68:0c:6e";
};
linkConfig = {
Name = "wan";
RxBufferSize = 4096;
TxBufferSize = 4096;
};
};
"10-lan" = lanLink;
};
# So we don't drop the IP we use to connect to NVMe-oF!
networks."60-lan-hi".networkConfig.KeepConfiguration = "static";
};
my = {
secrets = {
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP9uFa4z9WPuXRFVA+PClQSitQCSPckhKTxo1Hq585Oa";
};
server.enable = true;
deploy.node.hostname = "192.168.68.1";
};
};
};
};
}