dev/nixfiles
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

nixos/cellar: Working NVMe-oF
All checks were successful
CI / Check, build and cache Nix flake (push) Successful in 16m40s
Details

Browse Source
This commit is contained in:
Jack O'Sullivan
2023-12-10 02:29:53 +00:00
parent 9268256309
commit 33eded0626
54 changed files with 1421 additions and 846 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
nixos/boxes/home/palace/default.nix
View File

@@ -5,6 +5,8 @@ let
inherit (lib.my.c.home) domain vlans prefixes vips;
in
{
imports = [ ./vms ];
nixos.systems.palace = {
system = "x86_64-linux";
nixpkgs = "mine-stable";
@@ -47,7 +49,7 @@ in
kernelModules = [ "kvm-amd" ];
kernelParams = [ "amd_iommu=on" ];
initrd = {
availableKernelModules = [ "nvme" "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
availableKernelModules = [ "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
};
};
@@ -64,22 +66,29 @@ in
fsType = "vfat";
};
"/nix" = {
device = "/dev/disk/by-partuuid/71695225-b306-41e6-83f8-c5cde57c06f7";
device = "/dev/disk/by-uuid/450e1f72-238a-4160-98b8-b5e6d0d6fdf6";
fsType = "ext4";
};
"/persist" = {
device = "/dev/disk/by-partuuid/9991aec3-c062-41d1-971e-e056b63370f0";
device = "/dev/disk/by-uuid/9d6d53a8-dff8-49e0-9bc3-fb5f7c6760d0";
fsType = "ext4";
neededForBoot = true;
};
};
services = {
lvm = {
boot.thin.enable = true;
dmeventd.enable = true;
};
smartd = {
enable = true;
autodetect = true;
extraOptions = [ "-A /var/log/smartd/" "--interval=600" ];
};
udev.extraRules = ''
ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="3"
'';
};
environment.systemPackages = with pkgs; [
@@ -91,6 +100,7 @@ in
smartmontools
mstflint
ethtool
hwloc
];
networking.domain = "h.${pubDomain}";
@@ -100,25 +110,6 @@ in
"d /var/log/smartd 0755 root root"
];
services = {
setup-mlx5-vfs = {
description = "Enable 100G NIC VFs";
serviceConfig = {
Type = "oneshot";
};
script = ''
vfsFile=/sys/class/infiniband/mlx5_0/device/sriov_numvfs
until [ -f "$vfsFile" ]; do
sleep 0.2
done
echo 3 > "$vfsFile"
'';
wantedBy = [ "multi-user.target" ];
before = [ "network-pre.target" ];
};
};
network = {
links = {
"10-et1g0" = {
@@ -158,6 +149,14 @@ in
matchConfig.Name = "et100g";
vlan = [ "lan-hi" ];
networkConfig.IPv6AcceptRA = false;
extraConfig = ''
# cellar
[SR-IOV]
VirtualFunction=0
VLANId=${toString vlans.hi}
LinkState=yes
MACAddress=52:54:00:cc:3e:70
'';
};
"60-lan-hi" = mkMerge [
(networkdAssignment "lan-hi" assignments.hi)

128
nixos/boxes/home/palace/vms/cellar/default.nix Normal file
View File

@@ -0,0 +1,128 @@
{ lib, ... }:
let
inherit (lib.my) net;
inherit (lib.my.c) pubDomain;
inherit (lib.my.c.home) domain prefixes vips;
in
{
nixos.systems.cellar = {
system = "x86_64-linux";
nixpkgs = "mine";
assignments = {
hi = {
inherit domain;
ipv4 = {
address = net.cidr.host 80 prefixes.hi.v4;
mask = 22;
gateway = vips.hi.v4;
};
};
};
configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }:
let
inherit (lib) mkMerge;
inherit (lib.my) networkdAssignment;
spdk = pkgs.spdk.overrideAttrs (o: {
configureFlags = o.configureFlags ++ [ "--with-rdma" ];
});
in
{
imports = [
"${modulesPath}/profiles/qemu-guest.nix"
];
config = mkMerge [
{
boot = {
kernelParams = [ "console=ttyS0,115200n8" ];
blacklistedKernelModules = [ "nvme" ];
};
fileSystems = {
"/boot" = {
device = "/dev/disk/by-partuuid/f7562ee6-34c1-4e94-8ae7-c6e71794d563";
fsType = "vfat";
};
"/nix" = {
device = "/dev/disk/by-uuid/f31f6abd-0832-4014-a761-f3c3126d5739";
fsType = "ext4";
};
"/persist" = {
device = "/dev/disk/by-uuid/620364e3-3a30-4704-be80-8593516e7482";
fsType = "ext4";
neededForBoot = true;
};
};
environment.systemPackages = with pkgs; [
pciutils
spdk
(pkgs.writeShellScriptBin "spdk-rpc" ''
exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@"
'')
];
services = {
netdata.enable = true;
};
systemd.services = {
spdk-nvmf = {
description = "SPDK NVMe-oF target";
path = with pkgs; [
bash
python3
kmod
gawk
util-linux
];
after = [ "systemd-networkd-wait-online@lan-hi.service" ];
preStart = ''
${spdk.src}/scripts/setup.sh
'';
serviceConfig.ExecStart = "${spdk}/bin/spdk_tgt -c ${./spdk_nvmf.json}";
wantedBy = [ "multi-user.target" ];
};
};
systemd.network = {
links = {
"10-lan-hi" = {
matchConfig.PermanentMACAddress = "52:54:00:cc:3e:70";
linkConfig = {
Name = "lan-hi";
MTUBytes = "9000";
};
};
};
networks = {
"80-vms" = mkMerge [
(networkdAssignment "lan-hi" assignments.hi)
{
networkConfig.DNS = [
(allAssignments.stream.hi.ipv4.address)
# (allAssignments.river.hi.ipv4.address)
];
}
];
};
};
my = {
secrets.key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDcklmJp8xVRddNDU1DruKV+Ipim3Jtl6nE1oCWmpmZH";
server.enable = true;
deploy.node.hostname = "192.168.68.80";
firewall = {
tcp.allowed = [ 19999 ];
};
};
}
];
};
};
}

337
nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json Normal file
View File

@@ -0,0 +1,337 @@
{
"subsystems": [
{
"subsystem": "scheduler",
"config": [
{
"method": "framework_set_scheduler",
"params": {
"name": "static"
}
}
]
},
{
"subsystem": "vmd",
"config": []
},
{
"subsystem": "sock",
"config": [
{
"method": "sock_impl_set_options",
"params": {
"impl_name": "posix",
"recv_buf_size": 2097152,
"send_buf_size": 2097152,
"enable_recv_pipe": true,
"enable_quickack": false,
"enable_placement_id": 0,
"enable_zerocopy_send_server": true,
"enable_zerocopy_send_client": false,
"zerocopy_threshold": 0,
"tls_version": 0,
"enable_ktls": false
}
},
{
"method": "sock_impl_set_options",
"params": {
"impl_name": "ssl",
"recv_buf_size": 4096,
"send_buf_size": 4096,
"enable_recv_pipe": true,
"enable_quickack": false,
"enable_placement_id": 0,
"enable_zerocopy_send_server": true,
"enable_zerocopy_send_client": false,
"zerocopy_threshold": 0,
"tls_version": 0,
"enable_ktls": false
}
}
]
},
{
"subsystem": "iobuf",
"config": [
{
"method": "iobuf_set_options",
"params": {
"small_pool_count": 8192,
"large_pool_count": 1024,
"small_bufsize": 8192,
"large_bufsize": 135168
}
}
]
},
{
"subsystem": "accel",
"config": [
{
"method": "accel_set_options",
"params": {
"small_cache_size": 128,
"large_cache_size": 16,
"task_count": 2048,
"sequence_count": 2048,
"buf_count": 2048
}
}
]
},
{
"subsystem": "bdev",
"config": [
{
"method": "bdev_set_options",
"params": {
"bdev_io_pool_size": 65535,
"bdev_io_cache_size": 256,
"bdev_auto_examine": true
}
},
{
"method": "bdev_nvme_set_options",
"params": {
"action_on_timeout": "none",
"timeout_us": 0,
"timeout_admin_us": 0,
"keep_alive_timeout_ms": 10000,
"transport_retry_count": 4,
"arbitration_burst": 0,
"low_priority_weight": 0,
"medium_priority_weight": 0,
"high_priority_weight": 0,
"nvme_adminq_poll_period_us": 10000,
"nvme_ioq_poll_period_us": 0,
"io_queue_requests": 512,
"delay_cmd_submit": true,
"bdev_retry_count": 3,
"transport_ack_timeout": 0,
"ctrlr_loss_timeout_sec": 0,
"reconnect_delay_sec": 0,
"fast_io_fail_timeout_sec": 0,
"generate_uuids": false,
"transport_tos": 0,
"io_path_stat": false,
"allow_accel_sequence": false
}
},
{
"method": "bdev_nvme_attach_controller",
"params": {
"name": "NVMe0",
"trtype": "PCIe",
"traddr": "02:00.0",
"prchk_reftag": false,
"prchk_guard": false,
"ctrlr_loss_timeout_sec": 0,
"reconnect_delay_sec": 0,
"fast_io_fail_timeout_sec": 0,
"hostnqn": "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775",
"hdgst": false,
"ddgst": false
}
},
{
"method": "bdev_nvme_attach_controller",
"params": {
"name": "NVMe1",
"trtype": "PCIe",
"traddr": "03.00.0",
"prchk_reftag": false,
"prchk_guard": false,
"ctrlr_loss_timeout_sec": 0,
"reconnect_delay_sec": 0,
"fast_io_fail_timeout_sec": 0,
"hostnqn": "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775",
"hdgst": false,
"ddgst": false
}
},
{
"method": "bdev_nvme_attach_controller",
"params": {
"name": "NVMe2",
"trtype": "PCIe",
"traddr": "04.00.0",
"prchk_reftag": false,
"prchk_guard": false,
"ctrlr_loss_timeout_sec": 0,
"reconnect_delay_sec": 0,
"fast_io_fail_timeout_sec": 0,
"hostnqn": "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775",
"hdgst": false,
"ddgst": false
}
},
{
"method": "bdev_nvme_set_hotplug",
"params": {
"period_us": 100000,
"enable": false
}
},
{
"method": "bdev_raid_create",
"params": {
"name": "Raid0",
"strip_size_kb": 64,
"raid_level": "raid0",
"base_bdevs": [
"NVMe0n1",
"NVMe1n1",
"NVMe2n1"
]
}
},
{
"method": "bdev_wait_for_examine"
}
]
},
{
"subsystem": "nvmf",
"config": [
{
"method": "nvmf_set_config",
"params": {
"discovery_filter": "match_any",
"admin_cmd_passthru": {
"identify_ctrlr": false
}
}
},
{
"method": "nvmf_set_max_subsystems",
"params": {
"max_subsystems": 1024
}
},
{
"method": "nvmf_set_crdt",
"params": {
"crdt1": 0,
"crdt2": 0,
"crdt3": 0
}
},
{
"method": "nvmf_create_transport",
"params": {
"trtype": "RDMA",
"max_queue_depth": 128,
"max_io_qpairs_per_ctrlr": 127,
"in_capsule_data_size": 4096,
"max_io_size": 131072,
"io_unit_size": 8192,
"max_aq_depth": 128,
"num_shared_buffers": 4095,
"buf_cache_size": 4294967295,
"dif_insert_or_strip": false,
"zcopy": false,
"max_srq_depth": 4096,
"no_srq": false,
"acceptor_backlog": 100,
"no_wr_batching": false,
"abort_timeout_sec": 1
}
},
{
"method": "nvmf_create_subsystem",
"params": {
"nqn": "nqn.2016-06.io.spdk:cellar",
"allow_any_host": false,
"serial_number": "SPDK00000000000001",
"model_number": "SPDK bdev Controller",
"max_namespaces": 32,
"min_cntlid": 1,
"max_cntlid": 65519,
"ana_reporting": false
}
},
{
"method": "nvmf_subsystem_add_host",
"params": {
"nqn": "nqn.2016-06.io.spdk:cellar",
"host": "nqn.2014-08.org.nvmexpress:uuid:de712327-7fc8-6e54-15af-c87f546e170f"
}
},
{
"method": "nvmf_subsystem_add_ns",
"params": {
"nqn": "nqn.2016-06.io.spdk:cellar",
"namespace": {
"nsid": 1,
"bdev_name": "Raid0",
"nguid": "FA9D43CFD8D54885BB73F038BDC31B25",
"uuid": "fa9d43cf-d8d5-4885-bb73-f038bdc31b25"
}
}
},
{
"method": "nvmf_subsystem_add_listener",
"params": {
"nqn": "nqn.2016-06.io.spdk:cellar",
"listen_address": {
"trtype": "RDMA",
"adrfam": "IPv4",
"traddr": "192.168.68.80",
"trsvcid": "4420"
},
"secure_channel": false
}
}
]
},
{
"subsystem": "nbd",
"config": []
},
{
"subsystem": "vhost_blk",
"config": []
},
{
"subsystem": "scsi",
"config": null
},
{
"subsystem": "iscsi",
"config": [
{
"method": "iscsi_set_options",
"params": {
"node_base": "iqn.2016-06.io.spdk",
"max_sessions": 128,
"max_connections_per_session": 2,
"max_queue_depth": 64,
"default_time2wait": 2,
"default_time2retain": 20,
"first_burst_length": 8192,
"immediate_data": true,
"allow_duplicated_isid": false,
"error_recovery_level": 0,
"nop_timeout": 60,
"nop_in_interval": 30,
"disable_chap": false,
"require_chap": false,
"mutual_chap": false,
"chap_group": 0,
"max_large_datain_per_connection": 64,
"max_r2t_per_connection": 4,
"pdu_pool_size": 36864,
"immediate_data_pool_size": 16384,
"data_out_pool_size": 2048
}
}
]
},
{
"subsystem": "vhost_scsi",
"config": []
}
]
}

75
nixos/boxes/home/palace/vms/default.nix Normal file
View File

@@ -0,0 +1,75 @@
{
imports = [
./cellar
];
nixos.systems.palace.configuration = { lib, pkgs, config, systems, ... }:
let
inherit (lib) mkMerge;
inherit (lib.my) vm;
installerDisk = {
name = "installer";
backend = {
driver = "file";
filename = "/persist/home/dev/nixos-installer-devplayer0.iso";
read-only = "on";
};
format.driver = "raw";
frontend = "ide-cd";
frontendOpts = {
bootindex = 1;
};
};
in
{
systemd.services = {
"vm@cellar" = {
serviceConfig = {
CPUAffinity = "numa";
NUMAPolicy = "bind";
NUMAMask = "1";
};
};
};
my = {
vms = {
instances = {
cellar = {
uuid = "b126d135-9fc1-415a-b675-aaf727bf2f38";
cpu = "host,topoext";
smp = {
cpus = 8;
threads = 2;
};
memory = 32768;
drives = [
(mkMerge [ (vm.disk "cellar" "esp") { frontendOpts.bootindex = 0; } ])
(vm.disk "cellar" "nix")
(vm.disk "cellar" "persist")
];
hostDevices = {
et100g0vf0 = {
index = 0;
hostBDF = "44:00.1";
};
nvme0 = {
index = 1;
hostBDF = "41:00.0";
};
nvme1 = {
index = 2;
hostBDF = "42:00.0";
};
nvme2 = {
index = 3;
hostBDF = "43:00.0";
};
};
};
};
};
};
};
}