From 4b48d7e78802a1ab810b51016e635e62358e546e Mon Sep 17 00:00:00 2001 From: Jack O'Sullivan Date: Tue, 12 Dec 2023 01:34:37 +0000 Subject: [PATCH] nixos/nvme: Add module --- nixos/boxes/castle/default.nix | 1 + .../boxes/home/palace/vms/cellar/default.nix | 6 +- .../home/palace/vms/cellar/spdk_nvmf.json | 88 ++++++++++--------- nixos/boxes/home/palace/vms/default.nix | 11 ++- nixos/boxes/home/palace/vms/river.nix | 28 ++---- nixos/modules/_list.nix | 1 + nixos/modules/nvme/default.nix | 58 ++++++++++++ nixos/modules/nvme/libnvme-hostconf.patch | 15 ++++ 8 files changed, 140 insertions(+), 68 deletions(-) create mode 100644 nixos/modules/nvme/default.nix create mode 100644 nixos/modules/nvme/libnvme-hostconf.patch diff --git a/nixos/boxes/castle/default.nix b/nixos/boxes/castle/default.nix index 0ac9fd1..70f3d06 100644 --- a/nixos/boxes/castle/default.nix +++ b/nixos/boxes/castle/default.nix @@ -251,6 +251,7 @@ in secrets = { key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMlVuTzKObeaUuPocCF41IO/8X+443lzUJLuCIclt2vr"; }; + nvme.uuid = "2230b066-a674-4f45-a1dc-f7727b3a9e7b"; firewall = { enable = false; diff --git a/nixos/boxes/home/palace/vms/cellar/default.nix b/nixos/boxes/home/palace/vms/cellar/default.nix index 567574f..58df50a 100644 --- a/nixos/boxes/home/palace/vms/cellar/default.nix +++ b/nixos/boxes/home/palace/vms/cellar/default.nix @@ -57,8 +57,8 @@ in }; }; - environment.systemPackages = with pkgs; [ - pciutils + environment.systemPackages = [ + pkgs.pciutils spdk (pkgs.writeShellScriptBin "spdk-rpc" '' exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@" @@ -83,7 +83,7 @@ in preStart = '' ${spdk.src}/scripts/setup.sh ''; - serviceConfig.ExecStart = "${spdk}/bin/spdk_tgt -c ${./spdk_nvmf.json}"; + serviceConfig.ExecStart = "${spdk}/bin/spdk_tgt --cpumask 0xffff -c ${./spdk_nvmf.json}"; wantedBy = [ "multi-user.target" ]; }; }; diff --git a/nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json b/nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json index a2430f2..6942a64 100644 --- a/nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json +++ b/nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json @@ -239,6 +239,51 @@ "abort_timeout_sec": 1 } }, + { + "method": "nvmf_create_subsystem", + "params": { + "nqn": "nqn.2016-06.io.spdk:castle", + "allow_any_host": false, + "serial_number": "SPDK00000000000003", + "model_number": "SPDK bdev Controller", + "max_namespaces": 32, + "min_cntlid": 1, + "max_cntlid": 65519, + "ana_reporting": false + } + }, + { + "method": "nvmf_subsystem_add_host", + "params": { + "nqn": "nqn.2016-06.io.spdk:castle", + "host": "nqn.2014-08.org.nvmexpress:uuid:2230b066-a674-4f45-a1dc-f7727b3a9e7b" + } + }, + { + "method": "nvmf_subsystem_add_ns", + "params": { + "nqn": "nqn.2016-06.io.spdk:castle", + "namespace": { + "nsid": 1, + "bdev_name": "70f5af98-d685-42bd-9024-ca8c498a0ec2", + "nguid": "70F5AF98D68542BD9024CA8C498A0EC2", + "uuid": "70f5af98-d685-42bd-9024-ca8c498a0ec2" + } + } + }, + { + "method": "nvmf_subsystem_add_listener", + "params": { + "nqn": "nqn.2016-06.io.spdk:castle", + "listen_address": { + "trtype": "RDMA", + "adrfam": "IPv4", + "traddr": "192.168.68.80", + "trsvcid": "4420" + }, + "secure_channel": false + } + }, { "method": "nvmf_create_subsystem", "params": { @@ -327,49 +372,6 @@ { "subsystem": "nbd", "config": [] - }, - { - "subsystem": "vhost_blk", - "config": [] - }, - { - "subsystem": "scsi", - "config": null - }, - { - "subsystem": "iscsi", - "config": [ - { - "method": "iscsi_set_options", - "params": { - "node_base": "iqn.2016-06.io.spdk", - "max_sessions": 128, - "max_connections_per_session": 2, - "max_queue_depth": 64, - "default_time2wait": 2, - "default_time2retain": 20, - "first_burst_length": 8192, - "immediate_data": true, - "allow_duplicated_isid": false, - "error_recovery_level": 0, - "nop_timeout": 60, - "nop_in_interval": 30, - "disable_chap": false, - "require_chap": false, - "mutual_chap": false, - "chap_group": 0, - "max_large_datain_per_connection": 64, - "max_r2t_per_connection": 4, - "pdu_pool_size": 36864, - "immediate_data_pool_size": 16384, - "data_out_pool_size": 2048 - } - } - ] - }, - { - "subsystem": "vhost_scsi", - "config": [] } ] } diff --git a/nixos/boxes/home/palace/vms/default.nix b/nixos/boxes/home/palace/vms/default.nix index e0e10b9..f17515d 100644 --- a/nixos/boxes/home/palace/vms/default.nix +++ b/nixos/boxes/home/palace/vms/default.nix @@ -4,7 +4,7 @@ ./river.nix ]; - nixos.systems.palace.configuration = { lib, pkgs, config, systems, ... }: + nixos.systems.palace.configuration = { lib, pkgs, config, systems, allAssignments, ... }: let inherit (lib) mkMerge; inherit (lib.my) vm; @@ -70,6 +70,11 @@ { requires = [ vtapUnit ]; after = [ vtapUnit ]; + preStart = '' + until ${pkgs.netcat}/bin/nc -w1 -z ${allAssignments.cellar.hi.ipv4.address} 22; do + sleep 1 + done + ''; }; }; @@ -83,7 +88,8 @@ cpus = 8; threads = 2; }; - memory = 32768; + memory = 16384; + cleanShutdown.timeout = 120; drives = [ (mkMerge [ (vm.disk "cellar" "esp") { frontendOpts.bootindex = 0; } ]) (vm.disk "cellar" "nix") @@ -117,6 +123,7 @@ threads = 2; }; memory = 4096; + cleanShutdown.timeout = 120; networks = { et1g0 = { ifname = "vm-et1g0"; diff --git a/nixos/boxes/home/palace/vms/river.nix b/nixos/boxes/home/palace/vms/river.nix index ce93af4..b7908a6 100644 --- a/nixos/boxes/home/palace/vms/river.nix +++ b/nixos/boxes/home/palace/vms/river.nix @@ -43,7 +43,6 @@ extraBin = with pkgs; { dmesg = "${util-linux}/bin/dmesg"; ip = "${iproute2}/bin/ip"; - nvme = "${nvme-cli}/bin/nvme"; }; extraConfig = '' DefaultTimeoutStartSec=50 @@ -65,25 +64,6 @@ "30-lan-hi" = networkdAssignment "lan-hi" assignments.hi; }; }; - - services.connect-nvme = { - description = "Connect NVMe-oF"; - before = [ "initrd-root-device.target" ]; - after = [ "systemd-networkd-wait-online.service" ]; - requires = [ "systemd-networkd-wait-online.service" ]; - - serviceConfig = { - Type = "oneshot"; - Restart = "on-failure"; - RestartSec = 10; - }; - script = '' - ${pkgs.nvme-cli}/bin/nvme connect -t rdma -a 192.168.68.80 \ - -n nqn.2016-06.io.spdk:river -q nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9 - ''; - - wantedBy = [ "initrd-root-device.target" ]; - }; }; }; }; @@ -146,6 +126,14 @@ key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP9uFa4z9WPuXRFVA+PClQSitQCSPckhKTxo1Hq585Oa"; }; server.enable = true; + nvme = { + uuid = "12b52d80-ccb6-418d-9b2e-2be34bff3cd9"; + boot = { + nqn = "nqn.2016-06.io.spdk:river"; + address = "192.168.68.80"; + }; + }; + deploy.node.hostname = "192.168.68.1"; }; }; diff --git a/nixos/modules/_list.nix b/nixos/modules/_list.nix index 0de3fd7..37cd76c 100644 --- a/nixos/modules/_list.nix +++ b/nixos/modules/_list.nix @@ -17,5 +17,6 @@ gui = ./gui.nix; l2mesh = ./l2mesh.nix; borgthin = ./borgthin.nix; + nvme = ./nvme; }; } diff --git a/nixos/modules/nvme/default.nix b/nixos/modules/nvme/default.nix new file mode 100644 index 0000000..b2d4b04 --- /dev/null +++ b/nixos/modules/nvme/default.nix @@ -0,0 +1,58 @@ +{ lib, pkgs, config, ... }: +let + inherit (lib) mkIf; + inherit (lib.my) mkOpt'; + + cfg = config.my.nvme; + nvme-cli = pkgs.nvme-cli.override { + libnvme = pkgs.libnvme.overrideAttrs (o: { + patches = o.patches ++ [ ./libnvme-hostconf.patch ]; + }); + }; + + hostNQN = "nqn.2014-08.org.nvmexpress:uuid:${cfg.uuid}"; + etc = prefix: { + "${prefix}nvme/hostnqn".text = hostNQN; + "${prefix}nvme/hostid".text = cfg.uuid; + }; +in +{ + options.my.nvme = with lib.types; { + uuid = mkOpt' (nullOr str) null "NVMe host ID"; + boot = { + nqn = mkOpt' (nullOr str) null "NQN to connect to on boot"; + address = mkOpt' str null "Address of NVMe-oF target."; + }; + }; + + config = mkIf (cfg.uuid != null) { + environment = { + systemPackages = [ + nvme-cli + ]; + etc = etc ""; + }; + + boot.initrd.systemd = mkIf (cfg.boot.nqn != null) { + contents = etc "/etc/"; + extraBin.nvme = "${nvme-cli}/bin/nvme"; + + services.connect-nvme = { + description = "Connect NVMe-oF"; + before = [ "initrd-root-device.target" ]; + after = [ "systemd-networkd-wait-online.service" ]; + requires = [ "systemd-networkd-wait-online.service" ]; + + serviceConfig = { + Type = "oneshot"; + ExecStart = "${nvme-cli}/bin/nvme connect -t rdma -a ${cfg.boot.address} -n ${cfg.boot.nqn}"; + Restart = "on-failure"; + RestartSec = 10; + }; + + wantedBy = [ "initrd-root-device.target" ]; + }; + }; + }; +} + diff --git a/nixos/modules/nvme/libnvme-hostconf.patch b/nixos/modules/nvme/libnvme-hostconf.patch new file mode 100644 index 0000000..720ecf1 --- /dev/null +++ b/nixos/modules/nvme/libnvme-hostconf.patch @@ -0,0 +1,15 @@ +diff --git a/src/nvme/fabrics.c b/src/nvme/fabrics.c +index 21fb292..f9090d1 100644 +--- a/src/nvme/fabrics.c ++++ b/src/nvme/fabrics.c +@@ -41,8 +41,8 @@ + + #define NVMF_HOSTID_SIZE 37 + +-#define NVMF_HOSTNQN_FILE SYSCONFDIR "/nvme/hostnqn" +-#define NVMF_HOSTID_FILE SYSCONFDIR "/nvme/hostid" ++#define NVMF_HOSTNQN_FILE "/etc/nvme/hostnqn" ++#define NVMF_HOSTID_FILE "/etc/nvme/hostid" + + const char *nvmf_dev = "/dev/nvme-fabrics"; +