dev/nixfiles
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

nixos/nvme: Add module
All checks were successful
CI / Check, build and cache Nix flake (push) Successful in 17m30s
Details

Browse Source
This commit is contained in:
Jack O'Sullivan
2023-12-12 01:34:37 +00:00
parent 5686aa1a01
commit 4b48d7e788
8 changed files with 140 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
nixos/boxes/castle/default.nix
View File

@@ -251,6 +251,7 @@ in
secrets = {
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMlVuTzKObeaUuPocCF41IO/8X+443lzUJLuCIclt2vr";
};
nvme.uuid = "2230b066-a674-4f45-a1dc-f7727b3a9e7b";
firewall = {
enable = false;

6
nixos/boxes/home/palace/vms/cellar/default.nix
View File

@@ -57,8 +57,8 @@ in
};
};
environment.systemPackages = with pkgs; [
pciutils
environment.systemPackages = [
pkgs.pciutils
spdk
(pkgs.writeShellScriptBin "spdk-rpc" ''
exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@"
@@ -83,7 +83,7 @@ in
preStart = ''
${spdk.src}/scripts/setup.sh
'';
serviceConfig.ExecStart = "${spdk}/bin/spdk_tgt -c ${./spdk_nvmf.json}";
serviceConfig.ExecStart = "${spdk}/bin/spdk_tgt --cpumask 0xffff -c ${./spdk_nvmf.json}";
wantedBy = [ "multi-user.target" ];
};
};

88
nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json
View File

@@ -239,6 +239,51 @@
"abort_timeout_sec": 1
}
},
{
"method": "nvmf_create_subsystem",
"params": {
"nqn": "nqn.2016-06.io.spdk:castle",
"allow_any_host": false,
"serial_number": "SPDK00000000000003",
"model_number": "SPDK bdev Controller",
"max_namespaces": 32,
"min_cntlid": 1,
"max_cntlid": 65519,
"ana_reporting": false
}
},
{
"method": "nvmf_subsystem_add_host",
"params": {
"nqn": "nqn.2016-06.io.spdk:castle",
"host": "nqn.2014-08.org.nvmexpress:uuid:2230b066-a674-4f45-a1dc-f7727b3a9e7b"
}
},
{
"method": "nvmf_subsystem_add_ns",
"params": {
"nqn": "nqn.2016-06.io.spdk:castle",
"namespace": {
"nsid": 1,
"bdev_name": "70f5af98-d685-42bd-9024-ca8c498a0ec2",
"nguid": "70F5AF98D68542BD9024CA8C498A0EC2",
"uuid": "70f5af98-d685-42bd-9024-ca8c498a0ec2"
}
}
},
{
"method": "nvmf_subsystem_add_listener",
"params": {
"nqn": "nqn.2016-06.io.spdk:castle",
"listen_address": {
"trtype": "RDMA",
"adrfam": "IPv4",
"traddr": "192.168.68.80",
"trsvcid": "4420"
},
"secure_channel": false
}
},
{
"method": "nvmf_create_subsystem",
"params": {
@@ -327,49 +372,6 @@
{
"subsystem": "nbd",
"config": []
},
{
"subsystem": "vhost_blk",
"config": []
},
{
"subsystem": "scsi",
"config": null
},
{
"subsystem": "iscsi",
"config": [
{
"method": "iscsi_set_options",
"params": {
"node_base": "iqn.2016-06.io.spdk",
"max_sessions": 128,
"max_connections_per_session": 2,
"max_queue_depth": 64,
"default_time2wait": 2,
"default_time2retain": 20,
"first_burst_length": 8192,
"immediate_data": true,
"allow_duplicated_isid": false,
"error_recovery_level": 0,
"nop_timeout": 60,
"nop_in_interval": 30,
"disable_chap": false,
"require_chap": false,
"mutual_chap": false,
"chap_group": 0,
"max_large_datain_per_connection": 64,
"max_r2t_per_connection": 4,
"pdu_pool_size": 36864,
"immediate_data_pool_size": 16384,
"data_out_pool_size": 2048
}
}
]
},
{
"subsystem": "vhost_scsi",
"config": []
}
]
}

11
nixos/boxes/home/palace/vms/default.nix
View File

@@ -4,7 +4,7 @@
./river.nix
];
nixos.systems.palace.configuration = { lib, pkgs, config, systems, ... }:
nixos.systems.palace.configuration = { lib, pkgs, config, systems, allAssignments, ... }:
let
inherit (lib) mkMerge;
inherit (lib.my) vm;
@@ -70,6 +70,11 @@
{
requires = [ vtapUnit ];
after = [ vtapUnit ];
preStart = ''
until ${pkgs.netcat}/bin/nc -w1 -z ${allAssignments.cellar.hi.ipv4.address} 22; do
sleep 1
done
'';
};
};
@@ -83,7 +88,8 @@
cpus = 8;
threads = 2;
};
memory = 32768;
memory = 16384;
cleanShutdown.timeout = 120;
drives = [
(mkMerge [ (vm.disk "cellar" "esp") { frontendOpts.bootindex = 0; } ])
(vm.disk "cellar" "nix")
@@ -117,6 +123,7 @@
threads = 2;
};
memory = 4096;
cleanShutdown.timeout = 120;
networks = {
et1g0 = {
ifname = "vm-et1g0";

28
nixos/boxes/home/palace/vms/river.nix
View File

@@ -43,7 +43,6 @@
extraBin = with pkgs; {
dmesg = "${util-linux}/bin/dmesg";
ip = "${iproute2}/bin/ip";
nvme = "${nvme-cli}/bin/nvme";
};
extraConfig = ''
DefaultTimeoutStartSec=50
@@ -65,25 +64,6 @@
"30-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
};
};
services.connect-nvme = {
description = "Connect NVMe-oF";
before = [ "initrd-root-device.target" ];
after = [ "systemd-networkd-wait-online.service" ];
requires = [ "systemd-networkd-wait-online.service" ];
serviceConfig = {
Type = "oneshot";
Restart = "on-failure";
RestartSec = 10;
};
script = ''
${pkgs.nvme-cli}/bin/nvme connect -t rdma -a 192.168.68.80 \
-n nqn.2016-06.io.spdk:river -q nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9
'';
wantedBy = [ "initrd-root-device.target" ];
};
};
};
};
@@ -146,6 +126,14 @@
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP9uFa4z9WPuXRFVA+PClQSitQCSPckhKTxo1Hq585Oa";
};
server.enable = true;
nvme = {
uuid = "12b52d80-ccb6-418d-9b2e-2be34bff3cd9";
boot = {
nqn = "nqn.2016-06.io.spdk:river";
address = "192.168.68.80";
};
};
deploy.node.hostname = "192.168.68.1";
};
};

1
nixos/modules/_list.nix
View File

@@ -17,5 +17,6 @@
gui = ./gui.nix;
l2mesh = ./l2mesh.nix;
borgthin = ./borgthin.nix;
nvme = ./nvme;
};
}

58
nixos/modules/nvme/default.nix Normal file
View File

@@ -0,0 +1,58 @@
{ lib, pkgs, config, ... }:
let
inherit (lib) mkIf;
inherit (lib.my) mkOpt';
cfg = config.my.nvme;
nvme-cli = pkgs.nvme-cli.override {
libnvme = pkgs.libnvme.overrideAttrs (o: {
patches = o.patches ++ [ ./libnvme-hostconf.patch ];
});
};
hostNQN = "nqn.2014-08.org.nvmexpress:uuid:${cfg.uuid}";
etc = prefix: {
"${prefix}nvme/hostnqn".text = hostNQN;
"${prefix}nvme/hostid".text = cfg.uuid;
};
in
{
options.my.nvme = with lib.types; {
uuid = mkOpt' (nullOr str) null "NVMe host ID";
boot = {
nqn = mkOpt' (nullOr str) null "NQN to connect to on boot";
address = mkOpt' str null "Address of NVMe-oF target.";
};
};
config = mkIf (cfg.uuid != null) {
environment = {
systemPackages = [
nvme-cli
];
etc = etc "";
};
boot.initrd.systemd = mkIf (cfg.boot.nqn != null) {
contents = etc "/etc/";
extraBin.nvme = "${nvme-cli}/bin/nvme";
services.connect-nvme = {
description = "Connect NVMe-oF";
before = [ "initrd-root-device.target" ];
after = [ "systemd-networkd-wait-online.service" ];
requires = [ "systemd-networkd-wait-online.service" ];
serviceConfig = {
Type = "oneshot";
ExecStart = "${nvme-cli}/bin/nvme connect -t rdma -a ${cfg.boot.address} -n ${cfg.boot.nqn}";
Restart = "on-failure";
RestartSec = 10;
};
wantedBy = [ "initrd-root-device.target" ];
};
};
};
}

15
nixos/modules/nvme/libnvme-hostconf.patch Normal file
View File

@@ -0,0 +1,15 @@
diff --git a/src/nvme/fabrics.c b/src/nvme/fabrics.c
index 21fb292..f9090d1 100644
--- a/src/nvme/fabrics.c
+++ b/src/nvme/fabrics.c
@@ -41,8 +41,8 @@
#define NVMF_HOSTID_SIZE 37
-#define NVMF_HOSTNQN_FILE SYSCONFDIR "/nvme/hostnqn"
-#define NVMF_HOSTID_FILE SYSCONFDIR "/nvme/hostid"
+#define NVMF_HOSTNQN_FILE "/etc/nvme/hostnqn"
+#define NVMF_HOSTID_FILE "/etc/nvme/hostid"
const char *nvmf_dev = "/dev/nvme-fabrics";