nixos/multipath: add multipath module
The multipath-tools package had existed in Nixpkgs for some time but without a nixos module to configure/drive it. This module provides attributes to drive the majority of multipath configuration options and is being successfully used in stage-1 and stage-2 boot to mount /nix from a multipath-serviced iSCSI volume. Credit goes to @grahamc for early contributions to the module and authoring the NixOS module test.
This commit is contained in:
Michael Brantley
Michael Brantley
parent
c344317f18
commit
121cfd1998
@ -329,6 +329,13 @@
|
||||
controller support.
|
||||
</para>
|
||||
</listitem>
|
||||
<listitem>
|
||||
<para>
|
||||
<link xlink:href="https://github.com/opensvc/multipath-tools">multipath</link>,
|
||||
the device mapper multipath (DM-MP) daemon. Available as
|
||||
<link linkend="opt-services.multipath.enable">services.multipath</link>.
|
||||
</para>
|
||||
</listitem>
|
||||
</itemizedlist>
|
||||
</section>
|
||||
<section xml:id="sec-release-21.11-incompatibilities">
|
||||
|
||||
@ -101,6 +101,8 @@ In addition to numerous new and upgraded packages, this release has the followin
|
||||
|
||||
- [joycond](https://github.com/DanielOgorchock/joycond), a service that uses `hid-nintendo` to provide nintendo joycond pairing and better nintendo switch pro controller support.
|
||||
|
||||
- [multipath](https://github.com/opensvc/multipath-tools), the device mapper multipath (DM-MP) daemon. Available as [services.multipath](#opt-services.multipath.enable).
|
||||
|
||||
## Backward Incompatibilities {#sec-release-21.11-incompatibilities}
|
||||
|
||||
- The `security.wrappers` option now requires to always specify an owner, group and whether the setuid/setgid bit should be set.
|
||||
|
||||
@ -779,6 +779,7 @@
|
||||
./services/networking/mstpd.nix
|
||||
./services/networking/mtprotoproxy.nix
|
||||
./services/networking/mullvad-vpn.nix
|
||||
./services/networking/multipath.nix
|
||||
./services/networking/murmur.nix
|
||||
./services/networking/mxisd.nix
|
||||
./services/networking/namecoind.nix
|
||||
|
||||
572
nixos/modules/services/networking/multipath.nix
Normal file
572
nixos/modules/services/networking/multipath.nix
Normal file
@ -0,0 +1,572 @@
|
||||
{ config, lib, pkgs, ... }: with lib;
|
||||
|
||||
# See http://christophe.varoqui.free.fr/usage.html and
|
||||
# https://github.com/opensvc/multipath-tools/blob/master/multipath/multipath.conf.5
|
||||
|
||||
let
|
||||
cfg = config.services.multipath;
|
||||
|
||||
indentLines = n: str: concatStringsSep "\n" (
|
||||
map (line: "${fixedWidthString n " " " "}${line}") (
|
||||
filter ( x: x != "" ) ( splitString "\n" str )
|
||||
)
|
||||
);
|
||||
|
||||
addCheckDesc = desc: elemType: check: types.addCheck elemType check
|
||||
// { description = "${elemType.description} (with check: ${desc})"; };
|
||||
hexChars = stringToCharacters "0123456789abcdef";
|
||||
isHexString = s: all (c: elem c hexChars) (stringToCharacters (toLower s));
|
||||
hexStr = addCheckDesc "hexadecimal string" types.str isHexString;
|
||||
|
||||
in {
|
||||
|
||||
options.services.multipath = with types; {
|
||||
|
||||
enable = mkEnableOption "the device mapper multipath (DM-MP) daemon";
|
||||
|
||||
package = mkOption {
|
||||
type = package;
|
||||
description = "multipath-tools package to use";
|
||||
default = pkgs.multipath-tools;
|
||||
defaultText = "pkgs.multipath-tools";
|
||||
};
|
||||
|
||||
devices = mkOption {
|
||||
default = [ ];
|
||||
example = literalExpression ''
|
||||
[
|
||||
{
|
||||
vendor = "\"COMPELNT\"";
|
||||
product = "\"Compellent Vol\"";
|
||||
path_checker = "tur";
|
||||
no_path_retry = "queue";
|
||||
max_sectors_kb = 256;
|
||||
}, ...
|
||||
]
|
||||
'';
|
||||
description = ''
|
||||
This option allows you to define arrays for use in multipath
|
||||
groups.
|
||||
'';
|
||||
type = listOf (submodule {
|
||||
options = {
|
||||
|
||||
vendor = mkOption {
|
||||
type = str;
|
||||
example = "COMPELNT";
|
||||
description = "Regular expression to match the vendor name";
|
||||
};
|
||||
|
||||
product = mkOption {
|
||||
type = str;
|
||||
example = "Compellent Vol";
|
||||
description = "Regular expression to match the product name";
|
||||
};
|
||||
|
||||
revision = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "Regular expression to match the product revision";
|
||||
};
|
||||
|
||||
product_blacklist = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "Products with the given vendor matching this string are blacklisted";
|
||||
};
|
||||
|
||||
alias_prefix = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "The user_friendly_names prefix to use for this device type, instead of the default mpath";
|
||||
};
|
||||
|
||||
vpd_vendor = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "The vendor specific vpd page information, using the vpd page abbreviation";
|
||||
};
|
||||
|
||||
hardware_handler = mkOption {
|
||||
type = nullOr (enum [ "emc" "rdac" "hp_sw" "alua" "ana" ]);
|
||||
default = null;
|
||||
description = "The hardware handler to use for this device type";
|
||||
};
|
||||
|
||||
# Optional arguments
|
||||
path_grouping_policy = mkOption {
|
||||
type = nullOr (enum [ "failover" "multibus" "group_by_serial" "group_by_prio" "group_by_node_name" ]);
|
||||
default = null; # real default: "failover"
|
||||
description = "The default path grouping policy to apply to unspecified multipaths";
|
||||
};
|
||||
|
||||
uid_attribute = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "The udev attribute providing a unique path identifier (WWID)";
|
||||
};
|
||||
|
||||
getuid_callout = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = ''
|
||||
(Superseded by uid_attribute) The default program and args to callout
|
||||
to obtain a unique path identifier. Should be specified with an absolute path.
|
||||
'';
|
||||
};
|
||||
|
||||
path_selector = mkOption {
|
||||
type = nullOr (enum [
|
||||
''"round-robin 0"''
|
||||
''"queue-length 0"''
|
||||
''"service-time 0"''
|
||||
''"historical-service-time 0"''
|
||||
]);
|
||||
default = null; # real default: "service-time 0"
|
||||
description = "The default path selector algorithm to use; they are offered by the kernel multipath target";
|
||||
};
|
||||
|
||||
path_checker = mkOption {
|
||||
type = enum [ "readsector0" "tur" "emc_clariion" "hp_sw" "rdac" "directio" "cciss_tur" "none" ];
|
||||
default = "tur";
|
||||
description = "The default method used to determine the paths state";
|
||||
};
|
||||
|
||||
prio = mkOption {
|
||||
type = nullOr (enum [
|
||||
"none" "const" "sysfs" "emc" "alua" "ontap" "rdac" "hp_sw" "hds"
|
||||
"random" "weightedpath" "path_latency" "ana" "datacore" "iet"
|
||||
]);
|
||||
default = null; # real default: "const"
|
||||
description = "The name of the path priority routine";
|
||||
};
|
||||
|
||||
prio_args = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "Arguments to pass to to the prio function";
|
||||
};
|
||||
|
||||
features = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "Specify any device-mapper features to be used";
|
||||
};
|
||||
|
||||
failback = mkOption {
|
||||
type = nullOr str;
|
||||
default = null; # real default: "manual"
|
||||
description = "Tell multipathd how to manage path group failback. Quote integers as strings";
|
||||
};
|
||||
|
||||
rr_weight = mkOption {
|
||||
type = nullOr (enum [ "priorities" "uniform" ]);
|
||||
default = null; # real default: "uniform"
|
||||
description = ''
|
||||
If set to priorities the multipath configurator will assign path weights
|
||||
as "path prio * rr_min_io".
|
||||
'';
|
||||
};
|
||||
|
||||
no_path_retry = mkOption {
|
||||
type = nullOr str;
|
||||
default = null; # real default: "fail"
|
||||
description = "Specify what to do when all paths are down. Quote integers as strings";
|
||||
};
|
||||
|
||||
rr_min_io = mkOption {
|
||||
type = nullOr int;
|
||||
default = null; # real default: 1000
|
||||
description = ''
|
||||
Number of I/O requests to route to a path before switching to the next in the
|
||||
same path group. This is only for Block I/O (BIO) based multipath and
|
||||
only apply to round-robin path_selector.
|
||||
'';
|
||||
};
|
||||
|
||||
rr_min_io_rq = mkOption {
|
||||
type = nullOr int;
|
||||
default = null; # real default: 1
|
||||
description = ''
|
||||
Number of I/O requests to route to a path before switching to the next in the
|
||||
same path group. This is only for Request based multipath and
|
||||
only apply to round-robin path_selector.
|
||||
'';
|
||||
};
|
||||
|
||||
fast_io_fail_tmo = mkOption {
|
||||
type = nullOr str;
|
||||
default = null; # real default: 5
|
||||
description = ''
|
||||
Specify the number of seconds the SCSI layer will wait after a problem has been
|
||||
detected on a FC remote port before failing I/O to devices on that remote port.
|
||||
This should be smaller than dev_loss_tmo. Setting this to "off" will disable
|
||||
the timeout. Quote integers as strings.
|
||||
'';
|
||||
};
|
||||
|
||||
dev_loss_tmo = mkOption {
|
||||
type = nullOr str;
|
||||
default = null; # real default: 600
|
||||
description = ''
|
||||
Specify the number of seconds the SCSI layer will wait after a problem has
|
||||
been detected on a FC remote port before removing it from the system. This
|
||||
can be set to "infinity" which sets it to the max value of 2147483647
|
||||
seconds, or 68 years. It will be automatically adjusted to the overall
|
||||
retry interval no_path_retry * polling_interval
|
||||
if a number of retries is given with no_path_retry and the
|
||||
overall retry interval is longer than the specified dev_loss_tmo value.
|
||||
The Linux kernel will cap this value to 600 if fast_io_fail_tmo
|
||||
is not set.
|
||||
'';
|
||||
};
|
||||
|
||||
flush_on_last_del = mkOption {
|
||||
type = nullOr (enum [ "yes" "no" ]);
|
||||
default = null; # real default: "no"
|
||||
description = ''
|
||||
If set to "yes" multipathd will disable queueing when the last path to a
|
||||
device has been deleted.
|
||||
'';
|
||||
};
|
||||
|
||||
user_friendly_names = mkOption {
|
||||
type = nullOr (enum [ "yes" "no" ]);
|
||||
default = null; # real default: "no"
|
||||
description = ''
|
||||
If set to "yes", using the bindings file /etc/multipath/bindings
|
||||
to assign a persistent and unique alias to the multipath, in the
|
||||
form of mpath. If set to "no" use the WWID as the alias. In either
|
||||
case this be will be overridden by any specific aliases in the
|
||||
multipaths section.
|
||||
'';
|
||||
};
|
||||
|
||||
retain_attached_hw_handler = mkOption {
|
||||
type = nullOr (enum [ "yes" "no" ]);
|
||||
default = null; # real default: "yes"
|
||||
description = ''
|
||||
(Obsolete for kernels >= 4.3) If set to "yes" and the SCSI layer has
|
||||
already attached a hardware_handler to the device, multipath will not
|
||||
force the device to use the hardware_handler specified by mutipath.conf.
|
||||
If the SCSI layer has not attached a hardware handler, multipath will
|
||||
continue to use its configured hardware handler.
|
||||
|
||||
Important Note: Linux kernel 4.3 or newer always behaves as if
|
||||
"retain_attached_hw_handler yes" was set.
|
||||
'';
|
||||
};
|
||||
|
||||
detect_prio = mkOption {
|
||||
type = nullOr (enum [ "yes" "no" ]);
|
||||
default = null; # real default: "yes"
|
||||
description = ''
|
||||
If set to "yes", multipath will try to detect if the device supports
|
||||
SCSI-3 ALUA. If so, the device will automatically use the sysfs
|
||||
prioritizer if the required sysf attributes access_state and
|
||||
preferred_path are supported, or the alua prioritizer if not. If set
|
||||
to "no", the prioritizer will be selected as usual.
|
||||
'';
|
||||
};
|
||||
|
||||
detect_checker = mkOption {
|
||||
type = nullOr (enum [ "yes" "no" ]);
|
||||
default = null; # real default: "yes"
|
||||
description = ''
|
||||
If set to "yes", multipath will try to detect if the device supports
|
||||
SCSI-3 ALUA. If so, the device will automatically use the tur checker.
|
||||
If set to "no", the checker will be selected as usual.
|
||||
'';
|
||||
};
|
||||
|
||||
deferred_remove = mkOption {
|
||||
type = nullOr (enum [ "yes" "no" ]);
|
||||
default = null; # real default: "no"
|
||||
description = ''
|
||||
If set to "yes", multipathd will do a deferred remove instead of a
|
||||
regular remove when the last path device has been deleted. This means
|
||||
that if the multipath device is still in use, it will be freed when
|
||||
the last user closes it. If path is added to the multipath device
|
||||
before the last user closes it, the deferred remove will be canceled.
|
||||
'';
|
||||
};
|
||||
|
||||
san_path_err_threshold = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = ''
|
||||
If set to a value greater than 0, multipathd will watch paths and check
|
||||
how many times a path has been failed due to errors.If the number of
|
||||
failures on a particular path is greater then the san_path_err_threshold,
|
||||
then the path will not reinstate till san_path_err_recovery_time. These
|
||||
path failures should occur within a san_path_err_forget_rate checks, if
|
||||
not we will consider the path is good enough to reinstantate.
|
||||
'';
|
||||
};
|
||||
|
||||
san_path_err_forget_rate = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = ''
|
||||
If set to a value greater than 0, multipathd will check whether the path
|
||||
failures has exceeded the san_path_err_threshold within this many checks
|
||||
i.e san_path_err_forget_rate. If so we will not reinstante the path till
|
||||
san_path_err_recovery_time.
|
||||
'';
|
||||
};
|
||||
|
||||
san_path_err_recovery_time = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = ''
|
||||
If set to a value greater than 0, multipathd will make sure that when
|
||||
path failures has exceeded the san_path_err_threshold within
|
||||
san_path_err_forget_rate then the path will be placed in failed state
|
||||
for san_path_err_recovery_time duration. Once san_path_err_recovery_time
|
||||
has timeout we will reinstante the failed path. san_path_err_recovery_time
|
||||
value should be in secs.
|
||||
'';
|
||||
};
|
||||
|
||||
marginal_path_err_sample_time = mkOption {
|
||||
type = nullOr int;
|
||||
default = null;
|
||||
description = "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
|
||||
};
|
||||
|
||||
marginal_path_err_rate_threshold = mkOption {
|
||||
type = nullOr int;
|
||||
default = null;
|
||||
description = "The error rate threshold as a permillage (1/1000)";
|
||||
};
|
||||
|
||||
marginal_path_err_recheck_gap_time = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
|
||||
};
|
||||
|
||||
marginal_path_double_failed_time = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "One of the four parameters of supporting path check based on accounting IO error such as intermittent error";
|
||||
};
|
||||
|
||||
delay_watch_checks = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "This option is deprecated, and mapped to san_path_err_forget_rate";
|
||||
};
|
||||
|
||||
delay_wait_checks = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "This option is deprecated, and mapped to san_path_err_recovery_time";
|
||||
};
|
||||
|
||||
skip_kpartx = mkOption {
|
||||
type = nullOr (enum [ "yes" "no" ]);
|
||||
default = null; # real default: "no"
|
||||
description = "If set to yes, kpartx will not automatically create partitions on the device";
|
||||
};
|
||||
|
||||
max_sectors_kb = mkOption {
|
||||
type = nullOr int;
|
||||
default = null;
|
||||
description = "Sets the max_sectors_kb device parameter on all path devices and the multipath device to the specified value";
|
||||
};
|
||||
|
||||
ghost_delay = mkOption {
|
||||
type = nullOr int;
|
||||
default = null;
|
||||
description = "Sets the number of seconds that multipath will wait after creating a device with only ghost paths before marking it ready for use in systemd";
|
||||
};
|
||||
|
||||
all_tg_pt = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "Set the 'all targets ports' flag when registering keys with mpathpersist";
|
||||
};
|
||||
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
defaults = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = ''
|
||||
This section defines default values for attributes which are used
|
||||
whenever no values are given in the appropriate device or multipath
|
||||
sections.
|
||||
'';
|
||||
};
|
||||
|
||||
blacklist = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = ''
|
||||
This section defines which devices should be excluded from the
|
||||
multipath topology discovery.
|
||||
'';
|
||||
};
|
||||
|
||||
blacklist_exceptions = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = ''
|
||||
This section defines which devices should be included in the
|
||||
multipath topology discovery, despite being listed in the
|
||||
blacklist section.
|
||||
'';
|
||||
};
|
||||
|
||||
overrides = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = ''
|
||||
This section defines values for attributes that should override the
|
||||
device-specific settings for all devices.
|
||||
'';
|
||||
};
|
||||
|
||||
extraConfig = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "Lines to append to default multipath.conf";
|
||||
};
|
||||
|
||||
extraConfigFile = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
description = "Append an additional file's contents to /etc/multipath.conf";
|
||||
};
|
||||
|
||||
pathGroups = mkOption {
|
||||
example = literalExpression ''
|
||||
[
|
||||
{
|
||||
wwid = "360080e500043b35c0123456789abcdef";
|
||||
alias = 10001234;
|
||||
array = "bigarray.example.com";
|
||||
fsType = "zfs"; # optional
|
||||
options = "ro"; # optional
|
||||
}, ...
|
||||
]
|
||||
'';
|
||||
description = ''
|
||||
This option allows you to define multipath groups as described
|
||||
in http://christophe.varoqui.free.fr/usage.html.
|
||||
'';
|
||||
type = listOf (submodule {
|
||||
options = {
|
||||
|
||||
alias = mkOption {
|
||||
type = int;
|
||||
example = 1001234;
|
||||
description = "The name of the multipath device";
|
||||
};
|
||||
|
||||
wwid = mkOption {
|
||||
type = hexStr;
|
||||
example = "360080e500043b35c0123456789abcdef";
|
||||
description = "The identifier for the multipath device";
|
||||
};
|
||||
|
||||
array = mkOption {
|
||||
type = str;
|
||||
default = null;
|
||||
example = "bigarray.example.com";
|
||||
description = "The DNS name of the storage array";
|
||||
};
|
||||
|
||||
fsType = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
example = "zfs";
|
||||
description = "Type of the filesystem";
|
||||
};
|
||||
|
||||
options = mkOption {
|
||||
type = nullOr str;
|
||||
default = null;
|
||||
example = "ro";
|
||||
description = "Options used to mount the file system";
|
||||
};
|
||||
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
config = mkIf cfg.enable {
|
||||
environment.etc."multipath.conf".text =
|
||||
let
|
||||
inherit (cfg) defaults blacklist blacklist_exceptions overrides;
|
||||
|
||||
mkDeviceBlock = cfg: let
|
||||
nonNullCfg = lib.filterAttrs (k: v: v != null) cfg;
|
||||
attrs = lib.mapAttrsToList (name: value: " ${name} ${toString value}") nonNullCfg;
|
||||
in ''
|
||||
device {
|
||||
${lib.concatStringsSep "\n" attrs}
|
||||
}
|
||||
'';
|
||||
devices = lib.concatMapStringsSep "\n" mkDeviceBlock cfg.devices;
|
||||
|
||||
mkMultipathBlock = m: ''
|
||||
multipath {
|
||||
wwid ${m.wwid}
|
||||
alias ${toString m.alias}
|
||||
}
|
||||
'';
|
||||
multipaths = lib.concatMapStringsSep "\n" mkMultipathBlock cfg.pathGroups;
|
||||
|
||||
in ''
|
||||
devices {
|
||||
${indentLines 2 devices}
|
||||
}
|
||||
|
||||
${optionalString (!isNull defaults) ''
|
||||
defaults {
|
||||
${indentLines 2 defaults}
|
||||
multipath_dir ${cfg.package}/lib/multipath
|
||||
}
|
||||
''}
|
||||
${optionalString (!isNull blacklist) ''
|
||||
blacklist {
|
||||
${indentLines 2 blacklist}
|
||||
}
|
||||
''}
|
||||
${optionalString (!isNull blacklist_exceptions) ''
|
||||
blacklist_exceptions {
|
||||
${indentLines 2 blacklist_exceptions}
|
||||
}
|
||||
''}
|
||||
${optionalString (!isNull overrides) ''
|
||||
overrides {
|
||||
${indentLines 2 overrides}
|
||||
}
|
||||
''}
|
||||
multipaths {
|
||||
${indentLines 2 multipaths}
|
||||
}
|
||||
'';
|
||||
|
||||
systemd.packages = [ cfg.package ];
|
||||
|
||||
environment.systemPackages = [ cfg.package ];
|
||||
boot.kernelModules = [ "dm-multipath" "dm-service-time" ];
|
||||
|
||||
# We do not have systemd in stage-1 boot so must invoke `multipathd`
|
||||
# with the `-1` argument which disables systemd calls. Invoke `multipath`
|
||||
# to display the multipath mappings in the output of `journalctl -b`.
|
||||
boot.initrd.kernelModules = [ "dm-multipath" "dm-service-time" ];
|
||||
boot.initrd.postDeviceCommands = ''
|
||||
modprobe -a dm-multipath dm-service-time
|
||||
multipathd -s
|
||||
(set -x && sleep 1 && multipath -ll)
|
||||
'';
|
||||
};
|
||||
}
|
||||
@ -137,6 +137,14 @@ let
|
||||
copy_bin_and_libs ${pkgs.e2fsprogs}/sbin/resize2fs
|
||||
''}
|
||||
|
||||
# Copy multipath.
|
||||
${optionalString config.services.multipath.enable ''
|
||||
copy_bin_and_libs ${config.services.multipath.package}/bin/multipath
|
||||
copy_bin_and_libs ${config.services.multipath.package}/bin/multipathd
|
||||
# Copy lib/multipath manually.
|
||||
cp -rpv ${config.services.multipath.package}/lib/multipath $out/lib
|
||||
''}
|
||||
|
||||
# Copy secrets if needed.
|
||||
#
|
||||
# TODO: move out to a separate script; see #85000.
|
||||
@ -199,6 +207,10 @@ let
|
||||
$out/bin/dmsetup --version 2>&1 | tee -a log | grep -q "version:"
|
||||
LVM_SYSTEM_DIR=$out $out/bin/lvm version 2>&1 | tee -a log | grep -q "LVM"
|
||||
$out/bin/mdadm --version
|
||||
${optionalString config.services.multipath.enable ''
|
||||
($out/bin/multipath || true) 2>&1 | grep -q 'need to be root'
|
||||
($out/bin/multipathd || true) 2>&1 | grep -q 'need to be root'
|
||||
''}
|
||||
|
||||
${config.boot.initrd.extraUtilsCommandsTest}
|
||||
fi
|
||||
@ -338,6 +350,18 @@ let
|
||||
{ object = pkgs.kmod-debian-aliases;
|
||||
symlink = "/etc/modprobe.d/debian.conf";
|
||||
}
|
||||
] ++ lib.optionals config.services.multipath.enable [
|
||||
{ object = pkgs.runCommand "multipath.conf" {
|
||||
src = config.environment.etc."multipath.conf".text;
|
||||
preferLocalBuild = true;
|
||||
} ''
|
||||
target=$out
|
||||
printf "$src" > $out
|
||||
substituteInPlace $out \
|
||||
--replace ${config.services.multipath.package}/lib ${extraUtils}/lib
|
||||
'';
|
||||
symlink = "/etc/multipath.conf";
|
||||
}
|
||||
] ++ (lib.mapAttrsToList
|
||||
(symlink: options:
|
||||
{
|
||||
|
||||
267
nixos/tests/iscsi-multipath-root.nix
Normal file
267
nixos/tests/iscsi-multipath-root.nix
Normal file
@ -0,0 +1,267 @@
|
||||
import ./make-test-python.nix (
|
||||
{ pkgs, lib, ... }:
|
||||
let
|
||||
initiatorName = "iqn.2020-08.org.linux-iscsi.initiatorhost:example";
|
||||
targetName = "iqn.2003-01.org.linux-iscsi.target.x8664:sn.acf8fd9c23af";
|
||||
in
|
||||
{
|
||||
name = "iscsi";
|
||||
meta = {
|
||||
maintainers = pkgs.lib.teams.deshaw.members;
|
||||
};
|
||||
|
||||
nodes = {
|
||||
target = { config, pkgs, lib, ... }: {
|
||||
virtualisation.vlans = [ 1 2 ];
|
||||
services.target = {
|
||||
enable = true;
|
||||
config = {
|
||||
fabric_modules = [ ];
|
||||
storage_objects = [
|
||||
{
|
||||
dev = "/dev/vdb";
|
||||
name = "test";
|
||||
plugin = "block";
|
||||
write_back = true;
|
||||
wwn = "92b17c3f-6b40-4168-b082-ceeb7b495522";
|
||||
}
|
||||
];
|
||||
targets = [
|
||||
{
|
||||
fabric = "iscsi";
|
||||
tpgs = [
|
||||
{
|
||||
enable = true;
|
||||
attributes = {
|
||||
authentication = 0;
|
||||
generate_node_acls = 1;
|
||||
};
|
||||
luns = [
|
||||
{
|
||||
alias = "94dfe06967";
|
||||
alua_tg_pt_gp_name = "default_tg_pt_gp";
|
||||
index = 0;
|
||||
storage_object = "/backstores/block/test";
|
||||
}
|
||||
];
|
||||
node_acls = [
|
||||
{
|
||||
mapped_luns = [
|
||||
{
|
||||
alias = "d42f5bdf8a";
|
||||
index = 0;
|
||||
tpg_lun = 0;
|
||||
write_protect = false;
|
||||
}
|
||||
];
|
||||
node_wwn = initiatorName;
|
||||
}
|
||||
];
|
||||
portals = [
|
||||
{
|
||||
ip_address = "0.0.0.0";
|
||||
iser = false;
|
||||
offload = false;
|
||||
port = 3260;
|
||||
}
|
||||
];
|
||||
tag = 1;
|
||||
}
|
||||
];
|
||||
wwn = targetName;
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 3260 ];
|
||||
networking.firewall.allowedUDPPorts = [ 3260 ];
|
||||
|
||||
virtualisation.memorySize = 2048;
|
||||
virtualisation.emptyDiskImages = [ 2048 ];
|
||||
};
|
||||
|
||||
initiatorAuto = { nodes, config, pkgs, ... }: {
|
||||
virtualisation.vlans = [ 1 2 ];
|
||||
|
||||
services.multipath = {
|
||||
enable = true;
|
||||
defaults = ''
|
||||
find_multipaths yes
|
||||
user_friendly_names yes
|
||||
'';
|
||||
pathGroups = [
|
||||
{
|
||||
alias = 123456;
|
||||
wwid = "3600140592b17c3f6b404168b082ceeb7";
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
services.openiscsi = {
|
||||
enable = true;
|
||||
enableAutoLoginOut = true;
|
||||
discoverPortal = "target";
|
||||
name = initiatorName;
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
xfsprogs
|
||||
];
|
||||
|
||||
environment.etc."initiator-root-disk-closure".source = nodes.initiatorRootDisk.config.system.build.toplevel;
|
||||
|
||||
nix.binaryCaches = lib.mkForce [ ];
|
||||
nix.extraOptions = ''
|
||||
hashed-mirrors =
|
||||
connect-timeout = 1
|
||||
'';
|
||||
};
|
||||
|
||||
initiatorRootDisk = { config, pkgs, modulesPath, lib, ... }: {
|
||||
boot.initrd.network.enable = true;
|
||||
boot.loader.grub.enable = false;
|
||||
|
||||
boot.kernelParams = lib.mkOverride 5 (
|
||||
[
|
||||
"boot.shell_on_fail"
|
||||
"console=tty1"
|
||||
"ip=192.168.1.1:::255.255.255.0::ens9:none"
|
||||
"ip=192.168.2.1:::255.255.255.0::ens10:none"
|
||||
]
|
||||
);
|
||||
|
||||
# defaults to true, puts some code in the initrd that tries to mount an overlayfs on /nix/store
|
||||
virtualisation.writableStore = false;
|
||||
virtualisation.vlans = [ 1 2 ];
|
||||
|
||||
services.multipath = {
|
||||
enable = true;
|
||||
defaults = ''
|
||||
find_multipaths yes
|
||||
user_friendly_names yes
|
||||
'';
|
||||
pathGroups = [
|
||||
{
|
||||
alias = 123456;
|
||||
wwid = "3600140592b17c3f6b404168b082ceeb7";
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
fileSystems = lib.mkOverride 5 {
|
||||
"/" = {
|
||||
fsType = "xfs";
|
||||
device = "/dev/mapper/123456";
|
||||
options = [ "_netdev" ];
|
||||
};
|
||||
};
|
||||
|
||||
boot.initrd.extraFiles."etc/multipath/wwids".source = pkgs.writeText "wwids" "/3600140592b17c3f6b404168b082ceeb7/";
|
||||
|
||||
boot.iscsi-initiator = {
|
||||
discoverPortal = "target";
|
||||
name = initiatorName;
|
||||
target = targetName;
|
||||
extraIscsiCommands = ''
|
||||
iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
testScript = { nodes, ... }: ''
|
||||
target.start()
|
||||
target.wait_for_unit("iscsi-target.service")
|
||||
|
||||
initiatorAuto.start()
|
||||
|
||||
initiatorAuto.wait_for_unit("iscsid.service")
|
||||
initiatorAuto.wait_for_unit("iscsi.service")
|
||||
initiatorAuto.get_unit_info("iscsi")
|
||||
|
||||
# Expecting this to fail since we should already know about 192.168.1.3
|
||||
initiatorAuto.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
|
||||
# Expecting this to succeed since we don't yet know about 192.168.2.3
|
||||
initiatorAuto.succeed("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
|
||||
|
||||
# /dev/sda is provided by iscsi on target
|
||||
initiatorAuto.succeed("set -x; while ! test -e /dev/sda; do sleep 1; done")
|
||||
|
||||
initiatorAuto.succeed("mkfs.xfs /dev/sda")
|
||||
initiatorAuto.succeed("mkdir /mnt")
|
||||
|
||||
# Start by verifying /dev/sda and /dev/sdb are both the same disk
|
||||
initiatorAuto.succeed("mount /dev/sda /mnt")
|
||||
initiatorAuto.succeed("touch /mnt/hi")
|
||||
initiatorAuto.succeed("umount /mnt")
|
||||
|
||||
initiatorAuto.succeed("mount /dev/sdb /mnt")
|
||||
initiatorAuto.succeed("test -e /mnt/hi")
|
||||
initiatorAuto.succeed("umount /mnt")
|
||||
|
||||
initiatorAuto.succeed("systemctl restart multipathd")
|
||||
initiatorAuto.succeed("multipath -ll | systemd-cat")
|
||||
|
||||
# Install our RootDisk machine to 123456, the alias to the device that multipath is now managing
|
||||
initiatorAuto.succeed("mount /dev/mapper/123456 /mnt")
|
||||
initiatorAuto.succeed("mkdir -p /mnt/etc/{multipath,iscsi}")
|
||||
initiatorAuto.succeed("cp -r /etc/multipath/wwids /mnt/etc/multipath/wwids")
|
||||
initiatorAuto.succeed("cp -r /etc/iscsi/{nodes,send_targets} /mnt/etc/iscsi")
|
||||
initiatorAuto.succeed(
|
||||
"nixos-install --no-bootloader --no-root-passwd --system /etc/initiator-root-disk-closure"
|
||||
)
|
||||
initiatorAuto.succeed("umount /mnt")
|
||||
initiatorAuto.shutdown()
|
||||
|
||||
initiatorRootDisk.start()
|
||||
initiatorRootDisk.wait_for_unit("multi-user.target")
|
||||
initiatorRootDisk.wait_for_unit("iscsid")
|
||||
|
||||
# Log in over both nodes
|
||||
initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.1.3 --login")
|
||||
initiatorRootDisk.fail("iscsiadm -m discovery -o update -t sendtargets -p 192.168.2.3 --login")
|
||||
initiatorRootDisk.succeed("systemctl restart multipathd")
|
||||
initiatorRootDisk.succeed("multipath -ll | systemd-cat")
|
||||
|
||||
# Verify we can write and sync the root disk
|
||||
initiatorRootDisk.succeed("mkdir /scratch")
|
||||
initiatorRootDisk.succeed("touch /scratch/both-up")
|
||||
initiatorRootDisk.succeed("sync /scratch")
|
||||
|
||||
# Verify we can write to the root with ens9 (sda, 192.168.1.3) down
|
||||
initiatorRootDisk.succeed("ip link set ens9 down")
|
||||
initiatorRootDisk.succeed("touch /scratch/ens9-down")
|
||||
initiatorRootDisk.succeed("sync /scratch")
|
||||
initiatorRootDisk.succeed("ip link set ens9 up")
|
||||
|
||||
# todo: better way to wait until multipath notices the link is back
|
||||
initiatorRootDisk.succeed("sleep 5")
|
||||
initiatorRootDisk.succeed("touch /scratch/both-down")
|
||||
initiatorRootDisk.succeed("sync /scratch")
|
||||
|
||||
# Verify we can write to the root with ens10 (sdb, 192.168.2.3) down
|
||||
initiatorRootDisk.succeed("ip link set ens10 down")
|
||||
initiatorRootDisk.succeed("touch /scratch/ens10-down")
|
||||
initiatorRootDisk.succeed("sync /scratch")
|
||||
initiatorRootDisk.succeed("ip link set ens10 up")
|
||||
initiatorRootDisk.succeed("touch /scratch/ens10-down")
|
||||
initiatorRootDisk.succeed("sync /scratch")
|
||||
|
||||
initiatorRootDisk.succeed("ip link set ens9 up")
|
||||
initiatorRootDisk.succeed("ip link set ens10 up")
|
||||
initiatorRootDisk.shutdown()
|
||||
|
||||
# Verify we can boot with the target's eth1 down, forcing
|
||||
# it to multipath via the second link
|
||||
target.succeed("ip link set eth1 down")
|
||||
initiatorRootDisk.start()
|
||||
initiatorRootDisk.wait_for_unit("multi-user.target")
|
||||
initiatorRootDisk.wait_for_unit("iscsid")
|
||||
initiatorRootDisk.succeed("test -e /scratch/both-up")
|
||||
'';
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user