Initial networking VM
Also general improvements around VMs
This commit is contained in:
parent
5563d1be46
commit
009dec03cf
2
.envrc
2
.envrc
@ -1,2 +1,2 @@
|
||||
nix_direnv_watch_file devshell/{default,commands,install}.nix
|
||||
nix_direnv_watch_file devshell/{default,commands,install,vm-tasks}.nix
|
||||
use flake
|
||||
|
||||
@ -46,7 +46,13 @@ in
|
||||
name = "qemu-genmac";
|
||||
category = "utilities";
|
||||
help = "Generate MAC address suitable for QEMU";
|
||||
command = ''printf "52:54:00:ab:%02x:%02x\n" $((RANDOM%256)) $((RANDOM%256))'';
|
||||
command = ''printf "52:54:00:%02x:%02x:%02x\n" $((RANDOM%256)) $((RANDOM%256)) $((RANDOM%256))'';
|
||||
}
|
||||
{
|
||||
name = "ssh-get-ed25519";
|
||||
category = "utilities";
|
||||
help = "Print the ed25519 pubkey for a host";
|
||||
command = "${pkgs.openssh}/bin/ssh-keyscan -t ed25519 \"$1\" 2> /dev/null | awk '{ print $2 \" \" $3 }'";
|
||||
}
|
||||
|
||||
{
|
||||
|
||||
@ -3,7 +3,7 @@ let
|
||||
inherit (lib.my) attrsToNVList;
|
||||
in
|
||||
{
|
||||
imports = [ ./commands.nix ./install.nix ];
|
||||
imports = [ ./commands.nix ./install.nix ./vm-tasks.nix ];
|
||||
|
||||
env = attrsToNVList {
|
||||
# starship will show this
|
||||
@ -13,6 +13,8 @@ in
|
||||
''
|
||||
experimental-features = nix-command flakes ca-derivations
|
||||
'');
|
||||
|
||||
INSTALLER_SSH_OPTS = "-i .keys/deploy.key";
|
||||
};
|
||||
|
||||
packages = with pkgs; [
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{ lib, pkgs, config, ... }:
|
||||
let
|
||||
inherit (lib) mapAttrsToList concatMapStringsSep;
|
||||
inherit (lib.my) mkOpt' attrsToNVList;
|
||||
inherit (lib) mapAttrsToList;
|
||||
inherit (lib.my) mkOpt';
|
||||
|
||||
parseArgs = opts:
|
||||
''
|
||||
@ -45,10 +45,10 @@ let
|
||||
log "[\e[36;1minfo\e[0m]: \e[36m$*\e[0m"
|
||||
}
|
||||
warn() {
|
||||
log "[\e[33;1minfo\e[0m]: \e[33m$*\e[0m"
|
||||
log "[\e[33;1mwarn\e[0m]: \e[33m$*\e[0m"
|
||||
}
|
||||
error() {
|
||||
log "[\e[31;1minfo\e[0m]: \e[31m$*\e[0m"
|
||||
log "[\e[31;1merror\e[0m]: \e[31m$*\e[0m"
|
||||
}
|
||||
die() {
|
||||
error "$@"
|
||||
|
||||
128
devshell/vm-tasks.nix
Normal file
128
devshell/vm-tasks.nix
Normal file
@ -0,0 +1,128 @@
|
||||
{ lib, pkgs, config, ... }:
|
||||
let
|
||||
inherit (lib) mapAttrsToList;
|
||||
inherit (lib.my) mkOpt;
|
||||
|
||||
parseArgs = opts:
|
||||
''
|
||||
POSITIONAL_ARGS=()
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
# shellcheck disable=SC2221,SC2222
|
||||
case $1 in
|
||||
${opts}
|
||||
-*|--*)
|
||||
die "Unknown option $1"
|
||||
;;
|
||||
*)
|
||||
POSITIONAL_ARGS+=("$1") # save positional arg
|
||||
shift # past argument
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
set -- "''${POSITIONAL_ARGS[@]}" # restore positional parameters
|
||||
'';
|
||||
common = pkgs.writeShellApplication {
|
||||
name = "vm-tasks-common.sh";
|
||||
runtimeInputs = with pkgs; [
|
||||
openssh
|
||||
];
|
||||
text =
|
||||
''
|
||||
: "''${VM_SSH_OPTS:=}"
|
||||
|
||||
IFS=" " read -ra SSH_OPTS <<< "$VM_SSH_OPTS"
|
||||
SSH_OPTS+=(-N)
|
||||
|
||||
HOST="''${1:-}"
|
||||
VM="''${2:-}"
|
||||
if [ -z "$HOST" ] || [ -z "$VM" ]; then
|
||||
echo "usage: $0 <host> <vm> ..." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SOCKS_DIR="$(mktemp -d --tmpdir vm-socks.XXXXXX)"
|
||||
cleanup() {
|
||||
rm -rf "$SOCKS_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
SOCKS=()
|
||||
closeSocks() {
|
||||
for p in "''${SOCK_PIDS[@]}"; do
|
||||
kill "$p"
|
||||
done
|
||||
}
|
||||
openSock() {
|
||||
local s="$SOCKS_DIR"/"$1".sock
|
||||
ssh "''${SSH_OPTS[@]}" -L "$s":/run/vms/"$VM"/"$1".sock "$HOST" &
|
||||
SOCKS+=($!)
|
||||
echo "$s"
|
||||
}
|
||||
'';
|
||||
};
|
||||
vmTaskOpts = with lib.types; {
|
||||
options = {
|
||||
help = mkOpt str null;
|
||||
script = mkOpt lines "";
|
||||
packages = mkOpt (listOf package) [ ];
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
options.my.vmTasks = with lib.types;
|
||||
mkOpt (attrsOf (submodule vmTaskOpts)) { };
|
||||
|
||||
config = {
|
||||
my.vmTasks = {
|
||||
vm-tty = {
|
||||
help = "Access remote VM's TTY";
|
||||
packages = with pkgs; [ minicom ];
|
||||
script =
|
||||
''
|
||||
sock="$(openSock tty)"
|
||||
minicom -D unix#"$sock"
|
||||
closeSocks
|
||||
'';
|
||||
};
|
||||
vm-monitor = {
|
||||
help = "Access remote VM's QEMU monitor";
|
||||
packages = with pkgs; [ minicom ];
|
||||
script =
|
||||
''
|
||||
sock="$(openSock monitor)"
|
||||
minicom -D unix#"$sock"
|
||||
closeSocks
|
||||
'';
|
||||
};
|
||||
vm-viewer = {
|
||||
help = "Access remote VM's display with virt-viewer";
|
||||
packages = with pkgs; [ virt-viewer ];
|
||||
script =
|
||||
''
|
||||
sock=$(openSock spice)
|
||||
remote-viewer spice+unix://"$sock"
|
||||
closeSocks
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
commands = mapAttrsToList (name: cmd: {
|
||||
inherit name;
|
||||
inherit (cmd) help;
|
||||
category = "vm-tasks";
|
||||
package = pkgs.writeShellApplication {
|
||||
inherit name;
|
||||
runtimeInputs = cmd.packages;
|
||||
text =
|
||||
''
|
||||
# shellcheck disable=SC1091
|
||||
source "${common}/bin/vm-tasks-common.sh"
|
||||
|
||||
${cmd.script}
|
||||
'';
|
||||
};
|
||||
}) config.my.vmTasks;
|
||||
};
|
||||
}
|
||||
@ -95,6 +95,7 @@
|
||||
nixos/installer.nix
|
||||
|
||||
nixos/boxes/colony.nix
|
||||
nixos/vms/estuary.nix
|
||||
nixos/containers/vaultwarden.nix
|
||||
|
||||
# Homes
|
||||
@ -138,7 +139,7 @@
|
||||
} //
|
||||
(eachDefaultSystem (system:
|
||||
let
|
||||
pkgs = pkgs'.unstable.${system};
|
||||
pkgs = pkgs'.mine.${system};
|
||||
lib = pkgs.lib;
|
||||
in
|
||||
# Stuff for each platform
|
||||
|
||||
@ -6,43 +6,112 @@
|
||||
|
||||
configuration = { lib, pkgs, modulesPath, config, systems, ... }:
|
||||
let
|
||||
inherit (lib) mkIf;
|
||||
inherit (lib) mkIf mapAttrs;
|
||||
|
||||
wanBDF =
|
||||
if config.my.build.isDevVM then "00:02.0" else "01:00.0";
|
||||
in
|
||||
{
|
||||
imports = [ "${modulesPath}/profiles/qemu-guest.nix" ];
|
||||
|
||||
boot.kernelParams = [ "intel_iommu=on" ];
|
||||
boot.loader.systemd-boot.configurationLimit = 20;
|
||||
fileSystems = {
|
||||
"/boot" = {
|
||||
device = "/dev/disk/by-label/ESP";
|
||||
fsType = "vfat";
|
||||
};
|
||||
"/nix" = {
|
||||
device = "/dev/ssds/colony-nix";
|
||||
fsType = "ext4";
|
||||
};
|
||||
"/persist" = {
|
||||
device = "/dev/ssds/colony-persist";
|
||||
fsType = "ext4";
|
||||
neededForBoot = true;
|
||||
};
|
||||
};
|
||||
services = {
|
||||
lvm = {
|
||||
boot.thin.enable = true;
|
||||
dmeventd.enable = true;
|
||||
};
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
pciutils
|
||||
];
|
||||
|
||||
networking = {
|
||||
interfaces = mkIf (!config.my.build.isDevVM) {
|
||||
enp10s0.useDHCP = true;
|
||||
};
|
||||
};
|
||||
|
||||
systemd = {
|
||||
network = {
|
||||
netdevs."25-base-bridge".netdevConfig = {
|
||||
Name = "base";
|
||||
Kind = "bridge";
|
||||
};
|
||||
networks."80-base-bridge" = {
|
||||
matchConfig = {
|
||||
Name = "base";
|
||||
Driver = "bridge";
|
||||
};
|
||||
DHCP = "ipv4";
|
||||
networkConfig = {
|
||||
IPv6AcceptRA = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
services."vm@estuary" = rec {
|
||||
# Bind to the interface, networkd wait-online would deadlock...
|
||||
requires = [ "sys-subsystem-net-devices-base.device" ];
|
||||
bindsTo = requires;
|
||||
};
|
||||
};
|
||||
|
||||
#environment.etc."udev/udev.conf".text = "udev_log=debug";
|
||||
#systemd.services.systemd-networkd.environment.SYSTEMD_LOG_LEVEL = "debug";
|
||||
virtualisation = {
|
||||
cores = 8;
|
||||
memorySize = 8192;
|
||||
qemu.options = [
|
||||
"-machine q35"
|
||||
"-accel kvm,kernel-irqchip=split"
|
||||
"-device intel-iommu,intremap=on,caching-mode=on"
|
||||
];
|
||||
};
|
||||
|
||||
my = {
|
||||
#deploy.generate.system.mode = "boot";
|
||||
secrets = {
|
||||
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINkqdN5t3UKwrNOOPKlbnG1WYhnkV5H9luAzMotr8SbT";
|
||||
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKp5WDdDr/1NS3SJIDOKwcCNZDFOxqPAD7cbZWAP7EkX";
|
||||
files."test.txt" = {};
|
||||
};
|
||||
|
||||
firewall = {
|
||||
trustedInterfaces = [ "virtual" ];
|
||||
nat = {
|
||||
externalInterface = "eth0";
|
||||
forwardPorts = [
|
||||
{
|
||||
proto = "tcp";
|
||||
sourcePort = 2222;
|
||||
destination = "127.0.0.1:22";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
server.enable = true;
|
||||
|
||||
containers = {
|
||||
instances.vaultwarden = {
|
||||
networking.bridge = "virtual";
|
||||
network = {
|
||||
ipv6 = "2a0e:97c0:4d1:0::2";
|
||||
ipv4 = "10.110.0.2";
|
||||
};
|
||||
firewall = {
|
||||
trustedInterfaces = [ "base" ];
|
||||
};
|
||||
|
||||
#containers = {
|
||||
# instances.vaultwarden = {
|
||||
# networking.bridge = "virtual";
|
||||
# };
|
||||
#};
|
||||
vms = {
|
||||
instances.test = {
|
||||
instances.estuary = {
|
||||
uuid = "59f51efb-7e6d-477b-a263-ed9620dbc87b";
|
||||
networks.virtual.mac = "52:54:00:ab:f1:52";
|
||||
networks.base.mac = "52:54:00:ab:f1:52";
|
||||
drives = {
|
||||
disk = {
|
||||
installer = {
|
||||
backend = {
|
||||
driver = "file";
|
||||
filename = "${systems.installer.configuration.config.my.buildAs.iso}/iso/nixos.iso";
|
||||
@ -50,62 +119,31 @@
|
||||
};
|
||||
format.driver = "raw";
|
||||
frontend = "ide-cd";
|
||||
frontendOpts = {
|
||||
bootindex = 1;
|
||||
};
|
||||
};
|
||||
disk = {
|
||||
backend = {
|
||||
driver = "host_device";
|
||||
filename = "/dev/ssds/vm-estuary";
|
||||
# It appears this needs to be set on the backend _and_ the format
|
||||
discard = "unmap";
|
||||
};
|
||||
format = {
|
||||
driver = "raw";
|
||||
discard = "unmap";
|
||||
};
|
||||
frontend = "virtio-blk";
|
||||
frontendOpts = {
|
||||
bootindex = 0;
|
||||
};
|
||||
};
|
||||
};
|
||||
hostDevices."${wanBDF}" = { };
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
fileSystems = {
|
||||
"/boot" = {
|
||||
device = "/dev/disk/by-label/ESP";
|
||||
fsType = "vfat";
|
||||
};
|
||||
"/nix" = {
|
||||
device = "/dev/disk/by-label/nix";
|
||||
fsType = "ext4";
|
||||
};
|
||||
"/persist" = {
|
||||
device = "/dev/disk/by-label/persist";
|
||||
fsType = "ext4";
|
||||
neededForBoot = true;
|
||||
};
|
||||
};
|
||||
|
||||
networking = {
|
||||
interfaces = mkIf (!config.my.build.isDevVM) {
|
||||
enp1s0.useDHCP = true;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.network = {
|
||||
netdevs."25-virtual-bridge".netdevConfig = {
|
||||
Name = "virtual";
|
||||
Kind = "bridge";
|
||||
};
|
||||
networks."80-virtual-bridge" = {
|
||||
matchConfig = {
|
||||
Name = "virtual";
|
||||
Driver = "bridge";
|
||||
};
|
||||
networkConfig = {
|
||||
Address = "172.16.137.1/24";
|
||||
DHCPServer = true;
|
||||
# TODO: Configuration for routed IPv6 (and maybe IPv4)
|
||||
IPMasquerade = "both";
|
||||
IPv6SendRA = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
#systemd.services.systemd-networkd.environment.SYSTEMD_LOG_LEVEL = "debug";
|
||||
virtualisation = {
|
||||
cores = 8;
|
||||
memorySize = 8192;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@ -13,14 +13,14 @@
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
# Lots of kernel modules and firmware
|
||||
"${modulesPath}/profiles/all-hardware.nix"
|
||||
# Useful tools to have
|
||||
"${modulesPath}/profiles/base.nix"
|
||||
];
|
||||
|
||||
config = {
|
||||
my = {
|
||||
# Lots of kernel modules and firmware
|
||||
build.allHardware = true;
|
||||
# Whatever installer mechanism is chosen will provide an appropriate `/`
|
||||
tmproot.enable = false;
|
||||
firewall.nat.enable = false;
|
||||
@ -72,9 +72,6 @@
|
||||
'';
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
# We disable networking.useDHCP, so bring these in for the user
|
||||
# dhcpcd probably has more features, but dhclient actually seems a bit more simple
|
||||
(pkgs.writeShellScriptBin "dhclient" ''exec ${pkgs.dhcp}/bin/dhclient -v "$@"'')
|
||||
dhcpcd
|
||||
];
|
||||
|
||||
@ -99,6 +96,7 @@
|
||||
# download-using-manifests.pl from forking even if there is
|
||||
# plenty of free memory.
|
||||
boot.kernel.sysctl."vm.overcommit_memory" = "1";
|
||||
services.lvm.boot.thin.enable = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
@ -11,5 +11,6 @@
|
||||
secrets = ./secrets.nix;
|
||||
containers = ./containers.nix;
|
||||
vms = ./vms.nix;
|
||||
network = ./network.nix;
|
||||
};
|
||||
}
|
||||
|
||||
@ -80,6 +80,7 @@ in
|
||||
sharedDirectories = dummyOption;
|
||||
cores = dummyOption;
|
||||
memorySize = dummyOption;
|
||||
qemu.options = dummyOption;
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{ lib, pkgs, pkgs', inputs, config, ... }:
|
||||
let
|
||||
inherit (lib) flatten optional mkIf mkDefault mkMerge;
|
||||
inherit (lib) mkIf mkDefault mkMerge;
|
||||
inherit (lib.my) mkBoolOpt' dummyOption;
|
||||
in
|
||||
{
|
||||
@ -95,17 +95,11 @@ in
|
||||
};
|
||||
};
|
||||
|
||||
networking = {
|
||||
domain = mkDefault "int.nul.ie";
|
||||
useDHCP = false;
|
||||
enableIPv6 = mkDefault true;
|
||||
useNetworkd = mkDefault true;
|
||||
};
|
||||
|
||||
environment.systemPackages = with pkgs; [
|
||||
bash-completion
|
||||
vim
|
||||
ldns
|
||||
minicom
|
||||
];
|
||||
|
||||
programs = {
|
||||
@ -141,14 +135,6 @@ in
|
||||
})
|
||||
];
|
||||
})
|
||||
(mkIf config.my.build.isDevVM {
|
||||
networking.interfaces.eth0.useDHCP = mkDefault true;
|
||||
virtualisation = {
|
||||
forwardPorts = flatten [
|
||||
(optional config.services.openssh.openFirewall { from = "host"; host.port = 2222; guest.port = 22; })
|
||||
];
|
||||
};
|
||||
})
|
||||
];
|
||||
|
||||
meta.buildDocsInSandbox = false;
|
||||
|
||||
@ -1,11 +1,24 @@
|
||||
{ lib, pkgs, config, systems, ... }:
|
||||
let
|
||||
inherit (builtins) head attrNames;
|
||||
inherit (lib) mkMerge mkIf mkDefault optionalAttrs mapAttrs';
|
||||
inherit (lib) mkMerge mkIf mkDefault optionalAttrs mapAttrs' optionalString;
|
||||
inherit (lib.my) mkOpt' mkBoolOpt';
|
||||
|
||||
cfg = config.my.deploy;
|
||||
|
||||
# Based on https://github.com/serokell/deploy-rs/blob/master/flake.nix
|
||||
nixosActivate = mode: base: (pkgs.deploy-rs.lib.activate.custom // { dryActivate = "$PROFILE/bin/switch-to-configuration dry-activate"; }) base.config.system.build.toplevel ''
|
||||
# work around https://github.com/NixOS/nixpkgs/issues/73404
|
||||
cd /tmp
|
||||
|
||||
$PROFILE/bin/switch-to-configuration ${mode}
|
||||
|
||||
# https://github.com/serokell/deploy-rs/issues/31
|
||||
${with base.config.boot.loader;
|
||||
optionalString ((mode == "switch" || mode == "boot") && systemd-boot.enable)
|
||||
"sed -i '/^default /d' ${efi.efiSysMountPoint}/loader/loader.conf"}
|
||||
'';
|
||||
|
||||
ctrProfiles = optionalAttrs cfg.generate.containers.enable (mapAttrs' (n: c:
|
||||
let
|
||||
ctrConfig = systems."${n}".configuration.config;
|
||||
@ -34,7 +47,10 @@ in
|
||||
inherit (lib.my.deploy-rs) node;
|
||||
|
||||
generate = {
|
||||
system.enable = mkBoolOpt' true "Whether to generate a deploy-rs profile for this system's config.";
|
||||
system = {
|
||||
enable = mkBoolOpt' true "Whether to generate a deploy-rs profile for this system's config.";
|
||||
mode = mkOpt' str "switch" "switch-to-configuration mode.";
|
||||
};
|
||||
containers.enable = mkBoolOpt' true "Whether to generate deploy-rs profiles for this system's containers.";
|
||||
};
|
||||
};
|
||||
@ -49,7 +65,7 @@ in
|
||||
profilesOrder = [ "system" ] ++ (attrNames ctrProfiles);
|
||||
profiles = {
|
||||
system = mkIf cfg.generate.system.enable {
|
||||
path = pkgs.deploy-rs.lib.activate.nixos { inherit config; };
|
||||
path = nixosActivate cfg.generate.system.mode { inherit config; };
|
||||
|
||||
user = "root";
|
||||
};
|
||||
|
||||
35
nixos/modules/network.nix
Normal file
35
nixos/modules/network.nix
Normal file
@ -0,0 +1,35 @@
|
||||
{ lib, config, ... }:
|
||||
let
|
||||
inherit (lib) flatten optional mkIf mkDefault mkMerge;
|
||||
inherit (lib.my) mkOpt' mkBoolOpt';
|
||||
|
||||
cfg = config.my.network;
|
||||
in
|
||||
{
|
||||
options = with lib.types; {
|
||||
my.network = {
|
||||
ipv4 = mkOpt' str null "Internal network IPv4 address.";
|
||||
ipv6 = mkOpt' str null "Internal network IPv6 address.";
|
||||
};
|
||||
};
|
||||
|
||||
config = mkMerge [
|
||||
{
|
||||
networking = {
|
||||
domain = mkDefault "int.nul.ie";
|
||||
useDHCP = false;
|
||||
enableIPv6 = mkDefault true;
|
||||
useNetworkd = mkDefault true;
|
||||
};
|
||||
}
|
||||
|
||||
(mkIf config.my.build.isDevVM {
|
||||
networking.interfaces.eth0.useDHCP = mkDefault true;
|
||||
virtualisation = {
|
||||
forwardPorts = flatten [
|
||||
(optional config.services.openssh.openFirewall { from = "host"; host.port = 2222; guest.port = 22; })
|
||||
];
|
||||
};
|
||||
})
|
||||
];
|
||||
}
|
||||
@ -22,15 +22,14 @@ in
|
||||
# agenix sets this as a default but adding any custom extras will _replace_ the list (different priority)
|
||||
identityPaths =
|
||||
mkIf config.services.openssh.enable
|
||||
(map (e: e.path) (lib.filter (e: e.type == "rsa" || e.type == "ed25519") config.services.openssh.hostKeys));
|
||||
(map
|
||||
# Use the persit dir to grab the keys instead, otherwise they might not be ready. We can't really make
|
||||
# agenix depend on impermanence, since users depends on agenix (to decrypt passwords) and impermanence
|
||||
# depends on users
|
||||
(e: let pDir = config.my.tmproot.persistence.dir; in if pDir != null then "${pDir}/${e.path}" else e.path)
|
||||
(lib.filter (e: e.type == "rsa" || e.type == "ed25519") config.services.openssh.hostKeys));
|
||||
};
|
||||
}
|
||||
(mkIf (config.age.secrets != { }) {
|
||||
system.activationScripts.agenixMountSecrets.deps = mkIf (config.my.tmproot.persistence.dir != null) [
|
||||
# The key used to decrypt is not going to exist!
|
||||
"persist-files"
|
||||
];
|
||||
})
|
||||
(mkIf config.my.build.isDevVM {
|
||||
age.identityPaths = [ cfg.vmKeyPath ];
|
||||
})
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
{ lib, pkgs, config, ... }:
|
||||
let
|
||||
inherit (lib) optional optionals optionalString flatten concatStringsSep mapAttrsToList mapAttrs' mkIf mkDefault;
|
||||
inherit (builtins) filter any attrNames attrValues fetchGit;
|
||||
inherit (lib)
|
||||
unique optional optionals optionalString flatten concatStringsSep
|
||||
concatMapStringsSep mapAttrsToList mapAttrs' mkIf mkDefault;
|
||||
inherit (lib.my) mkOpt' mkBoolOpt';
|
||||
|
||||
flattenQEMUOpts = attrs:
|
||||
@ -38,6 +41,29 @@ let
|
||||
pass
|
||||
'';
|
||||
|
||||
# TODO: Upstream or something...
|
||||
vfio-pci-bind = pkgs.stdenv.mkDerivation rec {
|
||||
pname = "vfio-pci-bind";
|
||||
version = "b41e4545b21de434fc51a34a9bf1d72e3ac66cc8";
|
||||
|
||||
src = fetchGit {
|
||||
url = "https://github.com/andre-richter/vfio-pci-bind";
|
||||
rev = version;
|
||||
};
|
||||
|
||||
prePatch = ''
|
||||
substituteInPlace vfio-pci-bind.sh \
|
||||
--replace modprobe ${pkgs.kmod}/bin/modprobe
|
||||
substituteInPlace 25-vfio-pci-bind.rules \
|
||||
--replace vfio-pci-bind.sh "$out"/bin/vfio-pci-bind.sh
|
||||
'';
|
||||
installPhase = ''
|
||||
mkdir -p "$out"/bin/ "$out"/lib/udev/rules.d
|
||||
cp vfio-pci-bind.sh "$out"/bin/
|
||||
cp 25-vfio-pci-bind.rules "$out"/lib/udev/rules.d/
|
||||
'';
|
||||
};
|
||||
|
||||
cfg = config.my.vms;
|
||||
|
||||
netOpts = with lib.types; { name, ... }: {
|
||||
@ -61,6 +87,12 @@ let
|
||||
};
|
||||
};
|
||||
|
||||
hostDevOpts = with lib.types; {
|
||||
options = {
|
||||
bindVFIO = mkBoolOpt' true "Whether to automatically bind the device to vfio-pci.";
|
||||
};
|
||||
};
|
||||
|
||||
vmOpts = with lib.types; { name, ... }: {
|
||||
options = {
|
||||
qemuBin = mkOpt' path "${pkgs.qemu_kvm}/bin/qemu-kvm" "Path to QEMU executable.";
|
||||
@ -85,9 +117,16 @@ let
|
||||
spice.enable = mkBoolOpt' true "Whether to enable SPICE.";
|
||||
networks = mkOpt' (attrsOf (submodule netOpts)) { } "Networks to attach VM to.";
|
||||
drives = mkOpt' (attrsOf (submodule driveOpts)) { } "Drives to attach to VM.";
|
||||
hostDevices = mkOpt' (attrsOf (submodule hostDevOpts)) { } "Host PCI devices to pass to the VM.";
|
||||
};
|
||||
};
|
||||
|
||||
allHostDevs =
|
||||
flatten
|
||||
(map
|
||||
(i: mapAttrsToList (bdf: c: { inherit bdf; inherit (c) bindVFIO; }) i.hostDevices)
|
||||
(attrValues cfg.instances));
|
||||
|
||||
mkQemuCommand = n: i:
|
||||
let
|
||||
flags =
|
||||
@ -122,7 +161,8 @@ let
|
||||
"blockdev node-name=${dn}-backend,${c.backend}"
|
||||
"blockdev node-name=${dn}-format,${c.formatBackendProp}=${dn}-backend,${c.format}"
|
||||
("device ${c.frontend},id=${dn},drive=${dn}-format" + (extraQEMUOpts c.frontendOpts))
|
||||
]) i.drives));
|
||||
]) i.drives)) ++
|
||||
(map (bdf: "device vfio-pci,host=${bdf}") (attrNames i.hostDevices));
|
||||
args = map (v: "-${v}") flags;
|
||||
in
|
||||
concatStringsSep " " ([ i.qemuBin ] ++ args);
|
||||
@ -134,6 +174,30 @@ in
|
||||
};
|
||||
|
||||
config = mkIf (cfg.instances != { }) {
|
||||
assertions = [
|
||||
{
|
||||
assertion = let bdfs = map (d: d.bdf) allHostDevs; in (unique bdfs) == bdfs;
|
||||
message = "VMs cannot share host devices!";
|
||||
}
|
||||
];
|
||||
|
||||
services.udev = {
|
||||
packages =
|
||||
optionals
|
||||
(any (d: d.bindVFIO) allHostDevs)
|
||||
[
|
||||
vfio-pci-bind
|
||||
(pkgs.writeTextDir
|
||||
"etc/udev/rules.d/20-vfio-tags.rules"
|
||||
(concatMapStringsSep
|
||||
"\n"
|
||||
(d: ''ACTION=="add", SUBSYSTEM=="pci", KERNEL=="0000:${d.bdf}", TAG="vfio-pci-bind"'')
|
||||
(filter (d: d.bindVFIO) allHostDevs)))
|
||||
];
|
||||
};
|
||||
|
||||
my.tmproot.persistence.config.directories = [ "/var/lib/vms" ];
|
||||
|
||||
# qemu-bridge-helper will fail otherwise
|
||||
environment.etc."qemu/bridge.conf".text = "allow all";
|
||||
systemd = {
|
||||
|
||||
101
nixos/vms/estuary.nix
Normal file
101
nixos/vms/estuary.nix
Normal file
@ -0,0 +1,101 @@
|
||||
{
|
||||
nixos.systems.estuary = {
|
||||
system = "x86_64-linux";
|
||||
nixpkgs = "mine";
|
||||
home-manager = "unstable";
|
||||
|
||||
configuration = { lib, pkgs, modulesPath, config, systems, ... }:
|
||||
let
|
||||
inherit (lib) mkIf mkMerge;
|
||||
in
|
||||
{
|
||||
imports = [ "${modulesPath}/profiles/qemu-guest.nix" ];
|
||||
|
||||
config = mkMerge [
|
||||
{
|
||||
boot.kernelParams = [ "console=ttyS0,115200n8" ];
|
||||
fileSystems = {
|
||||
"/boot" = {
|
||||
device = "/dev/disk/by-label/ESP";
|
||||
fsType = "vfat";
|
||||
};
|
||||
"/nix" = {
|
||||
device = "/dev/main/nix";
|
||||
fsType = "ext4";
|
||||
};
|
||||
"/persist" = {
|
||||
device = "/dev/main/persist";
|
||||
fsType = "ext4";
|
||||
neededForBoot = true;
|
||||
};
|
||||
};
|
||||
services = {
|
||||
lvm = {
|
||||
dmeventd.enable = true;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.network = {
|
||||
links = {
|
||||
"10-wan" = {
|
||||
matchConfig.MACAddress = "52:54:00:a1:b2:5f";
|
||||
linkConfig.Name = "wan";
|
||||
};
|
||||
"10-base" = {
|
||||
matchConfig.MACAddress = "52:54:00:ab:f1:52";
|
||||
linkConfig.Name = "base";
|
||||
};
|
||||
};
|
||||
|
||||
networks = {
|
||||
#"80-wan" = {
|
||||
# matchConfig.Name = "wan";
|
||||
# address = [
|
||||
# "1.2.3.4/24"
|
||||
# "2a00::2/64"
|
||||
# ];
|
||||
#};
|
||||
"80-wan" = {
|
||||
matchConfig.Name = "wan";
|
||||
DHCP = "ipv4";
|
||||
};
|
||||
"80-base" = {
|
||||
matchConfig.Name = "base";
|
||||
address = with config.my.network; [ "${ipv4}/24" "${ipv6}/64" ];
|
||||
networkConfig = {
|
||||
DHCPServer = true;
|
||||
IPv6SendRA = true;
|
||||
IPMasquerade = "both";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
my = {
|
||||
server.enable = true;
|
||||
|
||||
network = {
|
||||
ipv6 = "2a0e:97c0:4d1:0::1";
|
||||
ipv4 = "10.110.0.1";
|
||||
};
|
||||
firewall = {
|
||||
trustedInterfaces = [ "base" ];
|
||||
nat = {
|
||||
enable = true;
|
||||
externalInterface = "wan";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
(mkIf config.my.build.isDevVM {
|
||||
systemd.network = {
|
||||
netdevs."05-dummy-base".netdevConfig = {
|
||||
Name = "base";
|
||||
Kind = "dummy";
|
||||
};
|
||||
};
|
||||
})
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Loading…
Reference in New Issue
Block a user