dev/nixfiles
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

nixos: Add working container VM (w/ vaultwarden)

Browse Source 
Also improve IPv6 addressing / routing
This commit is contained in:
Jack O'Sullivan
2022-05-29 03:30:40 +01:00
parent 38e8827487
commit 00493bf30f
15 changed files with 351 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
nixos/boxes/colony/default.nix
View File

@@ -11,7 +11,10 @@
altNames = [ "vm" ];
ipv4.address = "10.100.0.2";
#ipv6.address = "2a0e:97c0:4d1:0::2";
ipv6.address = "2a0e:97c0:4d0:bbb0::2";
ipv6 = rec {
iid = "::2";
address = "2a0e:97c0:4d0:bbb0${iid}";
};
};
vms = {
ipv4 = {
@@ -31,7 +34,7 @@
{
imports = [ "${modulesPath}/profiles/qemu-guest.nix" ];
networking.domain = lib.my.colonyDomain;
networking.domain = lib.my.colony.domain;
boot.kernelParams = [ "intel_iommu=on" ];
boot.loader.systemd-boot.configurationLimit = 20;
@@ -88,7 +91,7 @@
};
"80-vms" = mkMerge [
(networkdAssignment "base" assignments.vms)
(networkdAssignment "vms" assignments.vms)
{
networkConfig = {
IPv6AcceptRA = mkForce false;
@@ -101,7 +104,17 @@
ipv6Prefixes = [
{
#ipv6PrefixConfig.Prefix = "2a0e:97c0:4d1:1::/64";
ipv6PrefixConfig.Prefix = "2a0e:97c0:4d0:bbb1::/64";
ipv6PrefixConfig.Prefix = lib.my.colony.prefixes.vms.v6;
}
];
routes = map (r: { routeConfig = r; }) [
{
Gateway = allAssignments.shill.internal.ipv4.address;
Destination = lib.my.colony.prefixes.ctrs.v4;
}
{
Gateway = allAssignments.shill.internal.ipv6.address;
Destination = lib.my.colony.prefixes.ctrs.v6;
}
];
}
@@ -145,14 +158,8 @@
server.enable = true;
firewall = {
trustedInterfaces = [ "base" ];
trustedInterfaces = [ "base" "vms" ];
};
#containers = {
# instances.vaultwarden = {
# networking.bridge = "virtual";
# };
#};
};
};
};

93
nixos/boxes/colony/vms/default.nix
View File

@@ -1,12 +1,31 @@
{
imports = [
./estuary
./shill
];
nixos.systems.colony.configuration = { lib, pkgs, config, systems, ... }:
let
inherit (lib) mkMerge;
wanBDF =
if config.my.build.isDevVM then "00:02.0" else "01:00.0";
vmLVM = vm: lv: {
"${lv}" = {
backend = {
driver = "host_device";
filename = "/dev/ssds/vm-${vm}-${lv}";
# It appears this needs to be set on the backend _and_ the format
discard = "unmap";
};
format = {
driver = "raw";
discard = "unmap";
};
frontend = "virtio-blk";
};
};
in
{
systemd = {
@@ -28,40 +47,52 @@
my = {
vms = {
instances.estuary = {
uuid = "59f51efb-7e6d-477b-a263-ed9620dbc87b";
networks.base.mac = "52:54:00:ab:f1:52";
drives = {
installer = {
backend = {
driver = "file";
filename = "${systems.installer.configuration.config.my.buildAs.iso}/iso/nixos.iso";
read-only = "on";
};
format.driver = "raw";
frontend = "ide-cd";
frontendOpts = {
bootindex = 1;
};
};
disk = {
backend = {
driver = "host_device";
filename = "/dev/ssds/vm-estuary";
# It appears this needs to be set on the backend _and_ the format
discard = "unmap";
};
format = {
driver = "raw";
discard = "unmap";
};
frontend = "virtio-blk";
frontendOpts = {
bootindex = 0;
instances = {
estuary = {
uuid = "59f51efb-7e6d-477b-a263-ed9620dbc87b";
networks.base.mac = "52:54:00:ab:f1:52";
drives = {
# TODO: Split into separate LVs
disk = {
backend = {
driver = "host_device";
filename = "/dev/ssds/vm-estuary";
# It appears this needs to be set on the backend _and_ the format
discard = "unmap";
};
format = {
driver = "raw";
discard = "unmap";
};
frontend = "virtio-blk";
};
};
hostDevices."${wanBDF}" = { };
};
shill = {
uuid = "e34569ec-d24e-446b-aca8-a3b27abc1f9b";
networks.vms.mac = "52:54:00:85:b3:b1";
drives = mkMerge [
(vmLVM "shill" "esp")
(vmLVM "shill" "nix")
(vmLVM "shill" "persist")
{
installer = {
backend = {
driver = "file";
filename = "${systems.installer.configuration.config.my.buildAs.iso}/iso/nixos.iso";
read-only = "on";
};
format.driver = "raw";
frontend = "ide-cd";
frontendOpts = {
bootindex = 1;
};
};
esp.frontendOpts.bootindex = 0;
}
];
};
hostDevices."${wanBDF}" = { };
};
};
};

25
nixos/boxes/colony/vms/estuary/default.nix
View File

@@ -25,7 +25,7 @@
config = mkMerge [
{
networking.domain = lib.my.colonyDomain;
networking.domain = lib.my.colony.domain;
boot.kernelParams = [ "console=ttyS0,115200n8" ];
fileSystems = {
@@ -94,7 +94,26 @@
ipv6Prefixes = [
{
#ipv6PrefixConfig.Prefix = "2a0e:97c0:4d1:0::/64";
ipv6PrefixConfig.Prefix = "2a0e:97c0:4d0:bbb0::/64";
ipv6PrefixConfig.Prefix = lib.my.colony.prefixes.base.v6;
}
];
routes = map (r: { routeConfig = r; }) [
{
Gateway = allAssignments.colony.internal.ipv4.address;
Destination = lib.my.colony.prefixes.vms.v4;
}
{
Gateway = allAssignments.colony.internal.ipv6.address;
Destination = lib.my.colony.prefixes.vms.v6;
}
{
Gateway = allAssignments.colony.internal.ipv4.address;
Destination = lib.my.colony.prefixes.ctrs.v4;
}
{
Gateway = allAssignments.colony.internal.ipv6.address;
Destination = lib.my.colony.prefixes.ctrs.v6;
}
];
}
@@ -138,7 +157,7 @@
iifname wan meta l4proto { udp, tcp } th dport domain redirect to :5353
}
chain postrouting {
ip saddr 10.100.0.0/16 masquerade
ip saddr ${lib.my.colony.prefixes.all.v4} masquerade
}
}
'';

4
nixos/boxes/colony/vms/estuary/dns.nix
View File

@@ -21,9 +21,7 @@ in
];
allowFrom = [
"127.0.0.0/8" "::1/128"
"10.100.0.0/16" "2a0e:97c0:4d1::/48"
# TODO: Remove when moving to proper net!
"2a0e:97c0:4d0::/48"
lib.my.colony.prefixes.all.v4 lib.my.colony.prefixes.all.v6
];
};
forwardZones = genAttrs authZones (_: "127.0.0.1:5353");

5
nixos/boxes/colony/vms/shill/containers/default.nix Normal file
View File

@@ -0,0 +1,5 @@
{
imports = [
./vaultwarden.nix
];
}

39
nixos/containers/vaultwarden.nix → nixos/boxes/colony/vms/shill/containers/vaultwarden.nix
View File

@@ -1,11 +1,23 @@
{
nixos.systems.vaultwarden = {
system = "x86_64-linux";
nixpkgs = "unstable";
nixpkgs = "mine";
configuration = { lib, config, ... }:
assignments = {
internal = {
name = "vaultwarden-ctr";
ipv4.address = "10.100.2.2";
ipv6 = rec {
iid = "::2";
address = "2a0e:97c0:4d0:bbb2${iid}";
};
};
};
configuration = { lib, config, assignments, ... }:
let
inherit (lib) mkMerge mkIf mkForce;
inherit (lib.my) networkdAssignment;
vwData = "/var/lib/vaultwarden";
vwSecrets = "vaultwarden.env";
@@ -17,16 +29,28 @@
server.enable = true;
secrets = {
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMHoWhafCkLVggsO24fFWm3nmkY5t23GHbBafBVGijbQ";
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILakffcjRp6h6lxSOADOsTK5h2MCkt8hKDv0cvchM7iw";
files."${vwSecrets}" = {};
};
firewall = {
tcp.allowed = [ 80 3012 ];
};
tmproot.persistence.config.directories = [
{
directory = vwData;
user = config.users.users.vaultwarden.name;
group = config.users.groups.vaultwarden.name;
}
];
};
systemd = {
services.vaultwarden.serviceConfig.StateDirectory = mkForce "vaultwarden";
network.networks."80-container-host0" = networkdAssignment "host0" assignments.internal;
};
systemd.services.vaultwarden.serviceConfig.StateDirectory = mkForce "vaultwarden";
services = {
vaultwarden = {
enable = true;
@@ -43,13 +67,6 @@
};
}
(mkIf config.my.build.isDevVM {
my.tmproot.persistence.config.directories = [
{
directory = vwData;
user = config.users.users.vaultwarden.name;
group = config.users.groups.vaultwarden.name;
}
];
virtualisation = {
forwardPorts = [
{ from = "host"; host.port = 8080; guest.port = 80; }

109
nixos/boxes/colony/vms/shill/default.nix Normal file
View File

@@ -0,0 +1,109 @@
{
imports = [ ./containers ];
nixos.systems.shill = {
system = "x86_64-linux";
nixpkgs = "mine";
assignments = {
internal = {
name = "shill-vm";
altNames = [ "ctr" ];
ipv4.address = "10.100.1.2";
ipv6 = rec {
iid = "::2";
address = "2a0e:97c0:4d0:bbb1${iid}";
};
};
ctrs = {
ipv4 = {
address = "10.100.2.1";
gateway = null;
};
ipv6.address = "2a0e:97c0:4d0:bbb2::1";
};
};
configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }:
let
inherit (lib) mkIf mkMerge mkForce;
inherit (lib.my) networkdAssignment;
in
{
imports = [ "${modulesPath}/profiles/qemu-guest.nix" ];
config = mkMerge [
{
networking.domain = lib.my.colony.domain;
boot.kernelParams = [ "console=ttyS0,115200n8" ];
fileSystems = {
"/boot" = {
device = "/dev/disk/by-label/ESP";
fsType = "vfat";
};
"/nix" = {
device = "/dev/vdb";
fsType = "ext4";
};
"/persist" = {
device = "/dev/vdc";
fsType = "ext4";
neededForBoot = true;
};
};
systemd.network = {
links = {
"10-vms" = {
matchConfig.MACAddress = "52:54:00:85:b3:b1";
linkConfig.Name = "vms";
};
};
netdevs."25-ctrs".netdevConfig = {
Name = "ctrs";
Kind = "bridge";
};
networks = {
"80-vms" = networkdAssignment "vms" assignments.internal;
"80-ctrs" = mkMerge [
(networkdAssignment "ctrs" assignments.ctrs)
{
networkConfig = {
IPv6AcceptRA = mkForce false;
IPv6SendRA = true;
};
ipv6SendRAConfig = {
DNS = [ allAssignments.estuary.internal.ipv6.address ];
Domains = [ config.networking.domain ];
};
ipv6Prefixes = [
{
ipv6PrefixConfig.Prefix = lib.my.colony.prefixes.ctrs.v6;
}
];
}
];
};
};
my = {
secrets.key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMWi6iEcpKdWPiHPgQEeVVKfB3yWNXQbXbr8IXYL+6Cw";
server.enable = true;
firewall = {
trustedInterfaces = [ "vms" "ctrs" ];
};
containers = {
instances.vaultwarden = {
networking.bridge = "ctrs";
};
};
};
}
];
};
};
}

1
nixos/default.nix
View File

@@ -102,6 +102,7 @@ let
ipv6 = {
address = mkOpt' str null "IPv6 address.";
mask = mkOpt' ints.u8 64 "Network mask.";
iid = mkOpt' (nullOr str) null "SLAAC static address.";
gateway = mkOpt' (nullOr str) null "IPv6 gateway.";
};
};

7
nixos/installer.nix
View File

@@ -52,8 +52,11 @@
};
};
# Will be set dynamically
networking.hostName = "";
networking = {
# Will be set dynamically
hostName = "";
useNetworkd = false;
};
# This should be overridden by whatever boot mechanism is used
fileSystems."/" = mkDefault {

44
nixos/modules/containers.nix
View File

@@ -10,6 +10,40 @@ let
devVMKeyPath = "/run/dev.key";
ctrProfiles = n: "/nix/var/nix/profiles/per-container/${n}";
dummyReady = pkgs.runCommandCC "dummy-sd-ready" {
buildInputs = [ pkgs.systemd ];
passAsFile = [ "code" ];
code = ''
#include <stdio.h>
#include <signal.h>
#include <unistd.h>
#include <systemd/sd-daemon.h>
void handler(int signum) {
exit(0);
}
int main() {
// systemd sends this to PID 1 for an "orderly shutdown"
signal(SIGRTMIN+3, handler);
int ret =
sd_notifyf(0, "READY=1\n"
"STATUS=Dummy container, please deploy for real!\n"
"MAINPID=%lu",
(unsigned long)getpid());
if (ret <= 0) {
fprintf(stderr, "sd_notify() returned %d\n", ret);
return ret == 0 ? -1 : ret;
}
pause();
return 0;
};
'';
} ''
$CC -o "$out" -x c -lsystemd "$codePath"
'';
dummyProfile = pkgs.writeTextFile {
name = "dummy-init";
executable = true;
@@ -19,10 +53,7 @@ let
#!${pkgs.runtimeShell}
${pkgs.iproute2}/bin/ip link set dev host0 up
while true; do
echo "This is a dummy, please deploy the real container!"
${pkgs.coreutils}/bin/sleep 5
done
exec ${dummyReady}
'';
};
@@ -185,6 +216,7 @@ in
reload =
# `switch-to-configuration test` switches config without trying to update bootloader
''
# TODO: This still breaks on first deploy over the dummy...
[ -e "${system}"/bin/switch-to-configuration ] && \
systemd-run --pipe --machine ${n} -- "${containerSystem}"/bin/switch-to-configuration test
'';
@@ -250,13 +282,9 @@ in
Virtualization = "container";
};
networkConfig = {
DHCP = "yes";
LLDP = true;
EmitLLDP = "customer-bridge";
};
dhcpConfig = {
UseTimezone = true;
};
};
# If the host is a dev VM

2
nixos/modules/vms.nix
View File

@@ -113,7 +113,7 @@ let
threads = mkOpt' ints.unsigned 1 "Number of threads per core.";
};
memory = mkOpt' ints.unsigned 1024 "Amount of RAM (mebibytes).";
vga = mkOpt' str "qxl" "VGA card type.";
vga = mkOpt' str "virtio" "VGA card type.";
spice.enable = mkBoolOpt' true "Whether to enable SPICE.";
networks = mkOpt' (attrsOf (submodule netOpts)) { } "Networks to attach VM to.";
drives = mkOpt' (attrsOf (submodule driveOpts)) { } "Drives to attach to VM.";