440 lines
14 KiB
Nix
440 lines
14 KiB
Nix
{ lib, ... }:
|
|
let
|
|
inherit (lib.my) net;
|
|
inherit (lib.my.c.colony) domain prefixes custRouting firewallForwards;
|
|
in
|
|
{
|
|
imports = [ ./vms ];
|
|
|
|
nixos.systems.colony = {
|
|
system = "x86_64-linux";
|
|
nixpkgs = "mine-stable";
|
|
home-manager = "mine-stable";
|
|
|
|
assignments = {
|
|
routing = {
|
|
name = "colony-routing";
|
|
inherit domain;
|
|
ipv4.address = net.cidr.host 2 prefixes.base.v4;
|
|
};
|
|
internal = {
|
|
altNames = [ "vm" ];
|
|
inherit domain;
|
|
ipv4 = {
|
|
address = net.cidr.host 0 prefixes.vip1;
|
|
mask = 32;
|
|
gateway = null;
|
|
genPTR = false;
|
|
};
|
|
ipv6 = {
|
|
iid = "::2";
|
|
address = net.cidr.host 2 prefixes.base.v6;
|
|
};
|
|
};
|
|
vms = {
|
|
name = "colony-vms";
|
|
inherit domain;
|
|
ipv4 = {
|
|
address = net.cidr.host 1 prefixes.vms.v4;
|
|
gateway = null;
|
|
};
|
|
ipv6.address = net.cidr.host 1 prefixes.vms.v6;
|
|
};
|
|
};
|
|
|
|
configuration = { lib, pkgs, modulesPath, config, systems, assignments, allAssignments, ... }:
|
|
let
|
|
inherit (lib) mkIf mkMerge mkForce;
|
|
inherit (lib.my) networkdAssignment;
|
|
in
|
|
{
|
|
hardware = {
|
|
enableRedistributableFirmware = true;
|
|
cpu = {
|
|
amd.updateMicrocode = true;
|
|
};
|
|
rasdaemon.enable = true;
|
|
};
|
|
|
|
boot = {
|
|
kernelPackages = (lib.my.c.kernel.lts pkgs).extend (self: super: {
|
|
kernel = super.kernel.override {
|
|
structuredExtraConfig = with lib.kernel; {
|
|
ACPI_APEI_PCIEAER = yes;
|
|
PCIEAER = yes;
|
|
};
|
|
};
|
|
});
|
|
kernelModules = [ "kvm-amd" ];
|
|
kernelParams = [
|
|
"amd_iommu=on"
|
|
"console=ttyS0,115200n8" "console=ttyS1,115200n8" "console=tty0"
|
|
"systemd.setenv=SYSTEMD_SULOGIN_FORCE=1"
|
|
];
|
|
initrd = {
|
|
kernelModules = [ "dm-raid" ];
|
|
availableKernelModules = [ "nvme" "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
|
|
systemd = {
|
|
enable = true;
|
|
# Onlu activate volumes needed for boot to prevent thin check from getting killed while switching root
|
|
contents."/etc/lvm/lvm.conf".text = ''
|
|
activation/auto_activation_volume_list = [ "main/colony-nix" "main/colony-persist" ]
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
|
|
fileSystems = {
|
|
"/boot" = {
|
|
device = "/dev/disk/by-uuid/C1C9-9CBC";
|
|
fsType = "vfat";
|
|
};
|
|
"/nix" = {
|
|
device = "/dev/main/colony-nix";
|
|
fsType = "ext4";
|
|
};
|
|
"/persist" = {
|
|
device = "/dev/main/colony-persist";
|
|
fsType = "ext4";
|
|
neededForBoot = true;
|
|
};
|
|
"/mnt/backup" = {
|
|
device = "/dev/main/backup";
|
|
fsType = "ext4";
|
|
};
|
|
};
|
|
|
|
programs.ssh.knownHostsFiles = [
|
|
lib.my.c.sshKeyFiles.rsyncNet
|
|
];
|
|
|
|
services = {
|
|
fstrim = lib.my.c.colony.fstrimConfig;
|
|
lvm = {
|
|
boot.thin.enable = true;
|
|
dmeventd.enable = true;
|
|
};
|
|
|
|
netdata = {
|
|
enable = true;
|
|
config = {
|
|
# Ignore the VCCM sensor (RAM voltage is 1.35V with XMP enabled)
|
|
"plugin:freeipmi"."command options" = "ignore 5";
|
|
};
|
|
};
|
|
|
|
smartd = {
|
|
enable = true;
|
|
autodetect = true;
|
|
extraOptions = [ "-A /var/log/smartd/" "--interval=600" ];
|
|
};
|
|
};
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
pciutils
|
|
usbutils
|
|
partclone
|
|
lm_sensors
|
|
linuxPackages.cpupower
|
|
smartmontools
|
|
xfsprogs
|
|
];
|
|
|
|
systemd = {
|
|
tmpfiles.rules = [
|
|
"d /var/log/smartd 0755 root root"
|
|
];
|
|
|
|
services = {
|
|
"serial-getty@ttyS0".enable = true;
|
|
"serial-getty@ttyS1".enable = true;
|
|
lvm-activate-main = {
|
|
description = "Activate remaining LVs";
|
|
unitConfig.DefaultDependencies = false;
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
ExecStart = "${pkgs.lvm2.bin}/bin/vgchange -aay main";
|
|
};
|
|
wantedBy = [ "local-fs-pre.target" ];
|
|
};
|
|
|
|
rsync-lvm-meta = {
|
|
description = "rsync lvm metadata backups / archives to rsync.net";
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
|
|
# Only run when no other process is using CPU or disk
|
|
CPUSchedulingPolicy = "idle";
|
|
IOSchedulingClass = "idle";
|
|
};
|
|
script = ''
|
|
${pkgs.rsync}/bin/rsync -av --delete --delete-after \
|
|
-e "${pkgs.openssh}/bin/ssh -i ${config.age.secrets."colony/rsync.key".path}" \
|
|
/etc/lvm/{archive,backup} zh2855@zh2855.rsync.net:colony/lvm/
|
|
'';
|
|
wantedBy = [ "borgthin-job-main.service" ];
|
|
after = [ "borgthin-job-main.service" ];
|
|
};
|
|
borgthin-rsync = {
|
|
description = "rsync borgthin backups to rsync.net";
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
|
|
# Only run when no other process is using CPU or disk
|
|
CPUSchedulingPolicy = "idle";
|
|
IOSchedulingClass = "idle";
|
|
};
|
|
script = ''
|
|
${pkgs.rsync}/bin/rsync -av --delete --delete-after \
|
|
-e "${pkgs.openssh}/bin/ssh -i ${config.age.secrets."colony/rsync.key".path}" \
|
|
/mnt/backup/main/ zh2855@zh2855.rsync.net:borg/colony/
|
|
'';
|
|
wantedBy = [ "borgthin-job-main.service" ];
|
|
after = [ "borgthin-job-main.service" ];
|
|
};
|
|
};
|
|
|
|
network = {
|
|
links = {
|
|
"10-wan0" = {
|
|
matchConfig.MACAddress = "d0:50:99:fa:a7:99";
|
|
linkConfig.Name = "wan0";
|
|
};
|
|
"10-wan1" = {
|
|
matchConfig.MACAddress = "d0:50:99:fa:a7:9a";
|
|
linkConfig.Name = "wan1";
|
|
};
|
|
};
|
|
netdevs = {
|
|
"25-base".netdevConfig = {
|
|
Name = "base";
|
|
Kind = "bridge";
|
|
};
|
|
"30-base-dummy".netdevConfig = {
|
|
Name = "base0";
|
|
Kind = "dummy";
|
|
};
|
|
|
|
"25-vms".netdevConfig = {
|
|
Name = "vms";
|
|
Kind = "bridge";
|
|
};
|
|
"30-vms-dummy".netdevConfig = {
|
|
Name = "vms0";
|
|
Kind = "dummy";
|
|
};
|
|
};
|
|
|
|
networks = {
|
|
"80-base" = mkMerge [
|
|
(networkdAssignment "base" assignments.routing)
|
|
(networkdAssignment "base" assignments.internal)
|
|
];
|
|
"80-base-dummy" = {
|
|
matchConfig.Name = "base0";
|
|
networkConfig.Bridge = "base";
|
|
};
|
|
"80-base-ext" = {
|
|
matchConfig.Name = "base-ext";
|
|
networkConfig.Bridge = "base";
|
|
};
|
|
|
|
"80-vms" = mkMerge [
|
|
(networkdAssignment "vms" assignments.vms)
|
|
{
|
|
networkConfig = {
|
|
IPv6AcceptRA = mkForce false;
|
|
IPv6SendRA = true;
|
|
};
|
|
ipv6SendRAConfig = {
|
|
DNS = [ allAssignments.estuary.base.ipv6.address ];
|
|
Domains = [ config.networking.domain ];
|
|
};
|
|
ipv6Prefixes = [
|
|
{
|
|
Prefix = prefixes.vms.v6;
|
|
}
|
|
];
|
|
routes = [
|
|
{
|
|
Destination = prefixes.ctrs.v4;
|
|
Gateway = allAssignments.shill.routing.ipv4.address;
|
|
}
|
|
{
|
|
Destination = prefixes.ctrs.v6;
|
|
Gateway = allAssignments.shill.internal.ipv6.address;
|
|
}
|
|
|
|
{
|
|
Destination = allAssignments.shill.internal.ipv4.address;
|
|
Gateway = allAssignments.shill.routing.ipv4.address;
|
|
}
|
|
|
|
{
|
|
Destination = lib.my.c.tailscale.prefix.v4;
|
|
Gateway = allAssignments.shill.routing.ipv4.address;
|
|
}
|
|
{
|
|
Destination = lib.my.c.tailscale.prefix.v6;
|
|
Gateway = allAssignments.shill.internal.ipv6.address;
|
|
}
|
|
{
|
|
Destination = prefixes.qclk.v4;
|
|
Gateway = allAssignments.shill.routing.ipv4.address;
|
|
}
|
|
|
|
{
|
|
Destination = prefixes.jam.v6;
|
|
Gateway = allAssignments.shill.internal.ipv6.address;
|
|
}
|
|
|
|
{
|
|
Destination = prefixes.oci.v4;
|
|
Gateway = allAssignments.whale2.routing.ipv4.address;
|
|
}
|
|
{
|
|
Destination = prefixes.oci.v6;
|
|
Gateway = allAssignments.whale2.internal.ipv6.address;
|
|
}
|
|
{
|
|
Destination = allAssignments.whale2.internal.ipv4.address;
|
|
Gateway = allAssignments.whale2.routing.ipv4.address;
|
|
}
|
|
|
|
{
|
|
Destination = allAssignments.git.internal.ipv4.address;
|
|
Gateway = allAssignments.git.routing.ipv4.address;
|
|
}
|
|
];
|
|
}
|
|
];
|
|
# Just so the vms interface will come up (in networkd's eyes), allowing dependant VMs to start.
|
|
# Could tweak the `waitOnline` for a single VM, but this seems better overall?
|
|
"80-vms-dummy" = {
|
|
matchConfig.Name = "vms0";
|
|
networkConfig.Bridge = "vms";
|
|
};
|
|
|
|
"90-vm-mail" = {
|
|
matchConfig.Name = "vm-mail";
|
|
address = [
|
|
"${custRouting.mail-vm}/32"
|
|
prefixes.mail.v6
|
|
];
|
|
networkConfig = {
|
|
IPv6AcceptRA = false;
|
|
IPv6SendRA = true;
|
|
};
|
|
ipv6Prefixes = [
|
|
{
|
|
Prefix = prefixes.mail.v6;
|
|
}
|
|
];
|
|
routes = [
|
|
{
|
|
Destination = prefixes.mail.v4;
|
|
Scope = "link";
|
|
}
|
|
];
|
|
};
|
|
|
|
"90-vm-darts" = {
|
|
matchConfig.Name = "vm-darts";
|
|
address = [
|
|
"${custRouting.darts-vm}/32"
|
|
prefixes.darts.v6
|
|
];
|
|
networkConfig = {
|
|
IPv6AcceptRA = false;
|
|
IPv6SendRA = true;
|
|
};
|
|
ipv6Prefixes = [
|
|
{
|
|
Prefix = prefixes.darts.v6;
|
|
}
|
|
];
|
|
routes = [
|
|
{
|
|
Destination = prefixes.darts.v4;
|
|
Scope = "link";
|
|
}
|
|
];
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
#environment.etc."udev/udev.conf".text = "udev_log=debug";
|
|
#systemd.services.systemd-networkd.environment.SYSTEMD_LOG_LEVEL = "debug";
|
|
virtualisation = {
|
|
cores = 8;
|
|
memorySize = 8192;
|
|
qemu.options = [
|
|
"-machine q35"
|
|
"-accel kvm,kernel-irqchip=split"
|
|
"-device intel-iommu,intremap=on,caching-mode=on"
|
|
];
|
|
};
|
|
|
|
my = {
|
|
#deploy.generate.system.mode = "boot";
|
|
secrets = {
|
|
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPIijqzAWF6OxKr4aeCa1TAc5xGn4rdIjVTt0wAPU6uY";
|
|
files = {
|
|
"colony/borg-pass.txt" = {};
|
|
"colony/rsync.key" = {};
|
|
};
|
|
};
|
|
|
|
server.enable = true;
|
|
|
|
firewall = {
|
|
trustedInterfaces = [ "vms" ];
|
|
nat.forwardPorts."${allAssignments.estuary.internal.ipv4.address}" = firewallForwards allAssignments;
|
|
extraRules = ''
|
|
define cust = { vm-mail, vm-darts }
|
|
table inet filter {
|
|
chain forward {
|
|
# Trust that the outer firewall has done the filtering!
|
|
iifname base oifname { vms, $cust } accept
|
|
iifname $cust accept # trust for now...
|
|
}
|
|
}
|
|
'';
|
|
};
|
|
|
|
borgthin = {
|
|
enable = true;
|
|
jobs = {
|
|
main = {
|
|
repo = "/mnt/backup/main";
|
|
passFile = config.age.secrets."colony/borg-pass.txt".path;
|
|
lvs = map (lv: "main/${lv}") [
|
|
"colony-persist"
|
|
"vm-shill-persist"
|
|
"minio"
|
|
"oci"
|
|
"vm-estuary-persist"
|
|
"vm-whale2-persist"
|
|
"vm-mail-data"
|
|
"vm-git-persist"
|
|
"git"
|
|
];
|
|
compression = "zstd,5";
|
|
extraCreateArgs = [ "--stats" ];
|
|
prune.keep = {
|
|
last = 1;
|
|
within = "1d";
|
|
daily = 7;
|
|
weekly = 4;
|
|
monthly = 12;
|
|
yearly = -1;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|