nixos: Add borgthin module

flake.lock

@@ -21,6 +21,28 @@
         "type": "github"
       }
     },
+    "borgthin": {
+      "inputs": {
+        "devshell": "devshell",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": [
+          "nixpkgs-mine"
+        ]
+      },
+      "locked": {
+        "lastModified": 1676811450,
+        "narHash": "sha256-V9IttEuUbRiOfnn956xdCO80EmuVriI5WNhUtsqeVKY=",
+        "owner": "devplayer0",
+        "repo": "borg",
+        "rev": "f67a35995c006974bec2233622cd2558e5d3beda",
+        "type": "github"
+      },
+      "original": {
+        "owner": "devplayer0",
+        "repo": "borg",
+        "type": "github"
+      }
+    },
     "deploy-rs": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -46,6 +68,25 @@
     "devshell": {
       "inputs": {
         "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1671489820,
+        "narHash": "sha256-qoei5HDJ8psd1YUPD7DhbHdhLIT9L2nadscp4Qk37uk=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "5aa3a8039c68b4bf869327446590f4cdf90bb634",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
+    "devshell_2": {
+      "inputs": {
+        "flake-utils": "flake-utils_3",
         "nixpkgs": [
           "nixpkgs-unstable"
         ]
@@ -96,6 +137,36 @@
       }
     },
     "flake-utils_2": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "locked": {
+        "lastModified": 1642700792,
+        "narHash": "sha256-XqHrk7hFb+zBvRg6Ghl+AZDq03ov6OshJLiSWOoX5es=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "846b2ae0fc4cc943637d3d1def4454213e203cba",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_4": {
       "locked": {
         "lastModified": 1644229661,
         "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=",
@@ -110,7 +181,7 @@
         "type": "github"
       }
     },
-    "flake-utils_3": {
+    "flake-utils_5": {
       "locked": {
         "lastModified": 1667395993,
         "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
@@ -125,7 +196,7 @@
         "type": "github"
       }
     },
-    "flake-utils_4": {
+    "flake-utils_6": {
       "locked": {
         "lastModified": 1667395993,
         "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
@@ -196,6 +267,22 @@
         "type": "github"
       }
     },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1643381941,
+        "narHash": "sha256-pHTwvnN4tTsEKkWlXQ8JMY423epos8wUOhthpwJjtpc=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "5efc8ca954272c4376ac929f4c5ffefcc20551d5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "nixpkgs-mine": {
       "locked": {
         "lastModified": 1673114714,
@@ -261,7 +348,7 @@
     "ragenix": {
       "inputs": {
         "agenix": "agenix",
-        "flake-utils": "flake-utils_3",
+        "flake-utils": "flake-utils_5",
         "nixpkgs": [
           "nixpkgs-unstable"
         ],
@@ -283,9 +370,10 @@
     },
     "root": {
       "inputs": {
+        "borgthin": "borgthin",
         "deploy-rs": "deploy-rs",
-        "devshell": "devshell",
+        "devshell": "devshell_2",
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils_4",
         "home-manager-stable": "home-manager-stable",
         "home-manager-unstable": "home-manager-unstable",
         "impermanence": "impermanence",
@@ -324,7 +412,7 @@
     },
     "sharry": {
       "inputs": {
-        "flake-utils": "flake-utils_4",
+        "flake-utils": "flake-utils_6",
         "nixpkgs": [
           "nixpkgs-unstable"
         ]

flake.nix

@@ -28,6 +28,8 @@
     # Packages not in nixpkgs
     sharry.url = "github:eikek/sharry";
     sharry.inputs.nixpkgs.follows = "nixpkgs-unstable";
+    borgthin.url = "github:devplayer0/borg";
+    borgthin.inputs.nixpkgs.follows = "nixpkgs-mine";
   };
 
   outputs =

nixos/boxes/colony/default.nix

@@ -81,6 +81,10 @@
             fsType = "ext4";
             neededForBoot = true;
           };
+          "/mnt/backup" = {
+            device = "/dev/main/tmp-backup";
+            fsType = "ext4";
+          };
         };
 
         services = {
@@ -240,6 +244,9 @@
           #deploy.generate.system.mode = "boot";
           secrets = {
             key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPIijqzAWF6OxKr4aeCa1TAc5xGn4rdIjVTt0wAPU6uY";
+            files = {
+              "colony/borg-pass.txt" = {};
+            };
           };
 
           server.enable = true;
@@ -255,6 +262,34 @@
               }
             '';
           };
+
+          borgthin = {
+            enable = true;
+            jobs = {
+              main = {
+                repo = "/mnt/backup/main";
+                passFile = config.age.secrets."colony/borg-pass.txt".path;
+                lvs = map (lv: "main/${lv}") [
+                  "colony-persist"
+                  "vm-shill-persist"
+                  "minio"
+                  "oci"
+                  "vm-estuary-persist"
+                  "vm-whale2-persist"
+                ];
+                compression = "zstd,5";
+                extraCreateArgs = [ "--stats" ];
+                prune.keep = {
+                  last = 1;
+                  within = "1d";
+                  daily = 7;
+                  weekly = 4;
+                  monthly = 12;
+                  yearly = -1;
+                };
+              };
+            };
+          };
         };
       };
   };

nixos/modules/_list.nix

@@ -16,5 +16,6 @@
     nginx-sso = ./nginx-sso.nix;
     gui = ./gui.nix;
     l2mesh = ./l2mesh.nix;
+    borgthin = ./borgthin.nix;
   };
 }

nixos/modules/borgthin.nix

@@ -0,0 +1,150 @@
+{ lib, pkgs, config, ... }:
+let
+  inherit (builtins) substring match;
+  inherit (lib)
+    nameValuePair optional optionalString optionalAttrs mapAttrs' mapAttrsToList concatStringsSep
+    concatMapStringsSep mkIf;
+  inherit (lib.my) mkOpt' mkBoolOpt';
+
+  jobType = with lib.types; submodule ({ name, ... }@args:
+  let
+    cfg = args.config;
+  in
+  {
+    options = {
+      repo = mkOpt' str null "borg repository URL";
+      passFile = mkOpt' (nullOr str) null "Path to file containing passphrase";
+
+      archivePrefix = mkOpt' str "${config.networking.hostName}-${name}-" "Prefix to start new archives with";
+      dateFormat = mkOpt' str "+%Y-%m-%dT%H:%M:%S" "Format passed to the date command";
+      compression = mkOpt' str "zstd,3" "Compression options";
+      lvs = mkOpt' (listOf str) null "Thin LVs to backup (vg/lv format)";
+      prune = {
+        pattern = mkOpt' str "sh:${cfg.archivePrefix}*" "Borg pattern to select archives for pruning";
+        keep = mkOpt' (attrsOf (either int str)) { } "Borg pruning params";
+      };
+
+      extraArgs = mkOpt' (listOf str) [ "--iec" ] "Extra args to pass to all borg commands";
+      extraCreateArgs = mkOpt' (listOf str) [ ] "Extra args to pass to tcreate command";
+      environment = mkOpt' (attrsOf str) { } "Extra environment variables to pass to borg";
+
+      timer = {
+        at = mkOpt' (either str (listOf str)) "5:00" "systemd calendar time(s) to run backup at";
+        persistent = mkBoolOpt' false "Persistent systemd timer";
+      };
+    };
+  });
+
+  cfg' = config.my.borgthin;
+
+  isLocalPath = x:
+    substring 0 1 x == "/"      # absolute path
+    || substring 0 1 x == "."   # relative path
+    || match "[.*:.*]" == null; # not machine:path
+
+  argStr = concatMapStringsSep " " (a: ''"${a}"'');
+
+  mkEnv = name: cfg: (rec {
+    BORG_BASE_DIR = "/var/lib/borgthin";
+    BORG_CONFIG_DIR = "${BORG_BASE_DIR}/config";
+    BORG_CACHE_DIR = "/var/cache/borgthin";
+
+    BORG_REPO = cfg.repo;
+  }) //
+  (optionalAttrs (cfg.passFile != null) {
+    BORG_PASSCOMMAND = "cat ${cfg.passFile}";
+  }) //
+  cfg.environment;
+
+  # utility function around makeWrapper
+  mkWrapperDrv = {
+    original, name,
+    set ? { }, addFlags ? [ ],
+  }:
+  pkgs.runCommand "${name}-wrapper" {
+    nativeBuildInputs = [ pkgs.makeWrapper ];
+  } ''
+    makeWrapper "${original}" "$out/bin/${name}" \
+      ${concatStringsSep " \\\n " (
+        (mapAttrsToList (name: value: ''--set ${name} "${value}"'') set) ++
+        (map (f: ''--add-flags "${f}"'') addFlags)
+      )}
+  '';
+  mkBorgWrapper = name: cfg: mkWrapperDrv {
+    original = "${cfg'.package}/bin/borgthin";
+    name = "borgthin-job-${name}";
+    set = mkEnv name cfg;
+    addFlags = cfg.extraArgs;
+  };
+
+  mkKeepArgs = keep:
+    # If cfg.prune.keep e.g. has a yearly attribute,
+    # its content is passed on as --keep-yearly
+    concatStringsSep " "
+      (mapAttrsToList (x: y: "--keep-${x}=${toString y}") keep);
+
+  mkService = name: cfg: nameValuePair "borgthin-job-${name}" {
+    description = "borgthin backup job ${name}";
+    serviceConfig = {
+      Type = "oneshot";
+      StateDirectory = "borgthin";
+      CacheDirectory = "borgthin";
+
+      # Only run when no other process is using CPU or disk
+      CPUSchedulingPolicy = "idle";
+      IOSchedulingClass = "idle";
+    };
+    environment = mkEnv name cfg;
+
+    path = [ cfg'.lvmPackage cfg'.thinToolsPackage cfg'.package ];
+    script = ''
+      extraArgs="${argStr cfg.extraArgs}"
+      borgthin $extraArgs tcreate \
+        --compression "${cfg.compression}" \
+        ${argStr cfg.extraCreateArgs} \
+        "${cfg.archivePrefix}$(date "${cfg.dateFormat}")" \
+        ${concatStringsSep " " cfg.lvs}
+    '' + optionalString (cfg.prune.keep != { }) ''
+      borgthin $extraArgs prune \
+        --match-archives "${cfg.prune.pattern}" \
+        ${mkKeepArgs cfg.prune.keep}
+      borgthin $extraArgs compact
+    '';
+  };
+
+  mkTimer = name: cfg: nameValuePair "borgthin-job-${name}" {
+    description = "borgthin backup job ${name} timer";
+    after = optional (cfg.timer.persistent && !isLocalPath cfg.repo) "network-online.target";
+    timerConfig = {
+      Persistent = cfg.timer.persistent;
+      OnCalendar = cfg.timer.at;
+    };
+    wantedBy = [ "timers.target" ];
+  };
+in
+{
+  options.my.borgthin = with lib.types; {
+    enable = mkBoolOpt' false "Whether to enable borgthin jobs";
+    lvmPackage = mkOpt' package pkgs.lvm2 "Packge containing LVM tools";
+    thinToolsPackage = mkOpt' package pkgs.thin-provisioning-tools "Package containing thin-provisioning-tools";
+    package = mkOpt' package pkgs.borgthin "borgthin package";
+    jobs = mkOpt' (attrsOf jobType) { } "borgthin jobs";
+  };
+
+  config = mkIf cfg'.enable {
+    environment.systemPackages =
+      [ cfg'.package ] ++
+      (mapAttrsToList mkBorgWrapper cfg'.jobs);
+
+    systemd = {
+      services = mapAttrs' mkService cfg'.jobs;
+      timers = mapAttrs' mkTimer cfg'.jobs;
+    };
+
+    my.tmproot.persistence.config.directories = [
+      "/var/lib/borgthin"
+      "/var/cache/borgthin"
+    ];
+  };
+}
+

nixos/modules/common.nix

@@ -73,6 +73,7 @@ in
         overlays = [
           inputs.deploy-rs.overlay
           inputs.sharry.overlays.default
+          inputs.borgthin.overlays.default
         ];
         config = {
           allowUnfree = true;

secrets/colony/borg-pass.txt.age

@@ -0,0 +1,11 @@
+-----BEGIN AGE ENCRYPTED FILE-----
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IGo2N0ZYUSBhSjdP
+MzBuMWxsNm1zK25RMlRiSDJ1OTB4UllJSlVDeUtTOXpZTk1HMEFrCjFwRTJZZzFL
+UEdobndzdC83dE9UbFFETG9jbkUrSmRJdjFUUXJ5YWhERVkKLT4gWDI1NTE5IGRy
+VE0zSEQ5SjFlOVUzc0lQV0I3bVBWQ24rMFVJOXFaNEk1OGpDZTZYRXMKS04vbjd1
+YWdCblU2cENzMkNrakRPMUFaT3A5ZVRNTWtpeHpQVVY0YzFycwotPiBzLWdyZWFz
+ZSAuIGBec1smPy4KSENtN2VPRnFKVHVEcFdYMDRZbW1lZXYwdHJjCi0tLSBKZlhT
+eUwzOEJ3bVhiMnRvNUhFank0d2VzUW1wM2RES3JabFllNkpZVjBRChYyL/XCeTNL
+CpIcjr41mdbBITOTfjvMXR4c9Vaqs1T0svrniTE6uHNnkNK6kGh65RPuSpo6Wesm
+nOW4DjcxhLjXdQuMizkCjhnnMsRvJRotILvUcrYr9yowEwJE5LKLD1o=
+-----END AGE ENCRYPTED FILE-----