Merge branch 'staging-next' into staging

nixos/doc/manual/from_md/release-notes/rl-2111.section.xml

@@ -25,6 +25,15 @@
   <section xml:id="sec-release-21.11-new-services">
     <title>New Services</title>
     <itemizedlist>
+      <listitem>
+        <para>
+          <link xlink:href="https://digint.ch/btrbk/index.html">btrbk</link>,
+          a backup tool for btrfs subvolumes, taking advantage of btrfs
+          specific capabilities to create atomic snapshots and transfer
+          them incrementally to your backup locations. Available as
+          <link xlink:href="options.html#opt-services.brtbk.instances">services.btrbk</link>.
+        </para>
+      </listitem>
       <listitem>
         <para>
           <link xlink:href="https://github.com/maxmind/geoipupdate">geoipupdate</link>,
@@ -64,7 +73,7 @@
       <listitem>
         <para>
           The <literal>staticjinja</literal> package has been upgraded
-          from 1.0.4 to 2.0.0
+          from 1.0.4 to 3.0.1
         </para>
       </listitem>
       <listitem>

nixos/doc/manual/release-notes/rl-2111.section.md

@@ -10,6 +10,8 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 ## New Services {#sec-release-21.11-new-services}
 
+- [btrbk](https://digint.ch/btrbk/index.html), a backup tool for btrfs subvolumes, taking advantage of btrfs specific capabilities to create atomic snapshots and transfer them incrementally to your backup locations. Available as [services.btrbk](options.html#opt-services.brtbk.instances).
+
 - [geoipupdate](https://github.com/maxmind/geoipupdate), a GeoIP database updater from MaxMind. Available as [services.geoipupdate](options.html#opt-services.geoipupdate.enable).
 
 - [sourcehut](https://sr.ht), a collection of tools useful for software development. Available as [services.sourcehut](options.html#opt-services.sourcehut.enable).
@@ -20,7 +22,7 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 ## Backward Incompatibilities {#sec-release-21.11-incompatibilities}
 
-- The `staticjinja` package has been upgraded from 1.0.4 to 2.0.0
+- The `staticjinja` package has been upgraded from 1.0.4 to 3.0.1
 
 - `services.geoip-updater` was broken and has been replaced by [services.geoipupdate](options.html#opt-services.geoipupdate.enable).
 

nixos/modules/module-list.nix

@@ -260,6 +260,7 @@
   ./services/backup/bacula.nix
   ./services/backup/borgbackup.nix
   ./services/backup/borgmatic.nix
+  ./services/backup/btrbk.nix
   ./services/backup/duplicati.nix
   ./services/backup/duplicity.nix
   ./services/backup/mysql-backup.nix

nixos/modules/services/backup/btrbk.nix

@@ -0,0 +1,220 @@
+{ config, pkgs, lib, ... }:
+let
+  cfg = config.services.btrbk;
+  sshEnabled = cfg.sshAccess != [ ];
+  serviceEnabled = cfg.instances != { };
+  attr2Lines = attr:
+    let
+      pairs = lib.attrsets.mapAttrsToList (name: value: { inherit name value; }) attr;
+      isSubsection = value:
+        if builtins.isAttrs value then true
+        else if builtins.isString value then false
+        else throw "invalid type in btrbk config ${builtins.typeOf value}";
+      sortedPairs = lib.lists.partition (x: isSubsection x.value) pairs;
+    in
+    lib.flatten (
+      # non subsections go first
+      (
+        map (pair: [ "${pair.name} ${pair.value}" ]) sortedPairs.wrong
+      )
+      ++ # subsections go last
+      (
+        map
+          (
+            pair:
+            lib.mapAttrsToList
+              (
+                childname: value:
+                  [ "${pair.name} ${childname}" ] ++ (map (x: " " + x) (attr2Lines value))
+              )
+              pair.value
+          )
+          sortedPairs.right
+      )
+    )
+  ;
+  addDefaults = settings: { backend = "btrfs-progs-sudo"; } // settings;
+  mkConfigFile = settings: lib.concatStringsSep "\n" (attr2Lines (addDefaults settings));
+  mkTestedConfigFile = name: settings:
+    let
+      configFile = pkgs.writeText "btrbk-${name}.conf" (mkConfigFile settings);
+    in
+    pkgs.runCommand "btrbk-${name}-tested.conf" { } ''
+      mkdir foo
+      cp ${configFile} $out
+      if (set +o pipefail; ${pkgs.btrbk}/bin/btrbk -c $out ls foo 2>&1 | grep $out);
+      then
+      echo btrbk configuration is invalid
+      cat $out
+      exit 1
+      fi;
+    '';
+in
+{
+  options = {
+    services.btrbk = {
+      extraPackages = lib.mkOption {
+        description = "Extra packages for btrbk, like compression utilities for <literal>stream_compress</literal>";
+        type = lib.types.listOf lib.types.package;
+        default = [ ];
+        example = lib.literalExample "[ pkgs.xz ]";
+      };
+      niceness = lib.mkOption {
+        description = "Niceness for local instances of btrbk. Also applies to remote ones connecting via ssh when positive.";
+        type = lib.types.ints.between (-20) 19;
+        default = 10;
+      };
+      ioSchedulingClass = lib.mkOption {
+        description = "IO scheduling class for btrbk (see ionice(1) for a quick description). Applies to local instances, and remote ones connecting by ssh if set to idle.";
+        type = lib.types.enum [ "idle" "best-effort" "realtime" ];
+        default = "best-effort";
+      };
+      instances = lib.mkOption {
+        description = "Set of btrbk instances. The instance named <literal>btrbk</literal> is the default one.";
+        type = with lib.types;
+          attrsOf (
+            submodule {
+              options = {
+                onCalendar = lib.mkOption {
+                  type = lib.types.str;
+                  default = "daily";
+                  description = "How often this btrbk instance is started. See systemd.time(7) for more information about the format.";
+                };
+                settings = lib.mkOption {
+                  type = let t = lib.types.attrsOf (lib.types.either lib.types.str (t // { description = "instances of this type recursively"; })); in t;
+                  default = { };
+                  example = {
+                    snapshot_preserve_min = "2d";
+                    snapshot_preserve = "14d";
+                    volume = {
+                      "/mnt/btr_pool" = {
+                        target = "/mnt/btr_backup/mylaptop";
+                        subvolume = {
+                          "rootfs" = { };
+                          "home" = { snapshot_create = "always"; };
+                        };
+                      };
+                    };
+                  };
+                  description = "configuration options for btrbk. Nested attrsets translate to subsections.";
+                };
+              };
+            }
+          );
+        default = { };
+      };
+      sshAccess = lib.mkOption {
+        description = "SSH keys that should be able to make or push snapshots on this system remotely with btrbk";
+        type = with lib.types; listOf (
+          submodule {
+            options = {
+              key = lib.mkOption {
+                type = str;
+                description = "SSH public key allowed to login as user <literal>btrbk</literal> to run remote backups.";
+              };
+              roles = lib.mkOption {
+                type = listOf (enum [ "info" "source" "target" "delete" "snapshot" "send" "receive" ]);
+                example = [ "source" "info" "send" ];
+                description = "What actions can be performed with this SSH key. See ssh_filter_btrbk(1) for details";
+              };
+            };
+          }
+        );
+        default = [ ];
+      };
+    };
+
+  };
+  config = lib.mkIf (sshEnabled || serviceEnabled) {
+    environment.systemPackages = [ pkgs.btrbk ] ++ cfg.extraPackages;
+    security.sudo.extraRules = [
+      {
+        users = [ "btrbk" ];
+        commands = [
+          { command = "${pkgs.btrfs-progs}/bin/btrfs"; options = [ "NOPASSWD" ]; }
+          { command = "${pkgs.coreutils}/bin/mkdir"; options = [ "NOPASSWD" ]; }
+          { command = "${pkgs.coreutils}/bin/readlink"; options = [ "NOPASSWD" ]; }
+          # for ssh, they are not the same than the one hard coded in ${pkgs.btrbk}
+          { command = "/run/current-system/bin/btrfs"; options = [ "NOPASSWD" ]; }
+          { command = "/run/current-system/sw/bin/mkdir"; options = [ "NOPASSWD" ]; }
+          { command = "/run/current-system/sw/bin/readlink"; options = [ "NOPASSWD" ]; }
+        ];
+      }
+    ];
+    users.users.btrbk = {
+      isSystemUser = true;
+      # ssh needs a home directory
+      home = "/var/lib/btrbk";
+      createHome = true;
+      shell = "${pkgs.bash}/bin/bash";
+      group = "btrbk";
+      openssh.authorizedKeys.keys = map
+        (
+          v:
+          let
+            options = lib.concatMapStringsSep " " (x: "--" + x) v.roles;
+            ioniceClass = {
+              "idle" = 3;
+              "best-effort" = 2;
+              "realtime" = 1;
+            }.${cfg.ioSchedulingClass};
+          in
+          ''command="${pkgs.util-linux}/bin/ionice -t -c ${toString ioniceClass} ${lib.optionalString (cfg.niceness >= 1) "${pkgs.coreutils}/bin/nice -n ${toString cfg.niceness}"} ${pkgs.btrbk}/share/btrbk/scripts/ssh_filter_btrbk.sh --sudo ${options}" ${v.key}''
+        )
+        cfg.sshAccess;
+    };
+    users.groups.btrbk = { };
+    systemd.tmpfiles.rules = [
+      "d /var/lib/btrbk 0750 btrbk btrbk"
+      "d /var/lib/btrbk/.ssh 0700 btrbk btrbk"
+      "f /var/lib/btrbk/.ssh/config 0700 btrbk btrbk - StrictHostKeyChecking=accept-new"
+    ];
+    environment.etc = lib.mapAttrs'
+      (
+        name: instance: {
+          name = "btrbk/${name}.conf";
+          value.source = mkTestedConfigFile name instance.settings;
+        }
+      )
+      cfg.instances;
+    systemd.services = lib.mapAttrs'
+      (
+        name: _: {
+          name = "btrbk-${name}";
+          value = {
+            description = "Takes BTRFS snapshots and maintains retention policies.";
+            unitConfig.Documentation = "man:btrbk(1)";
+            path = [ "/run/wrappers" ] ++ cfg.extraPackages;
+            serviceConfig = {
+              User = "btrbk";
+              Group = "btrbk";
+              Type = "oneshot";
+              ExecStart = "${pkgs.btrbk}/bin/btrbk -c /etc/btrbk/${name}.conf run";
+              Nice = cfg.niceness;
+              IOSchedulingClass = cfg.ioSchedulingClass;
+              StateDirectory = "btrbk";
+            };
+          };
+        }
+      )
+      cfg.instances;
+
+    systemd.timers = lib.mapAttrs'
+      (
+        name: instance: {
+          name = "btrbk-${name}";
+          value = {
+            description = "Timer to take BTRFS snapshots and maintain retention policies.";
+            wantedBy = [ "timers.target" ];
+            timerConfig = {
+              OnCalendar = instance.onCalendar;
+              AccuracySec = "10min";
+              Persistent = true;
+            };
+          };
+        }
+      )
+      cfg.instances;
+  };
+
+}

nixos/tests/all-tests.nix

@@ -47,6 +47,7 @@ in
   boot-stage1 = handleTest ./boot-stage1.nix {};
   borgbackup = handleTest ./borgbackup.nix {};
   botamusique = handleTest ./botamusique.nix {};
+  btrbk = handleTest ./btrbk.nix {};
   buildbot = handleTest ./buildbot.nix {};
   buildkite-agents = handleTest ./buildkite-agents.nix {};
   caddy = handleTest ./caddy.nix {};

nixos/tests/btrbk.nix

@@ -0,0 +1,110 @@
+import ./make-test-python.nix ({ pkgs, ... }:
+
+  let
+    privateKey = ''
+      -----BEGIN OPENSSH PRIVATE KEY-----
+      b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+      QyNTUxOQAAACBx8UB04Q6Q/fwDFjakHq904PYFzG9pU2TJ9KXpaPMcrwAAAJB+cF5HfnBe
+      RwAAAAtzc2gtZWQyNTUxOQAAACBx8UB04Q6Q/fwDFjakHq904PYFzG9pU2TJ9KXpaPMcrw
+      AAAEBN75NsJZSpt63faCuaD75Unko0JjlSDxMhYHAPJk2/xXHxQHThDpD9/AMWNqQer3Tg
+      9gXMb2lTZMn0pelo8xyvAAAADXJzY2h1ZXR6QGt1cnQ=
+      -----END OPENSSH PRIVATE KEY-----
+    '';
+    publicKey = ''
+      ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHHxQHThDpD9/AMWNqQer3Tg9gXMb2lTZMn0pelo8xyv
+    '';
+  in
+  {
+    name = "btrbk";
+    meta = with pkgs.lib; {
+      maintainers = with maintainers; [ symphorien ];
+    };
+
+    nodes = {
+      archive = { ... }: {
+        environment.systemPackages = with pkgs; [ btrfs-progs ];
+        # note: this makes the privateKey world readable.
+        # don't do it with real ssh keys.
+        environment.etc."btrbk_key".text = privateKey;
+        services.btrbk = {
+          extraPackages = [ pkgs.lz4 ];
+          instances = {
+            remote = {
+              onCalendar = "minutely";
+              settings = {
+                ssh_identity = "/etc/btrbk_key";
+                ssh_user = "btrbk";
+                stream_compress = "lz4";
+                volume = {
+                  "ssh://main/mnt" = {
+                    target = "/mnt";
+                    snapshot_dir = "btrbk/remote";
+                    subvolume = "to_backup";
+                  };
+                };
+              };
+            };
+          };
+        };
+      };
+
+      main = { ... }: {
+        environment.systemPackages = with pkgs; [ btrfs-progs ];
+        services.openssh = {
+          enable = true;
+          passwordAuthentication = false;
+          challengeResponseAuthentication = false;
+        };
+        services.btrbk = {
+          extraPackages = [ pkgs.lz4 ];
+          sshAccess = [
+            {
+              key = publicKey;
+              roles = [ "source" "send" "info" "delete" ];
+            }
+          ];
+          instances = {
+            local = {
+              onCalendar = "minutely";
+              settings = {
+                volume = {
+                  "/mnt" = {
+                    snapshot_dir = "btrbk/local";
+                    subvolume = "to_backup";
+                  };
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+
+    testScript = ''
+      start_all()
+
+      # create btrfs partition at /mnt
+      for machine in (archive, main):
+        machine.succeed("dd if=/dev/zero of=/data_fs bs=120M count=1")
+        machine.succeed("mkfs.btrfs /data_fs")
+        machine.succeed("mkdir /mnt")
+        machine.succeed("mount /data_fs /mnt")
+
+      # what to backup and where
+      main.succeed("btrfs subvolume create /mnt/to_backup")
+      main.succeed("mkdir -p /mnt/btrbk/{local,remote}")
+
+      # check that local snapshots work
+      with subtest("local"):
+          main.succeed("echo foo > /mnt/to_backup/bar")
+          main.wait_until_succeeds("cat /mnt/btrbk/local/*/bar | grep foo")
+          main.succeed("echo bar > /mnt/to_backup/bar")
+          main.succeed("cat /mnt/btrbk/local/*/bar | grep foo")
+
+      # check that btrfs send/receive works and ssh access works
+      with subtest("remote"):
+          archive.wait_until_succeeds("cat /mnt/*/bar | grep bar")
+          main.succeed("echo baz > /mnt/to_backup/bar")
+          archive.succeed("cat /mnt/*/bar | grep bar")
+    '';
+  })

pkgs/applications/misc/gpxsee/default.nix

@@ -2,13 +2,13 @@
 
 mkDerivation rec {
   pname = "gpxsee";
-  version = "9.1";
+  version = "9.2";
 
   src = fetchFromGitHub {
     owner = "tumic0";
     repo = "GPXSee";
     rev = version;
-    sha256 = "sha256-szq1i2/NEtMK4paSkxtkKXc8yY8qGj2U/p6CzNIecAY=";
+    sha256 = "sha256-pU02Eaq6tB7X6EPOo8YAyryJRbSV3KebQv8VELxXaBw=";
   };
 
   patches = (substituteAll {

pkgs/applications/misc/hugo/default.nix

@@ -2,13 +2,13 @@
 
 buildGoModule rec {
   pname = "hugo";
-  version = "0.84.3";
+  version = "0.84.4";
 
   src = fetchFromGitHub {
     owner = "gohugoio";
     repo = pname;
     rev = "v${version}";
-    sha256 = "sha256-3SbF4JsanNup0JmtEoZlyu3SvMn01r+nhnPgIi/W8pA=";
+    sha256 = "sha256-nD2UBDSDG6OFfUvDBXCfhOCiJyFMP2pDXSlIESaEfqE=";
   };
 
   vendorSha256 = "sha256-ImXTOtN6kQL7Q8IBlmK7+i47cWtyZT0xcnQdCw3NvWM=";

pkgs/applications/networking/seafile-client/default.nix

@@ -10,7 +10,7 @@ mkDerivation rec {
     owner = "haiwen";
     repo = "seafile-client";
     rev = "v${version}";
-    sha256 = "lhdKbR19ScNeezICf7vwZaeJikPjwbqrz42bo4lhxJs=";
+    sha256 = "cG3OSqRhYnxlzfauQia6pM/1gu+iE5mtHTGk3kGMFH0=";
   };
 
   nativeBuildInputs = [ pkg-config cmake ];

pkgs/applications/science/chemistry/gwyddion/default.nix

@@ -35,10 +35,10 @@ in
 
 stdenv.mkDerivation rec {
   pname = "gwyddion";
-   version = "2.58";
+   version = "2.59";
   src = fetchurl {
     url = "mirror://sourceforge/gwyddion/gwyddion-${version}.tar.xz";
-    sha256 = "sha256-0xNnzYkuW3nEsO2o+0WEA+Z71XWoq6FYXm342OWO9Sw=";
+    sha256 = "sha256-APMOJeZt/zp8JvXghKZ5lQFRKWO/4TVDORok8qAgEBk=";
   };
 
   nativeBuildInputs = [ pkg-config file ];

pkgs/applications/version-management/mercurial/default.nix

@@ -1,95 +1,158 @@
-{ lib, stdenv, fetchurl, fetchpatch, python3Packages, makeWrapper, gettext
+{ lib, stdenv, fetchurl, fetchpatch, python3Packages, makeWrapper, gettext, installShellFiles
 , re2Support ? true
 , rustSupport ? stdenv.hostPlatform.isLinux, rustPlatform
-, guiSupport ? false, tk ? null
+, fullBuild ? false
+, gitSupport ? fullBuild
+, guiSupport ? fullBuild, tk
+, highlightSupport ? fullBuild
 , ApplicationServices
 }:
 
 let
-  inherit (python3Packages) docutils python fb-re2;
+  inherit (python3Packages) docutils python fb-re2 pygit2 pygments;
 
-in python3Packages.buildPythonApplication rec {
+  self = python3Packages.buildPythonApplication rec {
-  pname = "mercurial";
+    pname = "mercurial";
-  version = "5.8";
+    version = "5.8";
 
-  src = fetchurl {
+    src = fetchurl {
-    url = "https://mercurial-scm.org/release/mercurial-${version}.tar.gz";
+      url = "https://mercurial-scm.org/release/mercurial-${version}.tar.gz";
-    sha256 = "17rhlmmkqz5ll3k68jfzpcifg3nndbcbc2nx7kw8xn3qcj7nlpgw";
+      sha256 = "17rhlmmkqz5ll3k68jfzpcifg3nndbcbc2nx7kw8xn3qcj7nlpgw";
+    };
+
+    patches = [
+      # https://phab.mercurial-scm.org/D10638, needed for below patch to apply
+      (fetchpatch {
+        url = "https://www.mercurial-scm.org/repo/hg/raw-rev/c365850b611490a5fdb235eb1cea310a542c2f84";
+        sha256 = "1gn3xvahbjdhbglffqpmj559w1bkqqsk70wqcanwv7nh972aqy9g";
+      })
+      # https://phab.mercurial-scm.org/D10639, fixes https://bz.mercurial-scm.org/show_bug.cgi?id=6514
+      (fetchpatch {
+        url = "https://www.mercurial-scm.org/repo/hg/raw-rev/c8f62920f07a40af3403ba9aefa1dac8a97d53ea";
+        sha256 = "1kw0xjg2c4jby0ncarjvpa5qafsyl1wzbk6jxls4hnxlxdl53nmn";
+      })
+    ];
+
+    format = "other";
+
+    passthru = { inherit python; }; # pass it so that the same version can be used in hg2git
+
+    cargoDeps = if rustSupport then rustPlatform.fetchCargoTarball {
+      inherit src;
+      name = "${pname}-${version}";
+      sha256 = "1kc2giqvfwsdl5fb0qmz96ws1gdrs3skfdzvpiif2i8f7r4nqlhd";
+      sourceRoot = "${pname}-${version}/rust";
+    } else null;
+    cargoRoot = if rustSupport then "rust" else null;
+
+    propagatedBuildInputs = lib.optional re2Support fb-re2
+      ++ lib.optional gitSupport pygit2
+      ++ lib.optional highlightSupport pygments;
+    nativeBuildInputs = [ makeWrapper gettext installShellFiles ]
+      ++ lib.optionals rustSupport (with rustPlatform; [
+           cargoSetupHook
+           rust.cargo
+           rust.rustc
+         ]);
+    buildInputs = [ docutils ]
+      ++ lib.optionals stdenv.isDarwin [ ApplicationServices ];
+
+    makeFlags = [ "PREFIX=$(out)" ]
+      ++ lib.optional rustSupport "PURE=--rust";
+
+    postInstall = (lib.optionalString guiSupport ''
+      mkdir -p $out/etc/mercurial
+      cp contrib/hgk $out/bin
+      cat >> $out/etc/mercurial/hgrc << EOF
+      [extensions]
+      hgk=$out/lib/${python.libPrefix}/site-packages/hgext/hgk.py
+      EOF
+      # setting HG so that hgk can be run itself as well (not only hg view)
+      WRAP_TK=" --set TK_LIBRARY ${tk}/lib/${tk.libPrefix}
+                --set HG $out/bin/hg
+                --prefix PATH : ${tk}/bin "
+    '') + ''
+      for i in $(cd $out/bin && ls); do
+        wrapProgram $out/bin/$i \
+          $WRAP_TK
+      done
+
+      # copy hgweb.cgi to allow use in apache
+      mkdir -p $out/share/cgi-bin
+      cp -v hgweb.cgi contrib/hgweb.wsgi $out/share/cgi-bin
+      chmod u+x $out/share/cgi-bin/hgweb.cgi
+
+      installShellCompletion --cmd hg \
+        --bash contrib/bash_completion \
+        --zsh contrib/zsh_completion
+    '';
+
+    passthru.tests = {};
+
+    meta = with lib; {
+      inherit version;
+      description = "A fast, lightweight SCM system for very large distributed projects";
+      homepage = "https://www.mercurial-scm.org";
+      downloadPage = "https://www.mercurial-scm.org/release/";
+      license = licenses.gpl2Plus;
+      maintainers = with maintainers; [ eelco lukegb ];
+      updateWalker = true;
+      platforms = platforms.unix;
+    };
   };
+in
+  self.overridePythonAttrs (origAttrs: {
+    passthru = origAttrs.passthru // rec {
+      # withExtensions takes a function which takes the python packages set and
+      # returns a list of extensions to install.
+      #
+      # for instance: mercurial.withExtension (pm: [ pm.hg-evolve ])
+      withExtensions = f: let
+        python = self.python;
+        mercurialHighPrio = ps: (ps.toPythonModule self).overrideAttrs (oldAttrs: {
+          meta = oldAttrs.meta // {
+            priority = 50;
+          };
+        });
+        plugins = (f python.pkgs) ++ [ (mercurialHighPrio python.pkgs) ];
+        env = python.withPackages (ps: plugins);
+      in stdenv.mkDerivation {
+        pname = "${self.pname}-with-extensions";
 
-  patches = [
+        inherit (self) src version meta;
-    # https://phab.mercurial-scm.org/D10638, needed for below patch to apply
-    (fetchpatch {
-      url = "https://phab.mercurial-scm.org/file/data/l7p2r4zcctcr3pzlybv2/PHID-FILE-bwjzxlz6sbegk3s4irik/D10638.diff";
-      sha256 = "0mfi324is02l7cnd3j0gbmg5rpyyqn3afg3f73flnfwmz5njqa5f";
-    })
-    # https://phab.mercurial-scm.org/D10639, fixes https://bz.mercurial-scm.org/show_bug.cgi?id=6514
-    (fetchpatch {
-      url = "https://phab.mercurial-scm.org/file/data/v53nhburhtkhpccyecei/PHID-FILE-6v34oll6r2gkqo4ng5nt/D10639.diff";
-      sha256 = "0h5ilrd2x1789fr6sf4k1mcvxdh0xdyr94yawdacw87v3x12c8cb";
-    })
-  ];
 
-  format = "other";
+        buildInputs = self.buildInputs ++ self.propagatedBuildInputs;
+        nativeBuildInputs = self.nativeBuildInputs;
 
-  passthru = { inherit python; }; # pass it so that the same version can be used in hg2git
+        phases = [ "installPhase" "installCheckPhase" ];
 
-  cargoDeps = if rustSupport then rustPlatform.fetchCargoTarball {
+        installPhase = ''
-    inherit src;
+          runHook preInstall
-    name = "${pname}-${version}";
-    sha256 = "1kc2giqvfwsdl5fb0qmz96ws1gdrs3skfdzvpiif2i8f7r4nqlhd";
-    sourceRoot = "${pname}-${version}/rust";
-  } else null;
-  cargoRoot = if rustSupport then "rust" else null;
 
-  propagatedBuildInputs = lib.optional re2Support fb-re2;
+          mkdir -p $out/bin
-  nativeBuildInputs = [ makeWrapper gettext ]
-    ++ lib.optionals rustSupport (with rustPlatform; [
-         cargoSetupHook
-         rust.cargo
-         rust.rustc
-       ]);
-  buildInputs = [ docutils ]
-    ++ lib.optionals stdenv.isDarwin [ ApplicationServices ];
 
-  makeFlags = [ "PREFIX=$(out)" ]
+          for bindir in ${lib.concatStringsSep " " (map (d: "${lib.getBin d}/bin") plugins)}; do
-    ++ lib.optional rustSupport "PURE=--rust";
+            for bin in $bindir/*; do
+              ln -s ${env}/bin/$(basename $bin) $out/bin/
+            done
+          done
 
-  postInstall = (lib.optionalString guiSupport ''
+          ln -s ${self}/share $out/share
-    mkdir -p $out/etc/mercurial
-    cp contrib/hgk $out/bin
-    cat >> $out/etc/mercurial/hgrc << EOF
-    [extensions]
-    hgk=$out/lib/${python.libPrefix}/site-packages/hgext/hgk.py
-    EOF
-    # setting HG so that hgk can be run itself as well (not only hg view)
-    WRAP_TK=" --set TK_LIBRARY ${tk}/lib/${tk.libPrefix}
-              --set HG $out/bin/hg
-              --prefix PATH : ${tk}/bin "
-  '') + ''
-    for i in $(cd $out/bin && ls); do
-      wrapProgram $out/bin/$i \
-        $WRAP_TK
-    done
 
-    # copy hgweb.cgi to allow use in apache
+          runHook postInstall
-    mkdir -p $out/share/cgi-bin
+        '';
-    cp -v hgweb.cgi contrib/hgweb.wsgi $out/share/cgi-bin
-    chmod u+x $out/share/cgi-bin/hgweb.cgi
 
-    # install bash/zsh completions
+        installCheckPhase = ''
-    install -v -m644 -D contrib/bash_completion $out/share/bash-completion/completions/_hg
+          runHook preInstallCheck
-    install -v -m644 -D contrib/zsh_completion $out/share/zsh/site-functions/_hg
-  '';
 
-  meta = with lib; {
+          $out/bin/hg help >/dev/null || exit 1
-    inherit version;
+
-    description = "A fast, lightweight SCM system for very large distributed projects";
+          runHook postInstallCheck
-    homepage = "https://www.mercurial-scm.org";
+        '';
-    downloadPage = "https://www.mercurial-scm.org/release/";
+      };
-    license = licenses.gpl2Plus;
+
-    maintainers = with maintainers; [ eelco lukegb ];
+      tests = origAttrs.passthru.tests // {
-    updateWalker = true;
+        withExtensions = withExtensions (pm: [ pm.hg-evolve ]);
-    platforms = platforms.unix;
+      };
-  };
+    };
-}
+  })

pkgs/applications/version-management/tortoisehg/default.nix

@@ -1,25 +1,16 @@
 { lib, fetchurl, python3Packages
 , mercurial, qt5
-}@args:
+}:
 let
   tortoisehgSrc = fetchurl rec {
     meta.name = "tortoisehg-${meta.version}";
-    meta.version = "5.6";
+    meta.version = "5.8";
     url = "https://www.mercurial-scm.org/release/tortoisehg/targz/tortoisehg-${meta.version}.tar.gz";
-    sha256 = "031bafj88wggpvw0lgvl0djhlbhs9nls9vzwvni8yn0m0bgzc9gr";
+    sha256 = "154q7kyrdk045wx7rsblzx41k3wbvp2f40kzkxmiiaa5n35srsm3";
   };
 
-  tortoiseMercurial = (mercurial.override {
+  # Extension point for when thg's mercurial is lagging behind mainline.
-    rustSupport = false;
+  tortoiseMercurial = mercurial;
-    re2Support = lib.versionAtLeast tortoisehgSrc.meta.version "5.8";
-  }).overridePythonAttrs (old: rec {
-    inherit (tortoisehgSrc.meta) version;
-    src = fetchurl {
-      url = "https://mercurial-scm.org/release/mercurial-${version}.tar.gz";
-      sha256 = "1hk2y30zzdnlv8f71kabvh0xi9c7qhp28ksh20vpd0r712sv79yz";
-    };
-    patches = [];
-  });
 
 in python3Packages.buildPythonApplication {
     inherit (tortoisehgSrc.meta) name version;
@@ -49,7 +40,7 @@ in python3Packages.buildPythonApplication {
     meta = {
       description = "Qt based graphical tool for working with Mercurial";
       homepage = "https://tortoisehg.bitbucket.io/";
-      license = lib.licenses.gpl2;
+      license = lib.licenses.gpl2Only;
       platforms = lib.platforms.linux;
       maintainers = with lib.maintainers; [ danbst ];
     };

pkgs/development/libraries/opencolorio/default.nix

@@ -1,5 +1,5 @@
 {
-  stdenv, lib, fetchFromGitHub,
+  stdenv, lib, fetchFromGitHub, symlinkJoin,
   cmake, expat, libyamlcpp, ilmbase, pystring, # Base dependencies
 
   glew, freeglut, # Only required on Linux
@@ -25,8 +25,8 @@ stdenv.mkDerivation rec {
     sha256 = "194j9jp5c8ws0fryiz936wyinphnpzwpqnzvw9ryx6rbiwrba487";
   };
 
-  nativeBuildInputs = [ cmake ];
+  nativeBuildInputs = [ cmake (symlinkJoin { name = "expat"; paths = [ expat.out expat.dev ]; }) ];
-  buildInputs = [ expat libyamlcpp ilmbase pystring ]
+  buildInputs = [ expat.out libyamlcpp ilmbase pystring ]
     ++ lib.optionals stdenv.hostPlatform.isLinux [ glew freeglut ]
     ++ lib.optionals stdenv.hostPlatform.isDarwin [ Carbon GLUT Cocoa ]
     ++ lib.optionals pythonBindings [ python3Packages.python python3Packages.pybind11 ]

pkgs/development/python-modules/bugsnag/default.nix

@@ -8,12 +8,12 @@
 
 buildPythonPackage rec {
   pname = "bugsnag";
-  version = "4.0.3";
+  version = "4.1.0";
   disabled = pythonOlder "3.5";
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "0b70bc95e4e4f98b2eef7a3dadfdc50c1a40da7f50446adf43be05574a4b9f7c";
+    sha256 = "sha256-3L1ZzZ7eomzJLvtlGK7YOi81b4G/1azHML/iAvsnwcE=";
   };
 
   propagatedBuildInputs = [ six webob ];

pkgs/development/python-modules/fastimport/default.nix

@@ -1,9 +1,16 @@
-{ lib, buildPythonPackage, python, fetchPypi}:
+{ lib
+, pythonOlder
+, buildPythonPackage
+, fetchPypi
+, python
+}:
 
 buildPythonPackage rec {
   pname = "fastimport";
   version = "0.9.13";
 
+  disabled = pythonOlder "3.5";
+
   src = fetchPypi {
     inherit pname version;
     sha256 = "486135a39edb85808fdbbe2c8009197978800a4544fca56cc2074df32e1304f3";
@@ -13,8 +20,10 @@ buildPythonPackage rec {
     ${python.interpreter} -m unittest discover
   '';
 
+  pythonImportsCheck = [ "fastimport" ];
+
   meta = with lib; {
-    homepage = "https://launchpad.net/python-fastimport";
+    homepage = "https://github.com/jelmer/python-fastimport";
     description = "VCS fastimport/fastexport parser";
     maintainers = with maintainers; [ koral ];
     license = licenses.gpl2Plus;

pkgs/development/python-modules/georss-ign-sismologia-client/default.nix

@@ -8,14 +8,14 @@
 
 buildPythonPackage rec {
   pname = "georss-ign-sismologia-client";
-  version = "0.3";
+  version = "0.4";
   disabled = pythonOlder "3.7";
 
   src = fetchFromGitHub {
     owner = "exxamalte";
     repo = "python-georss-ign-sismologia-client";
     rev = "v${version}";
-    sha256 = "sha256-7Jj6uWb4HyPAh3/XtVTy0N23bk33mlIiqlt9z/PW+4Y=";
+    sha256 = "sha256-g7lZC5ZiJV8dNZJceLROqyBRZSuqaivGFhaQrKe4B7g=";
   };
 
   propagatedBuildInputs = [

pkgs/development/python-modules/pyls-black/default.nix

@@ -29,6 +29,6 @@ buildPythonPackage rec {
     homepage = "https://github.com/rupert/pyls-black";
     description = "Black plugin for the Python Language Server";
     license = licenses.mit;
-    maintainers = [ maintainers.mic92 ];
+    maintainers = [ ];
   };
 }

pkgs/development/python-modules/pyls-isort/default.nix

@@ -24,6 +24,6 @@ buildPythonPackage rec {
     homepage = "https://github.com/paradoxxxzero/pyls-isort";
     description = "Isort plugin for python-language-server";
     license = licenses.mit;
-    maintainers = [ maintainers.mic92 ];
+    maintainers = [ ];
   };
 }

pkgs/development/python-modules/pyls-mypy/default.nix

@@ -46,6 +46,6 @@ buildPythonPackage rec {
     homepage = "https://github.com/tomv564/pyls-mypy";
     description = "Mypy plugin for the Python Language Server";
     license = licenses.mit;
-    maintainers = [ maintainers.mic92 ];
+    maintainers = [ ];
   };
 }

pkgs/development/python-modules/pyls-spyder/default.nix

@@ -1,18 +1,29 @@
-{ lib, buildPythonPackage, fetchPypi, python-language-server }:
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, python-lsp-server
+, pytestCheckHook
+}:
 
 buildPythonPackage rec {
   pname = "pyls-spyder";
   version = "0.4.0";
 
-  src = fetchPypi {
+  src = fetchFromGitHub {
-    inherit pname version;
+    owner = "spyder-ide";
-    sha256 = "45a321c83f64267d82907492c55199fccabda45bc872dd23bf1efd08edac1b0b";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "11ajbsia60d4c9s6m6rbvaqp1d69fcdbq6a98lkzkkzv2b9pdhkk";
   };
 
-  propagatedBuildInputs = [ python-language-server ];
+  propagatedBuildInputs = [
+    python-lsp-server
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
 
-  # no tests
-  doCheck = false;
   pythonImportsCheck = [ "pyls_spyder" ];
 
   meta = with lib; {

pkgs/development/python-modules/python-language-server/default.nix

@@ -84,6 +84,6 @@ buildPythonPackage rec {
     homepage = "https://github.com/palantir/python-language-server";
     description = "An implementation of the Language Server Protocol for Python";
     license = licenses.mit;
-    maintainers = [ maintainers.mic92 ];
+    maintainers = [ ];
   };
 }

pkgs/development/python-modules/python-lsp-jsonrpc/default.nix

@@ -0,0 +1,41 @@
+{ lib
+, buildPythonPackage
+, fetchFromGitHub
+, pytestCheckHook
+, ujson
+}:
+
+buildPythonPackage rec {
+  pname = "python-lsp-jsonrpc";
+  version = "1.0.0";
+
+  src = fetchFromGitHub {
+    owner = "python-lsp";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "0h4bs8s4axcm0p02v59amz9sq3nr4zhzdgwq7iaw6awl27v1hd0i";
+  };
+
+  propagatedBuildInputs = [
+    ujson
+  ];
+
+  checkInputs = [
+    pytestCheckHook
+  ];
+
+  postPatch = ''
+    substituteInPlace setup.cfg \
+      --replace "--cov-report html --cov-report term --junitxml=pytest.xml" "" \
+      --replace "--cov pylsp_jsonrpc --cov test" ""
+  '';
+
+  pythonImportsCheck = [ "pylsp_jsonrpc" ];
+
+  meta = with lib; {
+    description = "Python server implementation of the JSON RPC 2.0 protocol.";
+    homepage = "https://github.com/python-lsp/python-lsp-jsonrpc";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+}

pkgs/development/python-modules/python-lsp-server/default.nix

@@ -0,0 +1,81 @@
+{ lib
+, autopep8
+, buildPythonPackage
+, fetchFromGitHub
+, flake8
+, flaky
+, jedi
+, matplotlib
+, mccabe
+, numpy
+, pandas
+, pluggy
+, pycodestyle
+, pydocstyle
+, pyflakes
+, pylint
+, pyqt5
+, pytestCheckHook
+, python-lsp-jsonrpc
+, pythonOlder
+, rope
+, ujson
+, yapf
+}:
+
+buildPythonPackage rec {
+  pname = "python-lsp-server";
+  version = "1.1.0";
+  disabled = pythonOlder "3.6";
+
+  src = fetchFromGitHub {
+    owner = "python-lsp";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "1akdpfnylqg2mcwpkqmdwcg6j6hab23slp5rfjfidhphig2f2yjv";
+  };
+
+  propagatedBuildInputs = [
+    autopep8
+    flake8
+    jedi
+    mccabe
+    pluggy
+    pycodestyle
+    pydocstyle
+    pyflakes
+    pylint
+    python-lsp-jsonrpc
+    rope
+    ujson
+    yapf
+  ];
+
+  checkInputs = [
+    flaky
+    matplotlib
+    numpy
+    pandas
+    pyqt5
+    pytestCheckHook
+  ];
+
+  postPatch = ''
+    substituteInPlace setup.cfg \
+      --replace "--cov-report html --cov-report term --junitxml=pytest.xml" "" \
+      --replace "--cov pylsp --cov test" ""
+  '';
+
+  preCheck = ''
+    export HOME=$(mktemp -d);
+  '';
+
+  pythonImportsCheck = [ "pylsp" ];
+
+  meta = with lib; {
+    description = "Python implementation of the Language Server Protocol";
+    homepage = "https://github.com/python-lsp/python-lsp-server";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+}

pkgs/development/python-modules/staticjinja/default.nix

@@ -14,7 +14,7 @@
 
 buildPythonPackage rec {
   pname = "staticjinja";
-  version = "2.1.0";
+  version = "3.0.1";
   format = "pyproject";
 
   # No tests in pypi
@@ -22,7 +22,7 @@ buildPythonPackage rec {
     owner = "staticjinja";
     repo = pname;
     rev = version;
-    sha256 = "sha256-VKsDvWEurBdckWbPG5hQLK3dzdM7XVbrp23fR5wp1xk=";
+    sha256 = "sha256-W4q0vG8Kl2gCmA8UnUbdiGRtghhdnWxIJXFIIa6BogA=";
   };
 
   nativeBuildInputs = [

pkgs/development/tools/misc/global/default.nix

@@ -6,11 +6,11 @@ let
   pygments = python3Packages.pygments;
 in stdenv.mkDerivation rec {
   pname = "global";
-  version = "6.6.6";
+  version = "6.6.7";
 
   src = fetchurl {
     url = "mirror://gnu/global/${pname}-${version}.tar.gz";
-    sha256 = "sha256-dYB4r/+Y1MBRxYeFx62j7Rl3+rt3+Il/9le3HMYtTV0=";
+    sha256 = "sha256-aaD3f1OCfFVoF2wdOCFm3zYedCY6BH8LMFiqLyrVijw=";
   };
 
   nativeBuildInputs = [ libtool makeWrapper ];

pkgs/games/warzone2100/default.nix

@@ -84,10 +84,15 @@ stdenv.mkDerivation rec {
   cmakeFlags = [
     "-DWZ_DISTRIBUTOR=NixOS"
     # The cmake builder automatically sets CMAKE_INSTALL_BINDIR to an absolute
-    # path, but this results in an error.
+    # path, but this results in an error:
-    # By resetting it, we let the CMakeLists set it to an accepted value
+    #
-    # based on prefix.
+    # > An absolute CMAKE_INSTALL_BINDIR path cannot be used if the following
-    "-DCMAKE_INSTALL_BINDIR="
+    # > are not also absolute paths: WZ_DATADIR
+    #
+    # WZ_DATADIR is based on CMAKE_INSTALL_DATAROOTDIR, so we set that.
+    #
+    # Alternatively, we could have set CMAKE_INSTALL_BINDIR to "bin".
+    "-DCMAKE_INSTALL_DATAROOTDIR=${placeholder "out"}/share"
   ];
 
   postInstall = lib.optionalString withVideos ''

pkgs/misc/seafile-shared/default.nix

@@ -19,7 +19,7 @@ stdenv.mkDerivation rec {
     owner = "haiwen";
     repo = "seafile";
     rev = "v${version}";
-    sha256 = "F6kLPWZb7FttyAP7pNEn+aRcAjvZlMNXrmuHMYa0Xig=";
+    sha256 = "QflLh3fj+jOq/8etr9aG8LGrvtIlB/htVkWbdO+GIbM=";
   };
 
   nativeBuildInputs = [

pkgs/os-specific/linux/kernel/perf.nix

@@ -1,12 +1,14 @@
 { lib, stdenv, kernel, elfutils, python2, python3, perl, newt, slang, asciidoc, xmlto, makeWrapper
 , docbook_xsl, docbook_xml_dtd_45, libxslt, flex, bison, pkg-config, libunwind, binutils
 , libiberty, audit, libbfd, libopcodes, openssl, systemtap, numactl
-, zlib, withGtk ? false, gtk2 ? null
+, zlib
+, withGtk ? false, gtk2
+, withZstd ? true, zstd
+, withLibcap ? true, libcap
 }:
 
 with lib;
 
-assert withGtk -> gtk2 != null;
 assert versionAtLeast kernel.version "3.12";
 
 stdenv.mkDerivation {
@@ -42,7 +44,9 @@ stdenv.mkDerivation {
     elfutils newt slang libunwind libbfd zlib openssl systemtap.stapBuild numactl
     libopcodes python3 perl
   ] ++ lib.optional withGtk gtk2
-    ++ (if (versionAtLeast kernel.version "4.19") then [ python3 ] else [ python2 ]);
+    ++ (if (versionAtLeast kernel.version "4.19") then [ python3 ] else [ python2 ])
+    ++ lib.optional withZstd zstd
+    ++ lib.optional withLibcap libcap;
 
   # Note: we don't add elfutils to buildInputs, since it provides a
   # bad `ld' and other stuff.
@@ -55,8 +59,7 @@ stdenv.mkDerivation {
   ];
 
   postPatch = ''
-    patchShebangs scripts/bpf_helpers_doc.py
+    patchShebangs scripts
-    patchShebangs scripts/bpf_doc.py
   '';
 
   doCheck = false; # requires "sparse"

pkgs/tools/backup/btrbk/default.nix

@@ -1,5 +1,5 @@
 { lib, stdenv, fetchurl, bash, btrfs-progs, openssh, perl, perlPackages
-, util-linux, asciidoc, asciidoctor, mbuffer, makeWrapper }:
+, util-linux, asciidoc, asciidoctor, mbuffer, makeWrapper, nixosTests }:
 
 stdenv.mkDerivation rec {
   pname = "btrbk";
@@ -35,6 +35,8 @@ stdenv.mkDerivation rec {
       --prefix PATH ':' "${lib.makeBinPath [ btrfs-progs bash mbuffer openssh ]}"
   '';
 
+  passthru.tests.btrbk = nixosTests.btrbk;
+
   meta = with lib; {
     description = "A backup tool for btrfs subvolumes";
     homepage = "https://digint.ch/btrbk";

pkgs/tools/misc/infracost/default.nix

@@ -2,16 +2,16 @@
 
 buildGoModule rec {
   pname = "infracost";
-  version = "0.9.1";
+  version = "0.9.2";
 
   src = fetchFromGitHub {
     owner = "infracost";
     rev = "v${version}";
     repo = "infracost";
-    sha256 = "sha256-3dR4NZ1PiMuHNO+xl3zxeBLPOZTLAbJ0VtYJNYpJuXI=";
+    sha256 = "sha256-TM+7Am5hoiRk/StAwCh5yAN1GKv3oPun38pvhArBoJg=";
   };
 
-  vendorSha256 = "sha256-YHewZsIiDPsgJVYwQX/FovlD+UzJflXy/0oglk8ZkKk=";
+  vendorSha256 = "sha256-6sMtM7MTFTDXwH8AKr5Dxq8rPqE92xzcWqBTixcPi+8=";
 
   checkInputs = [ terraform ];
   checkPhase = "make test";

pkgs/tools/security/terrascan/default.nix

@@ -1,22 +1,22 @@
-{ buildGoModule
+{ lib
+, buildGoModule
 , fetchFromGitHub
-, lib
 }:
 
 buildGoModule rec {
   pname = "terrascan";
-  version = "1.7.0";
+  version = "1.8.0";
 
   src = fetchFromGitHub {
     owner = "accurics";
     repo = pname;
     rev = "v${version}";
-    sha256 = "sha256-P16CS1W42Q/glsB9G0kagB5oSgwLb5cGMvKFc9jzd8s=";
+    sha256 = "sha256-eCkinYJtZNf5Fo+LXu01cHRInA9CfDONvt1OIs3XJSk=";
   };
 
-  vendorSha256 = "sha256-22T7C4/ph3z+O1c9uC1p2xzg0JFV+TEdfy4iiIS4Y40=";
+  vendorSha256 = "sha256-eez/g0Np/vnSO6uvUA8vtqR3DEaKlBo6lyd9t25LE7s=";
 
-  # tests want to download a vulnerable Terraform project
+  # Tests want to download a vulnerable Terraform project
   doCheck = false;
 
   meta = with lib; {

pkgs/tools/security/traitor/default.nix

@@ -5,13 +5,13 @@
 
 buildGoModule rec {
   pname = "traitor";
-  version = "0.0.7";
+  version = "0.0.8";
 
   src = fetchFromGitHub {
     owner = "liamg";
     repo = pname;
     rev = "v${version}";
-    sha256 = "sha256-UuWJe4nVr87ab3yskqKxnclMg9EywlcgaM/WOREXD/c=";
+    sha256 = "sha256-eUeKkjSpKel6XH3/VVw/WPCG/Nq8BcZwMNFG9z9FUuU=";
   };
 
   vendorSha256 = null;

pkgs/top-level/all-packages.nix

@@ -25414,7 +25414,7 @@ in
     inherit (darwin.apple_sdk.frameworks) ApplicationServices;
   };
 
-  mercurialFull = appendToName "full" (pkgs.mercurial.override { guiSupport = true; });
+  mercurialFull = appendToName "full" (pkgs.mercurial.override { fullBuild = true; });
 
   merkaartor = libsForQt5.callPackage ../applications/misc/merkaartor { };
 

pkgs/top-level/python-packages.nix

@@ -6996,6 +6996,10 @@ in {
 
   python-louvain = callPackage ../development/python-modules/python-louvain { };
 
+  python-lsp-jsonrpc = callPackage ../development/python-modules/python-lsp-jsonrpc { };
+
+  python-lsp-server = callPackage ../development/python-modules/python-lsp-server { };
+
   python-ly = callPackage ../development/python-modules/python-ly { };
 
   python-lz4 = callPackage ../development/python-modules/python-lz4 { };