Merge master into staging-next

.github/CODEOWNERS

@@ -39,7 +39,7 @@
 /pkgs/top-level/stage.nix                        @nbp @Ericson2314 @matthewbauer
 /pkgs/top-level/splice.nix                       @Ericson2314 @matthewbauer
 /pkgs/top-level/release-cross.nix                @Ericson2314 @matthewbauer
-/pkgs/stdenv/generic                             @Ericson2314 @matthewbauer
+/pkgs/stdenv/generic                             @Ericson2314 @matthewbauer @cab404
 /pkgs/stdenv/cross                               @Ericson2314 @matthewbauer
 /pkgs/build-support/cc-wrapper                   @Ericson2314 @orivej
 /pkgs/build-support/bintools-wrapper             @Ericson2314 @orivej

nixos/lib/test-driver/default.nix

@@ -0,0 +1,32 @@
+{ lib
+, python3Packages
+, enableOCR ? false
+, qemu_pkg ? qemu_test
+, coreutils
+, imagemagick_light
+, libtiff
+, netpbm
+, qemu_test
+, socat
+, tesseract4
+, vde2
+}:
+
+python3Packages.buildPythonApplication rec {
+  pname = "nixos-test-driver";
+  version = "1.0";
+  src = ./.;
+
+  propagatedBuildInputs = [ coreutils netpbm python3Packages.colorama python3Packages.ptpython qemu_pkg socat vde2 ]
+    ++ (lib.optionals enableOCR [ imagemagick_light tesseract4 ]);
+
+  doCheck = true;
+  checkInputs = with python3Packages; [ mypy pylint black ];
+  checkPhase = ''
+    mypy --disallow-untyped-defs \
+          --no-implicit-optional \
+          --ignore-missing-imports ${src}/test_driver
+    pylint --errors-only ${src}/test_driver
+    black --check --diff ${src}/test_driver
+  '';
+}

nixos/lib/test-driver/setup.py

@@ -0,0 +1,13 @@
+from setuptools import setup, find_packages
+
+setup(
+  name="nixos-test-driver",
+  version='1.0',
+  packages=find_packages(),
+  entry_points={
+    "console_scripts": [
+      "nixos-test-driver=test_driver:main",
+      "generate-driver-symbols=test_driver:generate_driver_symbols"
+    ]
+  },
+)

nixos/lib/test-driver/test_driver/__init__.py

@@ -0,0 +1,100 @@
+from pathlib import Path
+import argparse
+import ptpython.repl
+import os
+import time
+
+from test_driver.logger import rootlog
+from test_driver.driver import Driver
+
+
+class EnvDefault(argparse.Action):
+    """An argpars Action that takes values from the specified
+    environment variable as the flags default value.
+    """
+
+    def __init__(self, envvar, required=False, default=None, nargs=None, **kwargs):  # type: ignore
+        if not default and envvar:
+            if envvar in os.environ:
+                if nargs is not None and (nargs.isdigit() or nargs in ["*", "+"]):
+                    default = os.environ[envvar].split()
+                else:
+                    default = os.environ[envvar]
+                kwargs["help"] = (
+                    kwargs["help"] + f" (default from environment: {default})"
+                )
+        if required and default:
+            required = False
+        super(EnvDefault, self).__init__(
+            default=default, required=required, nargs=nargs, **kwargs
+        )
+
+    def __call__(self, parser, namespace, values, option_string=None):  # type: ignore
+        setattr(namespace, self.dest, values)
+
+
+def main() -> None:
+    arg_parser = argparse.ArgumentParser(prog="nixos-test-driver")
+    arg_parser.add_argument(
+        "-K",
+        "--keep-vm-state",
+        help="re-use a VM state coming from a previous run",
+        action="store_true",
+    )
+    arg_parser.add_argument(
+        "-I",
+        "--interactive",
+        help="drop into a python repl and run the tests interactively",
+        action="store_true",
+    )
+    arg_parser.add_argument(
+        "--start-scripts",
+        metavar="START-SCRIPT",
+        action=EnvDefault,
+        envvar="startScripts",
+        nargs="*",
+        help="start scripts for participating virtual machines",
+    )
+    arg_parser.add_argument(
+        "--vlans",
+        metavar="VLAN",
+        action=EnvDefault,
+        envvar="vlans",
+        nargs="*",
+        help="vlans to span by the driver",
+    )
+    arg_parser.add_argument(
+        "testscript",
+        action=EnvDefault,
+        envvar="testScript",
+        help="the test script to run",
+        type=Path,
+    )
+
+    args = arg_parser.parse_args()
+
+    if not args.keep_vm_state:
+        rootlog.info("Machine state will be reset. To keep it, pass --keep-vm-state")
+
+    with Driver(
+        args.start_scripts, args.vlans, args.testscript.read_text(), args.keep_vm_state
+    ) as driver:
+        if args.interactive:
+            ptpython.repl.embed(driver.test_symbols(), {})
+        else:
+            tic = time.time()
+            driver.run_tests()
+            toc = time.time()
+            rootlog.info(f"test script finished in {(toc-tic):.2f}s")
+
+
+def generate_driver_symbols() -> None:
+    """
+    This generates a file with symbols of the test-driver code that can be used
+    in user's test scripts. That list is then used by pyflakes to lint those
+    scripts.
+    """
+    d = Driver([], [], "")
+    test_symbols = d.test_symbols()
+    with open("driver-symbols", "w") as fp:
+        fp.write(",".join(test_symbols.keys()))

nixos/lib/test-driver/test_driver/driver.py

@@ -0,0 +1,161 @@
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Dict, Iterator, List
+import os
+import tempfile
+
+from test_driver.logger import rootlog
+from test_driver.machine import Machine, NixStartScript, retry
+from test_driver.vlan import VLan
+
+
+class Driver:
+    """A handle to the driver that sets up the environment
+    and runs the tests"""
+
+    tests: str
+    vlans: List[VLan]
+    machines: List[Machine]
+
+    def __init__(
+        self,
+        start_scripts: List[str],
+        vlans: List[int],
+        tests: str,
+        keep_vm_state: bool = False,
+    ):
+        self.tests = tests
+
+        tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir()))
+        tmp_dir.mkdir(mode=0o700, exist_ok=True)
+
+        with rootlog.nested("start all VLans"):
+            self.vlans = [VLan(nr, tmp_dir) for nr in vlans]
+
+        def cmd(scripts: List[str]) -> Iterator[NixStartScript]:
+            for s in scripts:
+                yield NixStartScript(s)
+
+        self.machines = [
+            Machine(
+                start_command=cmd,
+                keep_vm_state=keep_vm_state,
+                name=cmd.machine_name,
+                tmp_dir=tmp_dir,
+            )
+            for cmd in cmd(start_scripts)
+        ]
+
+    def __enter__(self) -> "Driver":
+        return self
+
+    def __exit__(self, *_: Any) -> None:
+        with rootlog.nested("cleanup"):
+            for machine in self.machines:
+                machine.release()
+
+    def subtest(self, name: str) -> Iterator[None]:
+        """Group logs under a given test name"""
+        with rootlog.nested(name):
+            try:
+                yield
+                return True
+            except Exception as e:
+                rootlog.error(f'Test "{name}" failed with error: "{e}"')
+                raise e
+
+    def test_symbols(self) -> Dict[str, Any]:
+        @contextmanager
+        def subtest(name: str) -> Iterator[None]:
+            return self.subtest(name)
+
+        general_symbols = dict(
+            start_all=self.start_all,
+            test_script=self.test_script,
+            machines=self.machines,
+            vlans=self.vlans,
+            driver=self,
+            log=rootlog,
+            os=os,
+            create_machine=self.create_machine,
+            subtest=subtest,
+            run_tests=self.run_tests,
+            join_all=self.join_all,
+            retry=retry,
+            serial_stdout_off=self.serial_stdout_off,
+            serial_stdout_on=self.serial_stdout_on,
+            Machine=Machine,  # for typing
+        )
+        machine_symbols = {m.name: m for m in self.machines}
+        # If there's exactly one machine, make it available under the name
+        # "machine", even if it's not called that.
+        if len(self.machines) == 1:
+            (machine_symbols["machine"],) = self.machines
+        vlan_symbols = {
+            f"vlan{v.nr}": self.vlans[idx] for idx, v in enumerate(self.vlans)
+        }
+        print(
+            "additionally exposed symbols:\n    "
+            + ", ".join(map(lambda m: m.name, self.machines))
+            + ",\n    "
+            + ", ".join(map(lambda v: f"vlan{v.nr}", self.vlans))
+            + ",\n    "
+            + ", ".join(list(general_symbols.keys()))
+        )
+        return {**general_symbols, **machine_symbols, **vlan_symbols}
+
+    def test_script(self) -> None:
+        """Run the test script"""
+        with rootlog.nested("run the VM test script"):
+            symbols = self.test_symbols()  # call eagerly
+            exec(self.tests, symbols, None)
+
+    def run_tests(self) -> None:
+        """Run the test script (for non-interactive test runs)"""
+        self.test_script()
+        # TODO: Collect coverage data
+        for machine in self.machines:
+            if machine.is_up():
+                machine.execute("sync")
+
+    def start_all(self) -> None:
+        """Start all machines"""
+        with rootlog.nested("start all VMs"):
+            for machine in self.machines:
+                machine.start()
+
+    def join_all(self) -> None:
+        """Wait for all machines to shut down"""
+        with rootlog.nested("wait for all VMs to finish"):
+            for machine in self.machines:
+                machine.wait_for_shutdown()
+
+    def create_machine(self, args: Dict[str, Any]) -> Machine:
+        rootlog.warning(
+            "Using legacy create_machine(), please instantiate the"
+            "Machine class directly, instead"
+        )
+        tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir()))
+        tmp_dir.mkdir(mode=0o700, exist_ok=True)
+
+        if args.get("startCommand"):
+            start_command: str = args.get("startCommand", "")
+            cmd = NixStartScript(start_command)
+            name = args.get("name", cmd.machine_name)
+        else:
+            cmd = Machine.create_startcommand(args)  # type: ignore
+            name = args.get("name", "machine")
+
+        return Machine(
+            tmp_dir=tmp_dir,
+            start_command=cmd,
+            name=name,
+            keep_vm_state=args.get("keep_vm_state", False),
+            allow_reboot=args.get("allow_reboot", False),
+        )
+
+    def serial_stdout_on(self) -> None:
+        rootlog._print_serial_logs = True
+
+    def serial_stdout_off(self) -> None:
+        rootlog._print_serial_logs = False

nixos/lib/test-driver/test_driver/logger.py

@@ -0,0 +1,101 @@
+from colorama import Style
+from contextlib import contextmanager
+from typing import Any, Dict, Iterator
+from queue import Queue, Empty
+from xml.sax.saxutils import XMLGenerator
+import codecs
+import os
+import sys
+import time
+import unicodedata
+
+
+class Logger:
+    def __init__(self) -> None:
+        self.logfile = os.environ.get("LOGFILE", "/dev/null")
+        self.logfile_handle = codecs.open(self.logfile, "wb")
+        self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8")
+        self.queue: "Queue[Dict[str, str]]" = Queue()
+
+        self.xml.startDocument()
+        self.xml.startElement("logfile", attrs={})
+
+        self._print_serial_logs = True
+
+    @staticmethod
+    def _eprint(*args: object, **kwargs: Any) -> None:
+        print(*args, file=sys.stderr, **kwargs)
+
+    def close(self) -> None:
+        self.xml.endElement("logfile")
+        self.xml.endDocument()
+        self.logfile_handle.close()
+
+    def sanitise(self, message: str) -> str:
+        return "".join(ch for ch in message if unicodedata.category(ch)[0] != "C")
+
+    def maybe_prefix(self, message: str, attributes: Dict[str, str]) -> str:
+        if "machine" in attributes:
+            return "{}: {}".format(attributes["machine"], message)
+        return message
+
+    def log_line(self, message: str, attributes: Dict[str, str]) -> None:
+        self.xml.startElement("line", attributes)
+        self.xml.characters(message)
+        self.xml.endElement("line")
+
+    def info(self, *args, **kwargs) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+
+    def warning(self, *args, **kwargs) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+
+    def error(self, *args, **kwargs) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+        sys.exit(1)
+
+    def log(self, message: str, attributes: Dict[str, str] = {}) -> None:
+        self._eprint(self.maybe_prefix(message, attributes))
+        self.drain_log_queue()
+        self.log_line(message, attributes)
+
+    def log_serial(self, message: str, machine: str) -> None:
+        self.enqueue({"msg": message, "machine": machine, "type": "serial"})
+        if self._print_serial_logs:
+            self._eprint(
+                Style.DIM + "{} # {}".format(machine, message) + Style.RESET_ALL
+            )
+
+    def enqueue(self, item: Dict[str, str]) -> None:
+        self.queue.put(item)
+
+    def drain_log_queue(self) -> None:
+        try:
+            while True:
+                item = self.queue.get_nowait()
+                msg = self.sanitise(item["msg"])
+                del item["msg"]
+                self.log_line(msg, item)
+        except Empty:
+            pass
+
+    @contextmanager
+    def nested(self, message: str, attributes: Dict[str, str] = {}) -> Iterator[None]:
+        self._eprint(self.maybe_prefix(message, attributes))
+
+        self.xml.startElement("nest", attrs={})
+        self.xml.startElement("head", attributes)
+        self.xml.characters(message)
+        self.xml.endElement("head")
+
+        tic = time.time()
+        self.drain_log_queue()
+        yield
+        self.drain_log_queue()
+        toc = time.time()
+        self.log("(finished: {}, in {:.2f} seconds)".format(message, toc - tic))
+
+        self.xml.endElement("nest")
+
+
+rootlog = Logger()

nixos/lib/test-driver/test_driver/machine.py

@@ -1,19 +1,11 @@
-#! /somewhere/python3
+from contextlib import _GeneratorContextManager
-from contextlib import contextmanager, _GeneratorContextManager
-from queue import Queue, Empty
-from typing import Tuple, Any, Callable, Dict, Iterator, Optional, List, Iterable
-from xml.sax.saxutils import XMLGenerator
-from colorama import Style
 from pathlib import Path
-import queue
+from queue import Queue
-import io
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
-import threading
-import argparse
 import base64
-import codecs
+import io
 import os
-import ptpython.repl
+import queue
-import pty
 import re
 import shlex
 import shutil
@@ -21,8 +13,10 @@ import socket
 import subprocess
 import sys
 import tempfile
+import threading
 import time
-import unicodedata
+
+from test_driver.logger import rootlog
 
 CHAR_TO_KEY = {
     "A": "shift-a",
@@ -88,115 +82,10 @@ CHAR_TO_KEY = {
 }
 
 
-class Logger:
-    def __init__(self) -> None:
-        self.logfile = os.environ.get("LOGFILE", "/dev/null")
-        self.logfile_handle = codecs.open(self.logfile, "wb")
-        self.xml = XMLGenerator(self.logfile_handle, encoding="utf-8")
-        self.queue: "Queue[Dict[str, str]]" = Queue()
-
-        self.xml.startDocument()
-        self.xml.startElement("logfile", attrs={})
-
-        self._print_serial_logs = True
-
-    @staticmethod
-    def _eprint(*args: object, **kwargs: Any) -> None:
-        print(*args, file=sys.stderr, **kwargs)
-
-    def close(self) -> None:
-        self.xml.endElement("logfile")
-        self.xml.endDocument()
-        self.logfile_handle.close()
-
-    def sanitise(self, message: str) -> str:
-        return "".join(ch for ch in message if unicodedata.category(ch)[0] != "C")
-
-    def maybe_prefix(self, message: str, attributes: Dict[str, str]) -> str:
-        if "machine" in attributes:
-            return "{}: {}".format(attributes["machine"], message)
-        return message
-
-    def log_line(self, message: str, attributes: Dict[str, str]) -> None:
-        self.xml.startElement("line", attributes)
-        self.xml.characters(message)
-        self.xml.endElement("line")
-
-    def info(self, *args, **kwargs) -> None:  # type: ignore
-        self.log(*args, **kwargs)
-
-    def warning(self, *args, **kwargs) -> None:  # type: ignore
-        self.log(*args, **kwargs)
-
-    def error(self, *args, **kwargs) -> None:  # type: ignore
-        self.log(*args, **kwargs)
-        sys.exit(1)
-
-    def log(self, message: str, attributes: Dict[str, str] = {}) -> None:
-        self._eprint(self.maybe_prefix(message, attributes))
-        self.drain_log_queue()
-        self.log_line(message, attributes)
-
-    def log_serial(self, message: str, machine: str) -> None:
-        self.enqueue({"msg": message, "machine": machine, "type": "serial"})
-        if self._print_serial_logs:
-            self._eprint(
-                Style.DIM + "{} # {}".format(machine, message) + Style.RESET_ALL
-            )
-
-    def enqueue(self, item: Dict[str, str]) -> None:
-        self.queue.put(item)
-
-    def drain_log_queue(self) -> None:
-        try:
-            while True:
-                item = self.queue.get_nowait()
-                msg = self.sanitise(item["msg"])
-                del item["msg"]
-                self.log_line(msg, item)
-        except Empty:
-            pass
-
-    @contextmanager
-    def nested(self, message: str, attributes: Dict[str, str] = {}) -> Iterator[None]:
-        self._eprint(self.maybe_prefix(message, attributes))
-
-        self.xml.startElement("nest", attrs={})
-        self.xml.startElement("head", attributes)
-        self.xml.characters(message)
-        self.xml.endElement("head")
-
-        tic = time.time()
-        self.drain_log_queue()
-        yield
-        self.drain_log_queue()
-        toc = time.time()
-        self.log("(finished: {}, in {:.2f} seconds)".format(message, toc - tic))
-
-        self.xml.endElement("nest")
-
-
-rootlog = Logger()
-
-
 def make_command(args: list) -> str:
     return " ".join(map(shlex.quote, (map(str, args))))
 
 
-def retry(fn: Callable, timeout: int = 900) -> None:
-    """Call the given function repeatedly, with 1 second intervals,
-    until it returns True or a timeout is reached.
-    """
-
-    for _ in range(timeout):
-        if fn(False):
-            return
-        time.sleep(1)
-
-    if not fn(True):
-        raise Exception(f"action timed out after {timeout} seconds")
-
-
 def _perform_ocr_on_screenshot(
     screenshot_path: str, model_ids: Iterable[int]
 ) -> List[str]:
@@ -228,6 +117,20 @@ def _perform_ocr_on_screenshot(
     return model_results
 
 
+def retry(fn: Callable, timeout: int = 900) -> None:
+    """Call the given function repeatedly, with 1 second intervals,
+    until it returns True or a timeout is reached.
+    """
+
+    for _ in range(timeout):
+        if fn(False):
+            return
+        time.sleep(1)
+
+    if not fn(True):
+        raise Exception(f"action timed out after {timeout} seconds")
+
+
 class StartCommand:
     """The Base Start Command knows how to append the necesary
     runtime qemu options as determined by a particular test driver
@@ -1066,286 +969,3 @@ class Machine:
         self.shell.close()
         self.monitor.close()
         self.serial_thread.join()
-
-
-class VLan:
-    """This class handles a VLAN that the run-vm scripts identify via its
-    number handles. The network's lifetime equals the object's lifetime.
-    """
-
-    nr: int
-    socket_dir: Path
-
-    process: subprocess.Popen
-    pid: int
-    fd: io.TextIOBase
-
-    def __repr__(self) -> str:
-        return f"<Vlan Nr. {self.nr}>"
-
-    def __init__(self, nr: int, tmp_dir: Path):
-        self.nr = nr
-        self.socket_dir = tmp_dir / f"vde{self.nr}.ctl"
-
-        # TODO: don't side-effect environment here
-        os.environ[f"QEMU_VDE_SOCKET_{self.nr}"] = str(self.socket_dir)
-
-        rootlog.info("start vlan")
-        pty_master, pty_slave = pty.openpty()
-
-        self.process = subprocess.Popen(
-            ["vde_switch", "-s", self.socket_dir, "--dirmode", "0700"],
-            stdin=pty_slave,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            shell=False,
-        )
-        self.pid = self.process.pid
-        self.fd = os.fdopen(pty_master, "w")
-        self.fd.write("version\n")
-
-        # TODO: perl version checks if this can be read from
-        # an if not, dies. we could hang here forever. Fix it.
-        assert self.process.stdout is not None
-        self.process.stdout.readline()
-        if not (self.socket_dir / "ctl").exists():
-            rootlog.error("cannot start vde_switch")
-
-        rootlog.info(f"running vlan (pid {self.pid})")
-
-    def __del__(self) -> None:
-        rootlog.info(f"kill vlan (pid {self.pid})")
-        self.fd.close()
-        self.process.terminate()
-
-
-class Driver:
-    """A handle to the driver that sets up the environment
-    and runs the tests"""
-
-    tests: str
-    vlans: List[VLan]
-    machines: List[Machine]
-
-    def __init__(
-        self,
-        start_scripts: List[str],
-        vlans: List[int],
-        tests: str,
-        keep_vm_state: bool = False,
-    ):
-        self.tests = tests
-
-        tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir()))
-        tmp_dir.mkdir(mode=0o700, exist_ok=True)
-
-        with rootlog.nested("start all VLans"):
-            self.vlans = [VLan(nr, tmp_dir) for nr in vlans]
-
-        def cmd(scripts: List[str]) -> Iterator[NixStartScript]:
-            for s in scripts:
-                yield NixStartScript(s)
-
-        self.machines = [
-            Machine(
-                start_command=cmd,
-                keep_vm_state=keep_vm_state,
-                name=cmd.machine_name,
-                tmp_dir=tmp_dir,
-            )
-            for cmd in cmd(start_scripts)
-        ]
-
-    def __enter__(self) -> "Driver":
-        return self
-
-    def __exit__(self, *_: Any) -> None:
-        with rootlog.nested("cleanup"):
-            for machine in self.machines:
-                machine.release()
-
-    def subtest(self, name: str) -> Iterator[None]:
-        """Group logs under a given test name"""
-        with rootlog.nested(name):
-            try:
-                yield
-                return True
-            except Exception as e:
-                rootlog.error(f'Test "{name}" failed with error: "{e}"')
-                raise e
-
-    def test_symbols(self) -> Dict[str, Any]:
-        @contextmanager
-        def subtest(name: str) -> Iterator[None]:
-            return self.subtest(name)
-
-        general_symbols = dict(
-            start_all=self.start_all,
-            test_script=self.test_script,
-            machines=self.machines,
-            vlans=self.vlans,
-            driver=self,
-            log=rootlog,
-            os=os,
-            create_machine=self.create_machine,
-            subtest=subtest,
-            run_tests=self.run_tests,
-            join_all=self.join_all,
-            retry=retry,
-            serial_stdout_off=self.serial_stdout_off,
-            serial_stdout_on=self.serial_stdout_on,
-            Machine=Machine,  # for typing
-        )
-        machine_symbols = {m.name: m for m in self.machines}
-        # If there's exactly one machine, make it available under the name
-        # "machine", even if it's not called that.
-        if len(self.machines) == 1:
-            (machine_symbols["machine"],) = self.machines
-        vlan_symbols = {
-            f"vlan{v.nr}": self.vlans[idx] for idx, v in enumerate(self.vlans)
-        }
-        print(
-            "additionally exposed symbols:\n    "
-            + ", ".join(map(lambda m: m.name, self.machines))
-            + ",\n    "
-            + ", ".join(map(lambda v: f"vlan{v.nr}", self.vlans))
-            + ",\n    "
-            + ", ".join(list(general_symbols.keys()))
-        )
-        return {**general_symbols, **machine_symbols, **vlan_symbols}
-
-    def test_script(self) -> None:
-        """Run the test script"""
-        with rootlog.nested("run the VM test script"):
-            symbols = self.test_symbols()  # call eagerly
-            exec(self.tests, symbols, None)
-
-    def run_tests(self) -> None:
-        """Run the test script (for non-interactive test runs)"""
-        self.test_script()
-        # TODO: Collect coverage data
-        for machine in self.machines:
-            if machine.is_up():
-                machine.execute("sync")
-
-    def start_all(self) -> None:
-        """Start all machines"""
-        with rootlog.nested("start all VMs"):
-            for machine in self.machines:
-                machine.start()
-
-    def join_all(self) -> None:
-        """Wait for all machines to shut down"""
-        with rootlog.nested("wait for all VMs to finish"):
-            for machine in self.machines:
-                machine.wait_for_shutdown()
-
-    def create_machine(self, args: Dict[str, Any]) -> Machine:
-        rootlog.warning(
-            "Using legacy create_machine(), please instantiate the"
-            "Machine class directly, instead"
-        )
-        tmp_dir = Path(os.environ.get("TMPDIR", tempfile.gettempdir()))
-        tmp_dir.mkdir(mode=0o700, exist_ok=True)
-
-        if args.get("startCommand"):
-            start_command: str = args.get("startCommand", "")
-            cmd = NixStartScript(start_command)
-            name = args.get("name", cmd.machine_name)
-        else:
-            cmd = Machine.create_startcommand(args)  # type: ignore
-            name = args.get("name", "machine")
-
-        return Machine(
-            tmp_dir=tmp_dir,
-            start_command=cmd,
-            name=name,
-            keep_vm_state=args.get("keep_vm_state", False),
-            allow_reboot=args.get("allow_reboot", False),
-        )
-
-    def serial_stdout_on(self) -> None:
-        rootlog._print_serial_logs = True
-
-    def serial_stdout_off(self) -> None:
-        rootlog._print_serial_logs = False
-
-
-class EnvDefault(argparse.Action):
-    """An argpars Action that takes values from the specified
-    environment variable as the flags default value.
-    """
-
-    def __init__(self, envvar, required=False, default=None, nargs=None, **kwargs):  # type: ignore
-        if not default and envvar:
-            if envvar in os.environ:
-                if nargs is not None and (nargs.isdigit() or nargs in ["*", "+"]):
-                    default = os.environ[envvar].split()
-                else:
-                    default = os.environ[envvar]
-                kwargs["help"] = (
-                    kwargs["help"] + f" (default from environment: {default})"
-                )
-        if required and default:
-            required = False
-        super(EnvDefault, self).__init__(
-            default=default, required=required, nargs=nargs, **kwargs
-        )
-
-    def __call__(self, parser, namespace, values, option_string=None):  # type: ignore
-        setattr(namespace, self.dest, values)
-
-
-if __name__ == "__main__":
-    arg_parser = argparse.ArgumentParser(prog="nixos-test-driver")
-    arg_parser.add_argument(
-        "-K",
-        "--keep-vm-state",
-        help="re-use a VM state coming from a previous run",
-        action="store_true",
-    )
-    arg_parser.add_argument(
-        "-I",
-        "--interactive",
-        help="drop into a python repl and run the tests interactively",
-        action="store_true",
-    )
-    arg_parser.add_argument(
-        "--start-scripts",
-        metavar="START-SCRIPT",
-        action=EnvDefault,
-        envvar="startScripts",
-        nargs="*",
-        help="start scripts for participating virtual machines",
-    )
-    arg_parser.add_argument(
-        "--vlans",
-        metavar="VLAN",
-        action=EnvDefault,
-        envvar="vlans",
-        nargs="*",
-        help="vlans to span by the driver",
-    )
-    arg_parser.add_argument(
-        "testscript",
-        action=EnvDefault,
-        envvar="testScript",
-        help="the test script to run",
-        type=Path,
-    )
-
-    args = arg_parser.parse_args()
-
-    if not args.keep_vm_state:
-        rootlog.info("Machine state will be reset. To keep it, pass --keep-vm-state")
-
-    with Driver(
-        args.start_scripts, args.vlans, args.testscript.read_text(), args.keep_vm_state
-    ) as driver:
-        if args.interactive:
-            ptpython.repl.embed(driver.test_symbols(), {})
-        else:
-            tic = time.time()
-            driver.run_tests()
-            toc = time.time()
-            rootlog.info(f"test script finished in {(toc-tic):.2f}s")

nixos/lib/test-driver/test_driver/vlan.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+import io
+import os
+import pty
+import subprocess
+
+from test_driver.logger import rootlog
+
+
+class VLan:
+    """This class handles a VLAN that the run-vm scripts identify via its
+    number handles. The network's lifetime equals the object's lifetime.
+    """
+
+    nr: int
+    socket_dir: Path
+
+    process: subprocess.Popen
+    pid: int
+    fd: io.TextIOBase
+
+    def __repr__(self) -> str:
+        return f"<Vlan Nr. {self.nr}>"
+
+    def __init__(self, nr: int, tmp_dir: Path):
+        self.nr = nr
+        self.socket_dir = tmp_dir / f"vde{self.nr}.ctl"
+
+        # TODO: don't side-effect environment here
+        os.environ[f"QEMU_VDE_SOCKET_{self.nr}"] = str(self.socket_dir)
+
+        rootlog.info("start vlan")
+        pty_master, pty_slave = pty.openpty()
+
+        self.process = subprocess.Popen(
+            ["vde_switch", "-s", self.socket_dir, "--dirmode", "0700"],
+            stdin=pty_slave,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            shell=False,
+        )
+        self.pid = self.process.pid
+        self.fd = os.fdopen(pty_master, "w")
+        self.fd.write("version\n")
+
+        # TODO: perl version checks if this can be read from
+        # an if not, dies. we could hang here forever. Fix it.
+        assert self.process.stdout is not None
+        self.process.stdout.readline()
+        if not (self.socket_dir / "ctl").exists():
+            rootlog.error("cannot start vde_switch")
+
+        rootlog.info(f"running vlan (pid {self.pid})")
+
+    def __del__(self) -> None:
+        rootlog.info(f"kill vlan (pid {self.pid})")
+        self.fd.close()
+        self.process.terminate()

nixos/lib/testing-python.nix

@@ -16,65 +16,6 @@ rec {
 
   inherit pkgs;
 
-  # Reifies and correctly wraps the python test driver for
-  # the respective qemu version and with or without ocr support
-  pythonTestDriver = {
-      qemu_pkg ? pkgs.qemu_test
-    , enableOCR ? false
-  }:
-    let
-      name = "nixos-test-driver";
-      testDriverScript = ./test-driver/test-driver.py;
-      ocrProg = tesseract4.override { enableLanguages = [ "eng" ]; };
-      imagemagick_tiff = imagemagick_light.override { inherit libtiff; };
-    in stdenv.mkDerivation {
-      inherit name;
-
-      nativeBuildInputs = [ makeWrapper ];
-      buildInputs = [ (python3.withPackages (p: [ p.ptpython p.colorama ])) ];
-      checkInputs = with python3Packages; [ pylint black mypy ];
-
-      dontUnpack = true;
-
-      preferLocalBuild = true;
-
-      buildPhase = ''
-        python <<EOF
-        from pydoc import importfile
-        with open('driver-symbols', 'w') as fp:
-          t = importfile('${testDriverScript}')
-          d = t.Driver([],[],"")
-          test_symbols = d.test_symbols()
-          fp.write(','.join(test_symbols.keys()))
-        EOF
-      '';
-
-      doCheck = true;
-      checkPhase = ''
-        mypy --disallow-untyped-defs \
-             --no-implicit-optional \
-             --ignore-missing-imports ${testDriverScript}
-        pylint --errors-only ${testDriverScript}
-        black --check --diff ${testDriverScript}
-      '';
-
-      installPhase =
-        ''
-          mkdir -p $out/bin
-          cp ${testDriverScript} $out/bin/nixos-test-driver
-          chmod u+x $out/bin/nixos-test-driver
-          # TODO: copy user script part into this file (append)
-
-          wrapProgram $out/bin/nixos-test-driver \
-            --argv0 ${name} \
-            --prefix PATH : "${lib.makeBinPath [ qemu_pkg vde2 netpbm coreutils socat ]}" \
-            ${lib.optionalString enableOCR
-              "--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \
-
-          install -m 0644 -vD driver-symbols $out/nix-support/driver-symbols
-        '';
-    };
-
   # Run an automated test suite in the given virtual network.
   runTests = { driver, pos }:
     stdenv.mkDerivation {
@@ -112,8 +53,15 @@ rec {
     , passthru ? {}
   }:
     let
-      # FIXME: get this pkg from the module system
+      # Reifies and correctly wraps the python test driver for
-      testDriver = pythonTestDriver { inherit qemu_pkg enableOCR;};
+      # the respective qemu version and with or without ocr support
+      testDriver = pkgs.callPackage ./test-driver {
+        inherit enableOCR;
+        qemu_pkg = qemu_test;
+        imagemagick_light = imagemagick_light.override { inherit libtiff; };
+        tesseract4 = tesseract4.override { enableLanguages = [ "eng" ]; };
+      };
+
 
       testDriverName =
         let
@@ -178,10 +126,11 @@ rec {
         echo -n "$testScript" > $out/test-script
         ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
 
+        ${testDriver}/bin/generate-driver-symbols
         ${lib.optionalString (!skipLint) ''
           PYFLAKES_BUILTINS="$(
             echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
-            < ${lib.escapeShellArg "${testDriver}/nix-support/driver-symbols"}
+            < ${lib.escapeShellArg "driver-symbols"}
           )" ${python3Packages.pyflakes}/bin/pyflakes $out/test-script
         ''}
 

pkgs/applications/blockchains/trezor-suite/default.nix

@@ -8,7 +8,7 @@
 
 let
   pname = "trezor-suite";
-  version = "21.11.2";
+  version = "21.12.2";
   name = "${pname}-${version}";
 
   suffix = {
@@ -19,8 +19,8 @@ let
   src = fetchurl {
     url = "https://github.com/trezor/${pname}/releases/download/v${version}/Trezor-Suite-${version}-${suffix}.AppImage";
     sha512 = { # curl -Lfs https://github.com/trezor/trezor-suite/releases/latest/download/latest-linux{-arm64,}.yml | grep ^sha512 | sed 's/: /-/'
-      aarch64-linux = "sha512-QX5Ak2F1aD846BuGNcP1K/2c77Ut3LK3UiXsUPqiSBGZ9YRgdzROqdGjCVnTBBhxeCfGYQDhWmpuOpNbLr4eYg==";
+      aarch64-linux = "sha512-LzcTFSNN/loYaTDt+QpW8QpSgOTw2097IYdc7mC57Mn4NR/X2hycYZ9ZfZjBh9QFfVu/4R3UN2sA177v6Inomg==";
-      x86_64-linux  = "sha512-ckMlZoLEq3aLzyhoWf2rRE3XxNQhqo6rUHF2NKoV08sXz+Zth2Lk+P3te1vwFQl+Efryl84RTwVGWKmloZ8k9A==";
+      x86_64-linux  = "sha512-W/voBZrXaJVDN4eSUDD6lyBR9BqboD2k2/azI1pWm1NFUmDZFM+OGzyiPB3n+6SziAhca32Ot5Wy27sfmIjh3g==";
     }.${stdenv.hostPlatform.system} or (throw "Unsupported system: ${stdenv.hostPlatform.system}");
   };
 

pkgs/applications/misc/crow-translate/default.nix

@@ -34,8 +34,8 @@ let
   qonlinetranslator = fetchFromGitHub {
     owner = "crow-translate";
     repo = "QOnlineTranslator";
-    rev = "df89083d2f680a8f856b1df00b8846f995cf1fae";
+    rev = "1.5.2";
-    sha256 = "sha256-I64KGInnYd/QdI5kANJERsF95wMvRlr8kgQhUqXXN/0=";
+    sha256 = "sha256-iGi25aKwff2hNNx6o4kHZV8gVbEQcMgpTTvop3CoLjM=";
   };
   circleflags = fetchFromGitHub {
     owner = "HatScripts";
@@ -52,13 +52,13 @@ let
 in
 mkDerivation rec {
   pname = "crow-translate";
-  version = "2.8.7";
+  version = "2.9.1";
 
   src = fetchFromGitHub {
     owner = "crow-translate";
     repo = pname;
     rev = version;
-    sha256 = "sha256-0bq9itbFyzdOhdNuUtdCYLTCIhc91MM+YRhJgXC5PPw=";
+    sha256 = "sha256-7Zb6PZO8eLeGPEZD37ja+LZydIQdsgy5gMAMtlS4k5Y=";
   };
 
   patches = [

pkgs/applications/misc/crow-translate/dont-fetch-external-libs.patch

@@ -1,8 +1,8 @@
 diff --git i/CMakeLists.txt w/CMakeLists.txt
-index faa9417..059b899 100644
+index 375b17c..106efa9 100644
 --- i/CMakeLists.txt
 +++ w/CMakeLists.txt
-@@ -101,13 +101,11 @@ qt5_add_translation(QM_FILES
+@@ -114,13 +114,11 @@ qt5_add_translation(QM_FILES
  )
  
  configure_file(src/cmake.h.in cmake.h)
@@ -19,7 +19,7 @@ index faa9417..059b899 100644
      src/addlanguagedialog.cpp
      src/addlanguagedialog.ui
 diff --git i/cmake/ExternalLibraries.cmake w/cmake/ExternalLibraries.cmake
-index e2501e1..e15ce6c 100644
+index c92e745..f265f03 100644
 --- i/cmake/ExternalLibraries.cmake
 +++ w/cmake/ExternalLibraries.cmake
 @@ -2,34 +2,28 @@ include(FetchContent)
@@ -46,7 +46,7 @@ index e2501e1..e15ce6c 100644
  
  FetchContent_Declare(QOnlineTranslator
 -    GIT_REPOSITORY https://github.com/crow-translate/QOnlineTranslator
--    GIT_TAG df89083d2f680a8f856b1df00b8846f995cf1fae
+-    GIT_TAG 1.5.2
 +    SOURCE_DIR @qonlinetranslator@
  )
  

pkgs/applications/misc/crow-translate/fix-qttranslations-path.patch

@@ -1,13 +1,13 @@
 diff --git i/src/settings/appsettings.cpp w/src/settings/appsettings.cpp
-index aa8b357..15e4f74 100644
+index a73371b..b9d66ca 100644
 --- i/src/settings/appsettings.cpp
 +++ w/src/settings/appsettings.cpp
-@@ -81,7 +81,7 @@ void AppSettings::applyLanguage(QLocale::Language lang)
+@@ -75,7 +75,7 @@ void AppSettings::applyLocale(const QLocale &locale)
-     QLocale::setDefault(locale);
+     const QLocale newLocale = locale == defaultLocale() ? QLocale::system() : locale;
- 
+     QLocale::setDefault(newLocale);
-     s_appTranslator.load(locale, QStringLiteral(PROJECT_NAME), QStringLiteral("_"), QStandardPaths::locate(QStandardPaths::AppDataLocation, QStringLiteral("translations"), QStandardPaths::LocateDirectory));
+     s_appTranslator.load(newLocale, QStringLiteral(PROJECT_NAME), QStringLiteral("_"), QStandardPaths::locate(QStandardPaths::AppDataLocation, QStringLiteral("translations"), QStandardPaths::LocateDirectory));
--    s_qtTranslator.load(locale, QStringLiteral("qtbase"), QStringLiteral("_"), QLibraryInfo::location(QLibraryInfo::TranslationsPath));
+-    s_qtTranslator.load(newLocale, QStringLiteral("qt"), QStringLiteral("_"), QLibraryInfo::location(QLibraryInfo::TranslationsPath));
-+    s_qtTranslator.load(locale, QStringLiteral("qtbase"), QStringLiteral("_"), QLatin1String("@qttranslations@/translations"));
++    s_qtTranslator.load(newLocale, QStringLiteral("qt"), QStringLiteral("_"), QLatin1String("@qttranslations@/translations"));
  }
  
- QLocale::Language AppSettings::defaultLanguage()
+ QLocale AppSettings::defaultLocale()

pkgs/applications/misc/gpsprune/default.nix

@@ -1,18 +1,21 @@
-{ fetchurl, lib, stdenv, makeDesktopItem, makeWrapper, unzip, jdk }:
+{ fetchurl, lib, stdenv, makeDesktopItem, makeWrapper, unzip, jre, copyDesktopItems }:
 
 stdenv.mkDerivation rec {
   pname = "gpsprune";
-  version = "20.4";
+  version = "21";
 
   src = fetchurl {
     url = "https://activityworkshop.net/software/gpsprune/gpsprune_${version}.jar";
-    sha256 = "sha256-ZTYkKyu0/axf2uLUmQHRW/2bQ6p2zK7xBF66ozbPS2c=";
+    sha256 = "sha256-FS6nf8K+qfEWDCQwmoxH1laJIONMtwmb/H89SVJtV1E=";
   };
 
-  nativeBuildInputs = [ makeWrapper ];
+  dontUnpack = true;
-  buildInputs = [ jdk ];
 
-  desktopItem = makeDesktopItem {
+  nativeBuildInputs = [ makeWrapper copyDesktopItems ];
+  buildInputs = [ jre ];
+
+  desktopItems = [
+    (makeDesktopItem {
       name = "gpsprune";
       exec = "gpsprune";
       icon = "gpsprune";
@@ -20,24 +23,26 @@ stdenv.mkDerivation rec {
       genericName = "GPS Data Editor";
       comment = meta.description;
       categories = "Education;Geoscience;";
-  };
+    })
+  ];
 
-  buildCommand = ''
+  installPhase = ''
-    mkdir -p $out/bin $out/share/java
+    runHook preInstall
-    cp -v $src $out/share/java/gpsprune.jar
+
-    makeWrapper ${jdk}/bin/java $out/bin/gpsprune \
+    install -Dm644 ${src} $out/share/java/gpsprune.jar
+    makeWrapper ${jre}/bin/java $out/bin/gpsprune \
       --add-flags "-jar $out/share/java/gpsprune.jar"
-    mkdir -p $out/share/applications
-    cp $desktopItem/share/applications"/"* $out/share/applications
     mkdir -p $out/share/pixmaps
     ${unzip}/bin/unzip -p $src tim/prune/gui/images/window_icon_64.png > $out/share/pixmaps/gpsprune.png
+
+    runHook postInstall
   '';
 
   meta = with lib; {
     description = "Application for viewing, editing and converting GPS coordinate data";
     homepage = "https://activityworkshop.net/software/gpsprune/";
     license = licenses.gpl2Plus;
-    maintainers = [ maintainers.rycee ];
+    maintainers = with maintainers; [ rycee ];
     platforms = platforms.all;
   };
 }

pkgs/applications/misc/jgmenu/default.nix

@@ -15,13 +15,13 @@
 
 stdenv.mkDerivation rec {
   pname = "jgmenu";
-  version = "4.3.0";
+  version = "4.4.0";
 
   src = fetchFromGitHub {
     owner = "johanmalm";
     repo = pname;
     rev = "v${version}";
-    sha256 = "sha256-+JO/A7+6/yeYz0tP7vxSi04cS1bEet+3sAs7CYXKxI8=";
+    sha256 = "sha256-REzrN4tz+cFmKyJeOPOkzjvthsJdY3GButF7RdnzviE=";
   };
 
   nativeBuildInputs = [

pkgs/applications/networking/browsers/chromium/upstream-info.json

@@ -18,9 +18,9 @@
     }
   },
   "beta": {
-    "version": "97.0.4692.36",
+    "version": "97.0.4692.45",
-    "sha256": "0p0f19svnymql8skx6alb6zy4fmc5115dc2avs8h2mca1q8n5r0s",
+    "sha256": "1x55hys3340inrwwp4lzw5vq3vrg56288m746d4p2ligvsy19byp",
-    "sha256bin64": "08p0rwn4jglrzma1vf4jnyqaffnk0c8xwc7jkgfpkasm43d72zim",
+    "sha256bin64": "1jyhq27fb4pzbxlg0ssfz66hza9g8cbsqyx70ydkjqs9sf4yqqcw",
     "deps": {
       "gn": {
         "version": "2021-11-03",

pkgs/applications/office/qownnotes/default.nix

@@ -5,13 +5,13 @@
 
 mkDerivation rec {
   pname = "qownnotes";
-  version = "21.11.4";
+  version = "21.12.1";
 
   src = fetchurl {
     url = "https://download.tuxfamily.org/${pname}/src/${pname}-${version}.tar.xz";
     # Fetch the checksum of current version with curl:
     # curl https://download.tuxfamily.org/qownnotes/src/qownnotes-<version>.tar.xz.sha256
-    sha256 = "3eb025873160cecd4fa35ae5079c150d4aa5dd3152fd58c5e216b592af43e8dc";
+    sha256 = "sha256-gTxt3U2/x3VMWYU9cb5FPRvqezdn2K6VGYwEKBi5xFk=";
   };
 
   nativeBuildInputs = [ qmake qttools ];

pkgs/applications/video/haruna/default.nix

@@ -27,13 +27,13 @@
 
 mkDerivation rec {
   pname = "haruna";
-  version = "0.7.2";
+  version = "0.7.3";
 
   src = fetchFromGitLab {
     owner = "multimedia";
     repo = "haruna";
     rev = "v${version}";
-    sha256 = "sha256-0s4v3YJhSssp2S9mppMXq0AtWXPIaqOYWPmJgKjXjDE=";
+    sha256 = "sha256-pFrmTaRvsqxJw34VULzfjx2k56kJgkB96nJtai2D1wY=";
     domain = "invent.kde.org";
   };
 

pkgs/desktops/gnome/misc/gpaste/default.nix

@@ -17,14 +17,14 @@
 }:
 
 stdenv.mkDerivation rec {
-  version = "3.42.1";
+  version = "3.42.2";
   pname = "gpaste";
 
   src = fetchFromGitHub {
     owner = "Keruspe";
     repo = "GPaste";
     rev = "v${version}";
-    sha256 = "sha256-yoJ/k9cXXF5ELKF0JXGtxsUjfQ/S1sccLRQOQG7YMXo=";
+    sha256 = "sha256-VWtq1jPwUHHIDpVaSYQ0FiihlfulRofFmacMyv/buMw=";
   };
 
   patches = [

pkgs/development/python-modules/ansible-lint/default.nix

@@ -49,7 +49,7 @@ buildPythonPackage rec {
   ];
 
   pytestFlagsArray = [
-    "--numprocesses" "auto"
+    "--numprocesses" "$NIX_BUILD_CORES"
   ];
 
   preCheck = ''

pkgs/development/python-modules/mahotas/default.nix

@@ -1,21 +1,43 @@
-{ buildPythonPackage, fetchFromGitHub, pillow, scipy, numpy, pytestCheckHook, imread, freeimage, lib, stdenv }:
+{ buildPythonPackage
+, fetchFromGitHub
+, fetchpatch
+, pillow
+, scipy
+, numpy
+, pytestCheckHook
+, imread
+, freeimage
+, lib
+, stdenv
+}:
 
 buildPythonPackage rec {
   pname = "mahotas";
-  version = "1.4.11";
+  version = "1.4.12";
 
   src = fetchFromGitHub {
     owner = "luispedro";
     repo = "mahotas";
     rev = "v${version}";
-    sha256 = "029gvy1fb855pvxvy8zwj44k4s7qpqi0161bg5wldfiprrysn1kw";
+    sha256 = "1n19yha1cqyx7hnlici1wkl7n68dh0vbpsyydfhign2c0w9jvg42";
   };
 
+  patches = [
+    (fetchpatch {
+      name = "fix-freeimage-tests.patch";
+      url = "https://github.com/luispedro/mahotas/commit/08cc4aa0cbd5dbd4c37580d52b822810c03b2c69.patch";
+      sha256 = "0389sz7fyl8h42phw8sn4pxl4wc3brcrj9d05yga21gzil9bfi23";
+      excludes = [ "ChangeLog" ];
+    })
+  ];
+
   propagatedBuildInputs = [ numpy imread pillow scipy freeimage ];
   checkInputs = [ pytestCheckHook ];
 
   postPatch = ''
-    substituteInPlace mahotas/io/freeimage.py --replace "/opt/local/lib" "${freeimage}/lib"
+    substituteInPlace mahotas/io/freeimage.py \
+      --replace "ctypes.util.find_library('freeimage')" 'True' \
+      --replace 'ctypes.CDLL(libname)' 'np.ctypeslib.load_library("libfreeimage", "${freeimage}/lib")'
   '';
 
   # tests must be run in the build directory
@@ -31,6 +53,11 @@ buildPythonPackage rec {
     "test_haralick3d"
   ];
 
+  pythonImportsCheck = [
+    "mahotas"
+    "mahotas.freeimage"
+  ];
+
   disabled = stdenv.isi686; # Failing tests
 
   meta = with lib; {

pkgs/development/python-modules/ms-active-directory/default.nix

@@ -0,0 +1,49 @@
+{ lib
+, buildPythonPackage
+, dnspython
+, fetchFromGitHub
+, ldap3
+, pyasn1
+, pycryptodome
+, pythonOlder
+, pytz
+, six
+}:
+
+buildPythonPackage rec {
+  pname = "ms-active-directory";
+  version = "1.12.1";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.8";
+
+  src = fetchFromGitHub {
+    owner = "zorn96";
+    repo = "ms_active_directory";
+    rev = "v${version}";
+    sha256 = "sha256-mErQib8xTgo29iPAtiLnhxLXyFboAzyEW9A/QMseM6k=";
+  };
+
+  propagatedBuildInputs = [
+    dnspython
+    ldap3
+    pyasn1
+    pycryptodome
+    pytz
+    six
+  ];
+
+  # Module has no tests
+  doCheck = false;
+
+  pythonImportsCheck = [
+    "ms_active_directory"
+  ];
+
+  meta = with lib; {
+    description = "Python module for integrating with Microsoft Active Directory domains";
+    homepage = "https://github.com/zorn96/ms_active_directory/";
+    license = with licenses; [ mit ];
+    maintainers = with maintainers; [ fab ];
+  };
+}

pkgs/games/crispy-doom/default.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, autoreconfHook, pkg-config, SDL2, SDL2_mixer, SDL2_net, fetchFromGitHub, python2 }:
+{ lib, stdenv, autoreconfHook, pkg-config, SDL2, SDL2_mixer, SDL2_net, fetchFromGitHub, python3 }:
 
 stdenv.mkDerivation rec {
   pname = "crispy-doom";
@@ -13,10 +13,10 @@ stdenv.mkDerivation rec {
 
   postPatch = ''
     sed -e 's#/games#/bin#g' -i src{,/setup}/Makefile.am
-    for script in $(grep -lr '^#!/usr/bin/env python$'); do patchShebangs $script; done
+    for script in $(grep -lr '^#!/usr/bin/env python3$'); do patchShebangs $script; done
   '';
 
-  nativeBuildInputs = [ autoreconfHook pkg-config python2 ];
+  nativeBuildInputs = [ autoreconfHook pkg-config python3 ];
   buildInputs = [ SDL2 SDL2_mixer SDL2_net ];
   enableParallelBuilding = true;
 

pkgs/servers/monitoring/munin/default.nix

@@ -3,14 +3,14 @@
 }:
 
 stdenv.mkDerivation rec {
-  version = "2.0.66";
+  version = "2.0.69";
   pname = "munin";
 
   src = fetchFromGitHub {
     owner = "munin-monitoring";
     repo = "munin";
     rev = version;
-    sha256 = "sha256-1aikMRY1YiSQNUnYqsw1Eew9D9JHbkX+BXNCof6YK50=";
+    sha256 = "sha256-p273O5JLFX1dA2caV3lVVL9YNTcGMSrC7DWieUfUmqI=";
   };
 
   buildInputs = [

pkgs/stdenv/generic/default.nix

@@ -1,6 +1,6 @@
-let lib = import ../../../lib; in lib.makeOverridable (
+let lib = import ../../../lib; stdenv-overridable = lib.makeOverridable (
 
-{ name ? "stdenv", preHook ? "", initialPath
+argsStdenv@{ name ? "stdenv", preHook ? "", initialPath
 
 , # If we don't have a C compiler, we might either have `cc = null` or `cc =
   # throw ...`, but if we do have a C compiler we should definiely have `cc !=
@@ -81,8 +81,10 @@ let
 
   defaultBuildInputs = extraBuildInputs;
 
+  stdenv = (stdenv-overridable argsStdenv);
+
   # The stdenv that we are producing.
-  stdenv =
+  in
     derivation (
     lib.optionalAttrs (allowedRequisites != null) {
       allowedRequisites = allowedRequisites
@@ -172,6 +174,5 @@ let
     # "lift" packages like curl from the final stdenv for Linux to
     # all-packages.nix for that platform (meaning that it has a line
     # like curl = if stdenv ? curl then stdenv.curl else ...).
-    // extraAttrs;
+    // extraAttrs
-
+); in stdenv-overridable
-in stdenv)

pkgs/tools/misc/code-minimap/default.nix

@@ -7,16 +7,16 @@
 
 rustPlatform.buildRustPackage rec {
   pname = "code-minimap";
-  version = "0.6.1";
+  version = "0.6.2";
 
   src = fetchFromGitHub {
     owner = "wfxr";
     repo = pname;
     rev = "v${version}";
-    sha256 = "sha256-eCHmMtndcQJqKmjxhkcLvjMUXApkSnH+7qyG7PDfcwo=";
+    sha256 = "sha256-nUEmlKqCskPEQCOS2NC6jF4yVDarJeb3p+BKZq/2qvw=";
   };
 
-  cargoSha256 = "sha256-wKCANWznOJMlQ8T2q39NNNRmgPYMpbkJhXpxojusNsE=";
+  cargoSha256 = "sha256-yjjoQYYWK9/9fOP5ICnhpuF/07SyCszB9GCDr0GJ0v0=";
 
   buildInputs = lib.optional stdenv.isDarwin libiconv;
 

pkgs/tools/networking/innernet/default.nix

@@ -1,16 +1,27 @@
-{ lib, stdenv, rustPlatform, fetchFromGitHub, llvmPackages, sqlite, installShellFiles, Security, libiconv }:
+{ lib
+, stdenv
+, rustPlatform
+, fetchFromGitHub
+, llvmPackages
+, sqlite
+, installShellFiles
+, Security
+, libiconv
+, innernet
+, testVersion
+}:
 
 rustPlatform.buildRustPackage rec {
   pname = "innernet";
-  version = "1.5.1";
+  version = "1.5.2";
 
   src = fetchFromGitHub {
     owner = "tonarino";
     repo = "innernet";
     rev = "v${version}";
-    sha256 = "1ja7khvc4cy317ckglnlf11wfmin62ihic061phdp6rmfv95cza0";
+    sha256 = "141zjfl125m5lrimam1dbpk40dqfq4vnaz42sbiq1v1avyg684fq";
   };
-  cargoSha256 = "18xpwav48xv7xm7r3w9qplmv2i18cg09pkahyvs5l4akdjgxyw10";
+  cargoSha256 = "0559d0ayysvqs4k46fhgd4r8wa89abgx6rvhlh0gnlnga8vacpw5";
 
   nativeBuildInputs = with llvmPackages; [
     llvm
@@ -28,21 +39,10 @@ rustPlatform.buildRustPackage rec {
     installShellCompletion doc/innernet-server.completions.{bash,fish,zsh}
   '';
 
-  doInstallCheck = true;
+  passthru.tests = {
-  installCheckPhase = ''
+    serverVersion = testVersion { package = innernet; command = "innernet-server --version"; };
-    if [[ "$("$out/bin/${pname}"-server --version)" == "${pname}-server ${version}" ]]; then
+    version = testVersion { package = innernet; command = "innernet --version"; };
-      echo '${pname}-server smoke check passed'
+  };
-    else
-      echo '${pname}-server smoke check failed'
-      return 1
-    fi
-    if [[ "$("$out/bin/${pname}" --version)" == "${pname} ${version}" ]]; then
-      echo '${pname} smoke check passed'
-    else
-      echo '${pname} smoke check failed'
-      return 1
-    fi
-  '';
 
   meta = with lib; {
     description = "A private network system that uses WireGuard under the hood";

pkgs/tools/package-management/micromamba/default.nix

@@ -38,6 +38,13 @@ let
       rev = "yaml-cpp-${version}";
       sha256 = "0ykkxzxcwwiv8l8r697gyqh1nl582krpvi7m7l6b40ijnk4pw30s";
     };
+
+    patches = [
+      (fetchpatch {
+        url = "https://github.com/jbeder/yaml-cpp/commit/4f48727b365962e31451cd91027bd797bc7d2ee7.patch";
+        sha256 = "sha256-jarZAh7NgwL3xXzxijDiAQmC/EC2WYfNMkYHEIQBPhM=";
+      })
+    ];
   });
 in
 stdenv.mkDerivation rec {

pkgs/top-level/python-packages.nix

@@ -5007,6 +5007,8 @@ in {
 
   mrkd = callPackage ../development/python-modules/mrkd { };
 
+  ms-active-directory = callPackage ../development/python-modules/ms-active-directory { };
+
   ms-cv = callPackage ../development/python-modules/ms-cv { };
 
   msal = callPackage ../development/python-modules/msal { };