nixos-container: fix nixpkgs container options being ignored

Since the introduction of option `containers.<name>.pkgs`, the
`nixpkgs.*` options (including `nixpkgs.pkgs`, `nixpkgs.config`, ...) were always
ignored in container configs, which broke existing containers.

This was due to `containers.<name>.pkgs` having two separate effects:
(1) It sets the source for the modules that are used to evaluate the container.
(2) It sets the `pkgs` arg (`_module.args.pkgs`) that is used inside the container
    modules.
    This happens even when the default value of `containers.<name>.pkgs` is unchanged, in which
    case the container `pkgs` arg is set to the pkgs of the host system.
    Previously, the `pkgs` arg was determined by the `containers.<name>.config.nixpkgs.*` options.

This commit reverts the breaking change (2) while adding a backwards-compatible way to achieve (1).
It removes option `pkgs` and adds option `nixpkgs` which implements (1).
Existing users of `pkgs` are informed by an error message to use option
`nixpkgs` or to achieve only (2) by setting option `containers.<name>.config.nixpkgs.pkgs`.

nixos/modules/virtualisation/nixos-containers.nix

@@ -463,7 +463,6 @@ in
         { config, options, name, ... }:
         {
           options = {
-
             config = mkOption {
               description = ''
                 A specification of the desired configuration of this
@@ -471,9 +470,8 @@ in
               '';
               type = lib.mkOptionType {
                 name = "Toplevel NixOS config";
-                merge = loc: defs: (import (config.pkgs.path + "/nixos/lib/eval-config.nix") {
+                merge = loc: defs: (import "${toString config.nixpkgs}/nixos/lib/eval-config.nix" {
                   inherit system;
-                  pkgs = config.pkgs;
                   modules =
                     let
                       extraConfig = {
@@ -522,12 +520,18 @@ in
               '';
             };
 
-            pkgs = mkOption {
-              type = types.attrs;
-              default = pkgs;
-              defaultText = "pkgs";
+            nixpkgs = mkOption {
+              type = types.path;
+              default = pkgs.path;
+              defaultText = "pkgs.path";
               description = ''
-                Customise which nixpkgs to use for this container.
+                A path to the nixpkgs that provide the modules, pkgs and lib for evaluating the container.
+
+                To only change the <literal>pkgs</literal> argument used inside the container modules,
+                set the <literal>nixpkgs.*</literal> options in the container <option>config</option>.
+                Setting <literal>config.nixpkgs.pkgs = pkgs</literal> speeds up the container evaluation
+                by reusing the system pkgs, but the <literal>nixpkgs.config</literal> option in the
+                container config is ignored in this case.
               '';
             };
 
@@ -668,14 +672,31 @@ in
               '';
             };
 
+            # Removed option. See `checkAssertion` below for the accompanying error message.
+            pkgs = mkOption { visible = false; };
           } // networkOptions;
 
-          config = mkMerge
-            [
-              (mkIf options.config.isDefined {
-                path = config.config.system.build.toplevel;
-              })
-            ];
+          config = let
+            # Throw an error when removed option `pkgs` is used.
+            # Because this is a submodule we cannot use `mkRemovedOptionModule` or option `assertions`.
+            optionPath = "containers.${name}.pkgs";
+            files = showFiles options.pkgs.files;
+            checkAssertion = if options.pkgs.isDefined then throw ''
+              The option definition `${optionPath}' in ${files} no longer has any effect; please remove it.
+
+              Alternatively, you can use the following options:
+              - containers.${name}.nixpkgs
+                This sets the nixpkgs (and thereby the modules, pkgs and lib) that
+                are used for evaluating the container.
+
+              - containers.${name}.config.nixpkgs.pkgs
+                This only sets the `pkgs` argument used inside the container modules.
+            ''
+            else null;
+          in {
+            path = builtins.seq checkAssertion
+              mkIf options.config.isDefined config.config.system.build.toplevel;
+          };
         }));
 
       default = {};

nixos/tests/containers-custom-pkgs.nix

@@ -1,33 +1,34 @@
 import ./make-test-python.nix ({ pkgs, lib, ...} : let
 
-  customPkgs = pkgs // {
-    hello = pkgs.hello.overrideAttrs(old: {
-      name = "custom-hello";
+  customPkgs = pkgs.appendOverlays [ (self: super: {
+    hello = super.hello.overrideAttrs (old: {
+       name = "custom-hello";
     });
-  };
+  }) ];
 
 in {
   name = "containers-custom-pkgs";
   meta = with lib.maintainers; {
-    maintainers = [ adisbladis ];
+    maintainers = [ adisbladis earvstedt ];
   };
 
-  machine = { ... }: {
+  machine = { config, ... }: {
+    assertions = let
+      helloName = (builtins.head config.containers.test.config.system.extraDependencies).name;
+    in [ {
+      assertion = helloName == "custom-hello";
+      message = "Unexpected value: ${helloName}";
+    } ];
+
     containers.test = {
       autoStart = true;
-      pkgs = customPkgs;
-      config = {pkgs, config, ... }: {
-        environment.systemPackages = [
-          pkgs.hello
-        ];
+      config = { pkgs, config, ... }: {
+        nixpkgs.pkgs = customPkgs;
+        system.extraDependencies = [ pkgs.hello ];
       };
     };
   };
 
-  testScript = ''
-    machine.wait_for_unit("default.target")
-    machine.succeed(
-        "test $(nixos-container run test -- readlink -f /run/current-system/sw/bin/hello) = ${customPkgs.hello}/bin/hello"
-    )
-  '';
+  # This test only consists of evaluating the test machine
+  testScript = "";
 })