From c32369676ba85d572758abaaa438513bbed92175 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sun, 2 Feb 2020 09:48:11 +0100 Subject: [PATCH 1/3] nixosTests.ec2: Port tests that depend on common/ec2.nix --- nixos/tests/common/ec2.nix | 58 ++++++++------ nixos/tests/ec2.nix | 133 ++++++++++++++++---------------- nixos/tests/openstack-image.nix | 70 +++++++++-------- 3 files changed, 141 insertions(+), 120 deletions(-) diff --git a/nixos/tests/common/ec2.nix b/nixos/tests/common/ec2.nix index ba087bb60090..502fe96231f2 100644 --- a/nixos/tests/common/ec2.nix +++ b/nixos/tests/common/ec2.nix @@ -20,30 +20,44 @@ with pkgs.lib; in makeTest { name = "ec2-" + name; nodes = {}; - testScript = - '' - my $imageDir = ($ENV{'TMPDIR'} // "/tmp") . "/vm-state-machine"; - mkdir $imageDir, 0700; - my $diskImage = "$imageDir/machine.qcow2"; - system("qemu-img create -f qcow2 -o backing_file=${image} $diskImage") == 0 or die; - system("qemu-img resize $diskImage 10G") == 0 or die; + testScript = '' + import os + import subprocess - # Note: we use net=169.0.0.0/8 rather than - # net=169.254.0.0/16 to prevent dhcpcd from getting horribly - # confused. (It would get a DHCP lease in the 169.254.* - # range, which it would then configure and prompty delete - # again when it deletes link-local addresses.) Ideally we'd - # turn off the DHCP server, but qemu does not have an option - # to do that. - my $startCommand = "qemu-kvm -m 1024"; - $startCommand .= " -device virtio-net-pci,netdev=vlan0"; - $startCommand .= " -netdev 'user,id=vlan0,net=169.0.0.0/8,guestfwd=tcp:169.254.169.254:80-cmd:${pkgs.micro-httpd}/bin/micro_httpd ${metaData}'"; - $startCommand .= " -drive file=$diskImage,if=virtio,werror=report"; - $startCommand .= " \$QEMU_OPTS"; + image_dir = os.path.join( + os.environ.get("TMPDIR", tempfile.gettempdir()), "tmp", "vm-state-machine" + ) + os.makedirs(image_dir, mode=0o700, exist_ok=True) + disk_image = os.path.join(image_dir, "machine.qcow2") + subprocess.check_call( + [ + "qemu-img", + "create", + "-f", + "qcow2", + "-o", + "backing_file=${image}", + disk_image, + ] + ) + subprocess.check_call(["qemu-img", "resize", disk_image, "10G"]) - my $machine = createMachine({ startCommand => $startCommand }); + # Note: we use net=169.0.0.0/8 rather than + # net=169.254.0.0/16 to prevent dhcpcd from getting horribly + # confused. (It would get a DHCP lease in the 169.254.* + # range, which it would then configure and prompty delete + # again when it deletes link-local addresses.) Ideally we'd + # turn off the DHCP server, but qemu does not have an option + # to do that. + start_command = ( + "qemu-kvm -m 1024" + + " -device virtio-net-pci,netdev=vlan0" + + " -netdev 'user,id=vlan0,net=169.0.0.0/8,guestfwd=tcp:169.254.169.254:80-cmd:${pkgs.micro-httpd}/bin/micro_httpd ${metaData}'" + + f" -drive file={disk_image},if=virtio,werror=report" + + " $QEMU_OPTS" + ) - ${script} - ''; + machine = create_machine({"startCommand": start_command}) + '' + script; }; } diff --git a/nixos/tests/ec2.nix b/nixos/tests/ec2.nix index 5a59d65e6026..ad3755bb6e0e 100644 --- a/nixos/tests/ec2.nix +++ b/nixos/tests/ec2.nix @@ -3,58 +3,58 @@ pkgs ? import ../.. { inherit system config; } }: -with import ../lib/testing.nix { inherit system pkgs; }; +with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; with import common/ec2.nix { inherit makeTest pkgs; }; let - imageCfg = - (import ../lib/eval-config.nix { - inherit system; - modules = [ - ../maintainers/scripts/ec2/amazon-image.nix - ../modules/testing/test-instrumentation.nix - ../modules/profiles/qemu-guest.nix - { ec2.hvm = true; + imageCfg = (import ../lib/eval-config.nix { + inherit system; + modules = [ + ../maintainers/scripts/ec2/amazon-image.nix + ../modules/testing/test-instrumentation.nix + ../modules/profiles/qemu-guest.nix + { + ec2.hvm = true; - # Hack to make the partition resizing work in QEMU. - boot.initrd.postDeviceCommands = mkBefore - '' - ln -s vda /dev/xvda - ln -s vda1 /dev/xvda1 - ''; + # Hack to make the partition resizing work in QEMU. + boot.initrd.postDeviceCommands = mkBefore '' + ln -s vda /dev/xvda + ln -s vda1 /dev/xvda1 + ''; - # Needed by nixos-rebuild due to the lack of network - # access. Determined by trial and error. - system.extraDependencies = - with pkgs; ( - [ - # Needed for a nixos-rebuild. - busybox - stdenv - stdenvNoCC - mkinitcpio-nfs-utils - unionfs-fuse - cloud-utils - desktop-file-utils - texinfo - libxslt.bin - xorg.lndir + # Needed by nixos-rebuild due to the lack of network + # access. Determined by trial and error. + system.extraDependencies = with pkgs; ( [ + # Needed for a nixos-rebuild. + busybox + cloud-utils + desktop-file-utils + libxslt.bin + mkinitcpio-nfs-utils + stdenv + stdenvNoCC + texinfo + unionfs-fuse + xorg.lndir - # These are used in the configure-from-userdata tests - # for EC2. Httpd and valgrind are requested by the - # configuration. - apacheHttpd apacheHttpd.doc apacheHttpd.man valgrind.doc - ] - ); - } - ]; - }).config; + # These are used in the configure-from-userdata tests + # for EC2. Httpd and valgrind are requested by the + # configuration. + apacheHttpd + apacheHttpd.doc + apacheHttpd.man + valgrind.doc + ]); + } + ]; + }).config; image = "${imageCfg.system.build.amazonImage}/${imageCfg.amazonImage.name}.vhd"; sshKeys = import ./ssh-keys.nix pkgs; snakeOilPrivateKey = sshKeys.snakeOilPrivateKey.text; + snakeOilPrivateKeyFile = pkgs.writeText "private-key" snakeOilPrivateKey; snakeOilPublicKey = sshKeys.snakeOilPublicKey; in { @@ -68,38 +68,41 @@ in { SSH_HOST_ED25519_KEY:${replaceStrings ["\n"] ["|"] snakeOilPrivateKey} ''; script = '' - $machine->start; - $machine->waitForFile("/etc/ec2-metadata/user-data"); - $machine->waitForUnit("sshd.service"); + machine.start() + machine.wait_for_file("/etc/ec2-metadata/user-data") + machine.wait_for_unit("sshd.service") - $machine->succeed("grep unknown /etc/ec2-metadata/ami-manifest-path"); + machine.succeed("grep unknown /etc/ec2-metadata/ami-manifest-path") # We have no keys configured on the client side yet, so this should fail - $machine->fail("ssh -o BatchMode=yes localhost exit"); + machine.fail("ssh -o BatchMode=yes localhost exit") # Let's install our client private key - $machine->succeed("mkdir -p ~/.ssh"); + machine.succeed("mkdir -p ~/.ssh") - $machine->succeed("echo '${snakeOilPrivateKey}' > ~/.ssh/id_ed25519"); - $machine->succeed("chmod 600 ~/.ssh/id_ed25519"); + machine.copy_from_host_via_shell( + "${snakeOilPrivateKeyFile}", "~/.ssh/id_ed25519" + ) + machine.succeed("chmod 600 ~/.ssh/id_ed25519") # We haven't configured the host key yet, so this should still fail - $machine->fail("ssh -o BatchMode=yes localhost exit"); + machine.fail("ssh -o BatchMode=yes localhost exit") # Add the host key; ssh should finally succeed - $machine->succeed("echo localhost,127.0.0.1 ${snakeOilPublicKey} > ~/.ssh/known_hosts"); - $machine->succeed("ssh -o BatchMode=yes localhost exit"); + machine.succeed( + "echo localhost,127.0.0.1 ${snakeOilPublicKey} > ~/.ssh/known_hosts" + ) + machine.succeed("ssh -o BatchMode=yes localhost exit") # Test whether the root disk was resized. - my $blocks = $machine->succeed("stat -c %b -f /"); - my $bsize = $machine->succeed("stat -c %S -f /"); - my $size = $blocks * $bsize; - die "wrong free space $size" if $size < 9.7 * 1024 * 1024 * 1024 || $size > 10 * 1024 * 1024 * 1024; + blocks, block_size = map(int, machine.succeed("stat -c %b:%S -f /").split(":")) + GB = 1024 ** 3 + assert 9.7 * GB <= blocks * block_size <= 10 * GB # Just to make sure resizing is idempotent. - $machine->shutdown; - $machine->start; - $machine->waitForFile("/etc/ec2-metadata/user-data"); + machine.shutdown() + machine.start() + machine.wait_for_file("/etc/ec2-metadata/user-data") ''; }; @@ -133,17 +136,17 @@ in { } ''; script = '' - $machine->start; + machine.start() # amazon-init must succeed. if it fails, make the test fail - # immediately instead of timing out in waitForFile. - $machine->waitForUnit('amazon-init.service'); + # immediately instead of timing out in wait_for_file. + machine.wait_for_unit("amazon-init.service") - $machine->waitForFile("/etc/testFile"); - $machine->succeed("cat /etc/testFile | grep -q 'whoa'"); + machine.wait_for_file("/etc/testFile") + assert "whoa" in machine.succeed("cat /etc/testFile") - $machine->waitForUnit("httpd.service"); - $machine->succeed("curl http://localhost | grep Valgrind"); + machine.wait_for_unit("httpd.service") + assert "Valgrind" in machine.succeed("curl http://localhost") ''; }; } diff --git a/nixos/tests/openstack-image.nix b/nixos/tests/openstack-image.nix index 97c9137fe1d6..0b57dfb8e7eb 100644 --- a/nixos/tests/openstack-image.nix +++ b/nixos/tests/openstack-image.nix @@ -3,30 +3,30 @@ pkgs ? import ../.. { inherit system config; } }: -with import ../lib/testing.nix { inherit system pkgs; }; +with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; with import common/ec2.nix { inherit makeTest pkgs; }; let - image = - (import ../lib/eval-config.nix { - inherit system; - modules = [ - ../maintainers/scripts/openstack/openstack-image.nix - ../modules/testing/test-instrumentation.nix - ../modules/profiles/qemu-guest.nix - { - # Needed by nixos-rebuild due to lack of network access. - system.extraDependencies = with pkgs; [ - stdenv - ]; - } - ]; - }).config.system.build.openstackImage + "/nixos.qcow2"; + image = (import ../lib/eval-config.nix { + inherit system; + modules = [ + ../maintainers/scripts/openstack/openstack-image.nix + ../modules/testing/test-instrumentation.nix + ../modules/profiles/qemu-guest.nix + { + # Needed by nixos-rebuild due to lack of network access. + system.extraDependencies = with pkgs; [ + stdenv + ]; + } + ]; + }).config.system.build.openstackImage + "/nixos.qcow2"; sshKeys = import ./ssh-keys.nix pkgs; snakeOilPrivateKey = sshKeys.snakeOilPrivateKey.text; + snakeOilPrivateKeyFile = pkgs.writeText "private-key" snakeOilPrivateKey; snakeOilPublicKey = sshKeys.snakeOilPublicKey; in { @@ -39,32 +39,36 @@ in { SSH_HOST_ED25519_KEY:${replaceStrings ["\n"] ["|"] snakeOilPrivateKey} ''; script = '' - $machine->start; - $machine->waitForFile("/etc/ec2-metadata/user-data"); - $machine->waitForUnit("sshd.service"); + machine.start() + machine.wait_for_file("/etc/ec2-metadata/user-data") + machine.wait_for_unit("sshd.service") - $machine->succeed("grep unknown /etc/ec2-metadata/ami-manifest-path"); + machine.succeed("grep unknown /etc/ec2-metadata/ami-manifest-path") # We have no keys configured on the client side yet, so this should fail - $machine->fail("ssh -o BatchMode=yes localhost exit"); + machine.fail("ssh -o BatchMode=yes localhost exit") # Let's install our client private key - $machine->succeed("mkdir -p ~/.ssh"); + machine.succeed("mkdir -p ~/.ssh") - $machine->succeed("echo '${snakeOilPrivateKey}' > ~/.ssh/id_ed25519"); - $machine->succeed("chmod 600 ~/.ssh/id_ed25519"); + machine.copy_from_host_via_shell( + "${snakeOilPrivateKeyFile}", "~/.ssh/id_ed25519" + ) + machine.succeed("chmod 600 ~/.ssh/id_ed25519") # We haven't configured the host key yet, so this should still fail - $machine->fail("ssh -o BatchMode=yes localhost exit"); + machine.fail("ssh -o BatchMode=yes localhost exit") # Add the host key; ssh should finally succeed - $machine->succeed("echo localhost,127.0.0.1 ${snakeOilPublicKey} > ~/.ssh/known_hosts"); - $machine->succeed("ssh -o BatchMode=yes localhost exit"); + machine.succeed( + "echo localhost,127.0.0.1 ${snakeOilPublicKey} > ~/.ssh/known_hosts" + ) + machine.succeed("ssh -o BatchMode=yes localhost exit") # Just to make sure resizing is idempotent. - $machine->shutdown; - $machine->start; - $machine->waitForFile("/etc/ec2-metadata/user-data"); + machine.shutdown() + machine.start() + machine.wait_for_file("/etc/ec2-metadata/user-data") ''; }; @@ -86,9 +90,9 @@ in { } ''; script = '' - $machine->start; - $machine->waitForFile("/etc/testFile"); - $machine->succeed("cat /etc/testFile | grep -q 'whoa'"); + machine.start() + machine.wait_for_file("/etc/testFile") + assert "whoa" in machine.succeed("cat /etc/testFile") ''; }; } From 20d09ec3d1b7a21ee3979027f886576ddc2b9342 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 13 Feb 2020 21:31:58 +0100 Subject: [PATCH 2/3] nixosTests.ec2-config: Add to all-tests.nix --- nixos/tests/all-tests.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 0ce5f89b27c7..58ee1c920ad6 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -86,8 +86,7 @@ in documize = handleTest ./documize.nix {}; dokuwiki = handleTest ./dokuwiki.nix {}; dovecot = handleTest ./dovecot.nix {}; - # ec2-config doesn't work in a sandbox as the simulated ec2 instance needs network access - #ec2-config = (handleTestOn ["x86_64-linux"] ./ec2.nix {}).boot-ec2-config or {}; + ec2-config = (handleTestOn ["x86_64-linux"] ./ec2.nix {}).boot-ec2-config or {}; ec2-nixops = (handleTestOn ["x86_64-linux"] ./ec2.nix {}).boot-ec2-nixops or {}; ecryptfs = handleTest ./ecryptfs.nix {}; ejabberd = handleTest ./xmpp/ejabberd.nix {}; From 97a32bcd082a32f8a4de68121f86f878b9b4bc9e Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Sun, 23 Aug 2020 10:31:58 +0200 Subject: [PATCH 3/3] nixosTests.ec2-config: mark as broken This test wants to download things from the internet while building the system. It can probably be fixed by ensuring these paths are present in the initial nix-store. --- nixos/tests/ec2.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nixos/tests/ec2.nix b/nixos/tests/ec2.nix index ad3755bb6e0e..df0672480168 100644 --- a/nixos/tests/ec2.nix +++ b/nixos/tests/ec2.nix @@ -108,6 +108,7 @@ in { boot-ec2-config = makeEc2Test { name = "config-userdata"; + meta.broken = true; # amazon-init wants to download from the internet while building the system inherit image; sshPublicKey = snakeOilPublicKey;