From 0dcad215fea89fb3af6c713e7a88ec87c4edb8b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Thu, 23 Apr 2020 22:42:11 +0200 Subject: [PATCH 1/4] install-grub.pl: Refactor: Extract `getList()` --- .../system/boot/loader/grub/install-grub.pl | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/nixos/modules/system/boot/loader/grub/install-grub.pl b/nixos/modules/system/boot/loader/grub/install-grub.pl index 918a66866e96..ceda809a147f 100644 --- a/nixos/modules/system/boot/loader/grub/install-grub.pl +++ b/nixos/modules/system/boot/loader/grub/install-grub.pl @@ -20,6 +20,16 @@ my $dom = XML::LibXML->load_xml(location => $ARGV[0]); sub get { my ($name) = @_; return $dom->findvalue("/expr/attrs/attr[\@name = '$name']/*/\@value"); } +sub getList { + my ($name) = @_; + my @list = (); + foreach my $entry ($dom->findnodes("/expr/attrs/attr[\@name = '$name']/list/string/\@value")) { + $entry = $entry->findvalue(".") or die; + push(@list, $entry); + } + return @list; +} + sub readFile { my ($fn) = @_; local $/ = undef; open FILE, "<$fn" or return undef; my $s = ; close FILE; @@ -616,15 +626,7 @@ sub readGrubState { return $grubState } -sub getDeviceTargets { - my @devices = (); - foreach my $dev ($dom->findnodes('/expr/attrs/attr[@name = "devices"]/list/string/@value')) { - $dev = $dev->findvalue(".") or die; - push(@devices, $dev); - } - return @devices; -} -my @deviceTargets = getDeviceTargets(); +my @deviceTargets = getList('devices'); my $prevGrubState = readGrubState(); my @prevDeviceTargets = split/,/, $prevGrubState->devices; From 81c15742ce8bbe698706713195f755d7bcf1314c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Fri, 24 Apr 2020 00:20:56 +0200 Subject: [PATCH 2/4] install-grub.pl: Write state file atomically. Other files were already written atomically, but not this one. --- nixos/modules/system/boot/loader/grub/install-grub.pl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nixos/modules/system/boot/loader/grub/install-grub.pl b/nixos/modules/system/boot/loader/grub/install-grub.pl index ceda809a147f..b55b82abb846 100644 --- a/nixos/modules/system/boot/loader/grub/install-grub.pl +++ b/nixos/modules/system/boot/loader/grub/install-grub.pl @@ -682,11 +682,18 @@ if (($requireNewInstall != 0) && ($efiTarget eq "only" || $efiTarget eq "both")) # update GRUB state file if ($requireNewInstall != 0) { - open FILE, ">$bootPath/grub/state" or die "cannot create $bootPath/grub/state: $!\n"; + # Temp file for atomic rename. + my $stateFile = "$bootPath/grub/state"; + my $stateFileTmp = $stateFile . ".tmp"; + + open FILE, ">$stateFileTmp" or die "cannot create $stateFileTmp: $!\n"; print FILE get("fullName"), "\n" or die; print FILE get("fullVersion"), "\n" or die; print FILE $efiTarget, "\n" or die; print FILE join( ",", @deviceTargets ), "\n" or die; print FILE $efiSysMountPoint, "\n" or die; close FILE or die; + + # Atomically switch to the new state file + rename $stateFileTmp, $stateFile or die "cannot rename $stateFileTmp to $stateFile\n"; } From 8665b5ab918607f5775061682874c70a54f17e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Thu, 23 Apr 2020 22:44:21 +0200 Subject: [PATCH 3/4] grub: Add `boot.loader.grub.extraGrubInstallArgs` option. Useful for when you need to build grub modules into your grub kernel to get a working boot, as shown in the added example. To store this new value, we switch to more structural JSON approach. Using one line per value to store in `/boot/grub/state` gets really messy when the values are arrays, or even worse, can contain newlines (escaping would be needed). Further, removing a value from the file would get extra messy (empty lines we'd have to keep for backwards compatibility). Thus, from now on we use JSON to store all values we'll need in the future. --- .../modules/system/boot/loader/grub/grub.nix | 30 ++++++++++++++- .../system/boot/loader/grub/install-grub.pl | 37 ++++++++++++++++--- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/nixos/modules/system/boot/loader/grub/grub.nix b/nixos/modules/system/boot/loader/grub/grub.nix index b760c3f96ddf..49e73588ed6e 100644 --- a/nixos/modules/system/boot/loader/grub/grub.nix +++ b/nixos/modules/system/boot/loader/grub/grub.nix @@ -61,6 +61,7 @@ let inherit (efi) canTouchEfiVariables; inherit (cfg) version extraConfig extraPerEntryConfig extraEntries forceInstall useOSProber + extraGrubInstallArgs extraEntriesBeforeNixOS extraPrepareConfig configurationLimit copyKernels default fsIdentifier efiSupport efiInstallAsRemovable gfxmodeEfi gfxmodeBios gfxpayloadEfi gfxpayloadBios; path = with pkgs; makeBinPath ( @@ -298,6 +299,33 @@ in ''; }; + extraGrubInstallArgs = mkOption { + default = [ ]; + example = [ "--modules=nativedisk ahci pata part_gpt part_msdos diskfilter mdraid1x lvm ext2" ]; + type = types.listOf types.str; + description = '' + Additional arguments passed to grub-install. + + A use case for this is to build specific GRUB2 modules + directly into the GRUB2 kernel image, so that they are available + and activated even in the grub rescue shell. + + They are also necessary when the BIOS/UEFI is bugged and cannot + correctly read large disks (e.g. above 2 TB), so GRUB2's own + nativedisk and related modules can be used + to use its own disk drivers. The example shows one such case. + This is also useful for booting from USB. + See the + + GRUB source code + + for which disk modules are available. + + The list elements are passed directly as argv + arguments to the grub-install program, in order. + ''; + }; + extraPerEntryConfig = mkOption { default = ""; example = "root (hd0)"; @@ -669,7 +697,7 @@ in in pkgs.writeScript "install-grub.sh" ('' #!${pkgs.runtimeShell} set -e - export PERL5LIB=${with pkgs.perlPackages; makePerlPath [ FileSlurp XMLLibXML XMLSAX XMLSAXBase ListCompare ]} + export PERL5LIB=${with pkgs.perlPackages; makePerlPath [ FileSlurp XMLLibXML XMLSAX XMLSAXBase ListCompare JSON ]} ${optionalString cfg.enableCryptodisk "export GRUB_ENABLE_CRYPTODISK=y"} '' + flip concatMapStrings cfg.mirroredBoots (args: '' ${pkgs.perl}/bin/perl ${install-grub-pl} ${grubConfig args} $@ diff --git a/nixos/modules/system/boot/loader/grub/install-grub.pl b/nixos/modules/system/boot/loader/grub/install-grub.pl index b55b82abb846..422ca81847cd 100644 --- a/nixos/modules/system/boot/loader/grub/install-grub.pl +++ b/nixos/modules/system/boot/loader/grub/install-grub.pl @@ -8,6 +8,7 @@ use File::stat; use File::Copy; use File::Slurp; use File::Temp; +use JSON; require List::Compare; use POSIX; use Cwd; @@ -606,9 +607,12 @@ struct(GrubState => { efi => '$', devices => '$', efiMountPoint => '$', + extraGrubInstallArgs => '@', }); +# If you add something to the state file, only add it to the end +# because it is read line-by-line. sub readGrubState { - my $defaultGrubState = GrubState->new(name => "", version => "", efi => "", devices => "", efiMountPoint => "" ); + my $defaultGrubState = GrubState->new(name => "", version => "", efi => "", devices => "", efiMountPoint => "", extraGrubInstallArgs => () ); open FILE, "<$bootPath/grub/state" or return $defaultGrubState; local $/ = "\n"; my $name = ; @@ -621,16 +625,34 @@ sub readGrubState { chomp($devices); my $efiMountPoint = ; chomp($efiMountPoint); + # Historically, arguments in the state file were one per each line, but that + # gets really messy when newlines are involved, structured arguments + # like lists are needed (they have to have a separator encoding), or even worse, + # when we need to remove a setting in the future. Thus, the 6th line is a JSON + # object that can store structured data, with named keys, and all new state + # should go in there. + my $jsonStateLine = ; + # For historical reasons we do not check the values above for un-definedness + # (that is, when the state file has too few lines and EOF is reached), + # because the above come from the first version of this logic and are thus + # guaranteed to be present. + $jsonStateLine = defined $jsonStateLine ? $jsonStateLine : '{}'; # empty JSON object + chomp($jsonStateLine); + my %jsonState = %{decode_json($jsonStateLine)}; + my @extraGrubInstallArgs = @{$jsonState{'extraGrubInstallArgs'}}; close FILE; - my $grubState = GrubState->new(name => $name, version => $version, efi => $efi, devices => $devices, efiMountPoint => $efiMountPoint ); + my $grubState = GrubState->new(name => $name, version => $version, efi => $efi, devices => $devices, efiMountPoint => $efiMountPoint, extraGrubInstallArgs => \@extraGrubInstallArgs ); return $grubState } my @deviceTargets = getList('devices'); my $prevGrubState = readGrubState(); my @prevDeviceTargets = split/,/, $prevGrubState->devices; +my @extraGrubInstallArgs = getList('extraGrubInstallArgs'); +my @prevExtraGrubInstallArgs = $prevGrubState->extraGrubInstallArgs; my $devicesDiffer = scalar (List::Compare->new( '-u', '-a', \@deviceTargets, \@prevDeviceTargets)->get_symmetric_difference()); +my $extraGrubInstallArgsDiffer = scalar (List::Compare->new( '-u', '-a', \@extraGrubInstallArgs, \@prevExtraGrubInstallArgs)->get_symmetric_difference()); my $nameDiffer = get("fullName") ne $prevGrubState->name; my $versionDiffer = get("fullVersion") ne $prevGrubState->version; my $efiDiffer = $efiTarget ne $prevGrubState->efi; @@ -639,7 +661,7 @@ if (($ENV{'NIXOS_INSTALL_GRUB'} // "") eq "1") { warn "NIXOS_INSTALL_GRUB env var deprecated, use NIXOS_INSTALL_BOOTLOADER"; $ENV{'NIXOS_INSTALL_BOOTLOADER'} = "1"; } -my $requireNewInstall = $devicesDiffer || $nameDiffer || $versionDiffer || $efiDiffer || $efiMountPointDiffer || (($ENV{'NIXOS_INSTALL_BOOTLOADER'} // "") eq "1"); +my $requireNewInstall = $devicesDiffer || $extraGrubInstallArgsDiffer || $nameDiffer || $versionDiffer || $efiDiffer || $efiMountPointDiffer || (($ENV{'NIXOS_INSTALL_BOOTLOADER'} // "") eq "1"); # install a symlink so that grub can detect the boot drive my $tmpDir = File::Temp::tempdir(CLEANUP => 1) or die "Failed to create temporary space"; @@ -650,7 +672,7 @@ if (($requireNewInstall != 0) && ($efiTarget eq "no" || $efiTarget eq "both")) { foreach my $dev (@deviceTargets) { next if $dev eq "nodev"; print STDERR "installing the GRUB $grubVersion boot loader on $dev...\n"; - my @command = ("$grub/sbin/grub-install", "--recheck", "--root-directory=$tmpDir", Cwd::abs_path($dev)); + my @command = ("$grub/sbin/grub-install", "--recheck", "--root-directory=$tmpDir", Cwd::abs_path($dev), @extraGrubInstallArgs); if ($forceInstall eq "true") { push @command, "--force"; } @@ -665,7 +687,7 @@ if (($requireNewInstall != 0) && ($efiTarget eq "no" || $efiTarget eq "both")) { # install EFI GRUB if (($requireNewInstall != 0) && ($efiTarget eq "only" || $efiTarget eq "both")) { print STDERR "installing the GRUB $grubVersion EFI boot loader into $efiSysMountPoint...\n"; - my @command = ("$grubEfi/sbin/grub-install", "--recheck", "--target=$grubTargetEfi", "--boot-directory=$bootPath", "--efi-directory=$efiSysMountPoint"); + my @command = ("$grubEfi/sbin/grub-install", "--recheck", "--target=$grubTargetEfi", "--boot-directory=$bootPath", "--efi-directory=$efiSysMountPoint", @extraGrubInstallArgs); if ($forceInstall eq "true") { push @command, "--force"; } @@ -692,6 +714,11 @@ if ($requireNewInstall != 0) { print FILE $efiTarget, "\n" or die; print FILE join( ",", @deviceTargets ), "\n" or die; print FILE $efiSysMountPoint, "\n" or die; + my %jsonState = ( + extraGrubInstallArgs => \@extraGrubInstallArgs + ); + my $jsonStateLine = encode_json(\%jsonState); + print FILE $jsonStateLine, "\n" or die; close FILE or die; # Atomically switch to the new state file From a90ae331ecc3b3410379d61cf95828dde2b5c4a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Thu, 2 Jul 2020 22:18:49 +0200 Subject: [PATCH 4/4] install-grub.pl: Add errno messages to all `or die` errors. For example, turns the error cannot copy /nix/store/g24xsmmsz46hzi6whv7qwwn17myn3jfq-grub-2.04/share/grub/unicode.pf2 to /boot into the more useful cannot copy /nix/store/g24xsmmsz46hzi6whv7qwwn17myn3jfq-grub-2.04/share/grub/unicode.pf2 to /boot: Read-only file system --- .../system/boot/loader/grub/install-grub.pl | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/nixos/modules/system/boot/loader/grub/install-grub.pl b/nixos/modules/system/boot/loader/grub/install-grub.pl index 422ca81847cd..b788e427dff6 100644 --- a/nixos/modules/system/boot/loader/grub/install-grub.pl +++ b/nixos/modules/system/boot/loader/grub/install-grub.pl @@ -252,7 +252,7 @@ if ($grubVersion == 1) { timeout $timeout "; if ($splashImage) { - copy $splashImage, "$bootPath/background.xpm.gz" or die "cannot copy $splashImage to $bootPath\n"; + copy $splashImage, "$bootPath/background.xpm.gz" or die "cannot copy $splashImage to $bootPath: $!\n"; $conf .= "splashimage " . ($grubBoot->path eq "/" ? "" : $grubBoot->path) . "/background.xpm.gz\n"; } } @@ -330,7 +330,7 @@ else { "; if ($font) { - copy $font, "$bootPath/converted-font.pf2" or die "cannot copy $font to $bootPath\n"; + copy $font, "$bootPath/converted-font.pf2" or die "cannot copy $font to $bootPath: $!\n"; $conf .= " insmod font if loadfont " . ($grubBoot->path eq "/" ? "" : $grubBoot->path) . "/converted-font.pf2; then @@ -358,7 +358,7 @@ else { background_color '$backgroundColor' "; } - copy $splashImage, "$bootPath/background$suffix" or die "cannot copy $splashImage to $bootPath\n"; + copy $splashImage, "$bootPath/background$suffix" or die "cannot copy $splashImage to $bootPath: $!\n"; $conf .= " insmod " . substr($suffix, 1) . " if background_image --mode '$splashMode' " . ($grubBoot->path eq "/" ? "" : $grubBoot->path) . "/background$suffix; then @@ -392,8 +392,8 @@ sub copyToKernelsDir { # kernels or initrd if this script is ever interrupted. if (! -e $dst) { my $tmp = "$dst.tmp"; - copy $path, $tmp or die "cannot copy $path to $tmp\n"; - rename $tmp, $dst or die "cannot rename $tmp to $dst\n"; + copy $path, $tmp or die "cannot copy $path to $tmp: $!\n"; + rename $tmp, $dst or die "cannot rename $tmp to $dst: $!\n"; } $copied{$dst} = 1; return ($grubBoot->path eq "/" ? "" : $grubBoot->path) . "/kernels/$name"; @@ -416,10 +416,10 @@ sub addEntry { # Make sure initrd is not world readable (won't work if /boot is FAT) umask 0137; my $initrdSecretsPathTemp = File::Temp::mktemp("$initrdSecretsPath.XXXXXXXX"); - system("$path/append-initrd-secrets", $initrdSecretsPathTemp) == 0 or die "failed to create initrd secrets\n"; + system("$path/append-initrd-secrets", $initrdSecretsPathTemp) == 0 or die "failed to create initrd secrets: $!\n"; # Check whether any secrets were actually added if (-e $initrdSecretsPathTemp && ! -z _) { - rename $initrdSecretsPathTemp, $initrdSecretsPath or die "failed to move initrd secrets into place\n"; + rename $initrdSecretsPathTemp, $initrdSecretsPath or die "failed to move initrd secrets into place: $!\n"; $copied{$initrdSecretsPath} = 1; $initrd .= " " . ($grubBoot->path eq "/" ? "" : $grubBoot->path) . "/kernels/$initrdName-secrets"; } else { @@ -586,7 +586,7 @@ if (get("useOSProber") eq "true") { } # Atomically switch to the new config -rename $tmpFile, $confFile or die "cannot rename $tmpFile to $confFile\n"; +rename $tmpFile, $confFile or die "cannot rename $tmpFile to $confFile: $!\n"; # Remove obsolete files from $bootPath/kernels. @@ -664,8 +664,8 @@ if (($ENV{'NIXOS_INSTALL_GRUB'} // "") eq "1") { my $requireNewInstall = $devicesDiffer || $extraGrubInstallArgsDiffer || $nameDiffer || $versionDiffer || $efiDiffer || $efiMountPointDiffer || (($ENV{'NIXOS_INSTALL_BOOTLOADER'} // "") eq "1"); # install a symlink so that grub can detect the boot drive -my $tmpDir = File::Temp::tempdir(CLEANUP => 1) or die "Failed to create temporary space"; -symlink "$bootPath", "$tmpDir/boot" or die "Failed to symlink $tmpDir/boot"; +my $tmpDir = File::Temp::tempdir(CLEANUP => 1) or die "Failed to create temporary space: $!"; +symlink "$bootPath", "$tmpDir/boot" or die "Failed to symlink $tmpDir/boot: $!"; # install non-EFI GRUB if (($requireNewInstall != 0) && ($efiTarget eq "no" || $efiTarget eq "both")) { @@ -679,7 +679,7 @@ if (($requireNewInstall != 0) && ($efiTarget eq "no" || $efiTarget eq "both")) { if ($grubTarget ne "") { push @command, "--target=$grubTarget"; } - (system @command) == 0 or die "$0: installation of GRUB on $dev failed\n"; + (system @command) == 0 or die "$0: installation of GRUB on $dev failed: $!\n"; } } @@ -698,7 +698,7 @@ if (($requireNewInstall != 0) && ($efiTarget eq "only" || $efiTarget eq "both")) push @command, "--removable" if $efiInstallAsRemovable eq "true"; } - (system @command) == 0 or die "$0: installation of GRUB EFI into $efiSysMountPoint failed\n"; + (system @command) == 0 or die "$0: installation of GRUB EFI into $efiSysMountPoint failed: $!\n"; } @@ -722,5 +722,5 @@ if ($requireNewInstall != 0) { close FILE or die; # Atomically switch to the new state file - rename $stateFileTmp, $stateFile or die "cannot rename $stateFileTmp to $stateFile\n"; + rename $stateFileTmp, $stateFile or die "cannot rename $stateFileTmp to $stateFile: $!\n"; }