firmware: Working Pi 3 firmware

firmware/.envrc

@@ -1 +1,2 @@
+watch_file default.nix
 use flake ..#firmware --override-input rootdir "file+file://"<(printf %s "$PWD")

firmware/.gitignore

@@ -1 +1,2 @@
 /result
+/*.img

firmware/.keys/.gitignore

@@ -0,0 +1 @@
+/*.key

firmware/.keys/management.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINd+aujm9F06jaQIAvk/0ptQNDHp57J429SqIquVmhbh qclk-management

firmware/base.nix

@@ -1,19 +1,85 @@
-{ lib, ... }:
+{ lib, pkgs, ... }:
 let
   inherit (lib) mkOption;
 in
 {
-  options.my.build = {
-    image = mkOption {
-      description = "Final output image for distribution.";
-      type = lib.types.unspecified;
-    };
-  };
-
   config = {
     system = {
       stateVersion = "24.11";
+      nixos = {
+        distroName = "qCLKOS";
+      };
+      name = "qclk";
+    };
+    documentation.nixos.enable = false;
+
+    time.timeZone = "Europe/Dublin";
+    i18n.defaultLocale = "en_IE.UTF-8";
+
+    boot = {
+      loader = {
+        grub.enable = false;
+      };
+      initrd = {
+        systemd = {
+          enable = true;
+          emergencyAccess = true;
+        };
+      };
+      consoleLogLevel = 7;
+    };
+
+    nix = {
+      # We're a flake-only gal
+      channel.enable = false;
+      settings = {
+        experimental-features = [ "nix-command" "flakes" "ca-derivations" ];
+        extra-substituters = [ "https://nix-cache.nul.ie" ];
+        extra-trusted-public-keys = [ "nix-cache.nul.ie-1:BzH5yMfF4HbzY1C977XzOxoPhEc9Zbu39ftPkUbH+m4=" ];
+        fallback = false;
+      };
+    };
+
+    users = {
+      users = {
+        root = {
+          openssh.authorizedKeys.keyFiles = [
+            .keys/management.pub
+          ];
+        };
+      };
+    };
+
+    environment = {
+      systemPackages = with pkgs; [
+        usbutils
+        tcpdump
+
+        (pkgs.vim.customize {
+          name = "vim";
+          vimrcConfig.packages.default = {
+            start = [ pkgs.vimPlugins.vim-nix ];
+          };
+          vimrcConfig.customRC = "syntax on";
+        })
+      ];
+    };
+
+    programs = {
+      command-not-found.enable = false;
+      htop.enable = true;
+    };
+
+    services = {
+      getty.autologinUser = "root";
+
+      openssh = {
+        enable = true;
+        settings = {
+          PermitRootLogin = "prohibit-password";
+          PasswordAuthentication = false;
+        };
+      };
     };
   };
 }
-

firmware/default.nix

@@ -1,8 +1,32 @@
-{ inputs, ... }:
+{ self, inputs, ... }:
 let
-  mkSystem = target: inputs.nixpkgs.lib.nixosSystem {
+  nixpkgsLib = inputs.nixpkgs.lib.extend (final: prev:
+    let
+      date = final.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101");
+      revCode = flake: flake.shortRev or "dirty";
+    in
+    {
+      trivial = prev.trivial // {
+        release = "24.08:u-${prev.trivial.release}";
+        codeName = "Alpha";
+        revisionWithDefault = default: self.rev or default;
+        versionSuffix = ".${date}.${revCode self}:u-${revCode inputs.nixpkgs}";
+      };
+    }
+  );
+
+  mkSystem = target: nixpkgsLib.nixosSystem {
     modules = [
+      {
+        imports = [
+          inputs.impermanence.nixosModules.impermanence
+        ];
+      }
+
       ./base.nix
+      ./disk.nix
+      ./network.nix
+
       target
     ];
   };
@@ -24,7 +48,43 @@ in
           nix build "..#nixosConfigurations.qclk-$1.config.system.build.toplevel"
         '';
         build-image.exec = ''
-          nix build "..#nixosConfigurations.qclk-$1.config.my.build.image"
+          set -e
+          export PATH="$PATH:${pkgs.util-linux}/bin:${pkgs.fakeroot}/bin:${pkgs.e2fsprogs}/bin"
+          die() {
+            echo "$1" >&2
+            exit 1
+          }
+
+          [ -z "$1" ] && die "Need to set target"
+
+          target=$1
+          out=qclkos-$target.img
+
+          nix build "..#nixosConfigurations.qclk-$target.config.my.disk.image"
+
+          persistRoot=$(mktemp --tmpdir -d qclkos-persist-XXXXX)
+          # TODO: bless with unique stuff (e.g. keys)
+          touch "$persistRoot"/test.txt
+
+          cp --sparse=always result/$out $out
+          chmod u+w $out
+
+          eval $(partx $out -o START,SECTORS --nr 2 --pairs)
+          persistImg=$(mktemp --tmpdir qclkos-persist-XXXXX.img)
+          truncate -s $((SECTORS * 512)) $persistImg
+          fakeroot mkfs.ext4 -L qclkos-persist -d $persistRoot $persistImg
+
+          dd conv=notrunc if=$persistImg of=$out seek=$START count=$SECTORS
+          rm -r "$persistRoot" "$persistImg"
+        '';
+
+        push-config.exec = ''
+          host=$1; shift
+          target=$1; shift
+          verb=$1; shift
+
+          export NIX_SSHOPTS="-i .keys/management.key"
+          nixos-rebuild $verb --flake ..#qclk-$target --target-host root@"$host" --use-substitutes "$@"
         '';
       };
     };

firmware/disk.nix

@@ -0,0 +1,205 @@
+# Based on `nixos/modules/installer/sd-card/sd-image.nix`
+{ lib, modulesPath, pkgs, config, ... }:
+let
+  inherit (lib) concatMap mkOption;
+
+  cfg = config.my.disk;
+
+  nixImage = (pkgs.callPackage "${modulesPath}/../lib/make-ext4-fs.nix" {
+    volumeLabel = "qclkos-nix";
+    uuid = "38bd3706-c049-430d-9e33-5cd0e437279b";
+    storePaths = config.system.build.toplevel;
+    compressImage = false;
+  }).overrideAttrs (o: {
+    buildCommand = ''
+      # HACK: `populateImageCommands` is executed in a subshell _before_ the paths are copied in...
+      shopt -s expand_aliases
+      nixUpThenMkfs() {
+        mv rootImage/{nix/store,}
+        rmdir rootImage/nix
+
+        faketime "$@"
+      }
+      alias faketime=nixUpThenMkfs
+
+      ${o.buildCommand}
+    '';
+  });
+in
+{
+  options.my.disk = with lib.types; {
+    bootSize = mkOption {
+      description = "/boot size (MiB).";
+      type = ints.unsigned;
+      default = 1024;
+    };
+    persistSize = mkOption {
+      description = "/persist size (MiB).";
+      type = ints.unsigned;
+      default = 8192;
+    };
+
+    populateBootCommands = mkOption {
+      description = ''
+        Shell commands to populate the ./boot directory.
+        All files in that directory are copied to the
+        /boot partition on the image.
+      '';
+    };
+
+    imageBaseName = mkOption {
+      description = "Prefix of the name of the generated image file.";
+      default = "qclkos";
+    };
+    image = mkOption {
+      description = "Output disk image.";
+      type = unspecified;
+    };
+
+    rootSize = mkOption {
+      description = "tmpfs root size.";
+      type = str;
+      default = "2G";
+    };
+  };
+
+  config = {
+    fileSystems = {
+      "/boot" = {
+        device = "/dev/disk/by-label/QCLKOS_BOOT";
+        fsType = "vfat";
+      };
+      "/persist" = {
+        device = "/dev/disk/by-label/qclkos-persist";
+        fsType = "ext4";
+        neededForBoot = true;
+      };
+      "/nix" = {
+        device = "/dev/disk/by-label/qclkos-nix";
+        fsType = "ext4";
+      };
+
+      "/" = {
+        device = "yeet";
+        fsType = "tmpfs";
+        options = [ "size=${cfg.rootSize}" "mode=755" ];
+      };
+    };
+
+    boot = {
+      postBootCommands = ''
+        # On the first boot do some maintenance tasks
+        if [ -f /nix/nix-path-registration ]; then
+          set -euo pipefail
+          set -x
+          # Figure out device names for the boot device and nix filesystem.
+          nixPart=$(${pkgs.util-linux}/bin/findmnt -n -o SOURCE /nix)
+          bootDevice=$(lsblk -npo PKNAME $nixPart)
+          partNum=$(lsblk -npo MAJ:MIN $nixPart | ${pkgs.gawk}/bin/awk -F: '{print $2}')
+
+          # Resize the nix partition and the filesystem to fit the disk
+          echo ",+," | sfdisk -N$partNum --no-reread $bootDevice
+          ${pkgs.parted}/bin/partprobe
+          ${pkgs.e2fsprogs}/bin/resize2fs $nixPart
+
+          # Register the contents of the initial Nix store
+          ${config.nix.package.out}/bin/nix-store --load-db < /nix/nix-path-registration
+
+          # nixos-rebuild also requires a "system" profile and an /etc/NIXOS tag.
+          touch /etc/NIXOS
+          ${config.nix.package.out}/bin/nix-env -p /nix/var/nix/profiles/system --set /run/current-system
+
+          # Prevents this from running on later boots.
+          rm -f /nix/nix-path-registration
+        fi
+      '';
+    };
+
+    users.mutableUsers = false;
+
+    environment = {
+      # This is based on parts of `nixfiles/nixos/modules/tmproot.nix`
+      persistence."/persist" = {
+        directories = [
+          "/var/lib/nixos"
+          "/var/lib/systemd"
+        ];
+        files = [
+          "/etc/machine-id"
+        ] ++ (concatMap (k: [ k.path "${k.path}.pub" ]) config.services.openssh.hostKeys);
+      };
+    };
+
+    services = {
+      journald.storage = "volatile";
+    };
+
+    my.disk.image = pkgs.callPackage ({
+      stdenv, dosfstools, e2fsprogs, mtools, libfaketime, util-linux
+    }: stdenv.mkDerivation {
+      name = "${cfg.imageBaseName}-${config.system.nixos.label}-${pkgs.stdenv.hostPlatform.system}";
+      nativeBuildInputs = [
+        dosfstools e2fsprogs libfaketime mtools util-linux
+      ];
+
+      buildCommand = ''
+        mkdir -p $out
+        export img=$out/${cfg.imageBaseName}.img
+
+        nix_fs=${nixImage}
+
+        # Gap in front of the first partition, in MiB
+        gap=8
+
+        # Create the image file sized to fit /boot and /nix, plus slack for the gap.
+        nixSizeBlocks=$(du -B 512 --apparent-size $nix_fs | awk '{ print $1 }')
+        bootSizeBlocks=$((${toString cfg.bootSize} * 1024 * 1024 / 512))
+        persistSizeBlocks=$((${toString cfg.persistSize} * 1024 * 1024 / 512))
+        imageSize=$((nixSizeBlocks * 512 + persistSizeBlocks * 512 + bootSizeBlocks * 512 + gap * 1024 * 1024))
+        truncate -s $imageSize $img
+
+        # type=b is 'W95 FAT32', type=83 is 'Linux'.
+        # The "bootable" partition is where u-boot will look file for the bootloader
+        # information (dtbs, extlinux.conf file).
+        sfdisk --no-reread --no-tell-kernel $img <<EOF
+          label: dos
+          label-id: 0xabb10d5a
+
+          start=''${gap}M, size=$bootSizeBlocks, type=b, bootable
+          start=$((gap + ${toString cfg.bootSize}))M, size=$persistSizeBlocks, type=83
+          start=$((gap + ${toString cfg.bootSize} + ${toString cfg.persistSize}))M, type=83
+        EOF
+
+        # Copy the nixfs into the image
+        eval $(partx $img -o START,SECTORS --nr 3 --pairs)
+        dd conv=notrunc if=$nix_fs of=$img seek=$START count=$SECTORS
+
+        # Create a FAT32 /boot partition of suitable size into boot_part.img
+        eval $(partx $img -o START,SECTORS --nr 1 --pairs)
+        truncate -s $((SECTORS * 512)) boot_part.img
+
+        mkfs.vfat --invariant -n QCLKOS_BOOT boot_part.img
+
+        # Populate the files intended for /boot
+        mkdir boot
+        ${cfg.populateBootCommands}
+
+        find boot -exec touch --date=2000-01-01 {} +
+        # Copy the populated /boot into the image
+        cd boot
+        # Force a fixed order in mcopy for better determinism, and avoid file globbing
+        for d in $(find . -type d -mindepth 1 | sort); do
+          faketime "2000-01-01 00:00:00" mmd -i ../boot_part.img "::/$d"
+        done
+        for f in $(find . -type f | sort); do
+          mcopy -pvm -i ../boot_part.img "$f" "::/$f"
+        done
+        cd ..
+
+        # Verify the FAT partition before copying it.
+        fsck.vfat -vn boot_part.img
+        dd conv=notrunc if=boot_part.img of=$img seek=$START count=$SECTORS
+      '';
+    }) { };
+  };
+}

firmware/network.nix

@@ -0,0 +1,18 @@
+{ lib, config, ... }: {
+  networking = {
+    hostName = config.system.name;
+    useDHCP = false;
+    useNetworkd = true;
+  };
+
+  systemd = {
+    network = {
+      networks = {
+        "10-ethernet" = {
+          matchConfig.Name = "ethernet";
+          DHCP = "yes";
+        };
+      };
+    };
+  };
+}

firmware/target/rpi3.nix

@@ -1,8 +1,85 @@
-{ pkgs, modulesPath, config, ... }: {
+{ pkgs, config, ... }: {
-  imports = [
+  nixpkgs.system = "aarch64-linux";
-    "${modulesPath}/installer/sd-card/sd-image-aarch64.nix"
+
+  boot = {
+    kernelParams = [
+      "console=ttyS1,115200n8"
+      "console=tty0"
     ];
 
-  nixpkgs.system = "aarch64-linux";
+    loader.generic-extlinux-compatible = {
-  my.build.image = config.system.build.sdImage;
+      enable = true;
+      configurationLimit = 3;
+    };
+  };
+
+  hardware = {
+    deviceTree.filter = "*rpi-3*.dtb";
+    enableRedistributableFirmware = true;
+  };
+
+  systemd.network.links = {
+    "10-ethernet" = {
+      matchConfig.Path = "platform-3f980000.usb-usb-0:1.1.1:1.0";
+      linkConfig.Name = "ethernet";
+    };
+    "10-wifi" = {
+      matchConfig.Path = "platform-3f300000.mmc";
+      linkConfig.Name = "wifi";
+    };
+  };
+
+  my = {
+    disk = {
+      bootSize = 512;
+      persistSize = 1024;
+      rootSize = "128M";
+      imageBaseName = "qclkos-rpi3";
+
+      # Based on `nixos/modules/installer/sd-card/sd-image-aarch64.nix`
+      populateBootCommands =
+      let
+        configTxt = pkgs.writeText "config.txt" ''
+          [pi3]
+          kernel=u-boot-rpi3.bin
+
+          [pi02]
+          kernel=u-boot-rpi3.bin
+
+          # Otherwise the resolution will be weird in most cases, compared to
+          # what the pi3 firmware does by default.
+          disable_overscan=1
+
+          # Supported in newer board revisions
+          arm_boost=1
+
+          [all]
+          # Boot in 64-bit mode.
+          arm_64bit=1
+
+          # U-Boot needs this to work, regardless of whether UART is actually used or not.
+          # Look in arch/arm/mach-bcm283x/Kconfig in the U-Boot tree to see if this is still
+          # a requirement in the future.
+          enable_uart=1
+
+          # Prevent the firmware from smashing the framebuffer setup done by the mainline kernel
+          # when attempting to show low-voltage or overtemperature warnings.
+          avoid_warnings=1
+        '';
+      in ''
+        (cd ${pkgs.raspberrypifw}/share/raspberrypi/boot && cp bootcode.bin fixup*.dat start*.elf $NIX_BUILD_TOP/boot/)
+
+        # Add the config
+        cp ${configTxt} boot/config.txt
+
+        # Add pi3 specific files
+        cp ${pkgs.ubootRaspberryPi3_64bit}/u-boot.bin boot/u-boot-rpi3.bin
+        cp ${pkgs.raspberrypifw}/share/raspberrypi/boot/bcm2710-rpi-2-b.dtb boot/
+        cp ${pkgs.raspberrypifw}/share/raspberrypi/boot/bcm2710-rpi-3-b.dtb boot/
+        cp ${pkgs.raspberrypifw}/share/raspberrypi/boot/bcm2710-rpi-3-b-plus.dtb boot/
+
+        ${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d ./boot
+      '';
+    };
+  };
 }

flake.lock

@@ -193,6 +193,21 @@
         "type": "github"
       }
     },
+    "impermanence": {
+      "locked": {
+        "lastModified": 1724489415,
+        "narHash": "sha256-ey8vhwY/6XCKoh7fyTn3aIQs7WeYSYtLbYEG87VCzX4=",
+        "owner": "nix-community",
+        "repo": "impermanence",
+        "rev": "c7f5b394397398c023000cf843986ee2571a1fd7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "impermanence",
+        "type": "github"
+      }
+    },
     "nix": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -417,6 +432,7 @@
       "inputs": {
         "devenv": "devenv",
         "flake-parts": "flake-parts",
+        "impermanence": "impermanence",
         "nixpkgs": "nixpkgs_2",
         "rootdir": "rootdir"
       }

flake.nix

@@ -4,6 +4,8 @@
     devenv.url = "github:cachix/devenv";
     devenv.inputs.nixpkgs.follows = "nixpkgs";
 
+    impermanence.url = "github:nix-community/impermanence";
+
     rootdir = {
       url = "file+file:///dev/null";
       flake = false;