playwright: Use pre-built browsers, update them...

via update.sh. This lets us support playwright with browsers
other than chromium on linux. Building them from source would
be one step further.

pkgs/development/python-modules/playwright/update.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -i bash -p curl gnused common-updater-scripts jq prefetch-npm-deps
+#!nix-shell -i bash -p curl gnused common-updater-scripts jq prefetch-npm-deps unzip
 set -euo pipefail
 
 root="$(dirname "$(readlink -f "$0")")"
@@ -11,18 +11,60 @@ version=$(curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s https://api.github.c
 setup_py_url="https://github.com/microsoft/playwright-python/raw/v${version}/setup.py"
 driver_version=$(curl -Ls "$setup_py_url" | grep '^driver_version =' | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+')
 
+# TODO: skip if update-source-version reported the same version
 update-source-version playwright-driver "$driver_version"
 update-source-version python3Packages.playwright "$version"
 
-# Update package-lock.json files for all npm deps that are built in playwright
+playwright_dir="$root/../../web/playwright"
-# TODO: skip if update-source-version reported the same version
+driver_file="$playwright_dir/driver.nix"
-driver_file="$root/../../web/playwright/driver.nix"
 repo_url_prefix="https://github.com/microsoft/playwright/raw"
-temp_dir=$(mktemp -d)
 
+temp_dir=$(mktemp -d)
 trap 'rm -rf "$temp_dir"' EXIT
 
 
+
+# update binaries of browsers, used by playwright.
+replace_sha() {
+  sed -i "s|$2 = \".\{44,52\}\"|$2 = \"$3\"|" "$1"
+}
+
+prefetch_browser() {
+    nix store prefetch-file --json --hash-type sha256 --unpack "$1" | jq -r .hash
+}
+
+update_browser() {
+    name="$1"
+    suffix="$2"
+    arm64_suffix="${3:-$2-arm64}"
+    revision="$(jq -r ".browsers.$name.revision" "$playwright_dir/browsers.json")"
+    replace_sha "$playwright_dir/$name.nix" "x86_64-linux" \
+        "$(prefetch_browser "https://playwright.azureedge.net/builds/$name/$revision/$name-$suffix.zip")"
+    replace_sha "$playwright_dir/$name.nix" "aarch64-linux" \
+        "$(prefetch_browser "https://playwright.azureedge.net/builds/$name/$revision/$name-$arm64_suffix.zip")"
+}
+
+curl -fsSl \
+    "https://raw.githubusercontent.com/microsoft/playwright/v${driver_version}/packages/playwright-core/browsers.json" \
+    | jq '
+      .comment = "This file is kept up to date via update.sh"
+      | .browsers |= (
+        [.[]
+          | select(.installByDefault) | del(.installByDefault)]
+          | map({(.name): . | del(.name)})
+          | add
+      )
+    ' > "$playwright_dir/browsers.json"
+
+# We currently use Chromium from nixpkgs, so we don't need to download it here
+# Likewise, darwin can be ignored here atm as we are using an impure install anyway.
+update_browser "firefox" "ubuntu-22.04"
+update_browser "webkit" "ubuntu-22.04"
+update_browser "ffmpeg" "linux"
+
+
+# Update package-lock.json files for all npm deps that are built in playwright
+
 # Function to download `package-lock.json` for a given source path and update hash
 update_hash() {
     local source_root_path="$1"
@@ -30,7 +72,6 @@ update_hash() {
 
     # Formulate download URL
     local download_url="${repo_url_prefix}/v${driver_version}${source_root_path}/package-lock.json"
-
     # Download package-lock.json to temporary directory
     curl -fsSL -o "${temp_dir}/package-lock.json" "$download_url"
 
@@ -45,7 +86,6 @@ update_hash() {
 while IFS= read -r source_root_line; do
     [[ "$source_root_line" =~ sourceRoot ]] || continue
     source_root_path=$(echo "$source_root_line" | sed -e 's/^.*"${src.name}\(.*\)";.*$/\1/')
-
     # Extract the current npmDepsHash for this sourceRoot
     existing_hash=$(grep -A1 "$source_root_line" "$driver_file" | grep 'npmDepsHash' | sed -e 's/^.*npmDepsHash = "\(.*\)";$/\1/')
 

pkgs/development/web/playwright/browsers.json

@@ -0,0 +1,28 @@
+{
+  "comment": "This file is kept up to date via update.sh",
+  "browsers": {
+    "chromium": {
+      "revision": "1134",
+      "browserVersion": "129.0.6668.29"
+    },
+    "firefox": {
+      "revision": "1463",
+      "browserVersion": "130.0"
+    },
+    "webkit": {
+      "revision": "2070",
+      "revisionOverrides": {
+        "mac10.14": "1446",
+        "mac10.15": "1616",
+        "mac11": "1816",
+        "mac11-arm64": "1816",
+        "mac12": "2009",
+        "mac12-arm64": "2009"
+      },
+      "browserVersion": "18.0"
+    },
+    "ffmpeg": {
+      "revision": "1010"
+    }
+  }
+}

pkgs/development/web/playwright/chromium.nix

@@ -0,0 +1,29 @@
+{
+  runCommand,
+  makeWrapper,
+  makeFontsConf,
+  chromium,
+  ...
+}:
+let
+  fontconfig = makeFontsConf {
+    fontDirectories = [ ];
+  };
+in
+runCommand "playwright-chromium"
+  {
+    nativeBuildInputs = [
+      makeWrapper
+    ];
+  }
+  ''
+    mkdir -p $out/chrome-linux
+
+    # See here for the Chrome options:
+    # https://github.com/NixOS/nixpkgs/issues/136207#issuecomment-908637738
+    # We add --disable-gpu to be able to run in gpu-less environments such
+    # as headless nixos test vms.
+    makeWrapper ${chromium}/bin/chromium $out/chrome-linux/chrome \
+      --set-default SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt \
+      --set-default FONTCONFIG_FILE ${fontconfig}
+  ''

pkgs/development/web/playwright/driver.nix

@@ -7,6 +7,8 @@
   jq,
   nodejs,
   fetchFromGitHub,
+  linkFarm,
+  callPackage,
   makeFontsConf,
   makeWrapper,
   runCommand,
@@ -16,6 +18,14 @@ let
   inherit (stdenv.hostPlatform) system;
 
   throwSystem = throw "Unsupported system: ${system}";
+  suffix =
+    {
+      x86_64-linux = "linux";
+      aarch64-linux = "linux-arm64";
+      x86_64-darwin = "mac";
+      aarch64-darwin = "mac-arm64";
+    }
+    .${system} or throwSystem;
 
   version = "1.47.0";
 
@@ -148,6 +158,7 @@ let
     '';
 
     passthru = {
+      browsersJSON = (lib.importJSON ./browsers.json).browsers;
       browsers =
         {
           x86_64-linux = browsers-linux { };
@@ -209,37 +220,36 @@ let
   browsers-linux =
     {
       withChromium ? true,
+      withFirefox ? true,
+      withWebkit ? true,
+      withFfmpeg ? true,
     }:
     let
-      fontconfig = makeFontsConf { fontDirectories = [ ]; };
+      browsers =
+        lib.optionals withChromium [ "chromium" ]
+        ++ lib.optionals withFirefox [ "firefox" ]
+        ++ lib.optionals withWebkit [ "webkit" ]
+        ++ lib.optionals withFfmpeg [ "ffmpeg" ];
     in
-    runCommand ("playwright-browsers" + lib.optionalString withChromium "-chromium")
+    linkFarm "playwright-browsers" (
-      {
+      lib.listToAttrs (
-        nativeBuildInputs = [
+        map (
-          makeWrapper
+          name:
-          jq
+          let
-        ];
+            value = playwright-core.passthru.browsersJSON.${name};
-      }
+          in
-      (
+          lib.nameValuePair
-        ''
+            # TODO check platform for revisionOverrides
-          BROWSERS_JSON=${playwright-core}/browsers.json
+            "${name}-${value.revision}"
-        ''
+            (
-        + lib.optionalString withChromium ''
+              callPackage ./${name}.nix {
-          CHROMIUM_REVISION=$(jq -r '.browsers[] | select(.name == "chromium").revision' $BROWSERS_JSON)
+                inherit suffix system throwSystem;
-          mkdir -p $out/chromium-$CHROMIUM_REVISION/chrome-linux
+                inherit (value) revision;
-
+              }
-          # See here for the Chrome options:
+            )
-          # https://github.com/NixOS/nixpkgs/issues/136207#issuecomment-908637738
+        ) browsers
-          makeWrapper ${chromium}/bin/chromium $out/chromium-$CHROMIUM_REVISION/chrome-linux/chrome \
+      )
-            --set SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt \
+    );
-            --set FONTCONFIG_FILE ${fontconfig}
-        ''
-        + ''
-          FFMPEG_REVISION=$(jq -r '.browsers[] | select(.name == "ffmpeg").revision' $BROWSERS_JSON)
-          mkdir -p $out/ffmpeg-$FFMPEG_REVISION
-          ln -s ${ffmpeg}/bin/ffmpeg $out/ffmpeg-$FFMPEG_REVISION/ffmpeg-linux
-        ''
-      );
 in
 {
   playwright-core = playwright-core;

pkgs/development/web/playwright/ffmpeg.nix

@@ -0,0 +1,17 @@
+{
+  fetchzip,
+  suffix,
+  revision,
+  system,
+  throwSystem,
+}:
+fetchzip {
+  url = "https://playwright.azureedge.net/builds/ffmpeg/${revision}/ffmpeg-${suffix}.zip";
+  stripRoot = false;
+  hash =
+    {
+      x86_64-linux = "sha256-FEm62UvMv0h6Sav93WmbPLw3CW1L1xg4nD26ca5ol38=";
+      aarch64-linux = "sha256-jtQ+NS++VHRiKoIV++PIxEnyVnYtVwUyNlSILKSH4A4=";
+    }
+    .${system} or throwSystem;
+}

pkgs/development/web/playwright/firefox.nix

@@ -0,0 +1,39 @@
+{
+  lib,
+  stdenv,
+  fetchzip,
+  firefox-bin,
+  suffix,
+  revision,
+  system,
+  throwSystem,
+}:
+let
+  suffix' =
+    if lib.hasPrefix "linux" suffix then "ubuntu-22.04" + (lib.removePrefix "linux" suffix) else suffix;
+in
+stdenv.mkDerivation {
+  name = "playwright-firefox";
+  src = fetchzip {
+    url = "https://playwright.azureedge.net/builds/firefox/${revision}/firefox-${suffix'}.zip";
+    hash =
+      {
+        x86_64-linux = "sha256-Hd9LlSRLW51gDoFyszqvg46Q/sMizLRsVKAN9atbwsw=";
+        aarch64-linux = "sha256-SEXH3gLOfNjOcnNWQjQ5gaaow47veVs0BoTYSgXw+24=";
+      }
+      .${system} or throwSystem;
+  };
+
+  inherit (firefox-bin.unwrapped)
+    nativeBuildInputs
+    buildInputs
+    runtimeDependencies
+    appendRunpaths
+    patchelfFlags
+    ;
+
+  buildPhase = ''
+    mkdir -p $out/firefox
+    cp -R . $out/firefox
+  '';
+}

pkgs/development/web/playwright/webkit.nix

@@ -0,0 +1,135 @@
+{
+  lib,
+  stdenv,
+  fetchzip,
+  fetchFromGitHub,
+  makeWrapper,
+  autoPatchelfHook,
+  patchelfUnstable,
+
+  at-spi2-atk,
+  cairo,
+  flite,
+  fontconfig,
+  freetype,
+  glib,
+  glib-networking,
+  gst_all_1,
+  harfbuzz,
+  harfbuzzFull,
+  icu70,
+  lcms,
+  libdrm,
+  libepoxy,
+  libevent,
+  libgcc,
+  libgcrypt,
+  libgpg-error,
+  libjpeg8,
+  libopus,
+  libpng,
+  libsoup_3,
+  libtasn1,
+  libvpx,
+  libwebp,
+  libwpe,
+  libwpe-fdo,
+  libxkbcommon,
+  libxml2,
+  libxslt,
+  mesa,
+  sqlite,
+  systemdLibs,
+  wayland-scanner,
+  woff2,
+  zlib,
+  suffix,
+  revision,
+  system,
+  throwSystem,
+}:
+let
+  suffix' =
+    if lib.hasPrefix "linux" suffix then "ubuntu-22.04" + (lib.removePrefix "linux" suffix) else suffix;
+  libvpx' = libvpx.overrideAttrs (
+    finalAttrs: previousAttrs: {
+      version = "1.12.0";
+      src = fetchFromGitHub {
+        owner = "webmproject";
+        repo = finalAttrs.pname;
+        rev = "v${finalAttrs.version}";
+        sha256 = "sha256-9SFFE2GfYYMgxp1dpmL3STTU2ea1R5vFKA1L0pZwIvQ=";
+      };
+    }
+  );
+
+in
+stdenv.mkDerivation {
+  name = "playwright-webkit";
+  src = fetchzip {
+    url = "https://playwright.azureedge.net/builds/webkit/${revision}/webkit-${suffix'}.zip";
+    stripRoot = false;
+    hash =
+      {
+        x86_64-linux = "sha256-pHYGQYwu47jdOAD+/mLrP6Dd+2aDMHENddVwAu0uEfI=";
+        aarch64-linux = "sha256-0UeYWjeFnQ8yVa3juWg7Z7VF1GDbP4pJ9OUJRbv1OJw=";
+      }
+      .${system} or throwSystem;
+  };
+
+  nativeBuildInputs = [
+    autoPatchelfHook
+    patchelfUnstable
+    makeWrapper
+  ];
+  buildInputs = [
+    at-spi2-atk
+    cairo
+    flite
+    fontconfig.lib
+    freetype
+    glib
+    gst_all_1.gst-plugins-bad
+    gst_all_1.gst-plugins-base
+    gst_all_1.gstreamer
+    harfbuzz
+    harfbuzzFull
+    icu70
+    lcms
+    libdrm
+    libepoxy
+    libevent
+    libgcc.lib
+    libgcrypt
+    libgpg-error
+    libjpeg8
+    libopus
+    libpng
+    libsoup_3
+    libtasn1
+    libwebp
+    libwpe
+    libwpe-fdo
+    libvpx'
+    libxml2
+    libxslt
+    mesa
+    sqlite
+    systemdLibs
+    wayland-scanner
+    woff2.lib
+    libxkbcommon
+    zlib
+  ];
+
+  patchelfFlags = [ "--no-clobber-old-sections" ];
+  buildPhase = ''
+    cp -R . $out
+
+    # remove unused gtk browser
+    rm -rf $out/minibrowser-gtk
+
+    wrapProgram $out/minibrowser-wpe/bin/MiniBrowser \
+      --prefix GIO_EXTRA_MODULES ":" "${glib-networking}/lib/gio/modules/"
+  '';
+}