From 67b5c21b5853a9dc70a69e11a1044791354ebf99 Mon Sep 17 00:00:00 2001 From: phaer Date: Fri, 6 Sep 2024 11:41:26 +0200 Subject: [PATCH] playwright: Use pre-built browsers, update them... via update.sh. This lets us support playwright with browsers other than chromium on linux. Building them from source would be one step further. --- .../python-modules/playwright/update.sh | 54 ++++++- pkgs/development/web/playwright/browsers.json | 28 ++++ pkgs/development/web/playwright/chromium.nix | 29 ++++ pkgs/development/web/playwright/driver.nix | 66 +++++---- pkgs/development/web/playwright/ffmpeg.nix | 17 +++ pkgs/development/web/playwright/firefox.nix | 39 +++++ pkgs/development/web/playwright/webkit.nix | 135 ++++++++++++++++++ 7 files changed, 333 insertions(+), 35 deletions(-) create mode 100644 pkgs/development/web/playwright/browsers.json create mode 100644 pkgs/development/web/playwright/chromium.nix create mode 100644 pkgs/development/web/playwright/ffmpeg.nix create mode 100644 pkgs/development/web/playwright/firefox.nix create mode 100644 pkgs/development/web/playwright/webkit.nix diff --git a/pkgs/development/python-modules/playwright/update.sh b/pkgs/development/python-modules/playwright/update.sh index eb04e619feb6..2b283d3b607a 100755 --- a/pkgs/development/python-modules/playwright/update.sh +++ b/pkgs/development/python-modules/playwright/update.sh @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p curl gnused common-updater-scripts jq prefetch-npm-deps +#!nix-shell -i bash -p curl gnused common-updater-scripts jq prefetch-npm-deps unzip set -euo pipefail root="$(dirname "$(readlink -f "$0")")" @@ -11,18 +11,60 @@ version=$(curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s https://api.github.c setup_py_url="https://github.com/microsoft/playwright-python/raw/v${version}/setup.py" driver_version=$(curl -Ls "$setup_py_url" | grep '^driver_version =' | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+') +# TODO: skip if update-source-version reported the same version update-source-version playwright-driver "$driver_version" update-source-version python3Packages.playwright "$version" -# Update package-lock.json files for all npm deps that are built in playwright -# TODO: skip if update-source-version reported the same version -driver_file="$root/../../web/playwright/driver.nix" +playwright_dir="$root/../../web/playwright" +driver_file="$playwright_dir/driver.nix" repo_url_prefix="https://github.com/microsoft/playwright/raw" -temp_dir=$(mktemp -d) +temp_dir=$(mktemp -d) trap 'rm -rf "$temp_dir"' EXIT + +# update binaries of browsers, used by playwright. +replace_sha() { + sed -i "s|$2 = \".\{44,52\}\"|$2 = \"$3\"|" "$1" +} + +prefetch_browser() { + nix store prefetch-file --json --hash-type sha256 --unpack "$1" | jq -r .hash +} + +update_browser() { + name="$1" + suffix="$2" + arm64_suffix="${3:-$2-arm64}" + revision="$(jq -r ".browsers.$name.revision" "$playwright_dir/browsers.json")" + replace_sha "$playwright_dir/$name.nix" "x86_64-linux" \ + "$(prefetch_browser "https://playwright.azureedge.net/builds/$name/$revision/$name-$suffix.zip")" + replace_sha "$playwright_dir/$name.nix" "aarch64-linux" \ + "$(prefetch_browser "https://playwright.azureedge.net/builds/$name/$revision/$name-$arm64_suffix.zip")" +} + +curl -fsSl \ + "https://raw.githubusercontent.com/microsoft/playwright/v${driver_version}/packages/playwright-core/browsers.json" \ + | jq ' + .comment = "This file is kept up to date via update.sh" + | .browsers |= ( + [.[] + | select(.installByDefault) | del(.installByDefault)] + | map({(.name): . | del(.name)}) + | add + ) + ' > "$playwright_dir/browsers.json" + +# We currently use Chromium from nixpkgs, so we don't need to download it here +# Likewise, darwin can be ignored here atm as we are using an impure install anyway. +update_browser "firefox" "ubuntu-22.04" +update_browser "webkit" "ubuntu-22.04" +update_browser "ffmpeg" "linux" + + +# Update package-lock.json files for all npm deps that are built in playwright + # Function to download `package-lock.json` for a given source path and update hash update_hash() { local source_root_path="$1" @@ -30,7 +72,6 @@ update_hash() { # Formulate download URL local download_url="${repo_url_prefix}/v${driver_version}${source_root_path}/package-lock.json" - # Download package-lock.json to temporary directory curl -fsSL -o "${temp_dir}/package-lock.json" "$download_url" @@ -45,7 +86,6 @@ update_hash() { while IFS= read -r source_root_line; do [[ "$source_root_line" =~ sourceRoot ]] || continue source_root_path=$(echo "$source_root_line" | sed -e 's/^.*"${src.name}\(.*\)";.*$/\1/') - # Extract the current npmDepsHash for this sourceRoot existing_hash=$(grep -A1 "$source_root_line" "$driver_file" | grep 'npmDepsHash' | sed -e 's/^.*npmDepsHash = "\(.*\)";$/\1/') diff --git a/pkgs/development/web/playwright/browsers.json b/pkgs/development/web/playwright/browsers.json new file mode 100644 index 000000000000..e7fb77faf27e --- /dev/null +++ b/pkgs/development/web/playwright/browsers.json @@ -0,0 +1,28 @@ +{ + "comment": "This file is kept up to date via update.sh", + "browsers": { + "chromium": { + "revision": "1134", + "browserVersion": "129.0.6668.29" + }, + "firefox": { + "revision": "1463", + "browserVersion": "130.0" + }, + "webkit": { + "revision": "2070", + "revisionOverrides": { + "mac10.14": "1446", + "mac10.15": "1616", + "mac11": "1816", + "mac11-arm64": "1816", + "mac12": "2009", + "mac12-arm64": "2009" + }, + "browserVersion": "18.0" + }, + "ffmpeg": { + "revision": "1010" + } + } +} diff --git a/pkgs/development/web/playwright/chromium.nix b/pkgs/development/web/playwright/chromium.nix new file mode 100644 index 000000000000..0da9e66641c1 --- /dev/null +++ b/pkgs/development/web/playwright/chromium.nix @@ -0,0 +1,29 @@ +{ + runCommand, + makeWrapper, + makeFontsConf, + chromium, + ... +}: +let + fontconfig = makeFontsConf { + fontDirectories = [ ]; + }; +in +runCommand "playwright-chromium" + { + nativeBuildInputs = [ + makeWrapper + ]; + } + '' + mkdir -p $out/chrome-linux + + # See here for the Chrome options: + # https://github.com/NixOS/nixpkgs/issues/136207#issuecomment-908637738 + # We add --disable-gpu to be able to run in gpu-less environments such + # as headless nixos test vms. + makeWrapper ${chromium}/bin/chromium $out/chrome-linux/chrome \ + --set-default SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt \ + --set-default FONTCONFIG_FILE ${fontconfig} + '' diff --git a/pkgs/development/web/playwright/driver.nix b/pkgs/development/web/playwright/driver.nix index 107bf6281a44..f05d5fd6c081 100644 --- a/pkgs/development/web/playwright/driver.nix +++ b/pkgs/development/web/playwright/driver.nix @@ -7,6 +7,8 @@ jq, nodejs, fetchFromGitHub, + linkFarm, + callPackage, makeFontsConf, makeWrapper, runCommand, @@ -16,6 +18,14 @@ let inherit (stdenv.hostPlatform) system; throwSystem = throw "Unsupported system: ${system}"; + suffix = + { + x86_64-linux = "linux"; + aarch64-linux = "linux-arm64"; + x86_64-darwin = "mac"; + aarch64-darwin = "mac-arm64"; + } + .${system} or throwSystem; version = "1.47.0"; @@ -148,6 +158,7 @@ let ''; passthru = { + browsersJSON = (lib.importJSON ./browsers.json).browsers; browsers = { x86_64-linux = browsers-linux { }; @@ -209,37 +220,36 @@ let browsers-linux = { withChromium ? true, + withFirefox ? true, + withWebkit ? true, + withFfmpeg ? true, }: let - fontconfig = makeFontsConf { fontDirectories = [ ]; }; + browsers = + lib.optionals withChromium [ "chromium" ] + ++ lib.optionals withFirefox [ "firefox" ] + ++ lib.optionals withWebkit [ "webkit" ] + ++ lib.optionals withFfmpeg [ "ffmpeg" ]; in - runCommand ("playwright-browsers" + lib.optionalString withChromium "-chromium") - { - nativeBuildInputs = [ - makeWrapper - jq - ]; - } - ( - '' - BROWSERS_JSON=${playwright-core}/browsers.json - '' - + lib.optionalString withChromium '' - CHROMIUM_REVISION=$(jq -r '.browsers[] | select(.name == "chromium").revision' $BROWSERS_JSON) - mkdir -p $out/chromium-$CHROMIUM_REVISION/chrome-linux - - # See here for the Chrome options: - # https://github.com/NixOS/nixpkgs/issues/136207#issuecomment-908637738 - makeWrapper ${chromium}/bin/chromium $out/chromium-$CHROMIUM_REVISION/chrome-linux/chrome \ - --set SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt \ - --set FONTCONFIG_FILE ${fontconfig} - '' - + '' - FFMPEG_REVISION=$(jq -r '.browsers[] | select(.name == "ffmpeg").revision' $BROWSERS_JSON) - mkdir -p $out/ffmpeg-$FFMPEG_REVISION - ln -s ${ffmpeg}/bin/ffmpeg $out/ffmpeg-$FFMPEG_REVISION/ffmpeg-linux - '' - ); + linkFarm "playwright-browsers" ( + lib.listToAttrs ( + map ( + name: + let + value = playwright-core.passthru.browsersJSON.${name}; + in + lib.nameValuePair + # TODO check platform for revisionOverrides + "${name}-${value.revision}" + ( + callPackage ./${name}.nix { + inherit suffix system throwSystem; + inherit (value) revision; + } + ) + ) browsers + ) + ); in { playwright-core = playwright-core; diff --git a/pkgs/development/web/playwright/ffmpeg.nix b/pkgs/development/web/playwright/ffmpeg.nix new file mode 100644 index 000000000000..f541031019ed --- /dev/null +++ b/pkgs/development/web/playwright/ffmpeg.nix @@ -0,0 +1,17 @@ +{ + fetchzip, + suffix, + revision, + system, + throwSystem, +}: +fetchzip { + url = "https://playwright.azureedge.net/builds/ffmpeg/${revision}/ffmpeg-${suffix}.zip"; + stripRoot = false; + hash = + { + x86_64-linux = "sha256-FEm62UvMv0h6Sav93WmbPLw3CW1L1xg4nD26ca5ol38="; + aarch64-linux = "sha256-jtQ+NS++VHRiKoIV++PIxEnyVnYtVwUyNlSILKSH4A4="; + } + .${system} or throwSystem; +} diff --git a/pkgs/development/web/playwright/firefox.nix b/pkgs/development/web/playwright/firefox.nix new file mode 100644 index 000000000000..0f404bee2899 --- /dev/null +++ b/pkgs/development/web/playwright/firefox.nix @@ -0,0 +1,39 @@ +{ + lib, + stdenv, + fetchzip, + firefox-bin, + suffix, + revision, + system, + throwSystem, +}: +let + suffix' = + if lib.hasPrefix "linux" suffix then "ubuntu-22.04" + (lib.removePrefix "linux" suffix) else suffix; +in +stdenv.mkDerivation { + name = "playwright-firefox"; + src = fetchzip { + url = "https://playwright.azureedge.net/builds/firefox/${revision}/firefox-${suffix'}.zip"; + hash = + { + x86_64-linux = "sha256-Hd9LlSRLW51gDoFyszqvg46Q/sMizLRsVKAN9atbwsw="; + aarch64-linux = "sha256-SEXH3gLOfNjOcnNWQjQ5gaaow47veVs0BoTYSgXw+24="; + } + .${system} or throwSystem; + }; + + inherit (firefox-bin.unwrapped) + nativeBuildInputs + buildInputs + runtimeDependencies + appendRunpaths + patchelfFlags + ; + + buildPhase = '' + mkdir -p $out/firefox + cp -R . $out/firefox + ''; +} diff --git a/pkgs/development/web/playwright/webkit.nix b/pkgs/development/web/playwright/webkit.nix new file mode 100644 index 000000000000..5c8062bb6f70 --- /dev/null +++ b/pkgs/development/web/playwright/webkit.nix @@ -0,0 +1,135 @@ +{ + lib, + stdenv, + fetchzip, + fetchFromGitHub, + makeWrapper, + autoPatchelfHook, + patchelfUnstable, + + at-spi2-atk, + cairo, + flite, + fontconfig, + freetype, + glib, + glib-networking, + gst_all_1, + harfbuzz, + harfbuzzFull, + icu70, + lcms, + libdrm, + libepoxy, + libevent, + libgcc, + libgcrypt, + libgpg-error, + libjpeg8, + libopus, + libpng, + libsoup_3, + libtasn1, + libvpx, + libwebp, + libwpe, + libwpe-fdo, + libxkbcommon, + libxml2, + libxslt, + mesa, + sqlite, + systemdLibs, + wayland-scanner, + woff2, + zlib, + suffix, + revision, + system, + throwSystem, +}: +let + suffix' = + if lib.hasPrefix "linux" suffix then "ubuntu-22.04" + (lib.removePrefix "linux" suffix) else suffix; + libvpx' = libvpx.overrideAttrs ( + finalAttrs: previousAttrs: { + version = "1.12.0"; + src = fetchFromGitHub { + owner = "webmproject"; + repo = finalAttrs.pname; + rev = "v${finalAttrs.version}"; + sha256 = "sha256-9SFFE2GfYYMgxp1dpmL3STTU2ea1R5vFKA1L0pZwIvQ="; + }; + } + ); + +in +stdenv.mkDerivation { + name = "playwright-webkit"; + src = fetchzip { + url = "https://playwright.azureedge.net/builds/webkit/${revision}/webkit-${suffix'}.zip"; + stripRoot = false; + hash = + { + x86_64-linux = "sha256-pHYGQYwu47jdOAD+/mLrP6Dd+2aDMHENddVwAu0uEfI="; + aarch64-linux = "sha256-0UeYWjeFnQ8yVa3juWg7Z7VF1GDbP4pJ9OUJRbv1OJw="; + } + .${system} or throwSystem; + }; + + nativeBuildInputs = [ + autoPatchelfHook + patchelfUnstable + makeWrapper + ]; + buildInputs = [ + at-spi2-atk + cairo + flite + fontconfig.lib + freetype + glib + gst_all_1.gst-plugins-bad + gst_all_1.gst-plugins-base + gst_all_1.gstreamer + harfbuzz + harfbuzzFull + icu70 + lcms + libdrm + libepoxy + libevent + libgcc.lib + libgcrypt + libgpg-error + libjpeg8 + libopus + libpng + libsoup_3 + libtasn1 + libwebp + libwpe + libwpe-fdo + libvpx' + libxml2 + libxslt + mesa + sqlite + systemdLibs + wayland-scanner + woff2.lib + libxkbcommon + zlib + ]; + + patchelfFlags = [ "--no-clobber-old-sections" ]; + buildPhase = '' + cp -R . $out + + # remove unused gtk browser + rm -rf $out/minibrowser-gtk + + wrapProgram $out/minibrowser-wpe/bin/MiniBrowser \ + --prefix GIO_EXTRA_MODULES ":" "${glib-networking}/lib/gio/modules/" + ''; +}