python3.pkgs.home-assistant-chip-*: build from source (#348149)

pkgs/by-name/za/zap-chip/dont-download-copyfiles-to-copy-files.patch

@@ -0,0 +1,42 @@
+diff --git a/src-script/pack-apack-win-linux.js b/src-script/pack-apack-win-linux.js
+index 0120f9f9..41fcf99c 100644
+--- a/src-script/pack-apack-win-linux.js
++++ b/src-script/pack-apack-win-linux.js
+@@ -6,12 +6,12 @@ exports.default = async function (context) {
+     context.electronPlatformName === 'win32' ||
+     context.electronPlatformName === 'linux'
+   ) {
+-    return scriptUtil.executeCmd({}, 'npx', [
+-      'copyfiles',
+-      '-V',
+-      '-f',
+-      path.resolve(context.outDir, '../apack.json'),
+-      context.appOutDir
++    return scriptUtil.executeCmd({}, 'install', [
++      '-m466',
++      '-v',
++      '-Dt',
++      context.appOutDir,
++      path.resolve(context.outDir, '../apack.json')
+     ])
+   }
+ }
+diff --git a/src-script/script-util.js b/src-script/script-util.js
+index 1897c2b8..9faa5f20 100644
+--- a/src-script/script-util.js
++++ b/src-script/script-util.js
+@@ -181,11 +181,9 @@ async function rebuildBackendIfNeeded() {
+   return scriptUtil
+     .executeCmd({}, 'npx', ['tsc', '--build', './tsconfig.json'])
+     .then(() =>
+-      scriptUtil.executeCmd({}, 'npx', [
+-        'copyfiles',
+-        './src-electron/**/*.sql',
+-        './src-electron/icons/*',
+-        './dist/'
++      scriptUtil.executeCmd({}, 'bash', [
++        '-xec',
++        'for i in ./src-electron/**/*.sql ./src-electron/icons/*; do install -m 644 -Dt dist/$(dirname $i) $i; done'
+       ])
+     )
+ }

pkgs/by-name/za/zap-chip/dont-get-version-from-git.patch

@@ -0,0 +1,28 @@
+diff --git a/src-script/script-util.js b/src-script/script-util.js
+index 1897c2b8..aaf9a08b 100644
+--- a/src-script/script-util.js
++++ b/src-script/script-util.js
+@@ -196,23 +194,6 @@ async function rebuildBackendIfNeeded() {
+  * ads the timestamp and saves it into .version.json
+  */
+ async function stampVersion() {
+-  try {
+-    let out = await getStdout('{"hash": null,"date": null}', 'git', [
+-      'log',
+-      '-1',
+-      '--format={"hash": "%H","timestamp": %ct}'
+-    ])
+-    let version = JSON.parse(out)
+-    let d = new Date(version.timestamp * 1000) // git gives seconds, Date needs milliseconds
+-    let result = await setPackageJsonVersion(d, 'real')
+-    version.date = d
+-    version.zapVersion = result.version
+-    let versionFile = path.join(__dirname, '../.version.json')
+-    console.log(`🔍 Git commit: ${version.hash} from ${version.date}`)
+-    await fsp.writeFile(versionFile, JSON.stringify(version))
+-  } catch (err) {
+-    console.log(`Error retrieving version: ${err}`)
+-  }
+ }
+ 
+ /**

pkgs/by-name/za/zap-chip/package.nix

@@ -0,0 +1,74 @@
+{
+  lib,
+  buildNpmPackage,
+  electron_31,
+  fetchFromGitHub,
+  writers,
+}:
+
+let
+  electron = electron_31;
+in
+buildNpmPackage rec {
+  pname = "zap-chip";
+  version = "2024.09.27";
+
+  src = fetchFromGitHub {
+    owner = "project-chip";
+    repo = "zap";
+    rev = "v${version}";
+    hash = "sha256-Dc5rU4jJ6aJpk8mwL+XNSmtisYxF86VzXd/Aacd4p0o=";
+  };
+
+  npmDepsHash = "sha256-ZFksGwKlXkz6XTs2QdalGB0hR16HfB69XQOFWI9X/KY=";
+
+  env.ELECTRON_SKIP_BINARY_DOWNLOAD = "1";
+  env.CYPRESS_INSTALL_BINARY = "0";
+
+  patches = [
+    # the build system creates a file `.version.json` from a git command
+    # as we don't build from a git repo, we create the file manually in postPatch
+    # and this patch disables the logic running git
+    ./dont-get-version-from-git.patch
+    # some files are installed via `npx copyfiles` which tries to download
+    # code from the internet. This fails in the sandbox. This patch replaces the
+    # logic by running "normal" commands instead of `npx copyfiles`
+    ./dont-download-copyfiles-to-copy-files.patch
+  ];
+
+  postPatch =
+    let
+      versionJson = {
+        hash = version;
+        timestamp = 1;
+        date = version;
+        zapVersion = version;
+      };
+    in
+    ''
+      cp ${writers.writeJSON "zapversion.json" versionJson} .version.json
+      cat .version.json
+    '';
+
+  postBuild = ''
+    npm exec electron-builder -- \
+      --dir \
+      -c.electronDist=${electron.dist} \
+      -c.electronVersion=${electron.version}
+  '';
+
+  postInstall = ''
+    # this file is also used at runtime
+    install -m644 .version.json $out/lib/node_modules/zap/
+    # home-assistant chip-* python packages need the executable under the name zap-cli
+    ln -s $out/bin/zap $out/bin/zap-cli
+  '';
+
+  meta = {
+    description = "Generic generation engine and user interface for applications and libraries based on Zigbee Cluster Library (ZCL)";
+    changelog = "https://github.com/project-chip/zap/releases/tag/v${version}";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ symphorien ];
+    mainProgram = "zap-cli";
+  };
+}

pkgs/development/python-modules/home-assistant-chip-clusters/default.nix

@@ -1,23 +1,22 @@
 {
   lib,
   buildPythonPackage,
-  fetchPypi,
+  home-assistant-chip-wheels,
   aenum,
   dacite,
 }:
 
 buildPythonPackage rec {
   pname = "home-assistant-chip-clusters";
-  version = "2024.9.0";
+  inherit (home-assistant-chip-wheels) version;
   format = "wheel";
 
-  src = fetchPypi {
-    inherit format version;
-    pname = "home_assistant_chip_clusters";
-    dist = "py3";
-    python = "py3";
-    hash = "sha256-h1umP5bgw1HByiZ0mAd6yWvGHPcvr//G5UhM/qIIkQE=";
-  };
+  src = home-assistant-chip-wheels;
+
+  # format=wheel needs src to be a wheel not a folder of wheels
+  preUnpack = ''
+    src=($src/home_assistant_chip_clusters*.whl)
+  '';
 
   propagatedBuildInputs = [
     aenum
@@ -38,6 +37,5 @@ buildPythonPackage rec {
     changelog = "https://github.com/home-assistant-libs/chip-wheels/releases/tag/${version}";
     license = licenses.asl20;
     maintainers = teams.home-assistant.members;
-    sourceProvenance = with sourceTypes; [ binaryNativeCode ];
   };
 }

pkgs/development/python-modules/home-assistant-chip-core/default.nix

@@ -1,70 +1,35 @@
 {
   lib,
-  stdenv,
   buildPythonPackage,
-  fetchPypi,
   pythonOlder,
-
-  # build
-  autoPatchelfHook,
-
-  # runtime
-  glib,
-  libnl,
-
-  # propagates
   aenum,
+  home-assistant-chip-wheels,
   coloredlogs,
   construct,
   cryptography,
   dacite,
-  ecdsa,
-  rich,
-  pyyaml,
-  ipdb,
   deprecation,
+  ecdsa,
+  ipdb,
   mobly,
   pygobject3,
+  pyyaml,
+  rich,
 }:
 
 buildPythonPackage rec {
   pname = "home-assistant-chip-core";
-  version = "2024.9.0";
+  inherit (home-assistant-chip-wheels) version;
   format = "wheel";
 
   disabled = pythonOlder "3.7";
 
-  src =
-    let
-      system =
-        {
-          "aarch64-linux" = {
-            name = "aarch64";
-            hash = "sha256-oVqPaScNCjf1xY2iWJTXTh2YqimZk5kVsiQN586j9mc=";
-          };
-          "x86_64-linux" = {
-            name = "x86_64";
-            hash = "sha256-hWH2UCYIuDMFS/W4FELW57iQft0jfGPt8aFT1k59sWA=";
-          };
-        }
-        .${stdenv.system} or (throw "Unsupported system: ${stdenv.system}");
-    in
-    fetchPypi {
-      pname = "home_assistant_chip_core";
-      inherit version format;
-      dist = "cp37";
-      python = "cp37";
-      abi = "abi3";
-      platform = "manylinux_2_31_${system.name}";
-      hash = system.hash;
-    };
+  src = home-assistant-chip-wheels;
 
-  nativeBuildInputs = [ autoPatchelfHook ];
-
-  buildInputs = [
-    glib
-    libnl
-  ];
+  # format=wheel needs src to be a wheel not a folder of wheels
+  preUnpack = ''
+    src=($src/home_assistant_chip_core*.whl)
+  '';
 
   propagatedBuildInputs = [
     aenum
@@ -104,10 +69,5 @@ buildPythonPackage rec {
     changelog = "https://github.com/home-assistant-libs/chip-wheels/releases/tag/${version}";
     license = licenses.asl20;
     maintainers = teams.home-assistant.members;
-    platforms = [
-      "aarch64-linux"
-      "x86_64-linux"
-    ];
-    sourceProvenance = with sourceTypes; [ binaryNativeCode ];
   };
 }

pkgs/development/python-modules/home-assistant-chip-wheels/default.nix

@@ -0,0 +1,289 @@
+{
+  appdirs,
+  appnope,
+  black,
+  build,
+  clang-tools,
+  click,
+  colorama,
+  coloredlogs,
+  coverage,
+  cryptography,
+  diskcache,
+  fetchFromGitHub,
+  glib,
+  gn,
+  googleapis-common-protos,
+  google-cloud-storage,
+  ipython,
+  jinja2,
+  json5,
+  jsonschema,
+  lark,
+  lib,
+  libnl,
+  mobly,
+  mypy,
+  mypy-extensions,
+  mypy-protobuf,
+  ninja,
+  openssl,
+  packaging,
+  parameterized,
+  pip-tools,
+  pkg-config,
+  prompt-toolkit,
+  protobuf,
+  psutil,
+  ptpython,
+  pyelftools,
+  pygments,
+  pykwalify,
+  pylint,
+  pyperclip,
+  pyserial,
+  python3,
+  python-daemon,
+  pyyaml,
+  requests,
+  setuptools,
+  six,
+  sphinx,
+  sphinx-argparse,
+  sphinx-design,
+  stdenv,
+  stringcase,
+  toml,
+  tornado,
+  types-protobuf,
+  types-pyyaml,
+  types-requests,
+  types-setuptools,
+  watchdog,
+  websockets,
+  wheel,
+  yapf,
+  zap-chip,
+}:
+
+stdenv.mkDerivation rec {
+  pname = "home-assistant-chip-wheels";
+  version = "2024.9.0";
+  src = fetchFromGitHub {
+    owner = "home-assistant-libs";
+    repo = "chip-wheels";
+    rev = version;
+    fetchSubmodules = false;
+    leaveDotGit = true;
+    hash = "sha256-T0G6mxb/5wFOxPLL92Ay34oP+9Xvk9w0YV9VSzWJuzw=";
+    postFetch = ''
+      cd $out
+      # Download connectedhomeip.
+      git fetch
+      git reset --hard HEAD
+      git submodule update --init --depth 1 connectedhomeip
+
+      # Initialize only necessary submodules.
+      cd connectedhomeip
+      ${python3}/bin/python3 scripts/checkout_submodules.py --platform linux --shallow
+
+      # Keep the output deterministic.
+      cd $out
+      # in case python decided to leave a .pyc file, for example
+      git clean -fxd
+      rm -rf .git/
+    '';
+  };
+
+  strictDeps = true;
+
+  nativeBuildInputs = [
+    gn
+    pkg-config
+    ninja
+    clang-tools
+    zap-chip
+    # gdbus-codegen
+    glib
+    python3
+    # dependencies of build scripts
+    click
+    jinja2
+    lark
+    setuptools
+    stringcase
+    build
+    pip-tools
+    black
+    yapf
+  ];
+
+  buildInputs = [
+    openssl
+    glib
+    libnl
+  ];
+
+  postPatch = ''
+    cd connectedhomeip
+    export HOME=$(mktemp -d)
+
+    patchShebangs --build scripts
+
+    for patch in ../*.patch; do
+      patch -p1 < $patch
+    done
+
+    # unpin dependencies
+    # there are many files to modify, in different formats
+    sed -i 's/==.*$//' third_party/pigweed/repo/pw_env_setup/py/pw_env_setup/virtualenv_setup/python_base_requirements.txt
+    sed -i 's/==[^;]*//' scripts/setup/constraints.txt
+    sed -i 's/\(^ \+[a-zA-Z0-9-]*\)[=~><]=[^;]*/\1/' third_party/pigweed/repo/pw_protobuf_compiler/py/setup.cfg third_party/pigweed/repo/pw_protobuf/py/setup.cfg third_party/pigweed/repo/pw_protobuf_compiler/py/setup.cfg
+    # remove a few dependencies not packaged in nixpkgs and which are apparently
+    # not needed to build the python bindings of chip
+    sed -i -e '/sphinxcontrib-mermaid/d' -e '/types-six/d' -e '/types-pygment/d' -e '/types-pyserial/d' third_party/pigweed/repo/*/py/setup.cfg
+
+    # obtained by running a build in nix-shell with internet access
+    cp ${./pigweed_environment.gni} build_overrides/pigweed_environment.gni
+
+    # some code is generated by a templating tool (zap-cli)
+    scripts/codepregen.py ./zzz_pregenerated/
+  '';
+
+  # the python parts of the build system work as follows
+  # gn calls pigweed to read a dozen different files to generate
+  # a file looking like requirements.txt. It then calls pip
+  # to install this computed list of dependencies into a virtualenv.
+  # Of course, pip fails in the sandbox, because it cannot download
+  # the python packages.
+  # The documented way of doing offline builds is to create a folder
+  # with wheel files for all dependencies and point pip to it
+  # via its configuration file or environment variables.
+  # https://pigweed.dev/python_build.html#installing-offline
+  # The wheel of a python package foo is available as foo.dist.
+  # So that would be easy, but we also need wheels for transitive dependencies.
+  # the function saturateDependencies below computes this transitive closure.
+  #
+  # yes this list of dependencies contains both build tools and proper dependencies.
+  env.PIP_NO_INDEX = "1";
+  env.PIP_FIND_LINKS =
+    let
+      dependencies = [
+        appdirs
+        appnope
+        black
+        build
+        colorama
+        coloredlogs
+        coverage
+        click
+        cryptography
+        diskcache
+        googleapis-common-protos
+        google-cloud-storage
+        ipython
+        jinja2
+        json5
+        jsonschema
+        lark
+        mobly
+        mypy
+        mypy-extensions
+        mypy-protobuf
+        packaging
+        parameterized
+        pip-tools
+        prompt-toolkit
+        protobuf
+        psutil
+        ptpython
+        pyelftools
+        pygments
+        pykwalify
+        pylint
+        pyperclip
+        pyserial
+        python-daemon
+        pyyaml
+        requests
+        setuptools
+        six
+        sphinx
+        sphinx-argparse
+        sphinx-design
+        stringcase
+        toml
+        tornado
+        types-protobuf
+        types-pyyaml
+        types-requests
+        types-setuptools
+        watchdog
+        websockets
+        wheel
+        yapf
+      ];
+      depListToAttrs =
+        list:
+        builtins.listToAttrs (
+          map (dep: {
+            name = dep.name;
+            value = dep;
+          }) list
+        );
+      saturateDependencies =
+        deps:
+        let
+          before = deps;
+          new = lib.mergeAttrsList (
+            map (dep: depListToAttrs (dep.propagatedBuildInputs or [ ])) (lib.attrValues before)
+          );
+          after = before // new;
+        in
+        if lib.attrNames before != lib.attrNames after then saturateDependencies after else before;
+      saturateDependencyList = list: lib.attrValues (saturateDependencies (depListToAttrs list));
+      saturatedDependencyList = lib.filter (drv: drv ? dist) (saturateDependencyList dependencies);
+    in
+    lib.concatMapStringsSep " " (dep: "file://${dep.dist}") saturatedDependencyList;
+
+  gnFlags = [
+    ''chip_project_config_include_dirs=["//.."]''
+    ''chip_crypto="openssl"''
+    ''enable_rtti=true''
+    ''chip_config_memory_debug_checks=false''
+    ''chip_config_memory_debug_dmalloc=false''
+    ''chip_mdns="minimal"''
+    ''chip_minmdns_default_policy="libnl"''
+    ''chip_python_version="${lib.versions.majorMinor python3.version}"''
+    ''chip_python_platform_tag="any"''
+    ''chip_python_package_prefix="home-assistant-chip"''
+    ''custom_toolchain="custom"''
+    ''target_cc="${stdenv.cc.targetPrefix}cc"''
+    ''target_cxx="${stdenv.cc.targetPrefix}c++"''
+    ''target_ar="${stdenv.cc.targetPrefix}ar"''
+  ];
+
+  preBuild = ''
+    export NIX_CFLAGS_COMPILE="$($PKG_CONFIG --cflags glib-2.0) -O2 -Wno-error"
+    export NIX_CFLAGS_LINK="$($PKG_CONFIG --libs gio-2.0) $($PKG_CONFIG --libs gobject-2.0) $($PKG_CONFIG --libs glib-2.0)"
+  '';
+
+  ninjaFlags = [ "chip-repl" ];
+
+  installPhase = ''
+    runHook preInstall
+
+    cp -r controller/python $out
+
+    runHook postInstall
+  '';
+
+  meta = {
+    description = "Python wheels for APIs and tools related to CHIP";
+    homepage = "https://github.com/home-assistant-libs/chip-wheels";
+    changelog = "https://github.com/home-assistant-libs/chip-wheels/releases/tag/${version}";
+    license = lib.licenses.asl20;
+    maintainers = lib.teams.home-assistant.members;
+  };
+
+}

pkgs/development/python-modules/home-assistant-chip-wheels/pigweed_environment.gni

@@ -0,0 +1,12 @@
+# This file is automatically generated by Pigweed's environment setup. Do not
+# edit it manually or check it in.
+
+# Relative paths are interpreted with respect to this file, which helps
+# determine the correct path even if the source root changes.
+declare_args() {
+  pw_env_setup_CIPD_ARM = get_path_info("../.environment/cipd/packages/arm", "abspath")
+  pw_env_setup_CIPD_PIGWEED = get_path_info("../.environment/cipd/packages/pigweed", "abspath")
+  pw_env_setup_CIPD_PYTHON = get_path_info("../.environment/cipd/packages/python", "abspath")
+  pw_env_setup_CIPD_ZAP = get_path_info("../.environment/cipd/packages/zap", "abspath")
+  pw_env_setup_VIRTUAL_ENV = get_path_info("../.environment/pigweed-venv", "abspath")
+}

pkgs/top-level/python-packages.nix

@@ -5796,6 +5796,8 @@ self: super: with self; {
 
   home-assistant-bluetooth = callPackage ../development/python-modules/home-assistant-bluetooth { };
 
+  home-assistant-chip-wheels = callPackage ../development/python-modules/home-assistant-chip-wheels { };
+
   home-assistant-chip-clusters = callPackage ../development/python-modules/home-assistant-chip-clusters { };
 
   home-assistant-chip-core = callPackage ../development/python-modules/home-assistant-chip-core { };