From aa148a31bd62bc7208d018ce7dc667896a6f292b Mon Sep 17 00:00:00 2001 From: Theodore Ni <3806110+tjni@users.noreply.github.com> Date: Thu, 13 Jul 2023 20:40:58 -0700 Subject: [PATCH 01/10] python310Packages.hologram: init at 0.0.16 Co-Authored-By: Mauricio Scheffer --- .../python-modules/hologram/default.nix | 45 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 47 insertions(+) create mode 100644 pkgs/development/python-modules/hologram/default.nix diff --git a/pkgs/development/python-modules/hologram/default.nix b/pkgs/development/python-modules/hologram/default.nix new file mode 100644 index 000000000000..0964bbf89ca4 --- /dev/null +++ b/pkgs/development/python-modules/hologram/default.nix @@ -0,0 +1,45 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, jsonschema +, pytestCheckHook +, python-dateutil +, setuptools +}: + +buildPythonPackage rec { + pname = "hologram"; + version = "0.0.16"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "dbt-labs"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-DboVCvByI8bTThamGBwSiQADGxIaEnTMmwmVI+4ARgc="; + }; + + nativeBuildInputs = [ + setuptools + ]; + + propagatedBuildInputs = [ + jsonschema + python-dateutil + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + pythonImportsCheck = [ + "hologram" + ]; + + meta = with lib; { + description = "A library for automatically generating Draft 7 JSON Schemas from Python dataclasses"; + homepage = "https://github.com/dbt-labs/hologram"; + license = licenses.mit; + maintainers = with maintainers; [ mausch tjni ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 87edab702ac9..d50ca9d8b1da 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -4754,6 +4754,8 @@ self: super: with self; { holidays = callPackage ../development/python-modules/holidays { }; + hologram = callPackage ../development/python-modules/hologram { }; + holoviews = callPackage ../development/python-modules/holoviews { }; home-assistant-bluetooth = callPackage ../development/python-modules/home-assistant-bluetooth { }; From 0b8b643e85613b550f6f9868bb414bf6959cdb64 Mon Sep 17 00:00:00 2001 From: Theodore Ni <3806110+tjni@users.noreply.github.com> Date: Thu, 13 Jul 2023 20:50:34 -0700 Subject: [PATCH 02/10] python310Packages.minimal-snowplow-tracker: init at 0.0.2 Co-Authored-By: Mauricio Scheffer --- .../minimal-snowplow-tracker/default.nix | 36 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 ++ 2 files changed, 38 insertions(+) create mode 100644 pkgs/development/python-modules/minimal-snowplow-tracker/default.nix diff --git a/pkgs/development/python-modules/minimal-snowplow-tracker/default.nix b/pkgs/development/python-modules/minimal-snowplow-tracker/default.nix new file mode 100644 index 000000000000..a15f81014cfa --- /dev/null +++ b/pkgs/development/python-modules/minimal-snowplow-tracker/default.nix @@ -0,0 +1,36 @@ +{ lib +, buildPythonPackage +, fetchPypi +, requests +, six +}: + +buildPythonPackage rec { + pname = "minimal-snowplow-tracker"; + version = "0.0.2"; + format = "setuptools"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-rKv3Vy2w5/XL9pg9SV7vVAgfcb45IzDrOq25zLOdqqQ="; + }; + + propagatedBuildInputs = [ + requests + six + ]; + + # has no tests + doCheck = false; + + pythonImportsCheck = [ + "snowplow_tracker" + ]; + + meta = with lib; { + description = "Minimal snowplow event tracker"; + homepage = "https://github.com/dbt-labs/snowplow-python-tracker"; + license = licenses.asl20; + maintainers = with maintainers; [ mausch tjni ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index d50ca9d8b1da..41af11c7ad78 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -6462,6 +6462,8 @@ self: super: with self; { minikerberos = callPackage ../development/python-modules/minikerberos { }; + minimal-snowplow-tracker = callPackage ../development/python-modules/minimal-snowplow-tracker { }; + minimock = callPackage ../development/python-modules/minimock { }; mininet-python = (toPythonModule (pkgs.mininet.override { From 9ae3fb40e31e819f4dbcca9a5f33e8f75e58dc9d Mon Sep 17 00:00:00 2001 From: Theodore Ni <3806110+tjni@users.noreply.github.com> Date: Thu, 13 Jul 2023 21:35:33 -0700 Subject: [PATCH 03/10] python310Packages.dbt-extractor: init at 0.4.1 Co-Authored-By: Mauricio Scheffer --- .../python-modules/dbt-extractor/Cargo.lock | 531 ++++++++++++++++++ .../python-modules/dbt-extractor/default.nix | 49 ++ pkgs/top-level/python-packages.nix | 2 + 3 files changed, 582 insertions(+) create mode 100644 pkgs/development/python-modules/dbt-extractor/Cargo.lock create mode 100644 pkgs/development/python-modules/dbt-extractor/default.nix diff --git a/pkgs/development/python-modules/dbt-extractor/Cargo.lock b/pkgs/development/python-modules/dbt-extractor/Cargo.lock new file mode 100644 index 000000000000..8b297123703a --- /dev/null +++ b/pkgs/development/python-modules/dbt-extractor/Cargo.lock @@ -0,0 +1,531 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cc" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "crossbeam-channel" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" +dependencies = [ + "cfg-if", + "lazy_static", +] + +[[package]] +name = "dbt-extractor" +version = "0.4.1" +dependencies = [ + "pyo3", + "quickcheck", + "quickcheck_macros", + "rayon", + "thiserror", + "tree-sitter", + "tree-sitter-jinja2", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "env_logger" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "indoc" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47741a8bc60fb26eb8d6e0238bbb26d8575ff623fdc97b1a2c00c050b9684ed8" +dependencies = [ + "indoc-impl", + "proc-macro-hack", +] + +[[package]] +name = "indoc-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce046d161f000fffde5f432a0d034d0341dc152643b2598ed5bfce44c4f3a8f0" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", + "unindent", +] + +[[package]] +name = "instant" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21" + +[[package]] +name = "lock_api" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "memoffset" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "paste" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + +[[package]] +name = "paste-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" +dependencies = [ + "proc-macro-hack", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7ed8b8c7b886ea3ed7dde405212185f423ab44682667c8c6dd14aa1d9f6612" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "pyo3" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cf01dbf1c05af0a14c7779ed6f3aa9deac9c3419606ac9de537a2d649005720" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "parking_lot", + "paste", + "pyo3-build-config", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf9e4d128bfbddc898ad3409900080d8d5095c379632fbbfbb9c8cfb1fb852b" +dependencies = [ + "once_cell", +] + +[[package]] +name = "pyo3-macros" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67701eb32b1f9a9722b4bc54b548ff9d7ebfded011c12daece7b9063be1fd755" +dependencies = [ + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f44f09e825ee49a105f2c7b23ebee50886a9aee0746f4dd5a704138a64b0218a" +dependencies = [ + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quickcheck" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" +dependencies = [ + "env_logger", + "log", + "rand", +] + +[[package]] +name = "quickcheck_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b22a693222d716a9587786f37ac3f6b4faedb5b80c23914e7303ff5a1d8016e9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + +[[package]] +name = "redox_syscall" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "syn" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f58f7e8eaa0009c5fec437aabf511bd9933e4b2d7407bd05273c01a8906ea7" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "thiserror" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tree-sitter" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad726ec26496bf4c083fff0f43d4eb3a2ad1bba305323af5ff91383c0b6ecac0" +dependencies = [ + "cc", + "regex", +] + +[[package]] +name = "tree-sitter-jinja2" +version = "0.1.0" +source = "git+https://github.com/dbt-labs/tree-sitter-jinja2?tag=v0.1.0#52da7b0b1480b23381ea84cf5ea3bf058dd6d8c4" +dependencies = [ + "cc", + "tree-sitter", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "unindent" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/pkgs/development/python-modules/dbt-extractor/default.nix b/pkgs/development/python-modules/dbt-extractor/default.nix new file mode 100644 index 000000000000..2656be91086d --- /dev/null +++ b/pkgs/development/python-modules/dbt-extractor/default.nix @@ -0,0 +1,49 @@ +{ stdenv +, lib +, buildPythonPackage +, fetchPypi +, rustPlatform +, libiconv +}: + +buildPythonPackage rec { + pname = "dbt-extractor"; + version = "0.4.1"; + format = "setuptools"; + + src = fetchPypi { + pname = "dbt_extractor"; + inherit version; + hash = "sha256-dbHGZWmewPH/zhuj13b3386AIVbyLnCnucjwtNfoD0I="; + }; + + cargoDeps = rustPlatform.importCargoLock { + lockFile = ./Cargo.lock; + outputHashes = { + "tree-sitter-jinja2-0.1.0" = "sha256-lzA2iq4AK0iNwkLvbIt7Jm5WGFbMPFDi6i4AFDm0FOU="; + }; + }; + + nativeBuildInputs = [ + rustPlatform.cargoSetupHook + rustPlatform.maturinBuildHook + ]; + + buildInputs = lib.optionals stdenv.isDarwin [ + libiconv + ]; + + # no python tests exist + doCheck = false; + + pythonImportsCheck = [ + "dbt_extractor" + ]; + + meta = with lib; { + description = "A tool that processes the most common jinja value templates in dbt model files"; + homepage = "https://github.com/dbt-labs/dbt-extractor"; + license = licenses.asl20; + maintainers = with maintainers; [ mausch tjni ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 41af11c7ad78..fa9cf3e4198b 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2465,6 +2465,8 @@ self: super: with self; { dbfread = callPackage ../development/python-modules/dbfread { }; + dbt-extractor = callPackage ../development/python-modules/dbt-extractor { }; + dbus-client-gen = callPackage ../development/python-modules/dbus-client-gen { }; dbus-deviation = callPackage ../development/python-modules/dbus-deviation { }; From e45ffebad224d8007df8cf4d5009e27c4447652e Mon Sep 17 00:00:00 2001 From: Theodore Ni <3806110+tjni@users.noreply.github.com> Date: Thu, 13 Jul 2023 21:52:46 -0700 Subject: [PATCH 04/10] python310Packages.dbt-core: init at 1.5.3 Co-Authored-By: Mauricio Scheffer --- .../python-modules/dbt-core/default.nix | 91 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 93 insertions(+) create mode 100644 pkgs/development/python-modules/dbt-core/default.nix diff --git a/pkgs/development/python-modules/dbt-core/default.nix b/pkgs/development/python-modules/dbt-core/default.nix new file mode 100644 index 000000000000..f522cb1d239d --- /dev/null +++ b/pkgs/development/python-modules/dbt-core/default.nix @@ -0,0 +1,91 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, agate +, cffi +, click +, colorama +, dbt-extractor +, hologram +, idna +, isodate +, jinja2 +, logbook +, mashumaro +, minimal-snowplow-tracker +, networkx +, packaging +, pathspec +, protobuf +, pythonRelaxDepsHook +, pytz +, pyyaml +, requests +, sqlparse +, typing-extensions +, urllib3 +, werkzeug +}: + +buildPythonPackage rec { + pname = "dbt-core"; + version = "1.5.3"; + format = "setuptools"; + + src = fetchFromGitHub { + owner = "dbt-labs"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-ZgP11fVMtXpzo9QaTkejvKl0LzCAkIyGBMcOquBirxQ="; + }; + + sourceRoot = "source/core"; + + nativeBuildInputs = [ + pythonRelaxDepsHook + ]; + + pythonRelaxDeps = [ + "agate" + "click" + "mashumaro" + "networkx" + ]; + + propagatedBuildInputs = [ + agate + cffi + click + colorama + dbt-extractor + hologram + idna + isodate + jinja2 + logbook + mashumaro + minimal-snowplow-tracker + networkx + packaging + pathspec + protobuf + pytz + pyyaml + requests + sqlparse + typing-extensions + urllib3 + werkzeug + ] ++ mashumaro.optional-dependencies.msgpack; + + # tests exist for the dbt tool but not for this package specifically + doCheck = false; + + meta = with lib; { + description = "Enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications"; + homepage = "https://github.com/dbt-labs/dbt-core"; + changelog = "https://github.com/dbt-labs/dbt-core/blob/v${version}/CHANGELOG.md"; + license = licenses.asl20; + maintainers = with maintainers; [ mausch tjni ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index fa9cf3e4198b..82dc7f00e975 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2465,6 +2465,8 @@ self: super: with self; { dbfread = callPackage ../development/python-modules/dbfread { }; + dbt-core = callPackage ../development/python-modules/dbt-core { }; + dbt-extractor = callPackage ../development/python-modules/dbt-extractor { }; dbus-client-gen = callPackage ../development/python-modules/dbus-client-gen { }; From 4f37dd2d18c6322fc5fd85b27060cd4efbe82f63 Mon Sep 17 00:00:00 2001 From: Theodore Ni <3806110+tjni@users.noreply.github.com> Date: Thu, 13 Jul 2023 22:02:06 -0700 Subject: [PATCH 05/10] python310Packages.dbt-postgres: init at 1.5.3 --- .../python-modules/dbt-postgres/default.nix | 37 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 39 insertions(+) create mode 100644 pkgs/development/python-modules/dbt-postgres/default.nix diff --git a/pkgs/development/python-modules/dbt-postgres/default.nix b/pkgs/development/python-modules/dbt-postgres/default.nix new file mode 100644 index 000000000000..f5be13ccff7a --- /dev/null +++ b/pkgs/development/python-modules/dbt-postgres/default.nix @@ -0,0 +1,37 @@ +{ lib +, buildPythonPackage +, agate +, dbt-core +, psycopg2 +}: + +buildPythonPackage { + pname = "dbt-postgres"; + format = "setuptools"; + + inherit (dbt-core) version src; + + sourceRoot = "source/plugins/postgres"; + + env.DBT_PSYCOPG2_NAME = "psycopg2"; + + propagatedBuildInputs = [ + agate + dbt-core + psycopg2 + ]; + + # tests exist for the dbt tool but not for this package specifically + doCheck = false; + + pythonImportsCheck = [ + "dbt.adapters.postgres" + ]; + + meta = with lib; { + description = "Plugin enabling dbt to work with a Postgres database"; + homepage = "https://github.com/dbt-labs/dbt-core"; + license = licenses.asl20; + maintainers = with maintainers; [ tjni ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 82dc7f00e975..35fb5d760d26 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2469,6 +2469,8 @@ self: super: with self; { dbt-extractor = callPackage ../development/python-modules/dbt-extractor { }; + dbt-postgres = callPackage ../development/python-modules/dbt-postgres { }; + dbus-client-gen = callPackage ../development/python-modules/dbus-client-gen { }; dbus-deviation = callPackage ../development/python-modules/dbus-deviation { }; From b02c1afb9b464e4f063d75679eb2a63527c20899 Mon Sep 17 00:00:00 2001 From: Theodore Ni <3806110+tjni@users.noreply.github.com> Date: Thu, 13 Jul 2023 22:20:38 -0700 Subject: [PATCH 06/10] python310Packages.dbt-bigquery: init at 1.5.3 --- .../python-modules/dbt-bigquery/default.nix | 60 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 62 insertions(+) create mode 100644 pkgs/development/python-modules/dbt-bigquery/default.nix diff --git a/pkgs/development/python-modules/dbt-bigquery/default.nix b/pkgs/development/python-modules/dbt-bigquery/default.nix new file mode 100644 index 000000000000..9d456c84d22c --- /dev/null +++ b/pkgs/development/python-modules/dbt-bigquery/default.nix @@ -0,0 +1,60 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, agate +, dbt-core +, google-cloud-bigquery +, google-cloud-storage +, google-cloud-dataproc +, pytestCheckHook +, pythonRelaxDepsHook +}: + +buildPythonPackage rec { + pname = "dbt-bigquery"; + version = "1.5.3"; + format = "setuptools"; + + src = fetchFromGitHub { + owner = "dbt-labs"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-mzqUdT+dI7U0tHHcr0joIB1UCfKXrDdqhtVFShjIKus="; + }; + + nativeBuildInputs = [ + pythonRelaxDepsHook + ]; + + pythonRelaxDeps = [ + "agate" + ]; + + propagatedBuildInputs = [ + agate + dbt-core + google-cloud-bigquery + google-cloud-storage + google-cloud-dataproc + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + pytestFlagsArray = [ + "tests/unit" + ]; + + pythonImportsCheck = [ + "dbt.adapters.bigquery" + ]; + + meta = with lib; { + description = "Plugin enabling dbt to operate on a BigQuery database"; + homepage = "https://github.com/dbt-labs/dbt-bigquery"; + changelog = "https://github.com/dbt-labs/dbt-bigquery/blob/${version}/CHANGELOG.md"; + license = licenses.asl20; + maintainers = with maintainers; [ tjni ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 35fb5d760d26..f23d293761d1 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2465,6 +2465,8 @@ self: super: with self; { dbfread = callPackage ../development/python-modules/dbfread { }; + dbt-bigquery = callPackage ../development/python-modules/dbt-bigquery { }; + dbt-core = callPackage ../development/python-modules/dbt-core { }; dbt-extractor = callPackage ../development/python-modules/dbt-extractor { }; From 354ecddc47463a02587d1cf290a4552eae64e209 Mon Sep 17 00:00:00 2001 From: Theodore Ni <3806110+tjni@users.noreply.github.com> Date: Thu, 13 Jul 2023 22:51:15 -0700 Subject: [PATCH 07/10] python310Packages.dbt-redshift: init at 1.5.8 --- .../python-modules/dbt-redshift/default.nix | 61 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 63 insertions(+) create mode 100644 pkgs/development/python-modules/dbt-redshift/default.nix diff --git a/pkgs/development/python-modules/dbt-redshift/default.nix b/pkgs/development/python-modules/dbt-redshift/default.nix new file mode 100644 index 000000000000..b757683e9516 --- /dev/null +++ b/pkgs/development/python-modules/dbt-redshift/default.nix @@ -0,0 +1,61 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, agate +, boto3 +, dbt-core +, dbt-postgres +, pytestCheckHook +, pythonRelaxDepsHook +, redshift-connector +}: + +buildPythonPackage rec { + pname = "dbt-redshift"; + version = "1.5.8"; + format = "setuptools"; + + src = fetchFromGitHub { + owner = "dbt-labs"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-T7cNszIroOT8TNfOzZpdJDR1+5ybhkXvyvvM5zokVgo="; + }; + + nativeBuildInputs = [ + pythonRelaxDepsHook + ]; + + pythonRelaxDeps = [ + "boto3" + "redshift-connector" + ]; + + propagatedBuildInputs = [ + agate + boto3 + dbt-core + dbt-postgres + redshift-connector + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + pytestFlagsArray = [ + "tests/unit" + ]; + + pythonImportsCheck = [ + "dbt.adapters.redshift" + ]; + + meta = with lib; { + description = "Plugin enabling dbt to work with Amazon Redshift"; + homepage = "https://github.com/dbt-labs/dbt-redshift"; + changelog = "https://github.com/dbt-labs/dbt-redshift/blob/${version}/CHANGELOG.md"; + license = licenses.asl20; + maintainers = with maintainers; [ tjni ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index f23d293761d1..b189243bdd21 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2473,6 +2473,8 @@ self: super: with self; { dbt-postgres = callPackage ../development/python-modules/dbt-postgres { }; + dbt-redshift = callPackage ../development/python-modules/dbt-redshift { }; + dbus-client-gen = callPackage ../development/python-modules/dbus-client-gen { }; dbus-deviation = callPackage ../development/python-modules/dbus-deviation { }; From c0310c08d755809400366624c9ac77ecfef4266a Mon Sep 17 00:00:00 2001 From: Theodore Ni <3806110+tjni@users.noreply.github.com> Date: Thu, 13 Jul 2023 22:57:29 -0700 Subject: [PATCH 08/10] python310Packages.snowflake-connector-python: add secure-local-storage extra --- .../python-modules/snowflake-connector-python/default.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/development/python-modules/snowflake-connector-python/default.nix b/pkgs/development/python-modules/snowflake-connector-python/default.nix index 4a89534c63cd..fbf000d21e1e 100644 --- a/pkgs/development/python-modules/snowflake-connector-python/default.nix +++ b/pkgs/development/python-modules/snowflake-connector-python/default.nix @@ -7,6 +7,7 @@ , fetchPypi , filelock , idna +, keyring , oscrypto , pycryptodomex , pyjwt @@ -53,6 +54,10 @@ buildPythonPackage rec { typing-extensions ]; + passthru.optional-dependencies = { + secure-local-storage = [ keyring ]; + }; + # Tests require encrypted secrets, see # https://github.com/snowflakedb/snowflake-connector-python/tree/master/.github/workflows/parameters doCheck = false; From 859e95e2bdd9889fd534d8edf64be99313a2361e Mon Sep 17 00:00:00 2001 From: Theodore Ni <3806110+tjni@users.noreply.github.com> Date: Thu, 13 Jul 2023 23:00:46 -0700 Subject: [PATCH 09/10] python310Packages.dbt-snowflake: init at 1.5.2 --- .../python-modules/dbt-snowflake/default.nix | 45 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 47 insertions(+) create mode 100644 pkgs/development/python-modules/dbt-snowflake/default.nix diff --git a/pkgs/development/python-modules/dbt-snowflake/default.nix b/pkgs/development/python-modules/dbt-snowflake/default.nix new file mode 100644 index 000000000000..b32f7fbd9a6b --- /dev/null +++ b/pkgs/development/python-modules/dbt-snowflake/default.nix @@ -0,0 +1,45 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, dbt-core +, pytestCheckHook +, snowflake-connector-python +}: + +buildPythonPackage rec { + pname = "dbt-snowflake"; + version = "1.5.2"; + format = "setuptools"; + + src = fetchFromGitHub { + owner = "dbt-labs"; + repo = pname; + rev = "refs/tags/v${version}"; + hash = "sha256-Oz9BDvxSG4kOteuwKiPccxlsmQtOEwDLVyirDSWy1kY="; + }; + + propagatedBuildInputs = [ + dbt-core + snowflake-connector-python + ] ++ snowflake-connector-python.optional-dependencies.secure-local-storage; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + pytestFlagsArray = [ + "tests/unit" + ]; + + pythonImportsCheck = [ + "dbt.adapters.snowflake" + ]; + + meta = with lib; { + description = "Plugin enabling dbt to work with Snowflake"; + homepage = "https://github.com/dbt-labs/dbt-snowflake"; + changelog = "https://github.com/dbt-labs/dbt-snowflake/blob/${version}/CHANGELOG.md"; + license = licenses.asl20; + maintainers = with maintainers; [ tjni ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index b189243bdd21..de0cbc3ab155 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2475,6 +2475,8 @@ self: super: with self; { dbt-redshift = callPackage ../development/python-modules/dbt-redshift { }; + dbt-snowflake = callPackage ../development/python-modules/dbt-snowflake { }; + dbus-client-gen = callPackage ../development/python-modules/dbus-client-gen { }; dbus-deviation = callPackage ../development/python-modules/dbus-deviation { }; From 4bfbc45869b358a32cbd5b47c26326a08f9e34a7 Mon Sep 17 00:00:00 2001 From: Theodore Ni <3806110+tjni@users.noreply.github.com> Date: Thu, 13 Jul 2023 23:40:30 -0700 Subject: [PATCH 10/10] dbt: init at 1.5.3 Add a top-level dbt application with a passthru function that should be used to extend it with adapters. --- .../python-modules/dbt-core/default.nix | 22 +++++++++++++++++++ .../python-modules/dbt-core/with-adapters.nix | 18 +++++++++++++++ pkgs/top-level/all-packages.nix | 2 ++ 3 files changed, 42 insertions(+) create mode 100644 pkgs/development/python-modules/dbt-core/with-adapters.nix diff --git a/pkgs/development/python-modules/dbt-core/default.nix b/pkgs/development/python-modules/dbt-core/default.nix index f522cb1d239d..bd346c3262d7 100644 --- a/pkgs/development/python-modules/dbt-core/default.nix +++ b/pkgs/development/python-modules/dbt-core/default.nix @@ -1,4 +1,5 @@ { lib +, python3 , buildPythonPackage , fetchFromGitHub , agate @@ -81,11 +82,32 @@ buildPythonPackage rec { # tests exist for the dbt tool but not for this package specifically doCheck = false; + passthru = { + withAdapters = python3.pkgs.callPackage ./with-adapters.nix { }; + }; + meta = with lib; { description = "Enables data analysts and engineers to transform their data using the same practices that software engineers use to build applications"; + longDescription = '' + The dbt tool needs adapters to data sources in order to work. The available + adapters are: + + dbt-bigquery + dbt-postgres + dbt-redshift + dbt-snowflake + + An example of building this package with a few adapters: + + dbt.withAdapters (adapters: [ + adapters.dbt-bigquery + adapters.dbt-postgres + ]) + ''; homepage = "https://github.com/dbt-labs/dbt-core"; changelog = "https://github.com/dbt-labs/dbt-core/blob/v${version}/CHANGELOG.md"; license = licenses.asl20; maintainers = with maintainers; [ mausch tjni ]; + mainProgram = "dbt"; }; } diff --git a/pkgs/development/python-modules/dbt-core/with-adapters.nix b/pkgs/development/python-modules/dbt-core/with-adapters.nix new file mode 100644 index 000000000000..006f4a96b853 --- /dev/null +++ b/pkgs/development/python-modules/dbt-core/with-adapters.nix @@ -0,0 +1,18 @@ +{ python3 +, dbt-bigquery +, dbt-core +, dbt-postgres +, dbt-redshift +, dbt-snowflake +}: +let + adapters = { + inherit dbt-bigquery dbt-postgres dbt-redshift dbt-snowflake; + }; +in +adapterFun: (python3.buildEnv.override { + extraLibs = [ dbt-core ] ++ (adapterFun adapters); + ignoreCollisions = true; +}).overrideAttrs { + meta.mainProgram = dbt-core.meta.mainProgram; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index a84f1600cef6..997e04158887 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -18752,6 +18752,8 @@ with pkgs; cwltool = callPackage ../applications/science/misc/cwltool { }; + dbt = with python3Packages; toPythonApplication dbt-core; + dprint = callPackage ../development/tools/dprint { inherit (darwin.apple_sdk.frameworks) Security; };