From ba656ad84ebb89cad244d23bc61002ceff332b15 Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Tue, 21 Nov 2023 17:45:45 -0500 Subject: [PATCH 1/5] prefetch-npm-deps: bump deps --- .../node/fetch-npm-deps/Cargo.lock | 120 +++++++++--------- .../node/fetch-npm-deps/Cargo.toml | 11 +- 2 files changed, 66 insertions(+), 65 deletions(-) diff --git a/pkgs/build-support/node/fetch-npm-deps/Cargo.lock b/pkgs/build-support/node/fetch-npm-deps/Cargo.lock index 4f6e177fd9ae..8ba72a7b76c4 100644 --- a/pkgs/build-support/node/fetch-npm-deps/Cargo.lock +++ b/pkgs/build-support/node/fetch-npm-deps/Cargo.lock @@ -47,9 +47,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.4" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "bitflags" @@ -59,9 +59,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "block-buffer" @@ -110,9 +110,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -177,9 +177,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.67+curl-8.3.0" +version = "0.4.68+curl-8.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cc35d066510b197a0f72de863736641539957628c8a42e70e27c66849e77c34" +checksum = "b4a0d18d88360e374b16b2273c832b5e57258ffc1d4aa4f96b108e0738d5752f" dependencies = [ "cc", "libc", @@ -208,9 +208,9 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "95b3f3e67048839cb0d0781f445682a35113da7121f7c949db0e2be96a4fbece" dependencies = [ "humantime", "is-terminal", @@ -221,9 +221,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" dependencies = [ "libc", "windows-sys", @@ -267,15 +267,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-lite" @@ -304,9 +304,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "libc", @@ -321,9 +321,9 @@ checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -399,9 +399,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libz-sys" @@ -417,9 +417,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "log" @@ -456,9 +456,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9" dependencies = [ "cc", "libc", @@ -468,9 +468,9 @@ dependencies = [ [[package]] name = "parking" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e52c774a4c39359c1d1c52e43f73dd91a75a614652c825408eec30c95a9b2067" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" [[package]] name = "percent-encoding" @@ -542,6 +542,7 @@ dependencies = [ "digest", "env_logger", "isahc", + "log", "rayon", "serde", "serde_json", @@ -622,18 +623,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.3.5" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "regex" -version = "1.9.6" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", @@ -643,9 +644,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", @@ -654,17 +655,17 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.5" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rustix" -version = "0.38.18" +version = "0.38.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", @@ -703,18 +704,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", @@ -723,9 +724,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -776,9 +777,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -786,9 +787,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -797,9 +798,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand 2.0.1", @@ -810,9 +811,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" dependencies = [ "winapi-util", ] @@ -834,11 +835,10 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -847,9 +847,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", @@ -858,9 +858,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] diff --git a/pkgs/build-support/node/fetch-npm-deps/Cargo.toml b/pkgs/build-support/node/fetch-npm-deps/Cargo.toml index 0f7735a6e827..ea121c510c95 100644 --- a/pkgs/build-support/node/fetch-npm-deps/Cargo.toml +++ b/pkgs/build-support/node/fetch-npm-deps/Cargo.toml @@ -8,15 +8,16 @@ edition = "2021" [dependencies] anyhow = "1.0.75" backoff = "0.4.0" -base64 = "0.21.4" +base64 = "0.21.5" digest = "0.10.7" -env_logger = "0.10.0" +env_logger = "0.10.1" isahc = { version = "1.7.2", default_features = false } +log = "0.4.20" rayon = "1.8.0" -serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.107" +serde = { version = "1.0.193", features = ["derive"] } +serde_json = "1.0.108" sha1 = "0.10.6" sha2 = "0.10.8" -tempfile = "3.8.0" +tempfile = "3.8.1" url = { version = "2.4.1", features = ["serde"] } walkdir = "2.4.0" From 81ed58b0fef21985242dea442e398b3bb81c9d67 Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Tue, 21 Nov 2023 18:18:26 -0500 Subject: [PATCH 2/5] prefetch-npm-deps: make cargo happy --- pkgs/build-support/node/fetch-npm-deps/src/cacache.rs | 9 +++++---- pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs b/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs index 75133b1b03ea..c49c094b85c6 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs @@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize}; use sha1::Sha1; use sha2::{Sha256, Sha512}; use std::{ + fmt::Write as FmtWrite, fs::{self, File}, io::Write, path::PathBuf, @@ -78,10 +79,10 @@ impl Cache { push_hash_segments( &mut p, - &hash - .into_iter() - .map(|n| format!("{n:02x}")) - .collect::(), + &hash.into_iter().fold(String::new(), |mut out, n| { + let _ = write!(out, "{n:02x}"); + out + }), ); p diff --git a/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs b/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs index f50a31651d0e..f7273d6a7888 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs @@ -182,6 +182,7 @@ impl fmt::Display for Hash { } } +#[allow(clippy::incorrect_partial_ord_impl_on_ord_type)] impl PartialOrd for Hash { fn partial_cmp(&self, other: &Hash) -> Option { let lhs = self.0.split_once('-')?.0; From daec4bf7347a779d5d4a25524e056987c46ea519 Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Tue, 21 Nov 2023 18:00:03 -0500 Subject: [PATCH 3/5] prefetch-npm-deps: instrument some logging --- pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs b/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs index 880a972e80e9..0bca33f03915 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, bail, Context}; use lock::UrlOrString; +use log::{debug, info}; use rayon::prelude::*; use serde_json::{Map, Value}; use std::{ @@ -19,6 +20,8 @@ pub fn lockfile( force_git_deps: bool, force_empty_cache: bool, ) -> anyhow::Result> { + debug!("parsing lockfile with contents:\n{content}"); + let mut packages = lock::packages(content) .context("failed to extract packages from lockfile")? .into_par_iter() @@ -46,6 +49,8 @@ pub fn lockfile( let path = dir.path().join("package"); + info!("recursively parsing lockfile for {} at {path:?}", pkg.name); + let lockfile_contents = fs::read_to_string(path.join("package-lock.json")); let package_json_path = path.join("package.json"); From 77571a847f4c6c744a2541b493b2b7e9751a13ce Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Tue, 21 Nov 2023 17:49:05 -0500 Subject: [PATCH 4/5] prefetch-npm-deps: use default value when lockfile has no deps --- .../node/fetch-npm-deps/src/parse/lock.rs | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs b/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs index f7273d6a7888..e3580cfca0d3 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs @@ -18,23 +18,20 @@ pub(super) fn packages(content: &str) -> anyhow::Result> { 1 => { let initial_url = get_initial_url()?; - lockfile - .dependencies - .map(|p| to_new_packages(p, &initial_url)) - .transpose()? + to_new_packages(lockfile.dependencies.unwrap_or_default(), &initial_url)? } - 2 | 3 => lockfile.packages.map(|pkgs| { - pkgs.into_iter() - .filter(|(n, p)| !n.is_empty() && matches!(p.resolved, Some(UrlOrString::Url(_)))) - .map(|(n, p)| Package { name: Some(n), ..p }) - .collect() - }), + 2 | 3 => lockfile + .packages + .unwrap_or_default() + .into_iter() + .filter(|(n, p)| !n.is_empty() && matches!(p.resolved, Some(UrlOrString::Url(_)))) + .map(|(n, p)| Package { name: Some(n), ..p }) + .collect(), _ => bail!( "We don't support lockfile version {}, please file an issue.", lockfile.version ), - } - .expect("lockfile should have packages"); + }; packages.par_sort_by(|x, y| { x.resolved From 09081aa859322fa382c1f106d989b5b4b3b37e74 Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Wed, 22 Nov 2023 15:22:35 -0500 Subject: [PATCH 5/5] fetchNpmDeps: add test case where empty default lockfile packages is needed --- pkgs/build-support/node/fetch-npm-deps/default.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkgs/build-support/node/fetch-npm-deps/default.nix b/pkgs/build-support/node/fetch-npm-deps/default.nix index cfa3366491f2..725f9ba3bb01 100644 --- a/pkgs/build-support/node/fetch-npm-deps/default.nix +++ b/pkgs/build-support/node/fetch-npm-deps/default.nix @@ -125,6 +125,18 @@ forceGitDeps = true; }; + + # This package has a lockfile v1 git dependency with no `dependencies` attribute, since it sementically has no dependencies. + jitsiMeet9111 = makeTest { + name = "jitsi-meet-9111"; + + src = fetchurl { + url = "https://raw.githubusercontent.com/jitsi/jitsi-meet/stable/jitsi-meet_9111/package-lock.json"; + hash = "sha256-NU+eQD4WZ4BMur8uX79uk8wUPsZvIT02KhPWHTmaihk="; + }; + + hash = "sha256-FhxlJ0HdJMPiWe7+n1HaGLWOr/2HJEPwiS65uqXZM8Y="; + }; }; meta = with lib; {