From caa2ec97f3b35a2540792fefd7924ba6103e3684 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli <4332460+Cammisuli@users.noreply.github.com> Date: Thu, 20 Feb 2025 16:16:19 -0500 Subject: [PATCH 01/11] chore(core): add nx_glob crate --- Cargo.lock | 2 +- Cargo.toml | 73 +++++++++- crates/nx_glob/.gitignore | 1 + crates/nx_glob/Cargo.toml | 12 ++ .../glob => crates/nx_glob/src}/glob_group.rs | 0 .../nx_glob/src}/glob_parser.rs | 6 +- .../glob.rs => crates/nx_glob/src/glob_set.rs | 129 ++++++++---------- .../nx_glob/src}/glob_transform.rs | 4 +- crates/nx_glob/src/lib.rs | 23 ++++ packages/nx/Cargo.toml | 107 +++++++-------- .../nx/src/native/cache/expand_outputs.rs | 7 +- .../nx/src/native/cache/validate_outputs.rs | 2 +- packages/nx/src/native/mod.rs | 1 - .../native/plugins/js/ts_import_locators.rs | 4 +- .../tasks/hashers/hash_project_files.rs | 6 +- .../native/tasks/hashers/hash_task_output.rs | 6 +- .../tasks/hashers/hash_workspace_files.rs | 6 +- .../native/utils/find_matching_projects.rs | 8 +- packages/nx/src/native/walker.rs | 6 +- .../nx/src/native/workspace/config_files.rs | 7 +- 20 files changed, 243 insertions(+), 167 deletions(-) create mode 100644 crates/nx_glob/.gitignore create mode 100644 crates/nx_glob/Cargo.toml rename {packages/nx/src/native/glob => crates/nx_glob/src}/glob_group.rs (100%) rename {packages/nx/src/native/glob => crates/nx_glob/src}/glob_parser.rs (98%) rename packages/nx/src/native/glob.rs => crates/nx_glob/src/glob_set.rs (79%) rename {packages/nx/src/native/glob => crates/nx_glob/src}/glob_transform.rs (99%) create mode 100644 crates/nx_glob/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 73bca8093eb8c..495b9467acfcd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "Inflector" diff --git a/Cargo.toml b/Cargo.toml index 217f75ad61044..63f3d2fb98218 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,76 @@ - [workspace] -resolver = '2' +resolver = "2" members = [ - 'packages/nx', + "crates/*", + "packages/nx", ] +[workspace.package] +edition = "2021" +license = "MIT" +version = "0.1.0" + +[workspace.dependencies] +nx_glob = { path = "crates/nx_glob" } + +anyhow = "1.0.95" +colored = "2" +crossbeam-channel = '0.5' +dashmap = { version = "5.5.3", features = ["rayon"] } +dunce = "1" +fs_extra = "1.3.0" +hashbrown = { version = "0.14.5", features = ["rayon", "rkyv"] } +ignore = '0.4' +itertools = "0.10.5" +globset = "0.4.10" +once_cell = "1.18.0" +parking_lot = { version = "0.12.1", features = ["send_guard"] } +napi = { version = '2.16.0', default-features = false, features = [ + 'anyhow', + 'napi4', + 'tokio_rt', +] } +napi-derive = '2.16.0' +nom = '7.1.3' +regex = "1.9.1" +rayon = "1.7.0" +rkyv = { version = "0.7", features = ["validation"] } +thiserror = "1.0.40" +tracing = "0.1.37" +tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } +walkdir = '2.3.3' +xxhash-rust = { version = '0.8.5', features = ['xxh3', 'xxh64'] } +swc_common = "0.31.16" +swc_ecma_parser = { version = "0.137.1", features = ["typescript"] } +swc_ecma_visit = "0.93.0" +swc_ecma_ast = "0.107.0" + +#windows dependencies +winapi = { version = "0.3", features = ["fileapi"] } + +#unix dependencies +mio = "0.8" + +#non-wasm dependencies +portable-pty = { git = "https://github.com/cammisuli/wezterm", rev = "b538ee29e1e89eeb4832fb35ae095564dce34c29" } +crossterm = "0.27.0" +ignore-files = "2.1.0" +fs4 = "0.12.0" +rusqlite = { version = "0.32.1", features = ["bundled", "array", "vtab"] } +watchexec = "3.0.1" +watchexec-events = "2.0.1" +watchexec-filterer-ignore = "3.0.0" +watchexec-signals = "2.1.0" +machine-uid = "0.5.2" + +# build dependencies +napi-build = '2.1.3' +assert_fs = "1.0.10" +# This is only used for unit tests +swc_ecma_dep_graph = "0.109.1" +tempfile = "3.13.0" + [profile.release] lto = true + + diff --git a/crates/nx_glob/.gitignore b/crates/nx_glob/.gitignore new file mode 100644 index 0000000000000..ea8c4bf7f35f6 --- /dev/null +++ b/crates/nx_glob/.gitignore @@ -0,0 +1 @@ +/target diff --git a/crates/nx_glob/Cargo.toml b/crates/nx_glob/Cargo.toml new file mode 100644 index 0000000000000..ab0e573afcc73 --- /dev/null +++ b/crates/nx_glob/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "nx_glob" +edition.workspace = true +license.workspace = true +version.workspace = true + +[dependencies] +globset = { workspace = true } +tracing = { workspace = true } +anyhow = { workspace = true } +itertools = { workspace = true } +nom = { workspace = true } diff --git a/packages/nx/src/native/glob/glob_group.rs b/crates/nx_glob/src/glob_group.rs similarity index 100% rename from packages/nx/src/native/glob/glob_group.rs rename to crates/nx_glob/src/glob_group.rs diff --git a/packages/nx/src/native/glob/glob_parser.rs b/crates/nx_glob/src/glob_parser.rs similarity index 98% rename from packages/nx/src/native/glob/glob_parser.rs rename to crates/nx_glob/src/glob_parser.rs index a1b499da19379..bb72d1e8151f7 100644 --- a/packages/nx/src/native/glob/glob_parser.rs +++ b/crates/nx_glob/src/glob_parser.rs @@ -1,4 +1,4 @@ -use crate::native::glob::glob_group::GlobGroup; +use crate::glob_group::GlobGroup; use nom::branch::alt; use nom::bytes::complete::{is_not, tag, take_till, take_until, take_while}; use nom::combinator::{eof, map, map_parser}; @@ -198,8 +198,8 @@ pub fn parse_glob(input: &str) -> anyhow::Result<(bool, Vec>)> { #[cfg(test)] mod test { - use crate::native::glob::glob_group::GlobGroup; - use crate::native::glob::glob_parser::{parse_glob, special_char_with_no_group}; + use crate::glob_group::GlobGroup; + use crate::glob_parser::{parse_glob, special_char_with_no_group}; #[test] fn invalid_groups() { diff --git a/packages/nx/src/native/glob.rs b/crates/nx_glob/src/glob_set.rs similarity index 79% rename from packages/nx/src/native/glob.rs rename to crates/nx_glob/src/glob_set.rs index 6f4e995815c53..148175f5ece7f 100644 --- a/packages/nx/src/native/glob.rs +++ b/crates/nx_glob/src/glob_set.rs @@ -1,20 +1,16 @@ -mod glob_group; -mod glob_parser; -pub mod glob_transform; - -use crate::native::glob::glob_transform::convert_glob; +use crate::glob_transform::convert_glob; use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; use std::fmt::Debug; use std::path::Path; use tracing::trace; -pub struct NxGlobSetBuilder { +struct NxGlobSetBuilder { included_globs: GlobSetBuilder, excluded_globs: GlobSetBuilder, } impl NxGlobSetBuilder { - pub fn new>(globs: &[S]) -> anyhow::Result { + fn new>(globs: &[S]) -> anyhow::Result { let mut glob_set_builder = NxGlobSetBuilder { included_globs: GlobSetBuilder::new(), excluded_globs: GlobSetBuilder::new(), @@ -27,7 +23,7 @@ impl NxGlobSetBuilder { Ok(glob_set_builder) } - pub fn add(&mut self, glob: &str) -> anyhow::Result<&mut NxGlobSetBuilder> { + fn add(&mut self, glob: &str) -> anyhow::Result<&mut NxGlobSetBuilder> { let negated = glob.starts_with('!'); let glob_string = glob.strip_prefix('!').unwrap_or(glob).to_string(); @@ -51,7 +47,7 @@ impl NxGlobSetBuilder { Ok(self) } - pub fn build(&self) -> anyhow::Result { + fn build(&self) -> anyhow::Result { Ok(NxGlobSet { excluded_globs: self.excluded_globs.build()?, included_globs: self.included_globs.build()?, @@ -64,7 +60,31 @@ pub struct NxGlobSet { included_globs: GlobSet, excluded_globs: GlobSet, } + impl NxGlobSet { + pub fn new + Debug>(globs: &[S]) -> anyhow::Result { + let result = globs + .iter() + .flat_map(|s| potential_glob_split(s.as_ref())) + .map(|glob| { + if glob.contains('!') + || glob.contains('|') + || glob.contains('(') + || glob.contains("{,") + { + convert_glob(glob) + } else { + Ok(vec![glob.to_string()]) + } + }) + .collect::>>()? + .concat(); + + trace!(?globs, ?result, "converted globs"); + + NxGlobSetBuilder::new(&result)?.build() + } + pub fn is_match>(&self, path: P) -> bool { if self.included_globs.is_empty() { !self.excluded_globs.is_match(path.as_ref()) @@ -77,7 +97,7 @@ impl NxGlobSet { } } -fn potential_glob_split( +pub fn potential_glob_split( glob: &str, ) -> itertools::Either, std::iter::Once<&str>> { use itertools::Either::*; @@ -88,88 +108,53 @@ fn potential_glob_split( } } -pub(crate) fn build_glob_set + Debug>(globs: &[S]) -> anyhow::Result { - let result = globs - .iter() - .flat_map(|s| potential_glob_split(s.as_ref())) - .map(|glob| { - if glob.contains('!') || glob.contains('|') || glob.contains('(') || glob.contains("{,") - { - convert_glob(glob) - } else { - Ok(vec![glob.to_string()]) - } - }) - .collect::>>()? - .concat(); - - trace!(?globs, ?result, "converted globs"); - - NxGlobSetBuilder::new(&result)?.build() -} - -pub(crate) fn contains_glob_pattern(value: &str) -> bool { - value.contains('!') - || value.contains('?') - || value.contains('@') - || value.contains('+') - || value.contains('*') - || value.contains('|') - || value.contains(',') - || value.contains('{') - || value.contains('}') - || value.contains('[') - || value.contains(']') - || value.contains('(') - || value.contains(')') -} #[cfg(test)] mod test { - use super::*; + use crate::glob_set::NxGlobSet; #[test] fn should_work_with_simple_globs() { - let glob_set = build_glob_set(&["**/*"]).unwrap(); + let glob_set = NxGlobSet::new(&["**/*"]).unwrap(); assert!(glob_set.is_match("packages/nx/package.json")); - let glob_set = build_glob_set(&["!test/*.spec.ts"]).unwrap(); + let glob_set = NxGlobSet::new(&["!test/*.spec.ts"]).unwrap(); assert!(!glob_set.is_match("test/file.spec.ts")); assert!(glob_set.is_match("test/file.ts")); - let glob_set = build_glob_set(&["test/*.spec.ts"]).unwrap(); + let glob_set = NxGlobSet::new(&["test/*.spec.ts"]).unwrap(); assert!(glob_set.is_match("test/file.spec.ts")); assert!(!glob_set.is_match("test/file.ts")); } #[test] fn should_detect_package_json() { - let glob_set = build_glob_set(&["packages/*/package.json"]).unwrap(); + let glob_set = NxGlobSet::new(&["packages/*/package.json"]).unwrap(); assert!(glob_set.is_match("packages/nx/package.json")) } #[test] fn should_not_detect_deeply_nested_package_json() { - let glob_set = build_glob_set(&["packages/*/package.json"]).unwrap(); + let glob_set = NxGlobSet::new(&["packages/*/package.json"]).unwrap(); assert!(!glob_set.is_match("packages/nx/test-files/package.json")) } #[test] fn should_detect_deeply_nested_package_json() { - let glob_set = build_glob_set(&["packages/**/package.json"]).unwrap(); + let glob_set = NxGlobSet::new(&["packages/**/package.json"]).unwrap(); assert!(glob_set.is_match("packages/nx/test-files/package.json")) } #[test] fn should_detect_node_modules() { - let glob_set = build_glob_set(&["**/node_modules"]).unwrap(); + let glob_set = NxGlobSet::new(&["**/node_modules"]).unwrap(); assert!(glob_set.is_match("node_modules")); assert!(glob_set.is_match("packages/nx/node_modules")); } #[test] fn should_not_detect_root_plugin_configs() { - let glob_set = build_glob_set(&["*/**/Cargo.toml"]).unwrap(); + let glob_set = NxGlobSet::new(&["*/**/Cargo.toml"]).unwrap(); assert!(glob_set.is_match("packages/a/Cargo.toml")); assert!(glob_set.is_match("a/Cargo.toml")); assert!(!glob_set.is_match("Cargo.toml")) @@ -177,20 +162,20 @@ mod test { #[test] fn should_handle_negated_globs() { - let glob_set = build_glob_set(&["!nested/ignore/", "nested/"]).unwrap(); + let glob_set = NxGlobSet::new(&["!nested/ignore/", "nested/"]).unwrap(); assert!(!glob_set.is_match("file.map")); assert!(!glob_set.is_match("nested/ignore/file.js")); assert!(!glob_set.is_match("another-nested/nested/file.ts")); assert!(glob_set.is_match("nested/file.js")); assert!(glob_set.is_match("nested/nested/file.ts")); - let glob_set = build_glob_set(&["nested/", "!nested/*.{css,map}"]).unwrap(); + let glob_set = NxGlobSet::new(&["nested/", "!nested/*.{css,map}"]).unwrap(); assert!(glob_set.is_match("nested/file.js")); assert!(glob_set.is_match("nested/file.ts")); assert!(!glob_set.is_match("nested/file.css")); assert!(!glob_set.is_match("nested/file.map")); - let glob_set = build_glob_set(&["!nested/**/ignore/", "nested/**"]).unwrap(); + let glob_set = NxGlobSet::new(&["!nested/**/ignore/", "nested/**"]).unwrap(); assert!(glob_set.is_match("nested/nested/file.js")); assert!(!glob_set.is_match("nested/ignore/file.ts")); assert!(!glob_set.is_match("nested/nested/ignore/file.ts")); @@ -198,7 +183,7 @@ mod test { #[test] fn should_handle_multiple_globs() { - let glob_set = build_glob_set(&["nested/", "doesnt-exist/"]).unwrap(); + let glob_set = NxGlobSet::new(&["nested/", "doesnt-exist/"]).unwrap(); assert!(glob_set.is_match("nested/file.js")); assert!(!glob_set.is_match("file.js")); } @@ -206,7 +191,7 @@ mod test { #[test] fn should_handle_complex_patterns() { let glob_set = - build_glob_set(&["dist/!(cache|cache2)/**/!(README|LICENSE).(txt|md)"]).unwrap(); + NxGlobSet::new(&["dist/!(cache|cache2)/**/!(README|LICENSE).(txt|md)"]).unwrap(); // matches assert!(glob_set.is_match("dist/nested/file.txt")); @@ -222,7 +207,7 @@ mod test { assert!(!glob_set.is_match("dist/LICENSE.md")); assert!(!glob_set.is_match("dist/README.txt")); - let glob_set = build_glob_set(&["dist/*.(js|ts)"]).unwrap(); + let glob_set = NxGlobSet::new(&["dist/*.(js|ts)"]).unwrap(); // matches assert!(glob_set.is_match("dist/file.js")); assert!(glob_set.is_match("dist/file.ts")); @@ -230,13 +215,13 @@ mod test { assert!(!glob_set.is_match("dist/file.txt")); assert!(!glob_set.is_match("dist/nested/file.js")); - let glob_set = build_glob_set(&["dist/**/!(main).(js|ts)"]).unwrap(); + let glob_set = NxGlobSet::new(&["dist/**/!(main).(js|ts)"]).unwrap(); // matches assert!(glob_set.is_match("dist/file.js")); //no matches assert!(!glob_set.is_match("dist/main.js")); - let glob_set = build_glob_set(&["dist/!(main|cache)/"]).unwrap(); + let glob_set = NxGlobSet::new(&["dist/!(main|cache)/"]).unwrap(); // matches assert!(glob_set.is_match("dist/nested/")); // no matches @@ -245,7 +230,7 @@ mod test { assert!(!glob_set.is_match("dist/cache/")); assert!(!glob_set.is_match("dist/main/")); - let glob_set = build_glob_set(&["**/*.spec.ts{,.snap}"]).unwrap(); + let glob_set = NxGlobSet::new(&["**/*.spec.ts{,.snap}"]).unwrap(); // matches assert!(glob_set.is_match("src/file.spec.ts")); assert!(glob_set.is_match("src/file.spec.ts.snap")); @@ -255,7 +240,7 @@ mod test { #[test] fn should_handle_negative_globs_with_one_directory() { - let glob_set = build_glob_set(&["packages/!(package-a)*"]).unwrap(); + let glob_set = NxGlobSet::new(&["packages/!(package-a)*"]).unwrap(); // matches assert!(glob_set.is_match("packages/package-b")); @@ -266,7 +251,7 @@ mod test { assert!(!glob_set.is_match("packages/package-a-b/nested")); assert!(!glob_set.is_match("packages/package-b/nested")); - let glob_set = build_glob_set(&["packages/!(package-a)*/package.json"]).unwrap(); + let glob_set = NxGlobSet::new(&["packages/!(package-a)*/package.json"]).unwrap(); assert!(glob_set.is_match("packages/package-b/package.json")); assert!(glob_set.is_match("packages/package-c/package.json")); assert!(!glob_set.is_match("packages/package-a/package.json")); @@ -275,7 +260,7 @@ mod test { #[test] fn should_handle_complex_extglob_patterns() { - let glob_set = build_glob_set(&["**/?(*.)+(spec|test).[jt]s?(x)?(.snap)"]).unwrap(); + let glob_set = NxGlobSet::new(&["**/?(*.)+(spec|test).[jt]s?(x)?(.snap)"]).unwrap(); // matches assert!(glob_set.is_match("packages/package-a/spec.jsx.snap")); assert!(glob_set.is_match("packages/package-a/spec.js.snap")); @@ -315,14 +300,14 @@ mod test { assert!(!glob_set.is_match("packages/package-a/spec.js.snapx")); assert!(!glob_set.is_match("packages/package-a/file.ts")); - let glob_set = build_glob_set(&["**/!(*.module).ts"]).unwrap(); + let glob_set = NxGlobSet::new(&["**/!(*.module).ts"]).unwrap(); //matches assert!(glob_set.is_match("test.ts")); assert!(glob_set.is_match("nested/comp.test.ts")); //no matches assert!(!glob_set.is_match("test.module.ts")); - let glob_set = build_glob_set(&["**/*.*(component,module).ts?(x)"]).unwrap(); + let glob_set = NxGlobSet::new(&["**/*.*(component,module).ts?(x)"]).unwrap(); //matches assert!(glob_set.is_match("test.component.ts")); assert!(glob_set.is_match("test.module.ts")); @@ -343,18 +328,18 @@ mod test { #[test] fn supports_brace_expansion() { - let glob_set = build_glob_set(&["{packages,apps}/*"]).unwrap(); + let glob_set = NxGlobSet::new(&["{packages,apps}/*"]).unwrap(); assert!(glob_set.is_match("packages/package-a")); assert!(glob_set.is_match("apps/app-a")); assert!(!glob_set.is_match("apps/app-a/nested")); - let glob_set = build_glob_set(&["{package-lock.json,yarn.lock,pnpm-lock.yaml}"]).unwrap(); + let glob_set = NxGlobSet::new(&["{package-lock.json,yarn.lock,pnpm-lock.yaml}"]).unwrap(); assert!(glob_set.is_match("package-lock.json")); assert!(glob_set.is_match("yarn.lock")); assert!(glob_set.is_match("pnpm-lock.yaml")); let glob_set = - build_glob_set(&["{packages/!(package-a)*/package.json,packages/*/package.json}"]) + NxGlobSet::new(&["{packages/!(package-a)*/package.json,packages/*/package.json}"]) .unwrap(); assert!(glob_set.is_match("packages/package-b/package.json")); assert!(glob_set.is_match("packages/package-c/package.json")); @@ -363,7 +348,7 @@ mod test { #[test] fn should_handle_invalid_group_globs() { - let glob_set = build_glob_set(&[ + let glob_set = NxGlobSet::new(&[ "libs/**/*", "!libs/**/?(*.)+spec.ts?(.snap)", "!libs/tsconfig.spec.json", diff --git a/packages/nx/src/native/glob/glob_transform.rs b/crates/nx_glob/src/glob_transform.rs similarity index 99% rename from packages/nx/src/native/glob/glob_transform.rs rename to crates/nx_glob/src/glob_transform.rs index 2644f72feb646..735ea9de07d55 100644 --- a/packages/nx/src/native/glob/glob_transform.rs +++ b/crates/nx_glob/src/glob_transform.rs @@ -1,5 +1,5 @@ -use crate::native::glob::glob_group::GlobGroup; -use crate::native::glob::glob_parser::parse_glob; +use crate::glob_group::GlobGroup; +use crate::glob_parser::parse_glob; use itertools::Itertools; use std::collections::HashSet; use itertools::Either::{Left, Right}; diff --git a/crates/nx_glob/src/lib.rs b/crates/nx_glob/src/lib.rs new file mode 100644 index 0000000000000..16a837bb1aeab --- /dev/null +++ b/crates/nx_glob/src/lib.rs @@ -0,0 +1,23 @@ +mod glob_group; +mod glob_parser; +mod glob_transform; +pub use glob_transform::*; +mod glob_set; +pub use glob_set::*; + +pub fn contains_glob_pattern(value: &str) -> bool { + value.contains('!') + || value.contains('?') + || value.contains('@') + || value.contains('+') + || value.contains('*') + || value.contains('|') + || value.contains(',') + || value.contains('{') + || value.contains('}') + || value.contains('[') + || value.contains(']') + || value.contains('(') + || value.contains(')') +} + diff --git a/packages/nx/Cargo.toml b/packages/nx/Cargo.toml index cb22ab44a2da0..efab814a96549 100644 --- a/packages/nx/Cargo.toml +++ b/packages/nx/Cargo.toml @@ -1,78 +1,67 @@ [package] name = 'nx' -version = '0.1.0' -edition = '2021' +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +crate-type = ['cdylib'] -[profile.release-wasi] -codegen-units = 16 -debug = 'full' -inherits = "release" -lto = "thin" -opt-level = "z" -strip = "none" [dependencies] -anyhow = "1.0.71" -colored = "2" -crossbeam-channel = '0.5' -dashmap = { version = "5.5.3", features = ["rayon"] } -dunce = "1" -fs_extra = "1.3.0" -globset = "0.4.10" -hashbrown = { version = "0.14.5", features = ["rayon", "rkyv"] } -ignore = '0.4' -itertools = "0.10.5" -once_cell = "1.18.0" -parking_lot = { version = "0.12.1", features = ["send_guard"] } -napi = { version = '2.16.0', default-features = false, features = [ - 'anyhow', - 'napi4', - 'tokio_rt', -] } -napi-derive = '2.16.0' -nom = '7.1.3' -regex = "1.9.1" -rayon = "1.7.0" -rkyv = { version = "0.7", features = ["validation"] } -thiserror = "1.0.40" -tracing = "0.1.37" -tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } -walkdir = '2.3.3' -xxhash-rust = { version = '0.8.5', features = ['xxh3', 'xxh64'] } -swc_common = "0.31.16" -swc_ecma_parser = { version = "0.137.1", features = ["typescript"] } -swc_ecma_visit = "0.93.0" -swc_ecma_ast = "0.107.0" +nx_glob = { workspace = true} +anyhow = { workspace = true } +colored = { workspace = true } +crossbeam-channel = { workspace = true } +dashmap = { workspace = true } +dunce = { workspace = true } +fs_extra = { workspace = true } +hashbrown = { workspace = true } +ignore = { workspace = true } +itertools = { workspace = true } +parking_lot = { workspace = true } +napi = { workspace = true } +napi-derive = { workspace = true } +regex = { workspace = true } +rayon = { workspace = true } +rkyv = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +walkdir = { workspace = true } +xxhash-rust = { workspace = true } +swc_common = { workspace = true } +swc_ecma_parser = { workspace = true } +swc_ecma_ast = { workspace = true } [target.'cfg(windows)'.dependencies] -winapi = { version = "0.3", features = ["fileapi"] } +winapi = { workspace = true } [target.'cfg(all(not(windows), not(target_family = "wasm")))'.dependencies] -mio = "0.8" +mio = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -portable-pty = { git = "https://github.com/cammisuli/wezterm", rev = "b538ee29e1e89eeb4832fb35ae095564dce34c29" } -crossterm = "0.27.0" -ignore-files = "2.1.0" -fs4 = "0.12.0" -rusqlite = { version = "0.32.1", features = ["bundled", "array", "vtab"] } -watchexec = "3.0.1" -watchexec-events = "2.0.1" -watchexec-filterer-ignore = "3.0.0" -watchexec-signals = "2.1.0" -machine-uid = "0.5.2" +portable-pty = { workspace = true } +crossterm = { workspace = true } +ignore-files = { workspace = true } +fs4 = { workspace = true } +rusqlite = { workspace = true } +watchexec = { workspace = true } +watchexec-events = { workspace = true } +watchexec-filterer-ignore = { workspace = true } +watchexec-signals = { workspace = true } +machine-uid = { workspace = true } + -[lib] -crate-type = ['cdylib'] [build-dependencies] -napi-build = '2.1.3' +napi-build = { workspace = true } [dev-dependencies] -assert_fs = "1.0.10" +assert_fs = { workspace = true } # This is only used for unit tests -swc_ecma_dep_graph = "0.109.1" -tempfile = "3.13.0" +swc_ecma_dep_graph = { workspace = true } +tempfile = { workspace = true } # We only explicitly use tokio for async tests -tokio = "1.38.0" +tokio = { workspace = true } diff --git a/packages/nx/src/native/cache/expand_outputs.rs b/packages/nx/src/native/cache/expand_outputs.rs index 2f51b5213ac9a..5f3a34beb35fb 100644 --- a/packages/nx/src/native/cache/expand_outputs.rs +++ b/packages/nx/src/native/cache/expand_outputs.rs @@ -2,7 +2,8 @@ use hashbrown::HashMap; use std::path::{Path, PathBuf}; use tracing::trace; -use crate::native::glob::{build_glob_set, contains_glob_pattern, glob_transform::partition_glob}; +use nx_glob::{contains_glob_pattern, partition_glob}; +use nx_glob::NxGlobSet; use crate::native::logger::enable_logger; use crate::native::utils::Normalize; use crate::native::walker::{nx_walker, nx_walker_sync}; @@ -58,7 +59,7 @@ where trace!(?negated_globs, ?regular_globs, "Expanding globs"); - let glob_set = build_glob_set(®ular_globs)?; + let glob_set = NxGlobSet::new(®ular_globs)?; let found_paths = nx_walker_sync(directory, Some(&negated_globs)) .filter_map(|path| { if glob_set.is_match(&path) { @@ -122,7 +123,7 @@ pub fn get_files_for_outputs( let partitioned_globs = partition_globs_into_map(globs)?; for (root, patterns) in partitioned_globs { let root_path = directory.join(&root); - let glob_set = build_glob_set(&patterns)?; + let glob_set = NxGlobSet::new(&patterns)?; trace!("walking directory: {:?}", root_path); let found_paths: Vec = nx_walker(&root_path, false) diff --git a/packages/nx/src/native/cache/validate_outputs.rs b/packages/nx/src/native/cache/validate_outputs.rs index 18171ab612e63..c4b348871f803 100644 --- a/packages/nx/src/native/cache/validate_outputs.rs +++ b/packages/nx/src/native/cache/validate_outputs.rs @@ -1,7 +1,7 @@ use itertools::Itertools; use regex::Regex; -use crate::native::glob::{contains_glob_pattern, glob_transform::partition_glob}; +use nx_glob::{contains_glob_pattern, partition_glob}; const ALLOWED_WORKSPACE_ROOT_OUTPUT_PREFIXES: [&str; 2] = ["!{workspaceRoot}", "{workspaceRoot}"]; diff --git a/packages/nx/src/native/mod.rs b/packages/nx/src/native/mod.rs index 1b21111a05677..990ed97221a8c 100644 --- a/packages/nx/src/native/mod.rs +++ b/packages/nx/src/native/mod.rs @@ -1,5 +1,4 @@ pub mod cache; -pub mod glob; pub mod hasher; mod logger; pub mod metadata; diff --git a/packages/nx/src/native/plugins/js/ts_import_locators.rs b/packages/nx/src/native/plugins/js/ts_import_locators.rs index c208ba25d9ea0..9ce74f5b64609 100644 --- a/packages/nx/src/native/plugins/js/ts_import_locators.rs +++ b/packages/nx/src/native/plugins/js/ts_import_locators.rs @@ -713,7 +713,7 @@ fn find_imports( #[cfg(test)] mod find_imports { use super::*; - use crate::native::glob::build_glob_set; + use nx_glob::NxGlobSet; use crate::native::walker::nx_walker; use assert_fs::prelude::*; use assert_fs::TempDir; @@ -1481,7 +1481,7 @@ import(myTag`react@${version}`); ancestors.next(); let root = PathBuf::from(ancestors.next().unwrap()); - let glob = build_glob_set(&["**/*.[jt]s"]).unwrap(); + let glob = NxGlobSet::new(&["**/*.[jt]s"]).unwrap(); let files = nx_walker(root.clone(), true) .filter(|file| glob.is_match(&file.full_path)) .map(|file| file.full_path) diff --git a/packages/nx/src/native/tasks/hashers/hash_project_files.rs b/packages/nx/src/native/tasks/hashers/hash_project_files.rs index 0da62150e681e..501bacbe7dc30 100644 --- a/packages/nx/src/native/tasks/hashers/hash_project_files.rs +++ b/packages/nx/src/native/tasks/hashers/hash_project_files.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use anyhow::*; use tracing::{trace, trace_span}; -use crate::native::glob::build_glob_set; +use nx_glob::NxGlobSet; use crate::native::types::FileData; pub fn hash_project_files( @@ -41,8 +41,8 @@ fn collect_files<'a>( }) .collect::>(); let now = std::time::Instant::now(); - let glob_set = build_glob_set(&globs)?; - trace!("build_glob_set for {:?}", now.elapsed()); + let glob_set = NxGlobSet::new(&globs)?; + trace!("build glob set for {:?}", now.elapsed()); project_file_map.get(project_name).map_or_else( || Err(anyhow!("project {} not found", project_name)), diff --git a/packages/nx/src/native/tasks/hashers/hash_task_output.rs b/packages/nx/src/native/tasks/hashers/hash_task_output.rs index f4ccf3d30069f..c8b4e43c4e79d 100644 --- a/packages/nx/src/native/tasks/hashers/hash_task_output.rs +++ b/packages/nx/src/native/tasks/hashers/hash_task_output.rs @@ -1,16 +1,16 @@ -use std::path::Path; use crate::native::cache::expand_outputs::get_files_for_outputs; -use crate::native::glob::build_glob_set; use crate::native::hasher::{hash_array, hash_file}; use anyhow::*; +use nx_glob::NxGlobSet; use rayon::prelude::*; +use std::path::Path; use tracing::trace; pub fn hash_task_output(workspace_root: &str, glob: &str, outputs: &[String]) -> Result { let now = std::time::Instant::now(); let output_files = get_files_for_outputs(workspace_root.to_string(), outputs.to_vec())?; trace!("get_files_for_outputs: {:?}", now.elapsed()); - let glob = build_glob_set(&[glob])?; + let glob = NxGlobSet::new(&[glob])?; let hashes = output_files .into_par_iter() .filter(|file| glob.is_match(file)) diff --git a/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs b/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs index 53d07ce720399..af6826ab64eca 100644 --- a/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs +++ b/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs @@ -3,9 +3,9 @@ use std::sync::Arc; use anyhow::*; use dashmap::DashMap; use tracing::{debug, debug_span, trace, warn}; - +use nx_glob::NxGlobSet; +use crate::native::hasher::hash; use crate::native::types::FileData; -use crate::native::{glob::build_glob_set, hasher::hash}; pub fn hash_workspace_files( workspace_file_sets: &[String], @@ -44,7 +44,7 @@ pub fn hash_workspace_files( return Ok(cache_results.clone()); } - let glob = build_glob_set(&globs)?; + let glob = NxGlobSet::new(&globs)?; let mut hasher = xxhash_rust::xxh3::Xxh3::new(); debug_span!("Hashing workspace fileset", cache_key).in_scope(|| { diff --git a/packages/nx/src/native/utils/find_matching_projects.rs b/packages/nx/src/native/utils/find_matching_projects.rs index 4dde0e6fea8a4..ee49968683345 100644 --- a/packages/nx/src/native/utils/find_matching_projects.rs +++ b/packages/nx/src/native/utils/find_matching_projects.rs @@ -1,7 +1,7 @@ -use crate::native::glob::{build_glob_set, NxGlobSet}; use crate::native::project_graph::types::{Project, ProjectGraph}; use hashbrown::HashSet; use std::collections::HashMap; +use nx_glob::NxGlobSet; struct ProjectPattern<'a> { exclude: bool, @@ -162,7 +162,7 @@ fn add_matching_projects_by_name<'a>( get_matching_strings( pattern.value, - &build_glob_set(&[pattern.value])?, + &NxGlobSet::new(&[pattern.value])?, project_names, ) .iter() @@ -182,7 +182,7 @@ fn add_matching_projects_by_directory<'a>( pattern: &ProjectPattern, matched_projects: &mut HashSet<&'a str>, ) -> anyhow::Result<()> { - let glob = build_glob_set(&[pattern.value])?; + let glob = NxGlobSet::new(&[pattern.value])?; for project_name in project_names { let Some(root) = projects.get(*project_name).map(|p| p.root.as_str()) else { continue; @@ -206,7 +206,7 @@ fn add_matching_projects_by_tag<'a>( pattern: &ProjectPattern, matched_projects: &mut HashSet<&'a str>, ) -> anyhow::Result<()> { - let glob = build_glob_set(&[pattern.value])?; + let glob = NxGlobSet::new(&[pattern.value])?; for project_name in project_names { let project_tags = projects .get(*project_name) diff --git a/packages/nx/src/native/walker.rs b/packages/nx/src/native/walker.rs index 0a1fe49be17ea..bb95dd5806a77 100644 --- a/packages/nx/src/native/walker.rs +++ b/packages/nx/src/native/walker.rs @@ -2,11 +2,11 @@ use ignore::WalkBuilder; use std::fmt::Debug; use std::path::{Path, PathBuf}; -use crate::native::glob::build_glob_set; use crate::native::logger::enable_logger; use crate::native::utils::{get_mod_time, Normalize}; use walkdir::WalkDir; +use nx_glob::NxGlobSet; #[derive(PartialEq, Debug, Ord, PartialOrd, Eq, Clone)] pub struct NxFile { @@ -40,7 +40,7 @@ where base_ignores.extend(additional_ignores.iter().map(|s| format!("**/{}", s))); }; - let ignore_glob_set = build_glob_set(&base_ignores).expect("Should be valid globs"); + let ignore_glob_set = NxGlobSet::new(&base_ignores).expect("Should be valid globs"); // Use WalkDir instead of ignore::WalkBuilder because it's faster WalkDir::new(&base_dir) @@ -164,7 +164,7 @@ where { let directory: PathBuf = directory.as_ref().into(); - let ignore_glob_set = build_glob_set(&[ + let ignore_glob_set = NxGlobSet::new(&[ "**/node_modules", "**/.git", "**/.nx/cache", diff --git a/packages/nx/src/native/workspace/config_files.rs b/packages/nx/src/native/workspace/config_files.rs index 146abf3831969..695521809f1f6 100644 --- a/packages/nx/src/native/workspace/config_files.rs +++ b/packages/nx/src/native/workspace/config_files.rs @@ -1,6 +1,5 @@ use rayon::prelude::*; - -use crate::native::glob::build_glob_set; +use nx_glob::NxGlobSet; use crate::native::types::FileData; /// Get workspace config files based on provided globs @@ -9,14 +8,14 @@ pub(super) fn glob_files( globs: Vec, exclude: Option>, ) -> napi::Result> { - let globs = build_glob_set(&globs)?; + let globs = NxGlobSet::new(&globs)?; let exclude_glob_set = match exclude { Some(exclude) => { if exclude.is_empty() { None } else { - Some(build_glob_set(&exclude)?) + Some(NxGlobSet::new(&exclude)?) } } None => None, From 133b2b054e197fb3f720c04cc195bcaf90f32871 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli <4332460+Cammisuli@users.noreply.github.com> Date: Tue, 28 Jan 2025 10:49:08 -0500 Subject: [PATCH 02/11] chore(core): add new Rust crates for Nx core functionality - Added new Rust crates for core Nx functionality: - nx_cache - nx_db - nx_hasher - nx_machine - nx_project_graph - nx_tasks - nx_terminal - nx_walker - nx_watch - nx_workspace - Updated Cargo.toml and Cargo.lock to include new crates - Refactored existing code to use new crate modules - Prepared infrastructure for future Nx core improvements --- Cargo.lock | 198 +++++++++++++++++- Cargo.toml | 19 +- crates/nx_cache/Cargo.toml | 23 ++ crates/nx_cache/src/lib.rs | 4 + crates/nx_db/Cargo.toml | 17 ++ .../db => crates/nx_db/src}/connection.rs | 3 +- .../db => crates/nx_db/src}/initialize.rs | 15 +- .../db/mod.rs => crates/nx_db/src/lib.rs | 24 +-- crates/nx_hasher/Cargo.toml | 13 ++ crates/nx_hasher/src/lib.rs | 98 +++++++++ crates/nx_machine/Cargo.toml | 12 ++ crates/nx_machine/src/lib.rs | 1 + crates/nx_project_graph/Cargo.toml | 25 +++ crates/nx_project_graph/src/lib.rs | 4 + crates/nx_tasks/Cargo.toml | 18 ++ crates/nx_tasks/src/lib.rs | 1 + crates/nx_terminal/Cargo.toml | 22 ++ crates/nx_terminal/src/lib.rs | 4 + crates/nx_walker/Cargo.toml | 17 ++ .../walker.rs => crates/nx_walker/src/lib.rs | 100 ++------- crates/nx_walker/src/utils/get_mod_time.rs | 25 +++ crates/nx_walker/src/utils/mod.rs | 5 + crates/nx_walker/src/utils/normalize_trait.rs | 17 ++ crates/nx_watch/Cargo.toml | 21 ++ crates/nx_watch/src/lib.rs | 4 + crates/nx_workspace/Cargo.toml | 20 ++ crates/nx_workspace/src/lib.rs | 4 + packages/nx/Cargo.toml | 13 +- packages/nx/src/native/cache/cache.rs | 2 +- .../nx/src/native/cache/expand_outputs.rs | 6 +- packages/nx/src/native/db.rs | 27 +++ packages/nx/src/native/hasher.rs | 9 - packages/nx/src/native/machine_id/mod.rs | 2 +- packages/nx/src/native/mod.rs | 7 +- .../native/plugins/js/ts_import_locators.rs | 6 +- packages/nx/src/native/tasks/details.rs | 2 +- .../nx/src/native/tasks/hashers/hash_env.rs | 3 +- .../src/native/tasks/hashers/hash_external.rs | 5 +- .../tasks/hashers/hash_project_config.rs | 4 +- .../tasks/hashers/hash_project_files.rs | 35 ++-- .../src/native/tasks/hashers/hash_runtime.rs | 2 +- .../native/tasks/hashers/hash_task_output.rs | 12 +- .../src/native/tasks/hashers/hash_tsconfig.rs | 2 +- .../tasks/hashers/hash_workspace_files.rs | 9 +- packages/nx/src/native/tasks/task_hasher.rs | 22 +- packages/nx/src/native/tasks/task_history.rs | 2 +- packages/nx/src/native/utils/mod.rs | 1 - packages/nx/src/native/watch/types.rs | 2 +- packages/nx/src/native/workspace/context.rs | 2 +- .../nx/src/native/workspace/files_hashing.rs | 10 +- 50 files changed, 715 insertions(+), 184 deletions(-) create mode 100644 crates/nx_cache/Cargo.toml create mode 100644 crates/nx_cache/src/lib.rs create mode 100644 crates/nx_db/Cargo.toml rename {packages/nx/src/native/db => crates/nx_db/src}/connection.rs (99%) rename {packages/nx/src/native/db => crates/nx_db/src}/initialize.rs (94%) rename packages/nx/src/native/db/mod.rs => crates/nx_db/src/lib.rs (61%) create mode 100644 crates/nx_hasher/Cargo.toml create mode 100644 crates/nx_hasher/src/lib.rs create mode 100644 crates/nx_machine/Cargo.toml create mode 100644 crates/nx_machine/src/lib.rs create mode 100644 crates/nx_project_graph/Cargo.toml create mode 100644 crates/nx_project_graph/src/lib.rs create mode 100644 crates/nx_tasks/Cargo.toml create mode 100644 crates/nx_tasks/src/lib.rs create mode 100644 crates/nx_terminal/Cargo.toml create mode 100644 crates/nx_terminal/src/lib.rs create mode 100644 crates/nx_walker/Cargo.toml rename packages/nx/src/native/walker.rs => crates/nx_walker/src/lib.rs (78%) create mode 100644 crates/nx_walker/src/utils/get_mod_time.rs create mode 100644 crates/nx_walker/src/utils/mod.rs create mode 100644 crates/nx_walker/src/utils/normalize_trait.rs create mode 100644 crates/nx_watch/Cargo.toml create mode 100644 crates/nx_watch/src/lib.rs create mode 100644 crates/nx_workspace/Cargo.toml create mode 100644 crates/nx_workspace/src/lib.rs create mode 100644 packages/nx/src/native/db.rs diff --git a/Cargo.lock b/Cargo.lock index 495b9467acfcd..b315594921c89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1494,8 +1494,19 @@ dependencies = [ "napi", "napi-build", "napi-derive", - "nom", - "once_cell", +"nom", +"once_cell", +"nx_cache", +"nx_db", +"nx_glob", +"nx_hasher", +"nx_machine", +"nx_project_graph", +"nx_tasks", +"nx_terminal", +"nx_walker", +"nx_watch", +"nx_workspace", "parking_lot", "portable-pty", "rayon", @@ -1521,6 +1532,171 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "nx_cache" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_fs", + "nx_db", + "nx_glob", + "nx_hasher", + "nx_walker", + "parking_lot", + "rayon", + "rkyv", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tracing", +] + +[[package]] +name = "nx_db" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_fs", + "fs4", + "parking_lot", + "rusqlite", + "tempfile", + "thiserror", + "tracing", +] + +[[package]] +name = "nx_glob" +version = "0.1.0" +dependencies = [ + "anyhow", + "globset", + "itertools", + "nom", + "tracing", +] + +[[package]] +name = "nx_hasher" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_fs", + "tracing", + "xxhash-rust", +] + +[[package]] +name = "nx_machine" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_fs", + "tracing", +] + +[[package]] +name = "nx_project_graph" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_fs", + "nx_glob", + "nx_hasher", + "nx_walker", + "parking_lot", + "rayon", + "serde", + "serde_json", + "swc_common", + "swc_ecma_ast", + "swc_ecma_dep_graph", + "swc_ecma_parser", + "tempfile", + "thiserror", + "tracing", +] + +[[package]] +name = "nx_tasks" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_fs", + "nx_db", + "parking_lot", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tracing", +] + +[[package]] +name = "nx_terminal" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_fs", + "crossterm", + "mio", + "parking_lot", + "portable-pty", + "tempfile", + "tracing", + "winapi", +] + +[[package]] +name = "nx_walker" +version = "0.1.0" +dependencies = [ + "assert_fs", + "crossbeam-channel", + "ignore", + "nx_glob", + "rayon", + "serde", + "serde_json", + "walkdir", +] + +[[package]] +name = "nx_watch" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_fs", + "ignore-files 2.1.0", + "nx_glob", + "nx_walker", + "parking_lot", + "tempfile", + "tracing", + "watchexec", + "watchexec-events", + "watchexec-filterer-ignore", + "watchexec-signals", +] + +[[package]] +name = "nx_workspace" +version = "0.1.0" +dependencies = [ + "anyhow", + "assert_fs", + "nx_hasher", + "nx_walker", + "parking_lot", + "rayon", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tracing", +] + [[package]] name = "object" version = "0.32.2" @@ -1943,6 +2119,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + [[package]] name = "same-file" version = "1.0.6" @@ -1996,6 +2178,18 @@ dependencies = [ "syn 2.0.53", ] +[[package]] +name = "serde_json" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + [[package]] name = "serial" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 63f3d2fb98218..98471c2f2ebfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,6 @@ [workspace] resolver = "2" -members = [ - "crates/*", - "packages/nx", -] +members = ["crates/*", "packages/nx"] [workspace.package] edition = "2021" @@ -12,6 +9,16 @@ version = "0.1.0" [workspace.dependencies] nx_glob = { path = "crates/nx_glob" } +nx_walker = { path = "crates/nx_walker" } +nx_db = { path = "crates/nx_db" } +nx_hasher = { path = "crates/nx_hasher" } +nx_tasks = { path = "crates/nx_tasks" } +nx_workspace = { path = "crates/nx_workspace" } +nx_project_graph = { path = "crates/nx_project_graph" } +nx_cache = { path = "crates/nx_cache" } +nx_terminal = { path = "crates/nx_terminal" } +nx_watch = { path = "crates/nx_watch" } +nx_machine = { path = "crates/nx_machine" } anyhow = "1.0.95" colored = "2" @@ -44,6 +51,8 @@ swc_common = "0.31.16" swc_ecma_parser = { version = "0.137.1", features = ["typescript"] } swc_ecma_visit = "0.93.0" swc_ecma_ast = "0.107.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" #windows dependencies winapi = { version = "0.3", features = ["fileapi"] } @@ -72,5 +81,3 @@ tempfile = "3.13.0" [profile.release] lto = true - - diff --git a/crates/nx_cache/Cargo.toml b/crates/nx_cache/Cargo.toml new file mode 100644 index 0000000000000..b8c19991267e9 --- /dev/null +++ b/crates/nx_cache/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "nx_cache" +version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +tracing = { workspace = true } +nx_hasher = { workspace = true } +nx_walker = { workspace = true } +nx_glob = { workspace = true } +nx_db = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +parking_lot = { workspace = true } +rayon = { workspace = true } +rkyv = { workspace = true } + +[dev-dependencies] +assert_fs = { workspace = true } +tempfile = { workspace = true } diff --git a/crates/nx_cache/src/lib.rs b/crates/nx_cache/src/lib.rs new file mode 100644 index 0000000000000..23b1f8ae70563 --- /dev/null +++ b/crates/nx_cache/src/lib.rs @@ -0,0 +1,4 @@ +/// Placeholder function for cache operations +pub fn cache_task_result(_task_id: &str, _result: &[u8]) -> anyhow::Result<()> { + Ok(()) +} diff --git a/crates/nx_db/Cargo.toml b/crates/nx_db/Cargo.toml new file mode 100644 index 0000000000000..8b3fa715ff936 --- /dev/null +++ b/crates/nx_db/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "nx_db" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +rusqlite = { workspace = true } +anyhow = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +parking_lot = { workspace = true } +fs4 = { workspace = true } + +[dev-dependencies] +assert_fs = { workspace = true } +tempfile = { workspace = true } diff --git a/packages/nx/src/native/db/connection.rs b/crates/nx_db/src/connection.rs similarity index 99% rename from packages/nx/src/native/db/connection.rs rename to crates/nx_db/src/connection.rs index ef134952982ab..323301a47d624 100644 --- a/packages/nx/src/native/db/connection.rs +++ b/crates/nx_db/src/connection.rs @@ -1,5 +1,4 @@ use anyhow::Result; - use rusqlite::{Connection, DatabaseName, Error, OptionalExtension, Params, Row, Statement, ToSql}; use std::thread; use std::time::Duration; @@ -53,7 +52,7 @@ macro_rules! retry_db_operation_when_busy { } impl NxDbConnection { - pub fn new(connection: Connection) -> Self { + pub(crate) fn new(connection: Connection) -> Self { Self { conn: Some(connection), } diff --git a/packages/nx/src/native/db/initialize.rs b/crates/nx_db/src/initialize.rs similarity index 94% rename from packages/nx/src/native/db/initialize.rs rename to crates/nx_db/src/initialize.rs index b45f8f64bd463..3acc94a0075dd 100644 --- a/packages/nx/src/native/db/initialize.rs +++ b/crates/nx_db/src/initialize.rs @@ -1,15 +1,15 @@ -use crate::native::db::connection::NxDbConnection; +use crate::connection::NxDbConnection; use rusqlite::{Connection, OpenFlags}; use std::fs::{remove_file, File}; use std::path::{Path, PathBuf}; use tracing::{debug, trace}; -pub(super) struct LockFile { +pub(crate) struct LockFile { file: File, path: PathBuf, } -pub(super) fn unlock_file(lock_file: &LockFile) { +pub(crate) fn unlock_file(lock_file: &LockFile) { if lock_file.path.exists() { fs4::fs_std::FileExt::unlock(&lock_file.file) .and_then(|_| remove_file(&lock_file.path)) @@ -17,7 +17,7 @@ pub(super) fn unlock_file(lock_file: &LockFile) { } } -pub(super) fn create_lock_file(db_path: &Path) -> anyhow::Result { +pub(crate) fn create_lock_file(db_path: &Path) -> anyhow::Result { let lock_file_path = db_path.with_extension("lock"); let lock_file = File::create(&lock_file_path) .map_err(|e| anyhow::anyhow!("Unable to create db lock file: {:?}", e))?; @@ -32,7 +32,7 @@ pub(super) fn create_lock_file(db_path: &Path) -> anyhow::Result { }) } -pub(super) fn initialize_db(nx_version: String, db_path: &Path) -> anyhow::Result { +pub(crate) fn initialize_db(nx_version: String, db_path: &Path) -> anyhow::Result { match open_database_connection(db_path) { Ok(mut c) => { trace!( @@ -135,8 +135,6 @@ fn configure_database(connection: &NxDbConnection) -> anyhow::Result<()> { #[cfg(test)] mod tests { - use crate::native::logger::enable_logger; - use super::*; #[test] @@ -181,10 +179,9 @@ mod tests { #[test] fn initialize_db_recreates_incompatible_db() -> anyhow::Result<()> { - enable_logger(); let temp_dir = tempfile::tempdir()?; let db_path = temp_dir.path().join("test.db"); - // + // Create initial db let _ = initialize_db("1.0.0".to_string(), &db_path)?; diff --git a/packages/nx/src/native/db/mod.rs b/crates/nx_db/src/lib.rs similarity index 61% rename from packages/nx/src/native/db/mod.rs rename to crates/nx_db/src/lib.rs index 01c340cdf21b7..d7c82ce20b485 100644 --- a/packages/nx/src/native/db/mod.rs +++ b/crates/nx_db/src/lib.rs @@ -1,28 +1,26 @@ pub mod connection; mod initialize; -use crate::native::logger::enable_logger; -use crate::native::machine_id::get_machine_id; -use crate::native::{db::connection::NxDbConnection, hasher::hash}; -use napi::bindgen_prelude::External; use std::fs::create_dir_all; use std::path::PathBuf; use std::{mem, process}; use tracing::{trace, trace_span}; -#[napi] +use connection::NxDbConnection; + +/// Connect to the Nx database pub fn connect_to_nx_db( cache_dir: String, nx_version: String, db_name: Option, -) -> anyhow::Result> { - enable_logger(); + machine_id: String, +) -> anyhow::Result { let cache_dir_buf = PathBuf::from(cache_dir); - let mut db_file_name = db_name.unwrap_or_else(get_machine_id); + let mut db_file_name = db_name.unwrap_or_else(|| machine_id); if db_file_name.is_empty() { trace!("Invalid db file name, using fallback name"); - db_file_name = hash(b"machine"); + db_file_name = "machine".to_string(); } let db_path = cache_dir_buf.join(format!("{}.db", db_file_name)); @@ -37,12 +35,12 @@ pub fn connect_to_nx_db( initialize::unlock_file(&lock_file); - Ok(External::new(c)) + Ok(c) }) } -#[napi] -pub fn close_db_connection(mut connection: External) -> anyhow::Result<()> { - let conn = mem::take(connection.as_mut()); +/// Close the database connection +pub fn close_db_connection(mut connection: NxDbConnection) -> anyhow::Result<()> { + let conn = mem::take(&mut connection); conn.close() } diff --git a/crates/nx_hasher/Cargo.toml b/crates/nx_hasher/Cargo.toml new file mode 100644 index 0000000000000..9fc8b5532af16 --- /dev/null +++ b/crates/nx_hasher/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "nx_hasher" +version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +tracing = { workspace = true } +xxhash-rust = { workspace = true } + +[dev-dependencies] +assert_fs = { workspace = true } diff --git a/crates/nx_hasher/src/lib.rs b/crates/nx_hasher/src/lib.rs new file mode 100644 index 0000000000000..b5e48a4214fae --- /dev/null +++ b/crates/nx_hasher/src/lib.rs @@ -0,0 +1,98 @@ +use std::path::Path; +use tracing::trace; +use xxhash_rust::xxh3; + +/// Hash raw bytes using xxHash +pub fn hash(content: &[u8]) -> String { + xxh3::xxh3_64(content).to_string() +} + +/// Hash an array of strings by joining them with commas and hashing the result +pub fn hash_array(input: Vec) -> String { + let joined = input.join(","); + let content = joined.as_bytes(); + hash(content) +} + +/// Hash a file's contents given its path as a string +pub fn hash_file(file: String) -> Option { + hash_file_path(file) +} + +/// Hash a file's contents given its path +pub fn hash_file_path>(path: P) -> Option { + let path = path.as_ref(); + trace!("Reading {:?} to hash", path); + let Ok(content) = std::fs::read(path) else { + trace!("Failed to read file: {:?}", path); + return None; + }; + trace!("Hashing {:?}", path); + let hash = hash(&content); + trace!("Hashed file {:?} - {:?}", path, hash); + + Some(hash) +} + +#[cfg(test)] +mod tests { + use super::*; + use assert_fs::prelude::*; + use assert_fs::TempDir; + + /// + /// Setup a temporary directory to do testing in + /// + fn setup_fs() -> TempDir { + let temp = TempDir::new().unwrap(); + temp.child("test.txt").write_str("content").unwrap(); + temp.child("foo.txt").write_str("content1").unwrap(); + temp.child("bar.txt").write_str("content2").unwrap(); + temp.child("baz") + .child("qux.txt") + .write_str("content@qux") + .unwrap(); + temp.child("node_modules") + .child("node-module-dep") + .write_str("content") + .unwrap(); + temp + } + + #[test] + fn test_hash() { + assert_eq!(hash(b"hello world"), "15296390279056496779"); + } + + #[test] + fn test_hash_array() { + let input = vec!["hello".to_string(), "world".to_string()]; + assert_eq!(hash_array(input), "2040998404227468622"); + } + + #[test] + fn test_hash_file() { + // Test non-existent file + assert!(hash_file("".into()).is_none()); + + // Test existing file + let temp_dir = setup_fs(); + let test_file_path = temp_dir.display().to_string() + "/test.txt"; + let content = hash_file(test_file_path); + assert_eq!(content.unwrap(), "6193209363630369380"); + } + + #[test] + fn it_hashes_a_file() { + // handle non existent files + let content = hash_file("".into()); + assert!(content.is_none()); + + let temp_dir = setup_fs(); + + let test_file_path = temp_dir.display().to_string() + "/test.txt"; + let content = hash_file(test_file_path); + + assert_eq!(content.unwrap(), "6193209363630369380"); + } +} diff --git a/crates/nx_machine/Cargo.toml b/crates/nx_machine/Cargo.toml new file mode 100644 index 0000000000000..7ac6d73a431c4 --- /dev/null +++ b/crates/nx_machine/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "nx_machine" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +assert_fs = { workspace = true } diff --git a/crates/nx_machine/src/lib.rs b/crates/nx_machine/src/lib.rs new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/crates/nx_machine/src/lib.rs @@ -0,0 +1 @@ + diff --git a/crates/nx_project_graph/Cargo.toml b/crates/nx_project_graph/Cargo.toml new file mode 100644 index 0000000000000..87e89590a0345 --- /dev/null +++ b/crates/nx_project_graph/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "nx_project_graph" +version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +tracing = { workspace = true } +nx_hasher = { workspace = true } +nx_walker = { workspace = true } +nx_glob = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +parking_lot = { workspace = true } +rayon = { workspace = true } +swc_common = { workspace = true } +swc_ecma_parser = { workspace = true } +swc_ecma_ast = { workspace = true } + +[dev-dependencies] +assert_fs = { workspace = true } +tempfile = { workspace = true } +swc_ecma_dep_graph = { workspace = true } diff --git a/crates/nx_project_graph/src/lib.rs b/crates/nx_project_graph/src/lib.rs new file mode 100644 index 0000000000000..8c2ca4daf0e38 --- /dev/null +++ b/crates/nx_project_graph/src/lib.rs @@ -0,0 +1,4 @@ +/// Placeholder function for project graph operations +pub fn build_project_graph() -> anyhow::Result<()> { + Ok(()) +} diff --git a/crates/nx_tasks/Cargo.toml b/crates/nx_tasks/Cargo.toml new file mode 100644 index 0000000000000..9efb211507fff --- /dev/null +++ b/crates/nx_tasks/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "nx_tasks" +version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +tracing = { workspace = true } +nx_db = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +parking_lot = { workspace = true } + +[dev-dependencies] +assert_fs = { workspace = true } +tempfile = { workspace = true } diff --git a/crates/nx_tasks/src/lib.rs b/crates/nx_tasks/src/lib.rs new file mode 100644 index 0000000000000..8b137891791fe --- /dev/null +++ b/crates/nx_tasks/src/lib.rs @@ -0,0 +1 @@ + diff --git a/crates/nx_terminal/Cargo.toml b/crates/nx_terminal/Cargo.toml new file mode 100644 index 0000000000000..4d6f11ac5080d --- /dev/null +++ b/crates/nx_terminal/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "nx_terminal" +version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +tracing = { workspace = true } +portable-pty = { workspace = true } +crossterm = { workspace = true } +parking_lot = { workspace = true } + +[target.'cfg(windows)'.dependencies] +winapi = { workspace = true } + +[target.'cfg(all(not(windows), not(target_family = "wasm")))'.dependencies] +mio = { workspace = true } + +[dev-dependencies] +assert_fs = { workspace = true } +tempfile = { workspace = true } diff --git a/crates/nx_terminal/src/lib.rs b/crates/nx_terminal/src/lib.rs new file mode 100644 index 0000000000000..ddfc4c434d61e --- /dev/null +++ b/crates/nx_terminal/src/lib.rs @@ -0,0 +1,4 @@ +/// Placeholder function for terminal operations +pub fn create_terminal() -> anyhow::Result<()> { + Ok(()) +} diff --git a/crates/nx_walker/Cargo.toml b/crates/nx_walker/Cargo.toml new file mode 100644 index 0000000000000..62aac82c0c25b --- /dev/null +++ b/crates/nx_walker/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "nx_walker" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +ignore = { workspace = true } +walkdir = { workspace = true } +rayon = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +nx_glob = { workspace = true } +crossbeam-channel = { workspace = true } + +[dev-dependencies] +assert_fs = { workspace = true } diff --git a/packages/nx/src/native/walker.rs b/crates/nx_walker/src/lib.rs similarity index 78% rename from packages/nx/src/native/walker.rs rename to crates/nx_walker/src/lib.rs index bb95dd5806a77..d053c90345918 100644 --- a/packages/nx/src/native/walker.rs +++ b/crates/nx_walker/src/lib.rs @@ -1,12 +1,12 @@ +mod utils; + use ignore::WalkBuilder; +use nx_glob::NxGlobSet; use std::fmt::Debug; use std::path::{Path, PathBuf}; - - -use crate::native::logger::enable_logger; -use crate::native::utils::{get_mod_time, Normalize}; use walkdir::WalkDir; -use nx_glob::NxGlobSet; + +use crate::utils::{get_mod_time, Normalize}; #[derive(PartialEq, Debug, Ord, PartialOrd, Eq, Clone)] pub struct NxFile { @@ -50,14 +50,13 @@ where !ignore_glob_set.is_match(path.as_ref()) }) .filter_map(move |entry| { - entry - .ok() - .and_then(|e| - e.path() - .strip_prefix(&base_dir).ok() + entry.ok().and_then(|e| { + e.path() + .strip_prefix(&base_dir) + .ok() .filter(|p| !p.to_string_lossy().is_empty()) .map(|p| p.to_owned()) - ) + }) }) } @@ -107,8 +106,6 @@ where use std::thread::available_parallelism; use crossbeam_channel::unbounded; - use tracing::trace; - enable_logger(); let directory = directory.as_ref(); let mut walker = create_walker(directory, use_ignores); @@ -117,8 +114,6 @@ where let (sender, receiver) = unbounded(); - trace!(?directory, "walking"); - let now = std::time::Instant::now(); walker.threads(cpus).build_parallel().run(|| { let tx = sender.clone(); @@ -131,7 +126,7 @@ where if dir_entry.file_type().is_some_and(|d| d.is_dir()) { return Continue; - } + }; let Ok(file_path) = dir_entry.path().strip_prefix(directory) else { return Continue; @@ -151,7 +146,6 @@ where Continue }) }); - trace!("walked in {:?}", now.elapsed()); let receiver_thread = thread::spawn(move || receiver.into_iter()); drop(sender); @@ -246,73 +240,23 @@ mod test { fn handles_nx_ignore() { let temp_dir = setup_fs(); - temp_dir - .child("nested") - .child("child.txt") - .write_str("data") - .unwrap(); - temp_dir - .child("nested") - .child("child-two") - .child("grand_child.txt") - .write_str("data") - .unwrap(); - temp_dir - .child("v1") - .child("packages") - .child("pkg-a") - .child("pkg-a.txt") - .write_str("data") - .unwrap(); - temp_dir - .child("v1") - .child("packages") - .child("pkg-b") - .child("pkg-b.txt") - .write_str("data") - .unwrap(); - temp_dir - .child("packages") - .child("pkg-c") - .child("pkg-c.txt") - .write_str("data") - .unwrap(); - - // add nxignore file temp_dir .child(".nxignore") - .write_str( - r"baz/ -nested/child.txt -nested/child-two/ - -# this should only ignore root level packages, not nested -/packages - ", - ) + .write_str("foo.txt\nbar.txt") .unwrap(); - let mut file_names = nx_walker(temp_dir, true) - .map( - |NxFile { - normalized_path: relative_path, - .. - }| relative_path, - ) + let mut content = nx_walker(&temp_dir, true).collect::>(); + content.sort(); + let content = content + .into_iter() + .map(|f| (f.full_path.into(), f.normalized_path.into())) .collect::>(); - - file_names.sort(); - assert_eq!( - file_names, - vec!( - ".nxignore", - "bar.txt", - "foo.txt", - "test.txt", - "v1/packages/pkg-a/pkg-a.txt", - "v1/packages/pkg-b/pkg-b.txt" - ) + content, + vec![ + (temp_dir.join("baz/qux.txt"), PathBuf::from("baz/qux.txt")), + (temp_dir.join("test.txt"), PathBuf::from("test.txt")), + ] ); } } diff --git a/crates/nx_walker/src/utils/get_mod_time.rs b/crates/nx_walker/src/utils/get_mod_time.rs new file mode 100644 index 0000000000000..9cf2efe157593 --- /dev/null +++ b/crates/nx_walker/src/utils/get_mod_time.rs @@ -0,0 +1,25 @@ +use std::fs::Metadata; + +#[cfg(target_os = "macos")] +pub fn get_mod_time(metadata: &Metadata) -> i64 { + use std::os::macos::fs::MetadataExt; + metadata.st_mtime() +} + +#[cfg(target_os = "windows")] +pub fn get_mod_time(metadata: &Metadata) -> i64 { + use std::os::windows::fs::MetadataExt; + metadata.last_write_time() as i64 +} + +#[cfg(any(target_os = "linux", target_os = "freebsd"))] +pub fn get_mod_time(metadata: &Metadata) -> i64 { + use std::os::unix::fs::MetadataExt; + metadata.mtime() +} + +#[cfg(target_os = "wasi")] +pub fn get_mod_time(metadata: &Metadata) -> i64 { + use std::os::wasi::fs::MetadataExt; + metadata.mtim() as i64 +} diff --git a/crates/nx_walker/src/utils/mod.rs b/crates/nx_walker/src/utils/mod.rs new file mode 100644 index 0000000000000..7a849acf0f9d3 --- /dev/null +++ b/crates/nx_walker/src/utils/mod.rs @@ -0,0 +1,5 @@ +mod get_mod_time; +mod normalize_trait; + +pub use get_mod_time::*; +pub use normalize_trait::Normalize; diff --git a/crates/nx_walker/src/utils/normalize_trait.rs b/crates/nx_walker/src/utils/normalize_trait.rs new file mode 100644 index 0000000000000..bf50b76011276 --- /dev/null +++ b/crates/nx_walker/src/utils/normalize_trait.rs @@ -0,0 +1,17 @@ +use std::path::{Path, PathBuf}; + +pub trait Normalize { + fn to_normalized_string(&self) -> String; +} + +impl Normalize for Path { + fn to_normalized_string(&self) -> String { + self.to_string_lossy().replace('\\', "/") + } +} + +impl Normalize for PathBuf { + fn to_normalized_string(&self) -> String { + self.as_path().to_normalized_string() + } +} diff --git a/crates/nx_watch/Cargo.toml b/crates/nx_watch/Cargo.toml new file mode 100644 index 0000000000000..012ef9bef19bd --- /dev/null +++ b/crates/nx_watch/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "nx_watch" +version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +tracing = { workspace = true } +nx_walker = { workspace = true } +nx_glob = { workspace = true } +watchexec = { workspace = true } +watchexec-events = { workspace = true } +watchexec-filterer-ignore = { workspace = true } +watchexec-signals = { workspace = true } +ignore-files = { workspace = true } +parking_lot = { workspace = true } + +[dev-dependencies] +assert_fs = { workspace = true } +tempfile = { workspace = true } diff --git a/crates/nx_watch/src/lib.rs b/crates/nx_watch/src/lib.rs new file mode 100644 index 0000000000000..5920c5c4ef00f --- /dev/null +++ b/crates/nx_watch/src/lib.rs @@ -0,0 +1,4 @@ +/// Placeholder function for file watching +pub fn watch_directory(_path: &str) -> anyhow::Result<()> { + Ok(()) +} diff --git a/crates/nx_workspace/Cargo.toml b/crates/nx_workspace/Cargo.toml new file mode 100644 index 0000000000000..c030e10012865 --- /dev/null +++ b/crates/nx_workspace/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "nx_workspace" +version = { workspace = true } +edition = { workspace = true } +license = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +tracing = { workspace = true } +nx_hasher = { workspace = true } +nx_walker = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +parking_lot = { workspace = true } +rayon = { workspace = true } + +[dev-dependencies] +assert_fs = { workspace = true } +tempfile = { workspace = true } diff --git a/crates/nx_workspace/src/lib.rs b/crates/nx_workspace/src/lib.rs new file mode 100644 index 0000000000000..9c265ce3a2b38 --- /dev/null +++ b/crates/nx_workspace/src/lib.rs @@ -0,0 +1,4 @@ +/// Placeholder function for workspace operations +pub fn get_workspace_root() -> anyhow::Result { + Ok("placeholder_workspace".to_string()) +} diff --git a/packages/nx/Cargo.toml b/packages/nx/Cargo.toml index efab814a96549..91ccbd955240c 100644 --- a/packages/nx/Cargo.toml +++ b/packages/nx/Cargo.toml @@ -9,7 +9,17 @@ crate-type = ['cdylib'] [dependencies] -nx_glob = { workspace = true} +nx_glob = { workspace = true } +nx_walker = { workspace = true } +nx_db = { workspace = true } +nx_machine = { workspace = true } +nx_hasher = { workspace = true } +nx_tasks = { workspace = true } +nx_workspace = { workspace = true } +nx_project_graph = { workspace = true } +nx_cache = { workspace = true } +nx_terminal = { workspace = true } +nx_watch = { workspace = true } anyhow = { workspace = true } colored = { workspace = true } crossbeam-channel = { workspace = true } @@ -54,7 +64,6 @@ watchexec-signals = { workspace = true } machine-uid = { workspace = true } - [build-dependencies] napi-build = { workspace = true } diff --git a/packages/nx/src/native/cache/cache.rs b/packages/nx/src/native/cache/cache.rs index 2def54185744c..f90deb6adbda6 100644 --- a/packages/nx/src/native/cache/cache.rs +++ b/packages/nx/src/native/cache/cache.rs @@ -10,8 +10,8 @@ use tracing::trace; use crate::native::cache::expand_outputs::_expand_outputs; use crate::native::cache::file_ops::_copy; -use crate::native::db::connection::NxDbConnection; use crate::native::utils::Normalize; +use nx_db::connection::NxDbConnection; #[napi(object)] #[derive(Default, Clone, Debug)] diff --git a/packages/nx/src/native/cache/expand_outputs.rs b/packages/nx/src/native/cache/expand_outputs.rs index 5f3a34beb35fb..710dfd1463c87 100644 --- a/packages/nx/src/native/cache/expand_outputs.rs +++ b/packages/nx/src/native/cache/expand_outputs.rs @@ -2,11 +2,11 @@ use hashbrown::HashMap; use std::path::{Path, PathBuf}; use tracing::trace; -use nx_glob::{contains_glob_pattern, partition_glob}; -use nx_glob::NxGlobSet; use crate::native::logger::enable_logger; use crate::native::utils::Normalize; -use crate::native::walker::{nx_walker, nx_walker_sync}; +use nx_glob::NxGlobSet; +use nx_glob::{contains_glob_pattern, partition_glob}; +use nx_walker::{nx_walker, nx_walker_sync}; #[napi] pub fn expand_outputs(directory: String, entries: Vec) -> anyhow::Result> { diff --git a/packages/nx/src/native/db.rs b/packages/nx/src/native/db.rs new file mode 100644 index 0000000000000..df626b6deed98 --- /dev/null +++ b/packages/nx/src/native/db.rs @@ -0,0 +1,27 @@ +use crate::native::logger::enable_logger; +use crate::native::machine_id::get_machine_id; +use napi::bindgen_prelude::External; +use std::fs::create_dir_all; +use std::path::PathBuf; + +pub use nx_db::connection::NxDbConnection; + +#[napi] +pub fn connect_to_nx_db( + cache_dir: String, + nx_version: String, + db_name: Option, +) -> anyhow::Result> { + enable_logger(); + let cache_dir_buf = PathBuf::from(&cache_dir); + create_dir_all(&cache_dir_buf)?; + + let machine_id = get_machine_id(); + let connection = nx_db::connect_to_nx_db(cache_dir, nx_version, db_name, machine_id)?; + Ok(External::new(connection)) +} + +#[napi] +pub fn close_db_connection(mut connection: External) -> anyhow::Result<()> { + nx_db::close_db_connection(std::mem::take(connection.as_mut())) +} diff --git a/packages/nx/src/native/hasher.rs b/packages/nx/src/native/hasher.rs index 872ff65cbb52e..3485bdc9bed43 100644 --- a/packages/nx/src/native/hasher.rs +++ b/packages/nx/src/native/hasher.rs @@ -1,12 +1,3 @@ -use std::path::Path; - -use tracing::trace; -use xxhash_rust::xxh3; - -pub fn hash(content: &[u8]) -> String { - xxh3::xxh3_64(content).to_string() -} - #[napi] pub fn hash_array(input: Vec>) -> String { let joined = input.iter().filter_map(|s| { diff --git a/packages/nx/src/native/machine_id/mod.rs b/packages/nx/src/native/machine_id/mod.rs index 6c471eb2c3ccb..e1843ec81201f 100644 --- a/packages/nx/src/native/machine_id/mod.rs +++ b/packages/nx/src/native/machine_id/mod.rs @@ -4,8 +4,8 @@ pub fn get_machine_id() -> String { #[cfg(target_arch = "wasm32")] { - use crate::native::hasher::hash; use crate::native::tasks::hashers::create_command_builder; + use nx_hasher::hash; use std::fs::read_to_string; hash( diff --git a/packages/nx/src/native/mod.rs b/packages/nx/src/native/mod.rs index 990ed97221a8c..99f3b4b4d73dd 100644 --- a/packages/nx/src/native/mod.rs +++ b/packages/nx/src/native/mod.rs @@ -1,19 +1,18 @@ pub mod cache; pub mod hasher; mod logger; +mod machine_id; pub mod metadata; pub mod plugins; pub mod project_graph; pub mod tasks; mod types; mod utils; -mod walker; pub mod workspace; -mod machine_id; +#[cfg(not(target_arch = "wasm32"))] +pub mod db; #[cfg(not(target_arch = "wasm32"))] pub mod pseudo_terminal; #[cfg(not(target_arch = "wasm32"))] pub mod watch; -#[cfg(not(target_arch = "wasm32"))] -pub mod db; diff --git a/packages/nx/src/native/plugins/js/ts_import_locators.rs b/packages/nx/src/native/plugins/js/ts_import_locators.rs index 9ce74f5b64609..c76756eba8d34 100644 --- a/packages/nx/src/native/plugins/js/ts_import_locators.rs +++ b/packages/nx/src/native/plugins/js/ts_import_locators.rs @@ -713,10 +713,10 @@ fn find_imports( #[cfg(test)] mod find_imports { use super::*; - use nx_glob::NxGlobSet; - use crate::native::walker::nx_walker; use assert_fs::prelude::*; use assert_fs::TempDir; + use nx_glob::NxGlobSet; + use nx_walker::nx_walker; use std::env; use std::path::PathBuf; use swc_common::comments::NoopComments; @@ -962,7 +962,7 @@ import('./dynamic-import.vue') - Vue’s + Vue's official documentation provides you with all information you need to get started. diff --git a/packages/nx/src/native/tasks/details.rs b/packages/nx/src/native/tasks/details.rs index ac972f90d6be6..2d628e5e9cfc3 100644 --- a/packages/nx/src/native/tasks/details.rs +++ b/packages/nx/src/native/tasks/details.rs @@ -1,5 +1,5 @@ -use crate::native::db::connection::NxDbConnection; use napi::bindgen_prelude::*; +use nx_db::connection::NxDbConnection; use rusqlite::params; use tracing::trace; diff --git a/packages/nx/src/native/tasks/hashers/hash_env.rs b/packages/nx/src/native/tasks/hashers/hash_env.rs index fc093f8057c18..497632e7b8687 100644 --- a/packages/nx/src/native/tasks/hashers/hash_env.rs +++ b/packages/nx/src/native/tasks/hashers/hash_env.rs @@ -1,6 +1,7 @@ -use crate::native::hasher::hash; use std::collections::HashMap; +use nx_hasher::hash; + pub fn hash_env(env_name: &str, env: &HashMap) -> String { let env_value = env.get(env_name).map(|s| s.as_str()).unwrap_or(""); hash(env_value.as_bytes()) diff --git a/packages/nx/src/native/tasks/hashers/hash_external.rs b/packages/nx/src/native/tasks/hashers/hash_external.rs index 2fa7490477f23..7b90d49956735 100644 --- a/packages/nx/src/native/tasks/hashers/hash_external.rs +++ b/packages/nx/src/native/tasks/hashers/hash_external.rs @@ -1,10 +1,11 @@ -use crate::native::hasher::{hash, hash_array}; -use crate::native::project_graph::types::ExternalNode; use std::collections::HashMap; use std::sync::Arc; use anyhow::*; use dashmap::DashMap; +use nx_hasher::{hash, hash_array}; + +use crate::native::project_graph::types::ExternalNode; pub fn hash_external( external_name: &str, diff --git a/packages/nx/src/native/tasks/hashers/hash_project_config.rs b/packages/nx/src/native/tasks/hashers/hash_project_config.rs index 4ea79d4209967..3b275f53cc74a 100644 --- a/packages/nx/src/native/tasks/hashers/hash_project_config.rs +++ b/packages/nx/src/native/tasks/hashers/hash_project_config.rs @@ -2,8 +2,8 @@ use std::collections::HashMap; use anyhow::*; use itertools::Itertools; +use nx_hasher::hash; -use crate::native::hasher::hash; use crate::native::project_graph::types::Project; use crate::native::types::Input; @@ -17,7 +17,6 @@ pub fn hash_project_config( let targets = project .targets .iter() - .map(|(k, v)| (k, v)) .sorted_by(|a, b| a.0.cmp(b.0)) .map(|(k, v)| { format!( @@ -40,7 +39,6 @@ pub fn hash_project_config( .map(|inputs| { inputs .iter() - .map(|(k, v)| (k, v)) .sorted_by(|a, b| a.0.cmp(b.0)) .map(|(_, v)| { v.iter() diff --git a/packages/nx/src/native/tasks/hashers/hash_project_files.rs b/packages/nx/src/native/tasks/hashers/hash_project_files.rs index 501bacbe7dc30..cf8979dc74dea 100644 --- a/packages/nx/src/native/tasks/hashers/hash_project_files.rs +++ b/packages/nx/src/native/tasks/hashers/hash_project_files.rs @@ -3,8 +3,8 @@ use std::collections::HashMap; use anyhow::*; use tracing::{trace, trace_span}; -use nx_glob::NxGlobSet; use crate::native::types::FileData; +use nx_glob::NxGlobSet; pub fn hash_project_files( project_name: &str, @@ -60,9 +60,8 @@ fn collect_files<'a>( } #[cfg(test)] mod tests { - use crate::native::hasher::hash; - use super::*; + use nx_hasher::hash; use std::collections::HashMap; #[test] @@ -158,12 +157,15 @@ mod tests { let hash_result = hash_project_files(proj_name, proj_root, file_sets, &file_map).unwrap(); assert_eq!( hash_result, - hash(&[ - file_data1.hash.as_bytes(), - file_data1.file.as_bytes(), - file_data3.hash.as_bytes(), - file_data3.file.as_bytes() - ].concat()) + hash( + &[ + file_data1.hash.as_bytes(), + file_data1.file.as_bytes(), + file_data3.hash.as_bytes(), + file_data3.file.as_bytes() + ] + .concat() + ) ); } @@ -205,12 +207,15 @@ mod tests { let hash_result = hash_project_files(proj_name, proj_root, file_sets, &file_map).unwrap(); assert_eq!( hash_result, - hash(&[ - file_data1.hash.as_bytes(), - file_data1.file.as_bytes(), - file_data3.hash.as_bytes(), - file_data3.file.as_bytes(), - ].concat()) + hash( + &[ + file_data1.hash.as_bytes(), + file_data1.file.as_bytes(), + file_data3.hash.as_bytes(), + file_data3.file.as_bytes(), + ] + .concat() + ) ); } } diff --git a/packages/nx/src/native/tasks/hashers/hash_runtime.rs b/packages/nx/src/native/tasks/hashers/hash_runtime.rs index bcb4bd04e0e4b..81b3d5975d1b2 100644 --- a/packages/nx/src/native/tasks/hashers/hash_runtime.rs +++ b/packages/nx/src/native/tasks/hashers/hash_runtime.rs @@ -1,5 +1,5 @@ -use crate::native::hasher::hash; use dashmap::DashMap; +use nx_hasher::hash; use std::collections::HashMap; use std::process::Command; use std::sync::Arc; diff --git a/packages/nx/src/native/tasks/hashers/hash_task_output.rs b/packages/nx/src/native/tasks/hashers/hash_task_output.rs index c8b4e43c4e79d..b0aed971a845c 100644 --- a/packages/nx/src/native/tasks/hashers/hash_task_output.rs +++ b/packages/nx/src/native/tasks/hashers/hash_task_output.rs @@ -1,7 +1,7 @@ use crate::native::cache::expand_outputs::get_files_for_outputs; -use crate::native::hasher::{hash_array, hash_file}; use anyhow::*; use nx_glob::NxGlobSet; +use nx_hasher::{hash_array, hash_file}; use rayon::prelude::*; use std::path::Path; use tracing::trace; @@ -14,7 +14,15 @@ pub fn hash_task_output(workspace_root: &str, glob: &str, outputs: &[String]) -> let hashes = output_files .into_par_iter() .filter(|file| glob.is_match(file)) - .filter_map(|file| hash_file(Path::new(workspace_root).join(file).to_str().expect("path contains invalid utf-8").to_owned())) + .filter_map(|file| { + hash_file( + Path::new(workspace_root) + .join(file) + .to_str() + .expect("path contains invalid utf-8") + .to_owned(), + ) + }) .collect::>(); Ok(hash_array(hashes.into_iter().map(Some).collect())) } diff --git a/packages/nx/src/native/tasks/hashers/hash_tsconfig.rs b/packages/nx/src/native/tasks/hashers/hash_tsconfig.rs index dc080cbea4044..ad7b3acb3862a 100644 --- a/packages/nx/src/native/tasks/hashers/hash_tsconfig.rs +++ b/packages/nx/src/native/tasks/hashers/hash_tsconfig.rs @@ -1,8 +1,8 @@ use std::collections::HashMap; use anyhow::*; +use nx_hasher::hash; -use crate::native::hasher::hash; use crate::native::project_graph::utils::find_project_for_path; pub fn hash_tsconfig_selectively( diff --git a/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs b/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs index af6826ab64eca..6c3b30bcfda8a 100644 --- a/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs +++ b/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs @@ -1,11 +1,11 @@ use std::sync::Arc; +use crate::native::types::FileData; use anyhow::*; use dashmap::DashMap; -use tracing::{debug, debug_span, trace, warn}; use nx_glob::NxGlobSet; -use crate::native::hasher::hash; -use crate::native::types::FileData; +use nx_hasher::hash; +use tracing::{debug, debug_span, trace, warn}; pub fn hash_workspace_files( workspace_file_sets: &[String], @@ -66,10 +66,9 @@ pub fn hash_workspace_files( #[cfg(test)] mod test { - use crate::native::hasher::hash; - use super::*; use dashmap::DashMap; + use nx_hasher::hash; use std::sync::Arc; #[test] diff --git a/packages/nx/src/native/tasks/task_hasher.rs b/packages/nx/src/native/tasks/task_hasher.rs index 4c52843eaa4f0..a5fd4b8488b03 100644 --- a/packages/nx/src/native/tasks/task_hasher.rs +++ b/packages/nx/src/native/tasks/task_hasher.rs @@ -1,30 +1,26 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::native::{ - hasher::hash, - project_graph::{types::ProjectGraph, utils::create_project_root_mappings}, - tasks::types::HashInstruction, - types::NapiDashMap, -}; -use crate::native::{ - project_graph::utils::ProjectRootMappings, - tasks::hashers::{hash_env, hash_runtime, hash_workspace_files}, -}; +use crate::native::project_graph::{types::ProjectGraph, utils::create_project_root_mappings}; use crate::native::{ tasks::hashers::{ - hash_all_externals, hash_external, hash_project_config, hash_project_files, - hash_task_output, hash_tsconfig_selectively, + hash_all_externals, hash_env, hash_external, hash_project_config, hash_project_files, + hash_runtime, hash_task_output, hash_tsconfig_selectively, hash_workspace_files, }, - types::FileData, + tasks::types::HashInstruction, + types::{FileData, NapiDashMap}, workspace::types::ProjectFiles, }; + use anyhow::anyhow; use dashmap::DashMap; use napi::bindgen_prelude::{Buffer, External}; +use nx_hasher::hash; use rayon::prelude::*; use tracing::{debug, trace, trace_span}; +use crate::native::project_graph::utils::ProjectRootMappings; + #[napi(object)] #[derive(Debug)] pub struct HashDetails { diff --git a/packages/nx/src/native/tasks/task_history.rs b/packages/nx/src/native/tasks/task_history.rs index 41adad17ef408..8262785e6b41c 100644 --- a/packages/nx/src/native/tasks/task_history.rs +++ b/packages/nx/src/native/tasks/task_history.rs @@ -1,6 +1,6 @@ -use crate::native::db::connection::NxDbConnection; use crate::native::tasks::types::TaskTarget; use napi::bindgen_prelude::*; +use nx_db::connection::NxDbConnection; use rusqlite::vtab::array; use rusqlite::{params, types::Value}; use std::collections::HashMap; diff --git a/packages/nx/src/native/utils/mod.rs b/packages/nx/src/native/utils/mod.rs index ddd46fee2fbf7..c068479c9ecb8 100644 --- a/packages/nx/src/native/utils/mod.rs +++ b/packages/nx/src/native/utils/mod.rs @@ -4,7 +4,6 @@ mod normalize_trait; pub mod path; pub use find_matching_projects::*; -pub use get_mod_time::*; pub use normalize_trait::Normalize; #[cfg_attr(not(target_arch = "wasm32"), path = "atomics/default.rs")] diff --git a/packages/nx/src/native/watch/types.rs b/packages/nx/src/native/watch/types.rs index 0df9f96cf2081..bd32fbe722c1f 100644 --- a/packages/nx/src/native/watch/types.rs +++ b/packages/nx/src/native/watch/types.rs @@ -123,9 +123,9 @@ pub fn transform_event_to_watch_events( #[cfg(all(not(target_os = "macos"), not(target_os = "windows")))] { - use crate::native::walker::nx_walker_sync; use ignore::gitignore::GitignoreBuilder; use ignore::Match; + use nx_walker::nx_walker_sync; if matches!(event_kind, FileEventKind::Create(CreateKind::Folder)) { let mut result = vec![]; diff --git a/packages/nx/src/native/workspace/context.rs b/packages/nx/src/native/workspace/context.rs index 90d82a601e573..5de621db10eae 100644 --- a/packages/nx/src/native/workspace/context.rs +++ b/packages/nx/src/native/workspace/context.rs @@ -3,7 +3,6 @@ use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::Arc; -use crate::native::hasher::hash; use crate::native::logger::enable_logger; use crate::native::project_graph::utils::{find_project_for_path, ProjectRootMappings}; use crate::native::types::FileData; @@ -15,6 +14,7 @@ use crate::native::workspace::types::{ }; use crate::native::workspace::{config_files, types::NxWorkspaceFiles, workspace_files}; use napi::bindgen_prelude::External; +use nx_hasher::hash; use rayon::prelude::*; use tracing::{trace, warn}; use xxhash_rust::xxh3; diff --git a/packages/nx/src/native/workspace/files_hashing.rs b/packages/nx/src/native/workspace/files_hashing.rs index 087d4cedd43fa..1a48f22c7bea0 100644 --- a/packages/nx/src/native/workspace/files_hashing.rs +++ b/packages/nx/src/native/workspace/files_hashing.rs @@ -5,9 +5,9 @@ use std::thread::available_parallelism; use rayon::prelude::*; use tracing::trace; -use crate::native::hasher::hash_file_path; -use crate::native::walker::{nx_walker, NxFile}; use crate::native::workspace::files_archive::{NxFileHashed, NxFileHashes}; +use nx_hasher::hash_file_path; +use nx_walker::{nx_walker, NxFile}; pub fn full_files_hash(workspace_root: &Path) -> NxFileHashes { let files = nx_walker(workspace_root, true).collect::>(); @@ -61,7 +61,11 @@ fn hash_files(files: Vec) -> Vec<(String, NxFileHashed)> { }) .collect::>() } else { - trace!("hashing workspace files in {} chunks of {}", num_parallelism, chunks); + trace!( + "hashing workspace files in {} chunks of {}", + num_parallelism, + chunks + ); files .par_chunks(chunks) .flat_map_iter(|chunks| { From e3344df569d525611b8ebe5ab487d9df5c022997 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli <4332460+Cammisuli@users.noreply.github.com> Date: Thu, 20 Feb 2025 16:43:30 -0500 Subject: [PATCH 03/11] feat(nx_utils): extract reusable utilities into new crate - Moved `get_mod_time` and normalization utilities from `nx` and `nx_walker` to a new crate `nx_utils`. - Updated references across the codebase to use `nx_utils` for these utilities. - Added `hash_array_optional` function in `nx_hasher` and refactored its usage in `nx`. - Updated `Cargo.toml` and `Cargo.lock` to include `nx_utils` and other dependency updates. - Removed unused utilities and cleaned up imports in various modules. --- Cargo.lock | 90 +++++++++---------- Cargo.toml | 2 + crates/nx_hasher/src/lib.rs | 17 ++++ crates/nx_utils/Cargo.toml | 7 ++ .../utils => nx_utils/src}/get_mod_time.rs | 0 crates/nx_utils/src/lib.rs | 8 ++ .../src/to_normalized_string.rs} | 6 +- crates/nx_walker/Cargo.toml | 1 + crates/nx_walker/src/lib.rs | 5 +- crates/nx_walker/src/utils/mod.rs | 5 -- packages/nx/Cargo.toml | 1 + packages/nx/src/native/hasher.rs | 71 +-------------- packages/nx/src/native/utils/get_mod_time.rs | 25 ------ packages/nx/src/native/utils/mod.rs | 4 +- .../nx/src/native/utils/normalize_trait.rs | 3 - packages/nx/src/native/utils/path.rs | 14 +-- .../nx/src/native/workspace/files_hashing.rs | 5 +- 17 files changed, 91 insertions(+), 173 deletions(-) create mode 100644 crates/nx_utils/Cargo.toml rename crates/{nx_walker/src/utils => nx_utils/src}/get_mod_time.rs (100%) create mode 100644 crates/nx_utils/src/lib.rs rename crates/{nx_walker/src/utils/normalize_trait.rs => nx_utils/src/to_normalized_string.rs} (73%) delete mode 100644 crates/nx_walker/src/utils/mod.rs delete mode 100644 packages/nx/src/native/utils/get_mod_time.rs delete mode 100644 packages/nx/src/native/utils/normalize_trait.rs diff --git a/Cargo.lock b/Cargo.lock index b315594921c89..7dc523b6e24a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,9 +73,9 @@ checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anyhow" -version = "1.0.81" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" [[package]] name = "assert_fs" @@ -375,7 +375,7 @@ dependencies = [ "bitflags 2.6.0", "crossterm_winapi", "libc", - "mio", + "mio 0.8.11", "parking_lot", "signal-hook", "signal-hook-mio", @@ -962,12 +962,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - [[package]] name = "home" version = "0.5.9" @@ -1130,9 +1124,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libloading" @@ -1272,6 +1266,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.52.0", +] + [[package]] name = "napi" version = "2.16.0" @@ -1401,7 +1406,7 @@ dependencies = [ "kqueue", "libc", "log", - "mio", + "mio 0.8.11", "walkdir", "windows-sys 0.48.0", ] @@ -1452,16 +1457,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "num_threads" version = "0.1.7" @@ -1484,29 +1479,27 @@ dependencies = [ "dunce", "fs4", "fs_extra", - "globset", "hashbrown 0.14.5", "ignore", "ignore-files 2.1.0", "itertools", "machine-uid", - "mio", + "mio 0.8.11", "napi", "napi-build", "napi-derive", -"nom", -"once_cell", -"nx_cache", -"nx_db", -"nx_glob", -"nx_hasher", -"nx_machine", -"nx_project_graph", -"nx_tasks", -"nx_terminal", -"nx_walker", -"nx_watch", -"nx_workspace", + "nx_cache", + "nx_db", + "nx_glob", + "nx_hasher", + "nx_machine", + "nx_project_graph", + "nx_tasks", + "nx_terminal", + "nx_utils", + "nx_walker", + "nx_watch", + "nx_workspace", "parking_lot", "portable-pty", "rayon", @@ -1517,7 +1510,6 @@ dependencies = [ "swc_ecma_ast", "swc_ecma_dep_graph", "swc_ecma_parser", - "swc_ecma_visit", "tempfile", "thiserror", "tokio", @@ -1640,7 +1632,7 @@ dependencies = [ "anyhow", "assert_fs", "crossterm", - "mio", + "mio 0.8.11", "parking_lot", "portable-pty", "tempfile", @@ -1648,6 +1640,10 @@ dependencies = [ "winapi", ] +[[package]] +name = "nx_utils" +version = "0.1.0" + [[package]] name = "nx_walker" version = "0.1.0" @@ -1656,6 +1652,7 @@ dependencies = [ "crossbeam-channel", "ignore", "nx_glob", + "nx_utils", "rayon", "serde", "serde_json", @@ -2286,7 +2283,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" dependencies = [ "libc", - "mio", + "mio 0.8.11", "signal-hook", ] @@ -2683,26 +2680,25 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", "libc", - "mio", - "num_cpus", + "mio 1.0.3", "pin-project-lite", "signal-hook-registry", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 98471c2f2ebfd..ed544ef15095a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ nx_cache = { path = "crates/nx_cache" } nx_terminal = { path = "crates/nx_terminal" } nx_watch = { path = "crates/nx_watch" } nx_machine = { path = "crates/nx_machine" } +nx_utils = { path = "crates/nx_utils"} anyhow = "1.0.95" colored = "2" @@ -78,6 +79,7 @@ assert_fs = "1.0.10" # This is only used for unit tests swc_ecma_dep_graph = "0.109.1" tempfile = "3.13.0" +tokio = "1.38.0" [profile.release] lto = true diff --git a/crates/nx_hasher/src/lib.rs b/crates/nx_hasher/src/lib.rs index b5e48a4214fae..836ee2afcfc1d 100644 --- a/crates/nx_hasher/src/lib.rs +++ b/crates/nx_hasher/src/lib.rs @@ -7,6 +7,16 @@ pub fn hash(content: &[u8]) -> String { xxh3::xxh3_64(content).to_string() } +pub fn hash_array_optional(input: Vec>) -> String { + let joined = input.iter().filter_map(|s| { + if s.is_none() { + trace!("Encountered None value in hash_array input: {:?}", input); + } + s.as_deref() + }).collect::>().join(","); + hash(joined.as_bytes()) +} + /// Hash an array of strings by joining them with commas and hashing the result pub fn hash_array(input: Vec) -> String { let joined = input.join(","); @@ -95,4 +105,11 @@ mod tests { assert_eq!(content.unwrap(), "6193209363630369380"); } + + #[test] + fn it_hashes_an_array() { + // Resilient to None values (e.g. null values passed from the JS side) + let content = hash_array_optional(vec![Some("foo".to_string()), None, Some("bar".to_string())]); + assert_eq!(content, "10292076446133652019"); + } } diff --git a/crates/nx_utils/Cargo.toml b/crates/nx_utils/Cargo.toml new file mode 100644 index 0000000000000..dd531d3ca6517 --- /dev/null +++ b/crates/nx_utils/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "nx_utils" +edition.workspace = true +license.workspace = true +version.workspace = true + +[dependencies] diff --git a/crates/nx_walker/src/utils/get_mod_time.rs b/crates/nx_utils/src/get_mod_time.rs similarity index 100% rename from crates/nx_walker/src/utils/get_mod_time.rs rename to crates/nx_utils/src/get_mod_time.rs diff --git a/crates/nx_utils/src/lib.rs b/crates/nx_utils/src/lib.rs new file mode 100644 index 0000000000000..8f7b3a3f866af --- /dev/null +++ b/crates/nx_utils/src/lib.rs @@ -0,0 +1,8 @@ +mod get_mod_time; +mod to_normalized_string; + +pub use get_mod_time::*; + +pub mod path { + pub use super::to_normalized_string::*; +} diff --git a/crates/nx_walker/src/utils/normalize_trait.rs b/crates/nx_utils/src/to_normalized_string.rs similarity index 73% rename from crates/nx_walker/src/utils/normalize_trait.rs rename to crates/nx_utils/src/to_normalized_string.rs index bf50b76011276..2113fdab6422b 100644 --- a/crates/nx_walker/src/utils/normalize_trait.rs +++ b/crates/nx_utils/src/to_normalized_string.rs @@ -1,16 +1,16 @@ use std::path::{Path, PathBuf}; -pub trait Normalize { +pub trait ToNormalizedString { fn to_normalized_string(&self) -> String; } -impl Normalize for Path { +impl ToNormalizedString for Path { fn to_normalized_string(&self) -> String { self.to_string_lossy().replace('\\', "/") } } -impl Normalize for PathBuf { +impl ToNormalizedString for PathBuf { fn to_normalized_string(&self) -> String { self.as_path().to_normalized_string() } diff --git a/crates/nx_walker/Cargo.toml b/crates/nx_walker/Cargo.toml index 62aac82c0c25b..33e7966da44ea 100644 --- a/crates/nx_walker/Cargo.toml +++ b/crates/nx_walker/Cargo.toml @@ -11,6 +11,7 @@ rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } nx_glob = { workspace = true } +nx_utils = { workspace = true} crossbeam-channel = { workspace = true } [dev-dependencies] diff --git a/crates/nx_walker/src/lib.rs b/crates/nx_walker/src/lib.rs index d053c90345918..5502f4546d6b5 100644 --- a/crates/nx_walker/src/lib.rs +++ b/crates/nx_walker/src/lib.rs @@ -1,12 +1,11 @@ -mod utils; use ignore::WalkBuilder; use nx_glob::NxGlobSet; use std::fmt::Debug; use std::path::{Path, PathBuf}; use walkdir::WalkDir; - -use crate::utils::{get_mod_time, Normalize}; +use nx_utils::get_mod_time; +use nx_utils::path::ToNormalizedString; #[derive(PartialEq, Debug, Ord, PartialOrd, Eq, Clone)] pub struct NxFile { diff --git a/crates/nx_walker/src/utils/mod.rs b/crates/nx_walker/src/utils/mod.rs deleted file mode 100644 index 7a849acf0f9d3..0000000000000 --- a/crates/nx_walker/src/utils/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod get_mod_time; -mod normalize_trait; - -pub use get_mod_time::*; -pub use normalize_trait::Normalize; diff --git a/packages/nx/Cargo.toml b/packages/nx/Cargo.toml index 91ccbd955240c..049e53ff39c46 100644 --- a/packages/nx/Cargo.toml +++ b/packages/nx/Cargo.toml @@ -20,6 +20,7 @@ nx_project_graph = { workspace = true } nx_cache = { workspace = true } nx_terminal = { workspace = true } nx_watch = { workspace = true } +nx_utils = { workspace = true} anyhow = { workspace = true } colored = { workspace = true } crossbeam-channel = { workspace = true } diff --git a/packages/nx/src/native/hasher.rs b/packages/nx/src/native/hasher.rs index 3485bdc9bed43..022e00f28d78d 100644 --- a/packages/nx/src/native/hasher.rs +++ b/packages/nx/src/native/hasher.rs @@ -1,78 +1,11 @@ #[napi] pub fn hash_array(input: Vec>) -> String { - let joined = input.iter().filter_map(|s| { - if s.is_none() { - trace!("Encountered None value in hash_array input: {:?}", input); - } - s.as_deref() - }).collect::>().join(","); - let content = joined.as_bytes(); - hash(content) + nx_hasher::hash_array_optional(input) } #[napi] pub fn hash_file(file: String) -> Option { - hash_file_path(file) + nx_hasher::hash_file(file) } -#[inline] -pub fn hash_file_path>(path: P) -> Option { - let path = path.as_ref(); - trace!("Reading {:?} to hash", path); - let Ok(content) = std::fs::read(path) else { - trace!("Failed to read file: {:?}", path); - return None; - }; - trace!("Hashing {:?}", path); - let hash = hash(&content); - trace!("Hashed file {:?} - {:?}", path, hash); - Some(hash) -} - -#[cfg(test)] -mod tests { - use crate::native::hasher::{hash_file, hash_array}; - use assert_fs::prelude::*; - use assert_fs::TempDir; - - /// - /// Setup a temporary directory to do testing in - /// - fn setup_fs() -> TempDir { - let temp = TempDir::new().unwrap(); - temp.child("test.txt").write_str("content").unwrap(); - temp.child("foo.txt").write_str("content1").unwrap(); - temp.child("bar.txt").write_str("content2").unwrap(); - temp.child("baz") - .child("qux.txt") - .write_str("content@qux") - .unwrap(); - temp.child("node_modules") - .child("node-module-dep") - .write_str("content") - .unwrap(); - temp - } - - #[test] - fn it_hashes_a_file() { - // handle non existent files - let content = hash_file("".into()); - assert!(content.is_none()); - - let temp_dir = setup_fs(); - - let test_file_path = temp_dir.display().to_string() + "/test.txt"; - let content = hash_file(test_file_path); - - assert_eq!(content.unwrap(), "6193209363630369380"); - } - - #[test] - fn it_hashes_an_array() { - // Resilient to None values (e.g. null values passed from the JS side) - let content = hash_array(vec![Some("foo".to_string()), None, Some("bar".to_string())]); - assert_eq!(content, "10292076446133652019"); - } -} diff --git a/packages/nx/src/native/utils/get_mod_time.rs b/packages/nx/src/native/utils/get_mod_time.rs deleted file mode 100644 index 9cf2efe157593..0000000000000 --- a/packages/nx/src/native/utils/get_mod_time.rs +++ /dev/null @@ -1,25 +0,0 @@ -use std::fs::Metadata; - -#[cfg(target_os = "macos")] -pub fn get_mod_time(metadata: &Metadata) -> i64 { - use std::os::macos::fs::MetadataExt; - metadata.st_mtime() -} - -#[cfg(target_os = "windows")] -pub fn get_mod_time(metadata: &Metadata) -> i64 { - use std::os::windows::fs::MetadataExt; - metadata.last_write_time() as i64 -} - -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -pub fn get_mod_time(metadata: &Metadata) -> i64 { - use std::os::unix::fs::MetadataExt; - metadata.mtime() -} - -#[cfg(target_os = "wasi")] -pub fn get_mod_time(metadata: &Metadata) -> i64 { - use std::os::wasi::fs::MetadataExt; - metadata.mtim() as i64 -} diff --git a/packages/nx/src/native/utils/mod.rs b/packages/nx/src/native/utils/mod.rs index c068479c9ecb8..38c51f992660a 100644 --- a/packages/nx/src/native/utils/mod.rs +++ b/packages/nx/src/native/utils/mod.rs @@ -1,10 +1,8 @@ mod find_matching_projects; -mod get_mod_time; -mod normalize_trait; pub mod path; pub use find_matching_projects::*; -pub use normalize_trait::Normalize; + #[cfg_attr(not(target_arch = "wasm32"), path = "atomics/default.rs")] #[cfg_attr(target_arch = "wasm32", path = "atomics/wasm.rs")] diff --git a/packages/nx/src/native/utils/normalize_trait.rs b/packages/nx/src/native/utils/normalize_trait.rs deleted file mode 100644 index ab4d4b402dd35..0000000000000 --- a/packages/nx/src/native/utils/normalize_trait.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub trait Normalize { - fn to_normalized_string(&self) -> String; -} diff --git a/packages/nx/src/native/utils/path.rs b/packages/nx/src/native/utils/path.rs index 409f70a78aa2b..c16ba953089f2 100644 --- a/packages/nx/src/native/utils/path.rs +++ b/packages/nx/src/native/utils/path.rs @@ -1,17 +1,7 @@ -use crate::native::{utils::normalize_trait::Normalize, types::FileData}; +use crate::native::{ types::FileData}; use std::path::{Path, PathBuf}; -impl Normalize for Path { - fn to_normalized_string(&self) -> String { - normalize_nx_path(self) - } -} -impl Normalize for PathBuf { - fn to_normalized_string(&self) -> String { - normalize_nx_path(self) - } -} fn normalize_nx_path

(path: P) -> String where @@ -74,4 +64,4 @@ mod test { "foo/child/bar", ]); } -} \ No newline at end of file +} diff --git a/packages/nx/src/native/workspace/files_hashing.rs b/packages/nx/src/native/workspace/files_hashing.rs index 1a48f22c7bea0..f53ecc355826b 100644 --- a/packages/nx/src/native/workspace/files_hashing.rs +++ b/packages/nx/src/native/workspace/files_hashing.rs @@ -89,8 +89,7 @@ fn hash_files(files: Vec) -> Vec<(String, NxFileHashed)> { mod tests { use assert_fs::prelude::*; use assert_fs::TempDir; - - use crate::native::utils::get_mod_time; + use nx_utils::get_mod_time; use crate::native::workspace::files_archive::{NxFileHashed, NxFileHashes}; fn setup_fs() -> TempDir { @@ -140,7 +139,7 @@ mod tests { get_mod_time(&temp.child("modified.txt").metadata().unwrap()) - 10, ), ), - // this file is does not exist on the fs, aka it was deleted + // this file does not exist on the fs, aka it was deleted ( String::from("baz/qux.txt"), NxFileHashed(String::from("hash5"), 0), From a4dfee072fd5670fb4953a3312b5a0baffc4deec Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli <4332460+Cammisuli@users.noreply.github.com> Date: Thu, 20 Feb 2025 16:58:45 -0500 Subject: [PATCH 04/11] feat(nx_logger): nx_logger module - Removed `nx_machine` crate and its dependencies. - Introduced a new `nx_logger` crate for logging functionality. - Updated all references to the logger to use `nx_logger`. - Adjusted `Cargo.toml` and `Cargo.lock` to reflect crate changes. - Migrated shared logging utilities to `nx_logger`. --- Cargo.lock | 26 +++++++++---------- Cargo.toml | 2 +- crates/nx_logger/Cargo.toml | 10 +++++++ .../mod.rs => crates/nx_logger/src/lib.rs | 2 +- crates/nx_machine/Cargo.toml | 12 --------- crates/nx_machine/src/lib.rs | 1 - packages/nx/Cargo.toml | 4 +-- .../nx/src/native/cache/expand_outputs.rs | 4 +-- packages/nx/src/native/db.rs | 4 +-- packages/nx/src/native/mod.rs | 1 - .../native/plugins/js/ts_import_locators.rs | 3 +-- packages/nx/src/native/pseudo_terminal/mac.rs | 3 ++- .../nx/src/native/pseudo_terminal/non_mac.rs | 2 +- packages/nx/src/native/tasks/hash_planner.rs | 2 +- packages/nx/src/native/workspace/context.rs | 5 ++-- 15 files changed, 37 insertions(+), 44 deletions(-) create mode 100644 crates/nx_logger/Cargo.toml rename packages/nx/src/native/logger/mod.rs => crates/nx_logger/src/lib.rs (99%) delete mode 100644 crates/nx_machine/Cargo.toml delete mode 100644 crates/nx_machine/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 7dc523b6e24a4..cdfb30a3db93b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1472,7 +1472,6 @@ version = "0.1.0" dependencies = [ "anyhow", "assert_fs", - "colored", "crossbeam-channel", "crossterm", "dashmap", @@ -1492,7 +1491,7 @@ dependencies = [ "nx_db", "nx_glob", "nx_hasher", - "nx_machine", + "nx_logger", "nx_project_graph", "nx_tasks", "nx_terminal", @@ -1515,7 +1514,6 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", - "walkdir", "watchexec", "watchexec-events", "watchexec-filterer-ignore", @@ -1580,12 +1578,12 @@ dependencies = [ ] [[package]] -name = "nx_machine" +name = "nx_logger" version = "0.1.0" dependencies = [ - "anyhow", - "assert_fs", + "colored", "tracing", + "tracing-subscriber", ] [[package]] @@ -2718,9 +2716,9 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -2730,9 +2728,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", @@ -2741,9 +2739,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -2762,9 +2760,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", diff --git a/Cargo.toml b/Cargo.toml index ed544ef15095a..4911a3f5f4aac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,8 @@ nx_project_graph = { path = "crates/nx_project_graph" } nx_cache = { path = "crates/nx_cache" } nx_terminal = { path = "crates/nx_terminal" } nx_watch = { path = "crates/nx_watch" } -nx_machine = { path = "crates/nx_machine" } nx_utils = { path = "crates/nx_utils"} +nx_logger = { path = "crates/nx_logger"} anyhow = "1.0.95" colored = "2" diff --git a/crates/nx_logger/Cargo.toml b/crates/nx_logger/Cargo.toml new file mode 100644 index 0000000000000..e2512d8a6a6a8 --- /dev/null +++ b/crates/nx_logger/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "nx_logger" +edition.workspace = true +license.workspace = true +version.workspace = true + +[dependencies] +tracing = { workspace = true} +tracing-subscriber = { workspace = true} +colored = { workspace = true} diff --git a/packages/nx/src/native/logger/mod.rs b/crates/nx_logger/src/lib.rs similarity index 99% rename from packages/nx/src/native/logger/mod.rs rename to crates/nx_logger/src/lib.rs index 6a0a4df4eca6d..64671435dc7b0 100644 --- a/packages/nx/src/native/logger/mod.rs +++ b/crates/nx_logger/src/lib.rs @@ -88,7 +88,7 @@ where /// - `NX_NATIVE_LOGGING=nx=trace` - enable all logs for the `nx` (this) crate /// - `NX_NATIVE_LOGGING=nx::native::tasks::hashers::hash_project_files=trace` - enable all logs for the `hash_project_files` module /// - `NX_NATIVE_LOGGING=[{project_name=project}]` - enable logs that contain the project in its span -pub(crate) fn enable_logger() { +pub fn enable_logger() { let env_filter = EnvFilter::try_from_env("NX_NATIVE_LOGGING").unwrap_or_else(|_| EnvFilter::new("ERROR")); _ = tracing_subscriber::fmt() diff --git a/crates/nx_machine/Cargo.toml b/crates/nx_machine/Cargo.toml deleted file mode 100644 index 7ac6d73a431c4..0000000000000 --- a/crates/nx_machine/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "nx_machine" -version.workspace = true -edition.workspace = true -license.workspace = true - -[dependencies] -anyhow = { workspace = true } -tracing = { workspace = true } - -[dev-dependencies] -assert_fs = { workspace = true } diff --git a/crates/nx_machine/src/lib.rs b/crates/nx_machine/src/lib.rs deleted file mode 100644 index 8b137891791fe..0000000000000 --- a/crates/nx_machine/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/packages/nx/Cargo.toml b/packages/nx/Cargo.toml index 049e53ff39c46..389132fda2026 100644 --- a/packages/nx/Cargo.toml +++ b/packages/nx/Cargo.toml @@ -12,8 +12,8 @@ crate-type = ['cdylib'] nx_glob = { workspace = true } nx_walker = { workspace = true } nx_db = { workspace = true } -nx_machine = { workspace = true } nx_hasher = { workspace = true } +nx_logger = { workspace = true} nx_tasks = { workspace = true } nx_workspace = { workspace = true } nx_project_graph = { workspace = true } @@ -22,7 +22,6 @@ nx_terminal = { workspace = true } nx_watch = { workspace = true } nx_utils = { workspace = true} anyhow = { workspace = true } -colored = { workspace = true } crossbeam-channel = { workspace = true } dashmap = { workspace = true } dunce = { workspace = true } @@ -39,7 +38,6 @@ rkyv = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } -walkdir = { workspace = true } xxhash-rust = { workspace = true } swc_common = { workspace = true } swc_ecma_parser = { workspace = true } diff --git a/packages/nx/src/native/cache/expand_outputs.rs b/packages/nx/src/native/cache/expand_outputs.rs index 710dfd1463c87..e569e92437895 100644 --- a/packages/nx/src/native/cache/expand_outputs.rs +++ b/packages/nx/src/native/cache/expand_outputs.rs @@ -2,10 +2,10 @@ use hashbrown::HashMap; use std::path::{Path, PathBuf}; use tracing::trace; -use crate::native::logger::enable_logger; -use crate::native::utils::Normalize; use nx_glob::NxGlobSet; use nx_glob::{contains_glob_pattern, partition_glob}; +use nx_logger::enable_logger; +use nx_utils::path::ToNormalizedString; use nx_walker::{nx_walker, nx_walker_sync}; #[napi] diff --git a/packages/nx/src/native/db.rs b/packages/nx/src/native/db.rs index df626b6deed98..ee8e3037369b0 100644 --- a/packages/nx/src/native/db.rs +++ b/packages/nx/src/native/db.rs @@ -1,10 +1,10 @@ -use crate::native::logger::enable_logger; use crate::native::machine_id::get_machine_id; use napi::bindgen_prelude::External; use std::fs::create_dir_all; use std::path::PathBuf; -pub use nx_db::connection::NxDbConnection; +use nx_db::connection::NxDbConnection; +use nx_logger::enable_logger; #[napi] pub fn connect_to_nx_db( diff --git a/packages/nx/src/native/mod.rs b/packages/nx/src/native/mod.rs index 99f3b4b4d73dd..049b3af4b8e20 100644 --- a/packages/nx/src/native/mod.rs +++ b/packages/nx/src/native/mod.rs @@ -1,6 +1,5 @@ pub mod cache; pub mod hasher; -mod logger; mod machine_id; pub mod metadata; pub mod plugins; diff --git a/packages/nx/src/native/plugins/js/ts_import_locators.rs b/packages/nx/src/native/plugins/js/ts_import_locators.rs index c76756eba8d34..b81e155f38a43 100644 --- a/packages/nx/src/native/plugins/js/ts_import_locators.rs +++ b/packages/nx/src/native/plugins/js/ts_import_locators.rs @@ -19,8 +19,7 @@ use swc_ecma_parser::token::Keyword::{ use swc_ecma_parser::token::Word::{Ident, Keyword}; use swc_ecma_parser::token::{BinOpToken, Token, TokenAndSpan}; use swc_ecma_parser::{Syntax, Tokens, TsConfig}; - -use crate::native::logger::enable_logger; +use nx_logger::enable_logger; #[napi] #[derive(Debug)] diff --git a/packages/nx/src/native/pseudo_terminal/mac.rs b/packages/nx/src/native/pseudo_terminal/mac.rs index 19468598d50f6..5e007a715e67d 100644 --- a/packages/nx/src/native/pseudo_terminal/mac.rs +++ b/packages/nx/src/native/pseudo_terminal/mac.rs @@ -5,7 +5,8 @@ use tracing::trace; use super::child_process::ChildProcess; use super::os; use super::pseudo_terminal::{create_pseudo_terminal, run_command}; -use crate::native::logger::enable_logger; + +use nx_logger::enable_logger; #[napi] pub struct RustPseudoTerminal {} diff --git a/packages/nx/src/native/pseudo_terminal/non_mac.rs b/packages/nx/src/native/pseudo_terminal/non_mac.rs index 8e7ff6695a797..8138eb6677ba0 100644 --- a/packages/nx/src/native/pseudo_terminal/non_mac.rs +++ b/packages/nx/src/native/pseudo_terminal/non_mac.rs @@ -5,7 +5,7 @@ use tracing::trace; use super::child_process::ChildProcess; use super::os; use super::pseudo_terminal::{create_pseudo_terminal, run_command, PseudoTerminal}; -use crate::native::logger::enable_logger; +use nx_logger::enable_logger; #[napi] pub struct RustPseudoTerminal { diff --git a/packages/nx/src/native/tasks/hash_planner.rs b/packages/nx/src/native/tasks/hash_planner.rs index e23ddf086070f..4833e1ef057bf 100644 --- a/packages/nx/src/native/tasks/hash_planner.rs +++ b/packages/nx/src/native/tasks/hash_planner.rs @@ -1,4 +1,4 @@ -use crate::native::logger::enable_logger; +use nx_logger::enable_logger; use crate::native::tasks::{ dep_outputs::get_dep_output, types::{HashInstruction, TaskGraph}, diff --git a/packages/nx/src/native/workspace/context.rs b/packages/nx/src/native/workspace/context.rs index 5de621db10eae..9d92901c50099 100644 --- a/packages/nx/src/native/workspace/context.rs +++ b/packages/nx/src/native/workspace/context.rs @@ -3,10 +3,9 @@ use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::Arc; -use crate::native::logger::enable_logger; use crate::native::project_graph::utils::{find_project_for_path, ProjectRootMappings}; use crate::native::types::FileData; -use crate::native::utils::{path::get_child_files, Normalize, NxCondvar, NxMutex}; +use crate::native::utils::{path::get_child_files, NxCondvar, NxMutex}; use crate::native::workspace::files_archive::{read_files_archive, write_files_archive}; use crate::native::workspace::files_hashing::{full_files_hash, selective_files_hash}; use crate::native::workspace::types::{ @@ -18,6 +17,8 @@ use nx_hasher::hash; use rayon::prelude::*; use tracing::{trace, warn}; use xxhash_rust::xxh3; +use nx_utils::path::ToNormalizedString; +use nx_logger::enable_logger; #[napi] pub struct WorkspaceContext { From 4253c769350dd5af8d64975cef706dea2341093e Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli <4332460+Cammisuli@users.noreply.github.com> Date: Thu, 20 Feb 2025 17:20:43 -0500 Subject: [PATCH 05/11] feat(nx-utils): migrate `ci` module to shared `nx_utils` crate - Moved `ci` module from `nx/native/utils` to `nx_utils` crate for reusability. - Refactored imports to use `nx_utils::ci` across affected files. - Simplified code by optimizing path normalization and hash operations. - Updated dependencies in `Cargo.toml` and `Cargo.lock` for latest versions. --- Cargo.lock | 24 +++++++++---------- Cargo.toml | 6 ++--- .../utils => crates/nx_utils/src}/ci.rs | 0 crates/nx_utils/src/lib.rs | 2 ++ packages/nx/src/native/cache/cache.rs | 2 +- .../src/native/tasks/hashers/hash_external.rs | 4 ++-- .../native/tasks/hashers/hash_task_output.rs | 2 +- packages/nx/src/native/utils/mod.rs | 1 - packages/nx/src/native/utils/path.rs | 5 ++-- 9 files changed, 23 insertions(+), 23 deletions(-) rename {packages/nx/src/native/utils => crates/nx_utils/src}/ci.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index cdfb30a3db93b..8d83808595e6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,9 +393,9 @@ dependencies = [ [[package]] name = "ctor" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad291aa74992b9b7a7e88c38acbbf6ad7e107f1d90ee8775b7bc1fc3394f485c" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", "syn 2.0.53", @@ -1279,9 +1279,9 @@ dependencies = [ [[package]] name = "napi" -version = "2.16.0" +version = "2.16.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a63d0570e4c3e0daf7a8d380563610e159f538e20448d6c911337246f40e84" +checksum = "839ae2ee5e62c6348669c50098b187c08115bd3cced658c9c0bf945fca0fec83" dependencies = [ "anyhow", "bitflags 2.6.0", @@ -1294,15 +1294,15 @@ dependencies = [ [[package]] name = "napi-build" -version = "2.1.3" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1c0f5d67ee408a4685b61f5ab7e58605c8ae3f2b4189f0127d804ff13d5560a" +checksum = "db836caddef23662b94e16bf1f26c40eceb09d6aee5d5b06a7ac199320b69b19" [[package]] name = "napi-derive" -version = "2.16.0" +version = "2.16.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05bb7c37e3c1dda9312fdbe4a9fc7507fca72288ba154ec093e2d49114e727ce" +checksum = "7cbe2585d8ac223f7d34f13701434b9d5f4eb9c332cccce8dee57ea18ab8ab0c" dependencies = [ "cfg-if", "convert_case", @@ -1314,9 +1314,9 @@ dependencies = [ [[package]] name = "napi-derive-backend" -version = "1.0.62" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f785a8b8d7b83e925f5aa6d2ae3c159d17fe137ac368dc185bef410e7acdaeb4" +checksum = "1639aaa9eeb76e91c6ae66da8ce3e89e921cd3885e99ec85f4abacae72fc91bf" dependencies = [ "convert_case", "once_cell", @@ -1329,9 +1329,9 @@ dependencies = [ [[package]] name = "napi-sys" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2503fa6af34dc83fb74888df8b22afe933b58d37daf7d80424b1c60c68196b8b" +checksum = "427802e8ec3a734331fec1035594a210ce1ff4dc5bc1950530920ab717964ea3" dependencies = [ "libloading", ] diff --git a/Cargo.toml b/Cargo.toml index 4911a3f5f4aac..8393238cfdd7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,12 +33,12 @@ itertools = "0.10.5" globset = "0.4.10" once_cell = "1.18.0" parking_lot = { version = "0.12.1", features = ["send_guard"] } -napi = { version = '2.16.0', default-features = false, features = [ +napi = { version = '2.16.16', default-features = false, features = [ 'anyhow', 'napi4', 'tokio_rt', ] } -napi-derive = '2.16.0' +napi-derive = '2.16.13' nom = '7.1.3' regex = "1.9.1" rayon = "1.7.0" @@ -74,7 +74,7 @@ watchexec-signals = "2.1.0" machine-uid = "0.5.2" # build dependencies -napi-build = '2.1.3' +napi-build = '2.1.4' assert_fs = "1.0.10" # This is only used for unit tests swc_ecma_dep_graph = "0.109.1" diff --git a/packages/nx/src/native/utils/ci.rs b/crates/nx_utils/src/ci.rs similarity index 100% rename from packages/nx/src/native/utils/ci.rs rename to crates/nx_utils/src/ci.rs diff --git a/crates/nx_utils/src/lib.rs b/crates/nx_utils/src/lib.rs index 8f7b3a3f866af..912398604ad6b 100644 --- a/crates/nx_utils/src/lib.rs +++ b/crates/nx_utils/src/lib.rs @@ -1,8 +1,10 @@ mod get_mod_time; mod to_normalized_string; +pub mod ci; pub use get_mod_time::*; + pub mod path { pub use super::to_normalized_string::*; } diff --git a/packages/nx/src/native/cache/cache.rs b/packages/nx/src/native/cache/cache.rs index f90deb6adbda6..8fe8d3b2904a5 100644 --- a/packages/nx/src/native/cache/cache.rs +++ b/packages/nx/src/native/cache/cache.rs @@ -10,8 +10,8 @@ use tracing::trace; use crate::native::cache::expand_outputs::_expand_outputs; use crate::native::cache::file_ops::_copy; -use crate::native::utils::Normalize; use nx_db::connection::NxDbConnection; +use nx_utils::path::ToNormalizedString; #[napi(object)] #[derive(Default, Clone, Debug)] diff --git a/packages/nx/src/native/tasks/hashers/hash_external.rs b/packages/nx/src/native/tasks/hashers/hash_external.rs index 7b90d49956735..6c970c76de935 100644 --- a/packages/nx/src/native/tasks/hashers/hash_external.rs +++ b/packages/nx/src/native/tasks/hashers/hash_external.rs @@ -38,8 +38,8 @@ pub fn hash_all_externals>( ) -> Result { let hashes = sorted_externals .iter() - .map(|name| hash_external(name.as_ref(), externals, Arc::clone(&cache)).map(Some)) - .collect::>>>()?; + .map(|name| hash_external(name.as_ref(), externals, Arc::clone(&cache))) + .collect::>>()?; Ok(hash_array(hashes)) } diff --git a/packages/nx/src/native/tasks/hashers/hash_task_output.rs b/packages/nx/src/native/tasks/hashers/hash_task_output.rs index b0aed971a845c..398fe78b9e33a 100644 --- a/packages/nx/src/native/tasks/hashers/hash_task_output.rs +++ b/packages/nx/src/native/tasks/hashers/hash_task_output.rs @@ -24,5 +24,5 @@ pub fn hash_task_output(workspace_root: &str, glob: &str, outputs: &[String]) -> ) }) .collect::>(); - Ok(hash_array(hashes.into_iter().map(Some).collect())) + Ok(hash_array(hashes)) } diff --git a/packages/nx/src/native/utils/mod.rs b/packages/nx/src/native/utils/mod.rs index 38c51f992660a..f7d704ea4332c 100644 --- a/packages/nx/src/native/utils/mod.rs +++ b/packages/nx/src/native/utils/mod.rs @@ -7,7 +7,6 @@ pub use find_matching_projects::*; #[cfg_attr(not(target_arch = "wasm32"), path = "atomics/default.rs")] #[cfg_attr(target_arch = "wasm32", path = "atomics/wasm.rs")] pub mod atomics; -pub mod ci; pub mod file_lock; pub use atomics::*; diff --git a/packages/nx/src/native/utils/path.rs b/packages/nx/src/native/utils/path.rs index c16ba953089f2..c2111d60208be 100644 --- a/packages/nx/src/native/utils/path.rs +++ b/packages/nx/src/native/utils/path.rs @@ -1,6 +1,5 @@ -use crate::native::{ types::FileData}; -use std::path::{Path, PathBuf}; - +use crate::native::types::FileData; +use std::path::Path; fn normalize_nx_path

(path: P) -> String From 4236404574031eb9b9290ddd7602035d158b8901 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli <4332460+Cammisuli@users.noreply.github.com> Date: Thu, 20 Feb 2025 17:31:34 -0500 Subject: [PATCH 06/11] refactor(core): move path normalization to nx_utils - Relocated `normalize_nx_path` function from `path.rs` to `nx_utils` - Updated usage of `normalize_nx_path` in both `Path` and `PathBuf` implementations - Simplified `path.rs` by removing redundant functionality --- crates/nx_utils/src/to_normalized_string.rs | 20 +++++++++++++++-- packages/nx/src/native/utils/path.rs | 25 ++------------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/crates/nx_utils/src/to_normalized_string.rs b/crates/nx_utils/src/to_normalized_string.rs index 2113fdab6422b..0a99a12bd06e4 100644 --- a/crates/nx_utils/src/to_normalized_string.rs +++ b/crates/nx_utils/src/to_normalized_string.rs @@ -6,12 +6,28 @@ pub trait ToNormalizedString { impl ToNormalizedString for Path { fn to_normalized_string(&self) -> String { - self.to_string_lossy().replace('\\', "/") + normalize_nx_path(self) } } impl ToNormalizedString for PathBuf { fn to_normalized_string(&self) -> String { - self.as_path().to_normalized_string() + normalize_nx_path(self) + } +} + +fn normalize_nx_path

(path: P) -> String +where + P: AsRef, +{ + if path.as_ref() == Path::new("") { + return ".".into(); + } + + // convert back-slashes in Windows paths, since the js expects only forward-slash path separators + if cfg!(windows) { + path.as_ref().display().to_string().replace('\\', "/") + } else { + path.as_ref().display().to_string() } } diff --git a/packages/nx/src/native/utils/path.rs b/packages/nx/src/native/utils/path.rs index c2111d60208be..211e424059bf9 100644 --- a/packages/nx/src/native/utils/path.rs +++ b/packages/nx/src/native/utils/path.rs @@ -1,23 +1,6 @@ use crate::native::types::FileData; use std::path::Path; - -fn normalize_nx_path

(path: P) -> String -where - P: AsRef, -{ - if path.as_ref() == Path::new("") { - return ".".into(); - } - - // convert back-slashes in Windows paths, since the js expects only forward-slash path separators - if cfg!(windows) { - path.as_ref().display().to_string().replace('\\', "/") - } else { - path.as_ref().display().to_string() - } -} - pub fn get_child_files>(directory: P, files: Vec) -> Vec { files .into_iter() @@ -54,13 +37,9 @@ mod test { FileData { file: "foo-other/not-child".into(), hash: "123".into(), - } + }, ]; let child_files = get_child_files(&directory, files); - assert_eq!(child_files, [ - "foo/bar", - "foo/baz", - "foo/child/bar", - ]); + assert_eq!(child_files, ["foo/bar", "foo/baz", "foo/child/bar",]); } } From d69a8607ac4a39a78e5187fcc3b7a7c35a0fada1 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli <4332460+Cammisuli@users.noreply.github.com> Date: Sat, 22 Feb 2025 12:09:44 -0500 Subject: [PATCH 07/11] fix(nx_db): remove unused dependencies - Removed `thiserror`, `parking_lot`, and `assert_fs` from `Cargo.toml` and `Cargo.lock`. - Cleaned up unused logger import in `hash_planner.rs`. - Corrected function declarations in `index.d.ts`, resolving duplication issues. - Adjusted `hash_planner.rs` to return `External` directly without environment handling. --- Cargo.lock | 3 --- crates/nx_db/Cargo.toml | 3 --- packages/nx/src/native/index.d.ts | 28 ++++++++++---------- packages/nx/src/native/tasks/hash_planner.rs | 9 +++---- 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d83808595e6a..a4b2cfd4715fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1547,12 +1547,9 @@ name = "nx_db" version = "0.1.0" dependencies = [ "anyhow", - "assert_fs", "fs4", - "parking_lot", "rusqlite", "tempfile", - "thiserror", "tracing", ] diff --git a/crates/nx_db/Cargo.toml b/crates/nx_db/Cargo.toml index 8b3fa715ff936..c6e97009ce616 100644 --- a/crates/nx_db/Cargo.toml +++ b/crates/nx_db/Cargo.toml @@ -7,11 +7,8 @@ license.workspace = true [dependencies] rusqlite = { workspace = true } anyhow = { workspace = true } -thiserror = { workspace = true } tracing = { workspace = true } -parking_lot = { workspace = true } fs4 = { workspace = true } [dev-dependencies] -assert_fs = { workspace = true } tempfile = { workspace = true } diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts index 709d0987077e4..80b94395dcfb1 100644 --- a/packages/nx/src/native/index.d.ts +++ b/packages/nx/src/native/index.d.ts @@ -114,11 +114,11 @@ export interface CachedResult { outputsPath: string } -export declare export function closeDbConnection(connection: ExternalObject): void +export declare export declare function closeDbConnection(connection: ExternalObject): void -export declare export function connectToNxDb(cacheDir: string, nxVersion: string, dbName?: string | undefined | null): ExternalObject +export declare export declare function connectToNxDb(cacheDir: string, nxVersion: string, dbName?: string | undefined | null): ExternalObject -export declare export function copy(src: string, dest: string): void +export declare export declare function copy(src: string, dest: string): void export interface DepsOutputsInput { dependentTasksOutputFiles: string @@ -135,7 +135,7 @@ export declare const enum EventType { create = 'create' } -export declare export function expandOutputs(directory: string, entries: Array): Array +export declare export declare function expandOutputs(directory: string, entries: Array): Array export interface ExternalDependenciesInput { externalDependencies: Array @@ -161,19 +161,19 @@ export interface FileSetInput { fileset: string } -export declare export function findImports(projectFileMap: Record>): Array +export declare export declare function findImports(projectFileMap: Record>): Array -export declare export function getBinaryTarget(): string +export declare export declare function getBinaryTarget(): string /** * Expands the given outputs into a list of existing files. * This is used when hashing outputs */ -export declare export function getFilesForOutputs(directory: string, entries: Array): Array +export declare export declare function getFilesForOutputs(directory: string, entries: Array): Array -export declare export function getTransformableOutputs(outputs: Array): Array +export declare export declare function getTransformableOutputs(outputs: Array): Array -export declare export function hashArray(input: Array): string +export declare export declare function hashArray(input: Array): string export interface HashDetails { value: string @@ -191,7 +191,7 @@ export interface HasherOptions { selectivelyHashTsConfig: boolean } -export declare export function hashFile(file: string): string | null +export declare export declare function hashFile(file: string): string | null export interface InputsInput { input: string @@ -231,7 +231,7 @@ export interface ProjectGraph { externalNodes: Record } -export declare export function remove(src: string): void +export declare export declare function remove(src: string): void export interface RuntimeInput { runtime: string @@ -273,20 +273,20 @@ export interface TaskTarget { configuration?: string } -export declare export function testOnlyTransferFileMap(projectFiles: Record>, nonProjectFiles: Array): NxWorkspaceFilesExternals +export declare export declare function testOnlyTransferFileMap(projectFiles: Record>, nonProjectFiles: Array): NxWorkspaceFilesExternals /** * Transfer the project graph from the JS world to the Rust world, so that we can pass the project graph via memory quicker * This wont be needed once the project graph is created in Rust */ -export declare export function transferProjectGraph(projectGraph: ProjectGraph): ExternalObject +export declare export declare function transferProjectGraph(projectGraph: ProjectGraph): ExternalObject export interface UpdatedWorkspaceFiles { fileMap: FileMap externalReferences: NxWorkspaceFilesExternals } -export declare export function validateOutputs(outputs: Array): void +export declare export declare function validateOutputs(outputs: Array): void export interface WatchEvent { path: string diff --git a/packages/nx/src/native/tasks/hash_planner.rs b/packages/nx/src/native/tasks/hash_planner.rs index 4833e1ef057bf..0935259c6685d 100644 --- a/packages/nx/src/native/tasks/hash_planner.rs +++ b/packages/nx/src/native/tasks/hash_planner.rs @@ -1,4 +1,3 @@ -use nx_logger::enable_logger; use crate::native::tasks::{ dep_outputs::get_dep_output, types::{HashInstruction, TaskGraph}, @@ -9,7 +8,7 @@ use crate::native::{ tasks::{inputs::SplitInputs, types::Task}, }; use napi::bindgen_prelude::External; -use napi::{Env, JsExternal}; +use nx_logger::enable_logger; use rayon::prelude::*; use std::collections::HashMap; use tracing::trace; @@ -102,13 +101,11 @@ impl HashPlanner { #[napi] pub fn get_plans_reference( &self, - env: Env, task_ids: Vec<&str>, task_graph: TaskGraph, - ) -> anyhow::Result { + ) -> anyhow::Result>>> { let plans = self.get_plans_internal(task_ids, task_graph)?; - env.create_external(plans, None) - .map_err(anyhow::Error::from) + Ok(External::new(plans)) } fn target_input<'a>( From a27ef88052421d2f4d01e5aaadbbfcd986f24e9c Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli <4332460+Cammisuli@users.noreply.github.com> Date: Sat, 22 Feb 2025 12:52:18 -0500 Subject: [PATCH 08/11] refactor(core): extract reusable types and functions to nx_core - Moved types related to project graph, inputs, and nx_json to a reusable crate, `nx_core`. - Removed redundant `transfer_project_graph` function and refactored imports to use `nx_core`. - Updated `nx_project_graph` and other crates to depend on `nx_core` for shared functionality. - Migrated utility functions like `find_matching_projects` to `nx_project_graph/utils`. - Refactored redundant imports and aligned with consolidated module structure. --- Cargo.lock | 25 ++++++++----------- Cargo.toml | 5 ++-- crates/nx_core/Cargo.toml | 9 +++++++ crates/nx_core/src/lib.rs | 2 ++ crates/nx_core/src/types.rs | 3 +++ .../nx_core/src}/types/inputs.rs | 5 ++-- .../nx_core/src}/types/nx_json.rs | 3 ++- .../nx_core/src/types/project_graph.rs | 3 ++- crates/nx_project_graph/Cargo.toml | 20 +++------------ crates/nx_project_graph/src/lib.rs | 5 +--- .../nx_project_graph/src}/utils.rs | 8 +++--- .../src}/utils/find_matching_projects.rs | 4 +-- .../src}/utils/find_project_for_path.rs | 10 ++++---- packages/nx/Cargo.toml | 5 ++-- packages/nx/src/native/project_graph/mod.rs | 12 ++++++--- .../project_graph/transfer_project_graph.rs | 11 -------- .../native/pseudo_terminal/pseudo_terminal.rs | 12 +++++++-- packages/nx/src/native/tasks/hash_planner.rs | 25 ++++++++----------- .../src/native/tasks/hashers/hash_external.rs | 5 ++-- .../tasks/hashers/hash_project_config.rs | 7 +++--- .../src/native/tasks/hashers/hash_tsconfig.rs | 9 +++---- packages/nx/src/native/tasks/inputs.rs | 5 ++-- packages/nx/src/native/tasks/task_hasher.rs | 5 ++-- packages/nx/src/native/tasks/utils.rs | 2 +- packages/nx/src/native/types.rs | 4 --- packages/nx/src/native/utils/mod.rs | 4 --- packages/nx/src/native/workspace/context.rs | 9 ++++--- 27 files changed, 105 insertions(+), 112 deletions(-) create mode 100644 crates/nx_core/Cargo.toml create mode 100644 crates/nx_core/src/lib.rs create mode 100644 crates/nx_core/src/types.rs rename {packages/nx/src/native => crates/nx_core/src}/types/inputs.rs (97%) rename {packages/nx/src/native => crates/nx_core/src}/types/nx_json.rs (76%) rename packages/nx/src/native/project_graph/types.rs => crates/nx_core/src/types/project_graph.rs (93%) rename {packages/nx/src/native/project_graph => crates/nx_project_graph/src}/utils.rs (88%) rename {packages/nx/src/native => crates/nx_project_graph/src}/utils/find_matching_projects.rs (99%) rename {packages/nx/src/native/project_graph => crates/nx_project_graph/src}/utils/find_project_for_path.rs (91%) delete mode 100644 packages/nx/src/native/project_graph/transfer_project_graph.rs diff --git a/Cargo.lock b/Cargo.lock index a4b2cfd4715fa..a51b0673be6f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1488,6 +1488,7 @@ dependencies = [ "napi-build", "napi-derive", "nx_cache", + "nx_core", "nx_db", "nx_glob", "nx_hasher", @@ -1542,6 +1543,14 @@ dependencies = [ "tracing", ] +[[package]] +name = "nx_core" +version = "0.1.0" +dependencies = [ + "napi", + "napi-derive", +] + [[package]] name = "nx_db" version = "0.1.0" @@ -1588,21 +1597,9 @@ name = "nx_project_graph" version = "0.1.0" dependencies = [ "anyhow", - "assert_fs", + "hashbrown 0.14.5", + "nx_core", "nx_glob", - "nx_hasher", - "nx_walker", - "parking_lot", - "rayon", - "serde", - "serde_json", - "swc_common", - "swc_ecma_ast", - "swc_ecma_dep_graph", - "swc_ecma_parser", - "tempfile", - "thiserror", - "tracing", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8393238cfdd7e..402e1c9426013 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,9 @@ nx_project_graph = { path = "crates/nx_project_graph" } nx_cache = { path = "crates/nx_cache" } nx_terminal = { path = "crates/nx_terminal" } nx_watch = { path = "crates/nx_watch" } -nx_utils = { path = "crates/nx_utils"} -nx_logger = { path = "crates/nx_logger"} +nx_utils = { path = "crates/nx_utils" } +nx_logger = { path = "crates/nx_logger" } +nx_core = { path = "crates/nx_core" } anyhow = "1.0.95" colored = "2" diff --git a/crates/nx_core/Cargo.toml b/crates/nx_core/Cargo.toml new file mode 100644 index 0000000000000..69582c73545bf --- /dev/null +++ b/crates/nx_core/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "nx_core" +edition.workspace = true +license.workspace = true +version.workspace = true + +[dependencies] +napi = { workspace = true} +napi-derive = { workspace = true} diff --git a/crates/nx_core/src/lib.rs b/crates/nx_core/src/lib.rs new file mode 100644 index 0000000000000..97fe92f17b8d9 --- /dev/null +++ b/crates/nx_core/src/lib.rs @@ -0,0 +1,2 @@ +pub mod types; + diff --git a/crates/nx_core/src/types.rs b/crates/nx_core/src/types.rs new file mode 100644 index 0000000000000..033231721e47c --- /dev/null +++ b/crates/nx_core/src/types.rs @@ -0,0 +1,3 @@ +pub mod inputs; +pub mod nx_json; +pub mod project_graph; diff --git a/packages/nx/src/native/types/inputs.rs b/crates/nx_core/src/types/inputs.rs similarity index 97% rename from packages/nx/src/native/types/inputs.rs rename to crates/nx_core/src/types/inputs.rs index 70e91a899a0f2..cf76f72dac0ae 100644 --- a/packages/nx/src/native/types/inputs.rs +++ b/crates/nx_core/src/types/inputs.rs @@ -1,5 +1,6 @@ use napi::bindgen_prelude::Either7; use napi::Either; +use napi_derive::napi; #[napi(object)] pub struct InputsInput { @@ -34,7 +35,7 @@ pub struct DepsOutputsInput { pub transitive: Option, } -pub(crate) type JsInputs = Either7< +pub type JsInputs = Either7< InputsInput, String, FileSetInput, @@ -88,7 +89,7 @@ impl<'a> From<&'a JsInputs> for Input<'a> { } #[derive(Debug)] -pub(crate) enum Input<'a> { +pub enum Input<'a> { Inputs { input: &'a str, dependencies: bool, diff --git a/packages/nx/src/native/types/nx_json.rs b/crates/nx_core/src/types/nx_json.rs similarity index 76% rename from packages/nx/src/native/types/nx_json.rs rename to crates/nx_core/src/types/nx_json.rs index bdd9a418babd6..6454aea624844 100644 --- a/packages/nx/src/native/types/nx_json.rs +++ b/crates/nx_core/src/types/nx_json.rs @@ -1,5 +1,6 @@ -use crate::native::types::JsInputs; use std::collections::HashMap; +use napi_derive::napi; +use crate::types::inputs::JsInputs; #[napi(object)] /// Stripped version of the NxJson interface for use in rust diff --git a/packages/nx/src/native/project_graph/types.rs b/crates/nx_core/src/types/project_graph.rs similarity index 93% rename from packages/nx/src/native/project_graph/types.rs rename to crates/nx_core/src/types/project_graph.rs index 99c6c07764b81..2ace7d74cd236 100644 --- a/packages/nx/src/native/project_graph/types.rs +++ b/crates/nx_core/src/types/project_graph.rs @@ -1,4 +1,5 @@ -use crate::native::types::JsInputs; +use crate::types::inputs::JsInputs; +use napi_derive::napi; use std::collections::HashMap; #[napi(object)] diff --git a/crates/nx_project_graph/Cargo.toml b/crates/nx_project_graph/Cargo.toml index 87e89590a0345..ee90fa97b1da0 100644 --- a/crates/nx_project_graph/Cargo.toml +++ b/crates/nx_project_graph/Cargo.toml @@ -5,21 +5,9 @@ edition = { workspace = true } license = { workspace = true } [dependencies] -anyhow = { workspace = true } -tracing = { workspace = true } -nx_hasher = { workspace = true } -nx_walker = { workspace = true } -nx_glob = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -thiserror = { workspace = true } -parking_lot = { workspace = true } -rayon = { workspace = true } -swc_common = { workspace = true } -swc_ecma_parser = { workspace = true } -swc_ecma_ast = { workspace = true } +nx_core = { workspace = true } +hashbrown = {workspace = true} +anyhow = { workspace = true} +nx_glob = { workspace = true} [dev-dependencies] -assert_fs = { workspace = true } -tempfile = { workspace = true } -swc_ecma_dep_graph = { workspace = true } diff --git a/crates/nx_project_graph/src/lib.rs b/crates/nx_project_graph/src/lib.rs index 8c2ca4daf0e38..fab870e3eae62 100644 --- a/crates/nx_project_graph/src/lib.rs +++ b/crates/nx_project_graph/src/lib.rs @@ -1,4 +1 @@ -/// Placeholder function for project graph operations -pub fn build_project_graph() -> anyhow::Result<()> { - Ok(()) -} +pub mod utils; \ No newline at end of file diff --git a/packages/nx/src/native/project_graph/utils.rs b/crates/nx_project_graph/src/utils.rs similarity index 88% rename from packages/nx/src/native/project_graph/utils.rs rename to crates/nx_project_graph/src/utils.rs index c2dc966fb3c6c..0d564bdb7779b 100644 --- a/packages/nx/src/native/project_graph/utils.rs +++ b/crates/nx_project_graph/src/utils.rs @@ -1,8 +1,10 @@ -use crate::native::project_graph::types::Project; -use std::collections::HashMap; - mod find_project_for_path; pub use find_project_for_path::*; +mod find_matching_projects; +pub use find_matching_projects::*; + +use nx_core::types::project_graph::Project; +use std::collections::HashMap; pub type ProjectRootMappings = HashMap; pub fn create_project_root_mappings(nodes: &HashMap) -> ProjectRootMappings { diff --git a/packages/nx/src/native/utils/find_matching_projects.rs b/crates/nx_project_graph/src/utils/find_matching_projects.rs similarity index 99% rename from packages/nx/src/native/utils/find_matching_projects.rs rename to crates/nx_project_graph/src/utils/find_matching_projects.rs index ee49968683345..36ab84d918b0e 100644 --- a/packages/nx/src/native/utils/find_matching_projects.rs +++ b/crates/nx_project_graph/src/utils/find_matching_projects.rs @@ -1,6 +1,6 @@ -use crate::native::project_graph::types::{Project, ProjectGraph}; -use hashbrown::HashSet; use std::collections::HashMap; +use hashbrown::HashSet; +use nx_core::types::project_graph::{Project, ProjectGraph}; use nx_glob::NxGlobSet; struct ProjectPattern<'a> { diff --git a/packages/nx/src/native/project_graph/utils/find_project_for_path.rs b/crates/nx_project_graph/src/utils/find_project_for_path.rs similarity index 91% rename from packages/nx/src/native/project_graph/utils/find_project_for_path.rs rename to crates/nx_project_graph/src/utils/find_project_for_path.rs index ce344d188d20f..eee563cbac19f 100644 --- a/packages/nx/src/native/project_graph/utils/find_project_for_path.rs +++ b/crates/nx_project_graph/src/utils/find_project_for_path.rs @@ -1,4 +1,4 @@ -use crate::native::project_graph::utils::{normalize_project_root, ProjectRootMappings}; +use crate::utils::{normalize_project_root, ProjectRootMappings}; use std::path::Path; pub fn find_project_for_path>( @@ -33,10 +33,10 @@ pub fn find_project_for_path>( #[cfg(test)] mod test { - use crate::native::project_graph::types::Project; - use crate::native::project_graph::utils::{ - create_project_root_mappings, find_project_for_path, - }; + + use crate::utils::create_project_root_mappings; + use crate::utils::find_project_for_path::find_project_for_path; + use nx_core::types::project_graph::Project; use std::collections::HashMap; #[test] diff --git a/packages/nx/Cargo.toml b/packages/nx/Cargo.toml index 389132fda2026..f8fb41793675d 100644 --- a/packages/nx/Cargo.toml +++ b/packages/nx/Cargo.toml @@ -9,18 +9,19 @@ crate-type = ['cdylib'] [dependencies] +nx_core = { workspace = true } nx_glob = { workspace = true } nx_walker = { workspace = true } nx_db = { workspace = true } nx_hasher = { workspace = true } -nx_logger = { workspace = true} +nx_logger = { workspace = true } nx_tasks = { workspace = true } nx_workspace = { workspace = true } nx_project_graph = { workspace = true } nx_cache = { workspace = true } nx_terminal = { workspace = true } nx_watch = { workspace = true } -nx_utils = { workspace = true} +nx_utils = { workspace = true } anyhow = { workspace = true } crossbeam-channel = { workspace = true } dashmap = { workspace = true } diff --git a/packages/nx/src/native/project_graph/mod.rs b/packages/nx/src/native/project_graph/mod.rs index db42c2a45c105..0a2d8e264603c 100644 --- a/packages/nx/src/native/project_graph/mod.rs +++ b/packages/nx/src/native/project_graph/mod.rs @@ -1,3 +1,9 @@ -pub mod transfer_project_graph; -pub mod types; -pub mod utils; +use napi::bindgen_prelude::External; +use nx_core::types::project_graph::ProjectGraph; + +#[napi] +/// Transfer the project graph from the JS world to the Rust world, so that we can pass the project graph via memory quicker +/// This wont be needed once the project graph is created in Rust +pub fn transfer_project_graph(project_graph: ProjectGraph) -> External { + External::new(project_graph) +} diff --git a/packages/nx/src/native/project_graph/transfer_project_graph.rs b/packages/nx/src/native/project_graph/transfer_project_graph.rs deleted file mode 100644 index b68bc78a07f11..0000000000000 --- a/packages/nx/src/native/project_graph/transfer_project_graph.rs +++ /dev/null @@ -1,11 +0,0 @@ -use napi::bindgen_prelude::External; - -use crate::native::project_graph::types::ProjectGraph; - -#[napi] -/// Transfer the project graph from the JS world to the Rust world, so that we can pass the project graph via memory quicker -/// This wont be needed once the project graph is created in Rust -pub fn transfer_project_graph(project_graph: ProjectGraph) -> External { - External::new(project_graph) -} - diff --git a/packages/nx/src/native/pseudo_terminal/pseudo_terminal.rs b/packages/nx/src/native/pseudo_terminal/pseudo_terminal.rs index 001e7357c2d15..e48120958c03a 100644 --- a/packages/nx/src/native/pseudo_terminal/pseudo_terminal.rs +++ b/packages/nx/src/native/pseudo_terminal/pseudo_terminal.rs @@ -255,8 +255,16 @@ mod tests { let pseudo_terminal = create_pseudo_terminal().unwrap(); while i < 10 { println!("Running {}", i); - let cp1 = - run_command(&pseudo_terminal, String::from("whoami"), None, None, None).unwrap(); + let cp1 = run_command( + &pseudo_terminal, + String::from("whoami"), + None, + None, + None, + None, + None, + ) + .unwrap(); cp1.wait_receiver.recv().unwrap(); i += 1; } diff --git a/packages/nx/src/native/tasks/hash_planner.rs b/packages/nx/src/native/tasks/hash_planner.rs index 0935259c6685d..e161418c3e536 100644 --- a/packages/nx/src/native/tasks/hash_planner.rs +++ b/packages/nx/src/native/tasks/hash_planner.rs @@ -1,24 +1,21 @@ +use crate::native::tasks::inputs::{ + expand_single_project_inputs, get_inputs, get_inputs_for_dependency, get_named_inputs, +}; +use crate::native::tasks::utils; use crate::native::tasks::{ dep_outputs::get_dep_output, types::{HashInstruction, TaskGraph}, }; -use crate::native::types::{Input, NxJson}; -use crate::native::{ - project_graph::types::ProjectGraph, - tasks::{inputs::SplitInputs, types::Task}, -}; -use napi::bindgen_prelude::External; +use crate::native::tasks::{inputs::SplitInputs, types::Task}; +use nx_core::types::inputs::Input; +use nx_core::types::nx_json::NxJson; +use nx_core::types::project_graph::ProjectGraph; use nx_logger::enable_logger; use rayon::prelude::*; use std::collections::HashMap; +use napi::bindgen_prelude::External; use tracing::trace; -use crate::native::tasks::inputs::{ - expand_single_project_inputs, get_inputs, get_inputs_for_dependency, get_named_inputs, -}; -use crate::native::tasks::utils; -use crate::native::utils::find_matching_projects; - #[napi] pub struct HashPlanner { nx_json: NxJson, @@ -171,7 +168,7 @@ impl HashPlanner { let Some(external_node_name) = external_node_name else { if self.project_graph.nodes.contains_key(dep) { let deps = self.project_graph.dependencies.get(project_name); - if deps.is_some_and(|deps| deps.contains(dep)) { + if deps.is_some_and(|deps| deps.contains(&dep.to_string())) { anyhow::bail!("The externalDependency '{dep}' for '{project_name}:{target_name}' is not an external node and is already a dependency. Please remove it from the externalDependency inputs.") } else { anyhow::bail!("The externalDependency '{dep}' for '{project_name}:{target_name}' is not an external node. If you believe this is a dependency, add an implicitDependency to '{project_name}'") @@ -405,7 +402,7 @@ impl HashPlanner { let Input::Projects { input, projects } = project else { continue; }; - let projects = find_matching_projects(projects, &self.project_graph)?; + let projects = nx_project_graph::utils::find_matching_projects(projects, &self.project_graph)?; for project in projects { let named_inputs = get_named_inputs(&self.nx_json, &self.project_graph.nodes[project]); diff --git a/packages/nx/src/native/tasks/hashers/hash_external.rs b/packages/nx/src/native/tasks/hashers/hash_external.rs index 6c970c76de935..51de9858a0391 100644 --- a/packages/nx/src/native/tasks/hashers/hash_external.rs +++ b/packages/nx/src/native/tasks/hashers/hash_external.rs @@ -3,10 +3,9 @@ use std::sync::Arc; use anyhow::*; use dashmap::DashMap; +use nx_core::types::project_graph::ExternalNode; use nx_hasher::{hash, hash_array}; -use crate::native::project_graph::types::ExternalNode; - pub fn hash_external( external_name: &str, externals: &HashMap, @@ -46,8 +45,8 @@ pub fn hash_all_externals>( #[cfg(test)] mod test { use super::*; - use crate::native::project_graph::types::ExternalNode; use dashmap::DashMap; + use nx_core::types::project_graph::ExternalNode; use std::sync::Arc; fn get_external_nodes_map() -> HashMap { diff --git a/packages/nx/src/native/tasks/hashers/hash_project_config.rs b/packages/nx/src/native/tasks/hashers/hash_project_config.rs index 3b275f53cc74a..a93a40d1e71f0 100644 --- a/packages/nx/src/native/tasks/hashers/hash_project_config.rs +++ b/packages/nx/src/native/tasks/hashers/hash_project_config.rs @@ -2,11 +2,10 @@ use std::collections::HashMap; use anyhow::*; use itertools::Itertools; +use nx_core::types::inputs::Input; +use nx_core::types::project_graph::Project; use nx_hasher::hash; -use crate::native::project_graph::types::Project; -use crate::native::types::Input; - pub fn hash_project_config( project_name: &str, projects: &HashMap, @@ -66,7 +65,7 @@ pub fn hash_project_config( #[cfg(test)] mod tests { use super::*; - use crate::native::project_graph::types::Target; + use nx_core::types::project_graph::{Project, Target}; use std::collections::HashMap; #[test] diff --git a/packages/nx/src/native/tasks/hashers/hash_tsconfig.rs b/packages/nx/src/native/tasks/hashers/hash_tsconfig.rs index ad7b3acb3862a..c8d5f24aab1e6 100644 --- a/packages/nx/src/native/tasks/hashers/hash_tsconfig.rs +++ b/packages/nx/src/native/tasks/hashers/hash_tsconfig.rs @@ -2,8 +2,7 @@ use std::collections::HashMap; use anyhow::*; use nx_hasher::hash; - -use crate::native::project_graph::utils::find_project_for_path; +use nx_project_graph::utils::find_project_for_path; pub fn hash_tsconfig_selectively( project_name: &str, @@ -42,11 +41,11 @@ fn remove_other_project_paths( #[cfg(test)] mod test { + + use nx_core::types::project_graph::Project; + use nx_project_graph::utils::create_project_root_mappings; use std::collections::HashMap; - use crate::native::project_graph::types::Project; - use crate::native::project_graph::utils::create_project_root_mappings; - use super::*; #[test] diff --git a/packages/nx/src/native/tasks/inputs.rs b/packages/nx/src/native/tasks/inputs.rs index ffeee9786113f..c5b22f76ac111 100644 --- a/packages/nx/src/native/tasks/inputs.rs +++ b/packages/nx/src/native/tasks/inputs.rs @@ -1,7 +1,8 @@ -use crate::native::project_graph::types::{Project, ProjectGraph}; use crate::native::tasks::types::Task; -use crate::native::types::{Input, NxJson}; +use nx_core::types::project_graph::{Project, ProjectGraph}; use std::collections::HashMap; +use nx_core::types::inputs::Input; +use nx_core::types::nx_json::NxJson; #[derive(Debug)] pub(super) struct SplitInputs<'a> { diff --git a/packages/nx/src/native/tasks/task_hasher.rs b/packages/nx/src/native/tasks/task_hasher.rs index a5fd4b8488b03..434bf02f43d3e 100644 --- a/packages/nx/src/native/tasks/task_hasher.rs +++ b/packages/nx/src/native/tasks/task_hasher.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use std::sync::Arc; -use crate::native::project_graph::{types::ProjectGraph, utils::create_project_root_mappings}; use crate::native::{ tasks::hashers::{ hash_all_externals, hash_env, hash_external, hash_project_config, hash_project_files, @@ -15,12 +14,12 @@ use crate::native::{ use anyhow::anyhow; use dashmap::DashMap; use napi::bindgen_prelude::{Buffer, External}; +use nx_core::types::project_graph::ProjectGraph; use nx_hasher::hash; +use nx_project_graph::utils::{ProjectRootMappings, create_project_root_mappings}; use rayon::prelude::*; use tracing::{debug, trace, trace_span}; -use crate::native::project_graph::utils::ProjectRootMappings; - #[napi(object)] #[derive(Debug)] pub struct HashDetails { diff --git a/packages/nx/src/native/tasks/utils.rs b/packages/nx/src/native/tasks/utils.rs index 036256080359d..4a0931252d835 100644 --- a/packages/nx/src/native/tasks/utils.rs +++ b/packages/nx/src/native/tasks/utils.rs @@ -1,5 +1,5 @@ -use crate::native::project_graph::types::ProjectGraph; use hashbrown::HashSet; +use nx_core::types::project_graph::ProjectGraph; pub(super) fn find_all_project_node_dependencies<'a>( parent_node_name: &str, diff --git a/packages/nx/src/native/types.rs b/packages/nx/src/native/types.rs index cb6a75c064c65..d78a0dad2a4a9 100644 --- a/packages/nx/src/native/types.rs +++ b/packages/nx/src/native/types.rs @@ -1,9 +1,5 @@ mod file_data; -mod inputs; mod napi_dashmap; -mod nx_json; pub use file_data::FileData; -pub use inputs::*; pub use napi_dashmap::NapiDashMap; -pub use nx_json::*; diff --git a/packages/nx/src/native/utils/mod.rs b/packages/nx/src/native/utils/mod.rs index f7d704ea4332c..ec54535a1bea2 100644 --- a/packages/nx/src/native/utils/mod.rs +++ b/packages/nx/src/native/utils/mod.rs @@ -1,9 +1,5 @@ -mod find_matching_projects; pub mod path; -pub use find_matching_projects::*; - - #[cfg_attr(not(target_arch = "wasm32"), path = "atomics/default.rs")] #[cfg_attr(target_arch = "wasm32", path = "atomics/wasm.rs")] pub mod atomics; diff --git a/packages/nx/src/native/workspace/context.rs b/packages/nx/src/native/workspace/context.rs index 9d92901c50099..d097003c1fa6f 100644 --- a/packages/nx/src/native/workspace/context.rs +++ b/packages/nx/src/native/workspace/context.rs @@ -1,11 +1,12 @@ +use nx_project_graph::utils::find_project_for_path; +use nx_project_graph::utils::ProjectRootMappings; use std::collections::{HashMap, HashSet}; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::Arc; -use crate::native::project_graph::utils::{find_project_for_path, ProjectRootMappings}; use crate::native::types::FileData; -use crate::native::utils::{path::get_child_files, NxCondvar, NxMutex}; +use crate::native::utils::{path::get_child_files, NxCondvar, NxMutex}; use crate::native::workspace::files_archive::{read_files_archive, write_files_archive}; use crate::native::workspace::files_hashing::{full_files_hash, selective_files_hash}; use crate::native::workspace::types::{ @@ -14,11 +15,11 @@ use crate::native::workspace::types::{ use crate::native::workspace::{config_files, types::NxWorkspaceFiles, workspace_files}; use napi::bindgen_prelude::External; use nx_hasher::hash; +use nx_logger::enable_logger; +use nx_utils::path::ToNormalizedString; use rayon::prelude::*; use tracing::{trace, warn}; use xxhash_rust::xxh3; -use nx_utils::path::ToNormalizedString; -use nx_logger::enable_logger; #[napi] pub struct WorkspaceContext { From f03f3545ff4ce2528fb0b6d6425de6381b540a22 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli <4332460+Cammisuli@users.noreply.github.com> Date: Sat, 22 Feb 2025 13:04:15 -0500 Subject: [PATCH 09/11] feat(core): modularize machine_id functionality - Moved `machine_id` module from `packages/nx` to `crates/nx_utils`. - Updated imports in affected files across the project. - Added `machine-uid` dependency to `nx_utils`. - Refactored code to utilize the new `nx_utils::machine_id`. - Simplified fallback logic in `nx_db` for database file naming. --- Cargo.lock | 3 +++ crates/nx_db/src/lib.rs | 2 +- crates/nx_utils/Cargo.toml | 1 + crates/nx_utils/src/lib.rs | 3 ++- .../nx/src/native => crates/nx_utils/src}/machine_id/mod.rs | 0 packages/nx/src/native/db.rs | 2 +- packages/nx/src/native/mod.rs | 1 - 7 files changed, 8 insertions(+), 4 deletions(-) rename {packages/nx/src/native => crates/nx_utils/src}/machine_id/mod.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index a51b0673be6f7..44c83eb924166 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1635,6 +1635,9 @@ dependencies = [ [[package]] name = "nx_utils" version = "0.1.0" +dependencies = [ + "machine-uid", +] [[package]] name = "nx_walker" diff --git a/crates/nx_db/src/lib.rs b/crates/nx_db/src/lib.rs index d7c82ce20b485..e1beda4b18d62 100644 --- a/crates/nx_db/src/lib.rs +++ b/crates/nx_db/src/lib.rs @@ -16,7 +16,7 @@ pub fn connect_to_nx_db( machine_id: String, ) -> anyhow::Result { let cache_dir_buf = PathBuf::from(cache_dir); - let mut db_file_name = db_name.unwrap_or_else(|| machine_id); + let mut db_file_name = db_name.unwrap_or(machine_id); if db_file_name.is_empty() { trace!("Invalid db file name, using fallback name"); diff --git a/crates/nx_utils/Cargo.toml b/crates/nx_utils/Cargo.toml index dd531d3ca6517..cb22a20c0ba47 100644 --- a/crates/nx_utils/Cargo.toml +++ b/crates/nx_utils/Cargo.toml @@ -5,3 +5,4 @@ license.workspace = true version.workspace = true [dependencies] +machine-uid = { workspace = true } \ No newline at end of file diff --git a/crates/nx_utils/src/lib.rs b/crates/nx_utils/src/lib.rs index 912398604ad6b..e7aab9f99d3bd 100644 --- a/crates/nx_utils/src/lib.rs +++ b/crates/nx_utils/src/lib.rs @@ -1,10 +1,11 @@ mod get_mod_time; mod to_normalized_string; + pub mod ci; +pub mod machine_id; pub use get_mod_time::*; - pub mod path { pub use super::to_normalized_string::*; } diff --git a/packages/nx/src/native/machine_id/mod.rs b/crates/nx_utils/src/machine_id/mod.rs similarity index 100% rename from packages/nx/src/native/machine_id/mod.rs rename to crates/nx_utils/src/machine_id/mod.rs diff --git a/packages/nx/src/native/db.rs b/packages/nx/src/native/db.rs index ee8e3037369b0..65581ab9feec3 100644 --- a/packages/nx/src/native/db.rs +++ b/packages/nx/src/native/db.rs @@ -1,10 +1,10 @@ -use crate::native::machine_id::get_machine_id; use napi::bindgen_prelude::External; use std::fs::create_dir_all; use std::path::PathBuf; use nx_db::connection::NxDbConnection; use nx_logger::enable_logger; +use nx_utils::machine_id::get_machine_id; #[napi] pub fn connect_to_nx_db( diff --git a/packages/nx/src/native/mod.rs b/packages/nx/src/native/mod.rs index 049b3af4b8e20..2181362581738 100644 --- a/packages/nx/src/native/mod.rs +++ b/packages/nx/src/native/mod.rs @@ -1,6 +1,5 @@ pub mod cache; pub mod hasher; -mod machine_id; pub mod metadata; pub mod plugins; pub mod project_graph; From 5637d099ea8113fc71272d52c79b6fb880659525 Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli <4332460+Cammisuli@users.noreply.github.com> Date: Sat, 22 Feb 2025 15:32:48 -0500 Subject: [PATCH 10/11] feat(nx-pty): extract pseudo-terminal logic into a new crate - Moved pseudo-terminal-related implementation from `packages/nx` to a new standalone `nx_pty` crate for better modularity and reuse. - Updated dependencies and Cargo project files to include `nx_pty`. - Adjusted imports and references in original `packages/nx` code to use `nx_pty`. - Fixed typings and adjusted method signatures to align with new structure. --- Cargo.lock | 20 ++++++++++++++++-- Cargo.toml | 1 + crates/nx_pty/Cargo.toml | 21 +++++++++++++++++++ crates/nx_pty/src/lib.rs | 1 + .../nx_pty/src}/pseudo_terminal.rs | 12 +++++++---- .../src}/pseudo_terminal/child_process.rs | 1 + .../src}/pseudo_terminal/command/unix.rs | 0 .../src}/pseudo_terminal/command/windows.rs | 0 packages/nx/Cargo.toml | 1 + packages/nx/src/native/index.d.ts | 2 +- packages/nx/src/native/pseudo_terminal/mac.rs | 13 ++++-------- packages/nx/src/native/pseudo_terminal/mod.rs | 9 -------- .../nx/src/native/pseudo_terminal/non_mac.rs | 18 ++++++++++------ 13 files changed, 68 insertions(+), 31 deletions(-) create mode 100644 crates/nx_pty/Cargo.toml create mode 100644 crates/nx_pty/src/lib.rs rename {packages/nx/src/native/pseudo_terminal => crates/nx_pty/src}/pseudo_terminal.rs (97%) rename {packages/nx/src/native => crates/nx_pty/src}/pseudo_terminal/child_process.rs (99%) rename {packages/nx/src/native => crates/nx_pty/src}/pseudo_terminal/command/unix.rs (100%) rename {packages/nx/src/native => crates/nx_pty/src}/pseudo_terminal/command/windows.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 44c83eb924166..95f2bc0e40f2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -334,9 +334,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] @@ -1494,6 +1494,7 @@ dependencies = [ "nx_hasher", "nx_logger", "nx_project_graph", + "nx_pty", "nx_tasks", "nx_terminal", "nx_utils", @@ -1602,6 +1603,21 @@ dependencies = [ "nx_glob", ] +[[package]] +name = "nx_pty" +version = "0.1.0" +dependencies = [ + "anyhow", + "crossbeam-channel", + "crossterm", + "mio 0.8.11", + "napi", + "napi-derive", + "portable-pty", + "tracing", + "winapi", +] + [[package]] name = "nx_tasks" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 402e1c9426013..c833efea48660 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ nx_watch = { path = "crates/nx_watch" } nx_utils = { path = "crates/nx_utils" } nx_logger = { path = "crates/nx_logger" } nx_core = { path = "crates/nx_core" } +nx_pty = { path = "crates/nx_pty"} anyhow = "1.0.95" colored = "2" diff --git a/crates/nx_pty/Cargo.toml b/crates/nx_pty/Cargo.toml new file mode 100644 index 0000000000000..3cefc727c1669 --- /dev/null +++ b/crates/nx_pty/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "nx_pty" +edition.workspace = true +license.workspace = true +version.workspace = true + +[dependencies] +napi = { workspace = true } +napi-derive = { workspace = true } +portable-pty = { workspace = true } +anyhow = { workspace = true } +crossbeam-channel = { workspace = true } +crossterm = { workspace = true } +tracing = { workspace = true } + +[target.'cfg(all(not(windows), not(target_family = "wasm")))'.dependencies] +mio = { workspace = true } + +[target.'cfg(windows)'.dependencies] +winapi = { workspace = true } + diff --git a/crates/nx_pty/src/lib.rs b/crates/nx_pty/src/lib.rs new file mode 100644 index 0000000000000..bdc2d2610777a --- /dev/null +++ b/crates/nx_pty/src/lib.rs @@ -0,0 +1 @@ +pub mod pseudo_terminal; diff --git a/packages/nx/src/native/pseudo_terminal/pseudo_terminal.rs b/crates/nx_pty/src/pseudo_terminal.rs similarity index 97% rename from packages/nx/src/native/pseudo_terminal/pseudo_terminal.rs rename to crates/nx_pty/src/pseudo_terminal.rs index e48120958c03a..05ed2d5042ede 100644 --- a/packages/nx/src/native/pseudo_terminal/pseudo_terminal.rs +++ b/crates/nx_pty/src/pseudo_terminal.rs @@ -1,3 +1,9 @@ +#[cfg_attr(windows, path = "pseudo_terminal/command/windows.rs")] +#[cfg_attr(not(windows), path = "pseudo_terminal/command/unix.rs")] +pub mod os; + +pub mod child_process; + use std::{ collections::HashMap, io::{Read, Write}, @@ -8,6 +14,7 @@ use std::{ time::Instant, }; +use crate::pseudo_terminal::child_process::ChildProcess; use anyhow::anyhow; use crossbeam_channel::{bounded, unbounded, Receiver}; use crossterm::{ @@ -16,10 +23,7 @@ use crossterm::{ tty::IsTty, }; use portable_pty::{CommandBuilder, NativePtySystem, PtyPair, PtySize, PtySystem}; -use tracing::log::trace; - -use super::os; -use crate::native::pseudo_terminal::child_process::ChildProcess; +use tracing::trace; pub struct PseudoTerminal { pub pty_pair: PtyPair, diff --git a/packages/nx/src/native/pseudo_terminal/child_process.rs b/crates/nx_pty/src/pseudo_terminal/child_process.rs similarity index 99% rename from packages/nx/src/native/pseudo_terminal/child_process.rs rename to crates/nx_pty/src/pseudo_terminal/child_process.rs index be4deb401ec13..912508606f872 100644 --- a/packages/nx/src/native/pseudo_terminal/child_process.rs +++ b/crates/nx_pty/src/pseudo_terminal/child_process.rs @@ -5,6 +5,7 @@ use napi::{ }, Env, JsFunction, }; +use napi_derive::napi; use portable_pty::ChildKiller; pub enum ChildProcessMessage { diff --git a/packages/nx/src/native/pseudo_terminal/command/unix.rs b/crates/nx_pty/src/pseudo_terminal/command/unix.rs similarity index 100% rename from packages/nx/src/native/pseudo_terminal/command/unix.rs rename to crates/nx_pty/src/pseudo_terminal/command/unix.rs diff --git a/packages/nx/src/native/pseudo_terminal/command/windows.rs b/crates/nx_pty/src/pseudo_terminal/command/windows.rs similarity index 100% rename from packages/nx/src/native/pseudo_terminal/command/windows.rs rename to crates/nx_pty/src/pseudo_terminal/command/windows.rs diff --git a/packages/nx/Cargo.toml b/packages/nx/Cargo.toml index f8fb41793675d..30c6c4fc51a8f 100644 --- a/packages/nx/Cargo.toml +++ b/packages/nx/Cargo.toml @@ -15,6 +15,7 @@ nx_walker = { workspace = true } nx_db = { workspace = true } nx_hasher = { workspace = true } nx_logger = { workspace = true } +nx_pty = { workspace = true } nx_tasks = { workspace = true } nx_workspace = { workspace = true } nx_project_graph = { workspace = true } diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts index 80b94395dcfb1..3d935e63ba91f 100644 --- a/packages/nx/src/native/index.d.ts +++ b/packages/nx/src/native/index.d.ts @@ -25,7 +25,7 @@ export declare class FileLock { export declare class HashPlanner { constructor(nxJson: NxJson, projectGraph: ExternalObject) getPlans(taskIds: Array, taskGraph: TaskGraph): Record - getPlansReference(taskIds: Array, taskGraph: TaskGraph): JsExternal + getPlansReference(taskIds: Array, taskGraph: TaskGraph): ExternalObject>> } export declare class ImportResult { diff --git a/packages/nx/src/native/pseudo_terminal/mac.rs b/packages/nx/src/native/pseudo_terminal/mac.rs index 5e007a715e67d..8881f78d710bf 100644 --- a/packages/nx/src/native/pseudo_terminal/mac.rs +++ b/packages/nx/src/native/pseudo_terminal/mac.rs @@ -2,24 +2,20 @@ use std::collections::HashMap; use tracing::trace; -use super::child_process::ChildProcess; -use super::os; -use super::pseudo_terminal::{create_pseudo_terminal, run_command}; - use nx_logger::enable_logger; +use nx_pty::pseudo_terminal::child_process::ChildProcess; +use nx_pty::pseudo_terminal::{create_pseudo_terminal, os, run_command}; + + -#[napi] pub struct RustPseudoTerminal {} -#[napi] impl RustPseudoTerminal { - #[napi(constructor)] pub fn new() -> napi::Result { enable_logger(); Ok(Self {}) } - #[napi] pub fn run_command( &self, command: String, @@ -43,7 +39,6 @@ impl RustPseudoTerminal { /// This allows us to run a pseudoterminal with a fake node ipc channel /// this makes it possible to be backwards compatible with the old implementation - #[napi] #[allow(clippy::too_many_arguments)] pub fn fork( &self, diff --git a/packages/nx/src/native/pseudo_terminal/mod.rs b/packages/nx/src/native/pseudo_terminal/mod.rs index 5f658411998b8..0f5de7f6e639d 100644 --- a/packages/nx/src/native/pseudo_terminal/mod.rs +++ b/packages/nx/src/native/pseudo_terminal/mod.rs @@ -1,12 +1,3 @@ -#[cfg_attr(windows, path = "command/windows.rs")] -#[cfg_attr(not(windows), path = "command/unix.rs")] -mod os; - -#[allow(clippy::module_inception)] -mod pseudo_terminal; - -pub mod child_process; - #[cfg_attr(target_os = "macos", path = "mac.rs")] #[cfg_attr(not(target_os = "macos"), path = "non_mac.rs")] pub mod rust_pseudo_terminal; diff --git a/packages/nx/src/native/pseudo_terminal/non_mac.rs b/packages/nx/src/native/pseudo_terminal/non_mac.rs index 8138eb6677ba0..72badf1e8fb4a 100644 --- a/packages/nx/src/native/pseudo_terminal/non_mac.rs +++ b/packages/nx/src/native/pseudo_terminal/non_mac.rs @@ -1,12 +1,11 @@ +use nx_pty::pseudo_terminal::{create_pseudo_terminal, os, run_command, PseudoTerminal}; use std::collections::HashMap; -use tracing::trace; - -use super::child_process::ChildProcess; -use super::os; -use super::pseudo_terminal::{create_pseudo_terminal, run_command, PseudoTerminal}; use nx_logger::enable_logger; +use nx_pty::pseudo_terminal::child_process::ChildProcess; +use tracing::trace; + #[napi] pub struct RustPseudoTerminal { pseudo_terminal: PseudoTerminal, @@ -65,6 +64,13 @@ impl RustPseudoTerminal { ); trace!("nx_fork command: {}", &command); - self.run_command(command, command_dir, js_env, exec_argv, Some(quiet), Some(true)) + self.run_command( + command, + command_dir, + js_env, + exec_argv, + Some(quiet), + Some(true), + ) } } From 407698fdeaf59f467d6d5a52bbe65c018735ee0c Mon Sep 17 00:00:00 2001 From: Jonathan Cammisuli <4332460+Cammisuli@users.noreply.github.com> Date: Sat, 22 Feb 2025 16:30:00 -0500 Subject: [PATCH 11/11] refactor(core): modularize native DB and pseudo-terminal logic - Removed inline native db and pseudo-terminal modules from `nx` package. - Moved `pseudo_terminal` modules to `nx_pty` crate. - Moved DB connection handling into `nx_db` crate under a new `napi` module. - Updated dependencies and adjusted imports for modular separation. - Dropped unused dependencies (`tokio`, `crossbeam-channel`, etc.) for cleanup. --- Cargo.lock | 20 +++++++++++++------ Cargo.toml | 2 +- crates/nx_db/Cargo.toml | 8 +++++++- crates/nx_db/src/lib.rs | 3 ++- .../db.rs => crates/nx_db/src/napi/mod.rs | 9 +++++---- crates/nx_pty/Cargo.toml | 7 +++++-- crates/nx_pty/src/lib.rs | 3 +++ crates/nx_pty/src/napi.rs | 3 +++ .../nx_pty/src/napi}/pseudo_terminal/mac.rs | 6 ++---- .../src/napi}/pseudo_terminal/non_mac.rs | 8 ++++---- crates/nx_pty/src/pseudo_terminal.rs | 1 + crates/nx_utils/Cargo.toml | 4 +++- packages/nx/Cargo.toml | 10 ++-------- packages/nx/src/lib.rs | 6 ++++++ packages/nx/src/native/mod.rs | 7 +++---- packages/nx/src/native/pseudo_terminal/mod.rs | 3 --- 16 files changed, 61 insertions(+), 39 deletions(-) rename packages/nx/src/native/db.rs => crates/nx_db/src/napi/mod.rs (75%) create mode 100644 crates/nx_pty/src/napi.rs rename {packages/nx/src/native => crates/nx_pty/src/napi}/pseudo_terminal/mac.rs (92%) rename {packages/nx/src/native => crates/nx_pty/src/napi}/pseudo_terminal/non_mac.rs (91%) delete mode 100644 packages/nx/src/native/pseudo_terminal/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 95f2bc0e40f2e..6ca88ac6791eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1472,8 +1472,6 @@ version = "0.1.0" dependencies = [ "anyhow", "assert_fs", - "crossbeam-channel", - "crossterm", "dashmap", "dunce", "fs4", @@ -1482,7 +1480,6 @@ dependencies = [ "ignore", "ignore-files 2.1.0", "itertools", - "machine-uid", "mio 0.8.11", "napi", "napi-build", @@ -1502,7 +1499,6 @@ dependencies = [ "nx_watch", "nx_workspace", "parking_lot", - "portable-pty", "rayon", "regex", "rkyv", @@ -1511,9 +1507,7 @@ dependencies = [ "swc_ecma_ast", "swc_ecma_dep_graph", "swc_ecma_parser", - "tempfile", "thiserror", - "tokio", "tracing", "tracing-subscriber", "watchexec", @@ -1558,6 +1552,10 @@ version = "0.1.0" dependencies = [ "anyhow", "fs4", + "napi", + "napi-derive", + "nx_logger", + "nx_utils", "rusqlite", "tempfile", "tracing", @@ -1593,6 +1591,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "nx_napi" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] + [[package]] name = "nx_project_graph" version = "0.1.0" @@ -1613,6 +1620,7 @@ dependencies = [ "mio 0.8.11", "napi", "napi-derive", + "nx_logger", "portable-pty", "tracing", "winapi", diff --git a/Cargo.toml b/Cargo.toml index c833efea48660..42e97000aa42e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ nx_watch = { path = "crates/nx_watch" } nx_utils = { path = "crates/nx_utils" } nx_logger = { path = "crates/nx_logger" } nx_core = { path = "crates/nx_core" } -nx_pty = { path = "crates/nx_pty"} +nx_pty = { path = "crates/nx_pty" } anyhow = "1.0.95" colored = "2" diff --git a/crates/nx_db/Cargo.toml b/crates/nx_db/Cargo.toml index c6e97009ce616..6a17045658c37 100644 --- a/crates/nx_db/Cargo.toml +++ b/crates/nx_db/Cargo.toml @@ -5,10 +5,16 @@ edition.workspace = true license.workspace = true [dependencies] -rusqlite = { workspace = true } +napi = { workspace = true } +napi-derive = { workspace = true } +nx_logger = { workspace = true } +nx_utils = { workspace = true } anyhow = { workspace = true } tracing = { workspace = true } fs4 = { workspace = true } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +rusqlite = { workspace = true } + [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/nx_db/src/lib.rs b/crates/nx_db/src/lib.rs index e1beda4b18d62..aae63322225f8 100644 --- a/crates/nx_db/src/lib.rs +++ b/crates/nx_db/src/lib.rs @@ -1,5 +1,6 @@ pub mod connection; mod initialize; +pub mod napi; use std::fs::create_dir_all; use std::path::PathBuf; @@ -9,7 +10,7 @@ use tracing::{trace, trace_span}; use connection::NxDbConnection; /// Connect to the Nx database -pub fn connect_to_nx_db( +pub fn open_db_connection( cache_dir: String, nx_version: String, db_name: Option, diff --git a/packages/nx/src/native/db.rs b/crates/nx_db/src/napi/mod.rs similarity index 75% rename from packages/nx/src/native/db.rs rename to crates/nx_db/src/napi/mod.rs index 65581ab9feec3..0e9b0575cfd59 100644 --- a/packages/nx/src/native/db.rs +++ b/crates/nx_db/src/napi/mod.rs @@ -1,9 +1,10 @@ +use crate::NxDbConnection; use napi::bindgen_prelude::External; +use napi_derive::*; +use nx_logger::enable_logger; use std::fs::create_dir_all; use std::path::PathBuf; -use nx_db::connection::NxDbConnection; -use nx_logger::enable_logger; use nx_utils::machine_id::get_machine_id; #[napi] @@ -17,11 +18,11 @@ pub fn connect_to_nx_db( create_dir_all(&cache_dir_buf)?; let machine_id = get_machine_id(); - let connection = nx_db::connect_to_nx_db(cache_dir, nx_version, db_name, machine_id)?; + let connection = crate::open_db_connection(cache_dir, nx_version, db_name, machine_id)?; Ok(External::new(connection)) } #[napi] pub fn close_db_connection(mut connection: External) -> anyhow::Result<()> { - nx_db::close_db_connection(std::mem::take(connection.as_mut())) + crate::close_db_connection(std::mem::take(connection.as_mut())) } diff --git a/crates/nx_pty/Cargo.toml b/crates/nx_pty/Cargo.toml index 3cefc727c1669..77a5dba7ba565 100644 --- a/crates/nx_pty/Cargo.toml +++ b/crates/nx_pty/Cargo.toml @@ -5,14 +5,17 @@ license.workspace = true version.workspace = true [dependencies] +nx_logger = { workspace = true } napi = { workspace = true } napi-derive = { workspace = true } -portable-pty = { workspace = true } anyhow = { workspace = true } crossbeam-channel = { workspace = true } -crossterm = { workspace = true } tracing = { workspace = true } +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +portable-pty = { workspace = true } +crossterm = { workspace = true } + [target.'cfg(all(not(windows), not(target_family = "wasm")))'.dependencies] mio = { workspace = true } diff --git a/crates/nx_pty/src/lib.rs b/crates/nx_pty/src/lib.rs index bdc2d2610777a..3c8dd835ba166 100644 --- a/crates/nx_pty/src/lib.rs +++ b/crates/nx_pty/src/lib.rs @@ -1 +1,4 @@ +pub mod napi; + +#[cfg(not(target_arch = "wasm32"))] pub mod pseudo_terminal; diff --git a/crates/nx_pty/src/napi.rs b/crates/nx_pty/src/napi.rs new file mode 100644 index 0000000000000..61f8246b1b170 --- /dev/null +++ b/crates/nx_pty/src/napi.rs @@ -0,0 +1,3 @@ +#[cfg_attr(target_os = "macos", path = "napi/pseudo_terminal/mac.rs")] +#[cfg_attr(not(target_os = "macos"), path = "napi/pseudo_terminal/non_mac.rs")] +pub mod rust_pseudo_terminal; diff --git a/packages/nx/src/native/pseudo_terminal/mac.rs b/crates/nx_pty/src/napi/pseudo_terminal/mac.rs similarity index 92% rename from packages/nx/src/native/pseudo_terminal/mac.rs rename to crates/nx_pty/src/napi/pseudo_terminal/mac.rs index 8881f78d710bf..752c3316f56ef 100644 --- a/packages/nx/src/native/pseudo_terminal/mac.rs +++ b/crates/nx_pty/src/napi/pseudo_terminal/mac.rs @@ -2,11 +2,9 @@ use std::collections::HashMap; use tracing::trace; +use crate::pseudo_terminal::child_process::ChildProcess; +use crate::pseudo_terminal::{create_pseudo_terminal, os, run_command}; use nx_logger::enable_logger; -use nx_pty::pseudo_terminal::child_process::ChildProcess; -use nx_pty::pseudo_terminal::{create_pseudo_terminal, os, run_command}; - - pub struct RustPseudoTerminal {} diff --git a/packages/nx/src/native/pseudo_terminal/non_mac.rs b/crates/nx_pty/src/napi/pseudo_terminal/non_mac.rs similarity index 91% rename from packages/nx/src/native/pseudo_terminal/non_mac.rs rename to crates/nx_pty/src/napi/pseudo_terminal/non_mac.rs index 72badf1e8fb4a..79123bc226e6d 100644 --- a/packages/nx/src/native/pseudo_terminal/non_mac.rs +++ b/crates/nx_pty/src/napi/pseudo_terminal/non_mac.rs @@ -1,11 +1,11 @@ -use nx_pty::pseudo_terminal::{create_pseudo_terminal, os, run_command, PseudoTerminal}; +use napi_derive::napi; use std::collections::HashMap; +use tracing::trace; +use crate::pseudo_terminal::child_process::ChildProcess; +use crate::pseudo_terminal::{create_pseudo_terminal, os, run_command, PseudoTerminal}; use nx_logger::enable_logger; -use nx_pty::pseudo_terminal::child_process::ChildProcess; -use tracing::trace; - #[napi] pub struct RustPseudoTerminal { pseudo_terminal: PseudoTerminal, diff --git a/crates/nx_pty/src/pseudo_terminal.rs b/crates/nx_pty/src/pseudo_terminal.rs index 05ed2d5042ede..811f9f76b9965 100644 --- a/crates/nx_pty/src/pseudo_terminal.rs +++ b/crates/nx_pty/src/pseudo_terminal.rs @@ -131,6 +131,7 @@ pub fn create_pseudo_terminal() -> napi::Result { printing_rx, }) } + pub fn run_command( pseudo_terminal: &PseudoTerminal, command: String, diff --git a/crates/nx_utils/Cargo.toml b/crates/nx_utils/Cargo.toml index cb22a20c0ba47..91a85c8647ec3 100644 --- a/crates/nx_utils/Cargo.toml +++ b/crates/nx_utils/Cargo.toml @@ -5,4 +5,6 @@ license.workspace = true version.workspace = true [dependencies] -machine-uid = { workspace = true } \ No newline at end of file + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +machine-uid = { workspace = true } diff --git a/packages/nx/Cargo.toml b/packages/nx/Cargo.toml index 30c6c4fc51a8f..e99440c5f2cd1 100644 --- a/packages/nx/Cargo.toml +++ b/packages/nx/Cargo.toml @@ -15,7 +15,6 @@ nx_walker = { workspace = true } nx_db = { workspace = true } nx_hasher = { workspace = true } nx_logger = { workspace = true } -nx_pty = { workspace = true } nx_tasks = { workspace = true } nx_workspace = { workspace = true } nx_project_graph = { workspace = true } @@ -24,7 +23,6 @@ nx_terminal = { workspace = true } nx_watch = { workspace = true } nx_utils = { workspace = true } anyhow = { workspace = true } -crossbeam-channel = { workspace = true } dashmap = { workspace = true } dunce = { workspace = true } fs_extra = { workspace = true } @@ -53,8 +51,7 @@ mio = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -portable-pty = { workspace = true } -crossterm = { workspace = true } +nx_pty = { workspace = true } ignore-files = { workspace = true } fs4 = { workspace = true } rusqlite = { workspace = true } @@ -62,7 +59,7 @@ watchexec = { workspace = true } watchexec-events = { workspace = true } watchexec-filterer-ignore = { workspace = true } watchexec-signals = { workspace = true } -machine-uid = { workspace = true } + [build-dependencies] @@ -72,6 +69,3 @@ napi-build = { workspace = true } assert_fs = { workspace = true } # This is only used for unit tests swc_ecma_dep_graph = { workspace = true } -tempfile = { workspace = true } -# We only explicitly use tokio for async tests -tokio = { workspace = true } diff --git a/packages/nx/src/lib.rs b/packages/nx/src/lib.rs index 6cebee94eb05b..a2e4d1d3b5917 100644 --- a/packages/nx/src/lib.rs +++ b/packages/nx/src/lib.rs @@ -4,3 +4,9 @@ extern crate napi_derive; pub mod native; + + +#[cfg(not(target_arch = "wasm32"))] +pub use nx_pty::napi::*; +#[cfg(not(target_arch = "wasm32"))] +pub use nx_db::napi::*; diff --git a/packages/nx/src/native/mod.rs b/packages/nx/src/native/mod.rs index 2181362581738..9c447b98539a3 100644 --- a/packages/nx/src/native/mod.rs +++ b/packages/nx/src/native/mod.rs @@ -8,9 +8,8 @@ mod types; mod utils; pub mod workspace; -#[cfg(not(target_arch = "wasm32"))] -pub mod db; -#[cfg(not(target_arch = "wasm32"))] -pub mod pseudo_terminal; + + + #[cfg(not(target_arch = "wasm32"))] pub mod watch; diff --git a/packages/nx/src/native/pseudo_terminal/mod.rs b/packages/nx/src/native/pseudo_terminal/mod.rs deleted file mode 100644 index 0f5de7f6e639d..0000000000000 --- a/packages/nx/src/native/pseudo_terminal/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -#[cfg_attr(target_os = "macos", path = "mac.rs")] -#[cfg_attr(not(target_os = "macos"), path = "non_mac.rs")] -pub mod rust_pseudo_terminal;