Bump rattler for new lockfile support, fail on pypi packages (#22)

Co-authored-by: Pavel Zwerschke <pavel.zwerschke@quantco.com>

Author: PaulKMueller

Cargo.lock

@@ -1,7 +1,7 @@
 [package]
 name = "conda-deny"
 description = "A CLI tool to check your project's dependencies for license compliance."
-version = "0.3.2"
+version = "0.3.3"
 edition = "2021"
 
 [features]
@@ -29,8 +29,8 @@ regex = "1.11.1"
 spdx = "0.10.7"
 colored = "2.1.0"
 async-trait = "0.1.83"
-rattler_conda_types = "0.28.3"
-rattler_lock = "0.22.28"
+rattler_conda_types = "0.29.3"
+rattler_lock = "0.22.32"
 anyhow = "1.0.93"
 clap-verbosity-flag = "3.0.1"
 env_logger = "0.11.5"

Cargo.toml

@@ -36,6 +36,10 @@ pub enum Commands {
 
         #[arg(short, long, action = ArgAction::SetTrue)]
         osi: bool,
+
+        /// Ignore when encountering pypi packages instead of failing.
+        #[arg(long, action = ArgAction::SetTrue)]
+        ignore_pypi: bool,
     },
     List {},
 }

src/cli.rs

@@ -19,26 +19,29 @@ use log::debug;
 use crate::conda_deny_config::CondaDenyConfig;
 use crate::license_info::LicenseInfos;
 
+// todo: refactor this
 pub type CliInput<'a> = (
     &'a CondaDenyConfig,
     &'a Vec<String>,
     &'a Vec<String>,
     &'a Vec<String>,
     &'a Vec<String>,
     bool,
+    bool,
 );
 pub type CheckOutput = (Vec<LicenseInfo>, Vec<LicenseInfo>);
 
 pub fn fetch_license_infos(cli_input: CliInput) -> Result<LicenseInfos> {
-    let (conda_deny_config, cli_lockfiles, cli_platforms, cli_environments, conda_prefixes, _) =
+    let (conda_deny_config, cli_lockfiles, cli_platforms, cli_environments, conda_prefixes, _, cli_ignore_pypi) =
         cli_input;
 
     if conda_prefixes.is_empty() {
         LicenseInfos::get_license_infos_from_config(
             conda_deny_config,
             cli_lockfiles,
             cli_platforms,
-            cli_environments,
+            cli_environments, 
+            cli_ignore_pypi,
         )
         .with_context(|| "Getting license information from config file failed.")
     } else {
@@ -55,7 +58,7 @@ pub fn list(cli_input: CliInput) -> Result<()> {
 }
 
 pub fn check_license_infos(cli_input: CliInput) -> Result<CheckOutput> {
-    let (conda_deny_config, _, _, _, _, osi) = cli_input;
+    let (conda_deny_config, _, _, _, _, osi, _) = cli_input;
 
     let license_infos =
         fetch_license_infos(cli_input).with_context(|| "Fetching license information failed.")?;

src/lib.rs

@@ -12,7 +12,7 @@ use crate::{
     expression_utils::{check_expression_safety, extract_license_texts, parse_expression},
     license_whitelist::ParsedLicenseWhitelist,
     list,
-    pixi_lock::get_package_records_for_pixi_lock,
+    pixi_lock::get_conda_packages_for_pixi_lock,
     CheckOutput,
 };
 
@@ -142,27 +142,30 @@ impl LicenseInfos {
         lockfiles: Vec<String>,
         platforms: Vec<String>,
         environment_specs: Vec<String>,
+        ignore_pypi: bool,
     ) -> Result<LicenseInfos> {
         let mut license_infos = Vec::new();
 
         let mut package_records = Vec::new();
 
         if lockfiles.is_empty() {
-            let package_records_for_lockfile = get_package_records_for_pixi_lock(
+            let package_records_for_lockfile = get_conda_packages_for_pixi_lock(
                 None,
                 environment_specs.clone(),
                 platforms.clone(),
+                ignore_pypi,
             )
             .with_context(|| "Failed to get package records for pixi.lock")?;
 
             package_records.extend(package_records_for_lockfile);
         } else {
             for lockfile in lockfiles {
                 let path = Path::new(&lockfile);
-                let package_records_for_lockfile = get_package_records_for_pixi_lock(
+                let package_records_for_lockfile = get_conda_packages_for_pixi_lock(
                     Some(path),
                     environment_specs.clone(),
                     platforms.clone(),
+                    ignore_pypi,
                 )
                 .with_context(|| {
                     format!("Failed to get package records from lockfile: {}", &lockfile)
@@ -220,6 +223,7 @@ impl LicenseInfos {
         cli_lockfiles: &[String],
         cli_platforms: &[String],
         cli_environments: &[String],
+        cli_ignore_pypi: bool,
     ) -> Result<LicenseInfos> {
         let mut platforms = config.get_platform_spec().map_or(vec![], |p| p);
         let mut lockfiles = config.get_lockfile_spec();
@@ -229,7 +233,7 @@ impl LicenseInfos {
         lockfiles.extend(cli_lockfiles.to_owned());
         environment_specs.extend(cli_environments.to_owned());
 
-        LicenseInfos::from_pixi_lockfiles(lockfiles, platforms, environment_specs)
+        LicenseInfos::from_pixi_lockfiles(lockfiles, platforms, environment_specs, cli_ignore_pypi)
     }
 
     pub fn check(&self, license_whitelist: &ParsedLicenseWhitelist) -> CheckOutput {
@@ -469,7 +473,7 @@ mod tests {
             "tests/test_pyproject_toml_files/"
         );
         let config = CondaDenyConfig::from_path(&test_file_path).expect("Failed to read config");
-        let license_infos = LicenseInfos::get_license_infos_from_config(&config, &[], &[], &[]);
+        let license_infos = LicenseInfos::get_license_infos_from_config(&config, &[], &[], &[], false);
         assert_eq!(license_infos.unwrap().license_infos.len(), 396);
     }
 }

src/license_info.rs

@@ -18,6 +18,12 @@ fn main() -> Result<()> {
         _ => false,
     };
 
+    // todo: add an end-to-end test for this
+    let ignore_pypi = match cli.command {
+        Commands::Check { ignore_pypi, .. } => ignore_pypi,
+        _ => false,
+    };
+
     let mut config = CondaDenyConfig::empty();
 
     if !osi {
@@ -59,6 +65,7 @@ fn main() -> Result<()> {
         &cli_environments,
         &conda_prefixes,
         osi,
+        ignore_pypi,
     );
 
     debug!("CLI input for platforms: {:?}", cli_platforms);
@@ -73,6 +80,7 @@ fn main() -> Result<()> {
         Commands::Check {
             include_safe,
             osi: _,
+            ignore_pypi: _
         } => {
             debug!("Check command called.");
 

src/main.rs

@@ -2,8 +2,9 @@ use std::path::Path;
 
 use anyhow::{Context, Result};
 
+use log::warn;
 use rattler_conda_types::{PackageRecord, Platform};
-use rattler_lock::LockFile;
+use rattler_lock::{LockFile, LockedPackageRef};
 
 fn _get_environment_names(pixi_lock_path: &Path) -> Vec<String> {
     let lock = LockFile::from_path(pixi_lock_path).unwrap();
@@ -14,10 +15,11 @@ fn _get_environment_names(pixi_lock_path: &Path) -> Vec<String> {
     environment_names
 }
 
-pub fn get_package_records_for_pixi_lock(
+pub fn get_conda_packages_for_pixi_lock(
     pixi_lock_path: Option<&Path>,
     mut environment_spec: Vec<String>,
     platform_spec: Vec<String>,
+    ignore_pypi: bool,
 ) -> Result<Vec<PackageRecord>> {
     let pixi_lock_path = pixi_lock_path.unwrap_or(Path::new("pixi.lock"));
 
@@ -30,15 +32,25 @@ pub fn get_package_records_for_pixi_lock(
 
     let mut package_records = Vec::new();
 
+    // todo: refactor this
     if platform_spec.is_empty() {
         for environment_name in environment_spec {
             if let Some(environment) = lock.environment(&environment_name) {
                 for platform in environment.platforms() {
                     if let Some(packages) = environment.packages(platform) {
                         for package in packages {
-                            if let Some(conda_package) = package.into_conda() {
-                                let package_record = conda_package.package_record();
-                                package_records.push(package_record.to_owned());
+                            match package {
+                                LockedPackageRef::Conda(conda_package) => {
+                                    let package_record = conda_package.record();
+                                    package_records.push(package_record.to_owned());
+                                }
+                                LockedPackageRef::Pypi(package_data, _) => {
+                                    if !ignore_pypi {
+                                        return Err(anyhow::anyhow!("Pypi packages are not supported: {}", package_data.name));
+                                    } else {
+                                        warn!("Ignoring pypi package: {}", package_data.name);
+                                    }
+                                }
                             }
                         }
                     }
@@ -52,9 +64,18 @@ pub fn get_package_records_for_pixi_lock(
                     if let Some(environment) = lock.environment(&environment_name) {
                         if let Some(packages) = environment.packages(platform) {
                             for package in packages {
-                                if let Some(conda_package) = package.into_conda() {
-                                    let package_record = conda_package.package_record();
-                                    package_records.push(package_record.to_owned());
+                                match package {
+                                    LockedPackageRef::Conda(conda_package) => {
+                                        let package_record = conda_package.record();
+                                        package_records.push(package_record.to_owned());
+                                    }
+                                    LockedPackageRef::Pypi(package_data, _) => {
+                                        if !ignore_pypi {
+                                            return Err(anyhow::anyhow!("Pypi packages are not supported: {}", package_data.name));
+                                        } else {
+                                            warn!("Ignoring pypi package: {}", package_data.name);
+                                        }
+                                    }
                                 }
                             }
                         }
@@ -83,17 +104,18 @@ mod tests {
     #[test]
     fn test_get_packages_for_pixi_lock() {
         let path = Path::new("tests/test_pixi_lock_files/valid1_pixi.lock");
-        let package_records = get_package_records_for_pixi_lock(Some(path), vec![], vec![]);
+        let package_records = get_conda_packages_for_pixi_lock(Some(path), vec![], vec![], false);
         assert_eq!(package_records.unwrap().len(), 758);
 
         let package_records =
-            get_package_records_for_pixi_lock(Some(path), vec!["lint".to_string()], vec![]);
+            get_conda_packages_for_pixi_lock(Some(path), vec!["lint".to_string()], vec![], false);
         assert_eq!(package_records.unwrap().len(), 219);
 
-        let package_records = get_package_records_for_pixi_lock(
+        let package_records = get_conda_packages_for_pixi_lock(
             Some(path),
             vec!["lint".to_string()],
             vec!["linux-64".to_string()],
+            false,
         );
         assert_eq!(package_records.unwrap().len(), 48);
     }

src/pixi_lock.rs

@@ -0,0 +1,10 @@
+[tool.conda-deny]
+safe-licenses = [
+    "Apache-2.0",
+    "Unlicense",
+    "WTFPL",
+    "MIT",
+]
+ignore-packages = [
+    {package="_libgcc_mutex"}
+]