SQL-2272: sanitize deps (mongodb#237)

* SQL-2272: sanitize deps

* join arrays in case there are multiple

* send output to stderr

Author: terakilobyte

Cargo.lock

@@ -18,10 +18,7 @@ regex = "1.6.0"
 serde = { version = "1", features = ["derive"] }
 itertools = "0.10.4"
 lazy_static = "1.4.0"
-likely_stable = "0.1.2"
 num-traits = "0.2.14"
-num-derive = "0.3.3"
-chrono = "0.4.24"
 cstr = { path = "../cstr" }
 fancy-regex = "0.11.0"
 shared_sql_utils = { path = "../shared_sql_utils" }

core/Cargo.toml

@@ -19,3 +19,6 @@ utf32 = []
 
 [lints]
 workspace = true
+
+[package.metadata.cargo-machete]
+ignored = ["num-traits"]

cstr/Cargo.toml

@@ -17,3 +17,6 @@ iodbc = []
 odbc_version_3_50 = []
 odbc_version_3_80 = ["odbc_version_3_50"]
 odbc_version_4 = ["odbc_version_3_80"]
+
+[package.metadata.cargo-machete]
+ignored = ["num-traits"]

definitions/Cargo.toml

@@ -177,6 +177,24 @@ functions:
           ${prepare_shell}
           cargo fmt --all --  --check
 
+  "check unused dependencies":
+    - command: shell.exec
+      type: test
+      params:
+        shell: bash
+        working_dir: mongosql-odbc-driver
+        script: |
+          ${prepare_shell}
+          cargo install cargo-machete
+          set +e
+          cargo machete
+          RETURN=$?
+          set -e
+          if [ $RETURN -ne 0 ]; then
+            >&2 echo "Unused dependencies found"
+            >&2 cargo machete
+          fi
+
   "generate SBOM":
     - command: shell.exec
       type: test
@@ -253,7 +271,7 @@ functions:
           chmod +x ./$SBOM_DIR/jq
           echo "------------------------------------"
           echo "<<<< Done installing SBOM tools"
-  
+
           echo ">>>> Generate SBOM..."
           echo "--  Generating SBOMs with the licenses information --"
           cargo cyclonedx --target all -v -f json
@@ -270,18 +288,18 @@ functions:
           echo "------------------------------------"
 
           echo "-- Merging the SBOMs with the licenses information and the SBOM with the  vulnerabilities information in $SBOM_FINAL --"
-          
+
           temp_output="temp_output.json"
           if [[ -f "$temp_output" ]] ; then
               rm "$temp_output"
           fi
           touch $temp_output
-          
+
           while IFS= read -r line
           do
             if [[ "$line" == *"purl"* ]]; then
               bash_purl=$(echo $line | cut -d '"' -f4)
-              command=$(echo "./$SBOM_DIR/jq '.components[] | select(.purl == \"$bash_purl\").licenses' $SBOM_LICENSES")
+              command=$(echo "./$SBOM_DIR/jq '.components[] | select(.purl == \"$bash_purl\").licenses' $SBOM_LICENSES | ./$SBOM_DIR/jq -s 'flatten(1)'")
               # Add the license information back in the augmented SBOM.
               licenseInfo=$(eval " $command")
               if [[ -z "$licenseInfo" ]]; then
@@ -294,12 +312,12 @@ functions:
 
           done < $SBOM_VULN
           echo "------------------------------------"
-          
+
           echo "--  Adding the name of the team responsible for each dependency as required by Silk and format the json file --"
           echo "./$SBOM_DIR/jq '.components[].properties += [{\"name\": \"internal:team_responsible\", \"value\": \"Atlas SQL\"}]' $temp_output > $SBOM_FINAL"
           ./$SBOM_DIR/jq '.components[].properties += [{"name": "internal:team_responsible", "value": "Atlas SQL"}]' $temp_output > $SBOM_FINAL
           echo "------------------------------------"
-          
+
           echo "-- Adding VEX info for vulnerabilities still present in SBOM--"
           IFS=','; for vuln_id in $ALLOW_VULNS; do
             echo "-- Updating SBOM with VEX info for vulnerability with id $vuln_id--"
@@ -329,7 +347,7 @@ functions:
         working_dir: mongosql-odbc-driver
         script: |
           ${prepare_shell}
-          
+
           echo ">>>> Scan SBOM for vulnerabilities..."
           if [[ "$ALLOW_VULNS" != "" ]]; then
             echo "Vulnerability ids to ignore : $ALLOW_VULNS"
@@ -345,12 +363,12 @@ functions:
             done
             echo "------------------------------------"
           fi
-          
+
           echo "-- Scanning dependency for vulnerabilities --"
           ./$SBOM_DIR/grype sbom:$SBOM_LICENSES --fail-on low
           echo "---------------------------------------------"
           echo "<<<< Done scanning SBOM"
-  
+
   "generate compliance report":
     - command: shell.exec
       params:
@@ -424,9 +442,9 @@ functions:
           SILK_CLIENT_ID=${SILK_CLIENT_ID}
           SILK_CLIENT_SECRET=${SILK_CLIENT_SECRET}
           EOF
-          
+
           echo "SBOM_FINAL = $SBOM_FINAL"
-          
+
           echo "-- Uploading initial SBOM Lite to Silk --"
           docker run -i --platform="linux/amd64" --rm -v "$PWD":/pwd \
           --env-file silkbomb.env \
@@ -447,7 +465,7 @@ functions:
           SILK_CLIENT_ID=${SILK_CLIENT_ID}
           SILK_CLIENT_SECRET=${SILK_CLIENT_SECRET}
           EOF
-          
+
           echo "-- Downloading augmented SBOM --"
           docker run -i --platform="linux/amd64" --rm -v "$PWD":/pwd \
           --env-file silkbomb.env \
@@ -463,7 +481,6 @@ functions:
         content_type: application/json
         bucket: mciuploads
         permissions: public-read
-  
 
   "publish augmented SBOM":
     - command: s3.get
@@ -534,7 +551,7 @@ functions:
             -v $(pwd):$(pwd) -w $(pwd) \
             ${garasign_jsign_image} \
             /bin/bash -c "jsign -a mongo-authenticode-2021 --replace --tsaurl http://timestamp.digicert.com -d SHA-256 ${MSI_FILENAME}"
-          
+
           # Generating checksums
           if [ -e $msi_filename ]; then
             shasum -a 1 ${MSI_FILENAME} | tee ${MSI_FILENAME}.sha1
@@ -1877,14 +1894,14 @@ functions:
         content_type: text/plain
         bucket: mciuploads
         permissions: public-read
-  
+
   "publish static code analysis":
     - command: s3.get
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
         local_file: mongosql-odbc-driver/${STATIC_CODE_ANALYSIS_NAME}
-        remote_file:  mongosql-odbc-driver/artifacts/${version_id}/ssdlc/${STATIC_CODE_ANALYSIS_NAME}
+        remote_file: mongosql-odbc-driver/artifacts/${version_id}/ssdlc/${STATIC_CODE_ANALYSIS_NAME}
         content_type: application/json
         bucket: mciuploads
     - command: s3.put
@@ -1915,6 +1932,11 @@ tasks:
       - func: "install rust toolchain"
       - func: "check rustfmt"
 
+  - name: unused-deps
+    commands:
+      - func: "install rust toolchain"
+      - func: "check unused dependencies"
+
   - name: sbom
     commands:
       - func: "install rust toolchain"
@@ -2158,7 +2180,6 @@ tasks:
       - func: "generate compliance report"
       - func: "publish compliance report"
 
-
 task_groups:
   - name: windows-windows-test-unit-group
     setup_group_can_fail_task: false
@@ -2205,6 +2226,7 @@ buildvariants:
     tasks:
       - name: clippy
       - name: rustfmt
+      - name: unused-deps
       - name: asan-compile
 
   - name: code-quality-security

evergreen.yml

@@ -17,7 +17,6 @@ logger = { path = "../logger" }
 log = "0.4"
 regex = "1"
 num-traits = "0.2.14"
-num-derive = "0.3.3"
 mongodb = { version = "3", features = ["aws-auth", "dns-resolver"] }
 tailcall = "1.0"
 # Do NOT change these features without consulting with other team members.

integration_test/Cargo.toml

@@ -8,7 +8,6 @@ edition = "2021"
 [dependencies]
 log = "0.4.17"
 log4rs = { version = "1.2.0", features = ["background_rotation"] }
-cstr = { path = "../cstr" }
 shared_sql_utils = { path = "../shared_sql_utils" }
 constants = { path = "../constants" }
 directories = "5.0"

logger/Cargo.toml

@@ -1,12 +1,13 @@
 [package]
 name = "macos_postinstall"
 version = "0.0.0"
-authors = [
-  "Patrick Meredith <pmeredit@protonmail.com>",
-]
+authors = ["Patrick Meredith <pmeredit@protonmail.com>"]
 edition = "2021"
 
 [dependencies]
 rust-ini = "0.20.0"
 itertools = "0.10.5"
 lazy_static = "1"
+
+[package.metadata.cargo-machete]
+ignored = ["rust-ini"]

macos_postinstall/Cargo.toml

@@ -11,11 +11,9 @@ authors = [
 edition = "2021"
 
 [dependencies]
-async-std = { version = "1.12.0", features = ["attributes"] }
 thiserror = "1"
 lazy_static = "1.4.0"
 num-traits = "0.2.14"
-num-derive = "0.3.3"
 regex = "1.6.0"
 chrono = "0.4.24"
 constants = { path = "../constants" }
@@ -31,21 +29,12 @@ log = "0.4.17"
 # The features are used to control the behavior of tokio. Tokio is unsafe to use
 # across ABI boundaries in any other runtime but current_thread
 tokio = { version = "1", features = ["rt", "sync", "io-util", "macros", "net"] }
-mongodb = { version="3", features = ["aws-auth", "dns-resolver"] }
+mongodb = { version = "3", features = ["aws-auth", "dns-resolver"] }
 
 [dependencies.bson]
 version = "2"
 features = ["chrono-0_4"]
 
-[dependencies.windows]
-version = "0.*"
-features = [
-    "Win32_Foundation",
-    "Win32_System_SystemServices",
-    "Win32_UI_WindowsAndMessaging",
-    "Win32_System_LibraryLoader",
-]
-
 
 [dev-dependencies]
 serde = { version = "1", features = ["derive"] }