Parallelize make to reduce build time (#2006)

Author: naveentatikonda

.github/workflows/CI.yml

@@ -76,10 +76,10 @@ jobs:
           if lscpu  | grep -i avx2
           then
             echo "avx2 available on system"
-            su `id -un 1000` -c "whoami && java -version && ./gradlew build"
+            su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc`"
           else
             echo "avx2 not available on system"
-            su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dsimd.enabled=false"
+            su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dsimd.enabled=false -Dnproc.count=`nproc`"
           fi  
 
 
@@ -117,15 +117,16 @@ jobs:
           brew reinstall gcc
           export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran
 
+      # TODO: Detect processor count and set the value of nproc.count
       - name: Run build
         run: |
           if sysctl -n machdep.cpu.features machdep.cpu.leaf7_features | grep -i AVX2
           then
               echo "avx2 available on system"
-              ./gradlew build 
+              ./gradlew build -Dnproc.count=3
           else
               echo "avx2 not available on system"
-              ./gradlew build -Dsimd.enabled=false
+              ./gradlew build -Dsimd.enabled=false -Dnproc.count=3
           fi
 
   Build-k-NN-Windows:
@@ -183,6 +184,7 @@ jobs:
           rm .\OpenBLAS-0.3.21-x64.zip
           rm -r .\OpenBLAS\
 
+      # TODO: Detect processor count and set the value of nproc.count
       - name: Run build
         run: |
-          ./gradlew.bat build -D'simd.enabled=false'
+          ./gradlew.bat build -D'simd.enabled=false' -D'nproc.count=4'

.github/workflows/test_security.yml

@@ -70,4 +70,4 @@ jobs:
         # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
         run: |
           chown -R 1000:1000 `pwd`
-          su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true -Dsimd.enabled=true"
+          su `id -un 1000` -c "whoami && java -version && ./gradlew integTest -Dsecurity.enabled=true -Dsimd.enabled=true -Dnproc.count=`nproc`"

CHANGELOG.md

@@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * Fix graph merge stats size calculation [#1844](https://github.com/opensearch-project/k-NN/pull/1844)
 * Disallow a vector field to have an invalid character for a physical file name. [#1936](https://github.com/opensearch-project/k-NN/pull/1936)
 ### Infrastructure
+* Parallelize make to reduce build time [#2006] (https://github.com/opensearch-project/k-NN/pull/2006)
 ### Documentation
 ### Maintenance
 * Fix a flaky unit test:testMultiFieldsKnnIndex, which was failing due to inconsistent merge behaviors [#1924](https://github.com/opensearch-project/k-NN/pull/1924)

DEVELOPER_GUIDE.md

@@ -11,6 +11,7 @@
   - [Build](#build)
     - [JNI Library](#jni-library)
     - [JNI Library Artifacts](#jni-library-artifacts)
+    - [Parallelize make](#parallelize-make)
     - [Enable SIMD Optimization](#enable-simd-optimization)
   - [Run OpenSearch k-NN](#run-opensearch-k-nn)
     - [Run Single-node Cluster Locally](#run-single-node-cluster-locally)
@@ -215,7 +216,7 @@ To build the JNI Library manually, follow these steps:
 cd jni
 cmake .
 
-# To build everything, including tests
+# To build everything, including tests. If your computer has multiple cores you can speed it up by building in parallel using make -j 2 (or a higher number for more parallelism)
 make
 
 # To just build the libraries
@@ -263,6 +264,23 @@ these in your environment, you can disable committing the changes to the library
 not committed, then the full library build process will run each time `cmake` is invoked. In a development environment, 
 it is recommended to setup the user git configuration to avoid this cost.
 
+### Parallelize make
+When we are building the plugin for the first time, it takes some time to build the JNI libraries. We can parallelize make and speed up the build time by setting and passing
+this flag to gradle, `nproc.count` if your computer has more number of cores (greater than or equal to 2).
+```
+# While building OpenSearch k-NN
+./gradlew build -Dnproc.count=4
+
+# While running OpenSearch k-NN
+./gradlew run -Dnproc.count=4
+
+# When building the JNI library manually
+cd jni
+cmake .
+# Pass the processor count with make using `-j`
+make -j 4 
+```
+
 ### Enable SIMD Optimization
 SIMD(Single Instruction/Multiple Data) Optimization is enabled by default on Linux and Mac which boosts the performance
 by enabling `AVX2` on `x86 architecture` and `NEON` on `ARM64 architecture` while building the Faiss library. But to enable SIMD, the underlying processor

build.gradle

@@ -18,6 +18,7 @@ buildscript {
         opensearch_group = "org.opensearch"
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         simd_enabled = System.getProperty("simd.enabled", "true")
+        nproc_count = System.getProperty("nproc.count", "1")
         // This flag determines whether the CMake build system should apply a custom patch. It prevents build failures
         // when the cmakeJniLib task is run multiple times. If the build.lib.commit_patches is true, the CMake build
         // system skips applying the patch if the patches have been applied already. If build.lib.commit_patches is
@@ -331,7 +332,7 @@ task cmakeJniLib(type:Exec) {
 task buildJniLib(type:Exec) {
     dependsOn cmakeJniLib
     workingDir 'jni'
-    commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common'
+    commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}"
 }
 
 test {