[Backport to main] Backport multiple commits from 2.x to main branch

.github/workflows/CI-workflow.yml

@@ -1,19 +1,27 @@
 name: Build and Test ml-commons
 # This workflow is triggered on pull requests and push to any branches
 on:
-  pull_request:
-    branches:
-      - "*"
   push:
-    branches:
-      - "*"
+    branches-ignore:
+      - 'backport/**'
+      - 'create-pull-request/**'
+      - 'dependabot/**'
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   Build-ml:
     strategy:
       matrix:
         java: [11, 17, 20]
 
     name: Build and Test MLCommons Plugin
+    if: github.repository == 'opensearch-project/ml-commons'
+    environment: ml-commons-cicd-env
     runs-on: ubuntu-latest
 
     steps:
@@ -22,19 +30,34 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
 
-      # ml-commons
+      - uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.ML_ROLE }}
+          aws-region: us-west-2
+
       - name: Checkout MLCommons
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Build and Run Tests
         run: |
+          export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
+          export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
+          echo "::add-mask::$OPENAI_KEY"
+          echo "::add-mask::$COHERE_KEY"
           ./gradlew build
       - name: Publish to Maven Local
         run: |
           ./gradlew publishToMavenLocal
       - name: Multi Nodes Integration Testing
         run: |
+          export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
+          export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
+          echo "::add-mask::$OPENAI_KEY"
+          echo "::add-mask::$COHERE_KEY"
           ./gradlew integTest -PnumNodes=3
+
       - name: Pull and Run Docker
         run: |
           plugin=`basename $(ls plugin/build/distributions/*.zip)`
@@ -71,6 +94,10 @@ jobs:
         if: env.imagePresent == 'true'
         run: |
           security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opensearch-security|wc -l`
+          export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
+          export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
+          echo "::add-mask::$OPENAI_KEY"
+          echo "::add-mask::$COHERE_KEY"
           if [ $security -gt 0 ]
           then
             echo "Security plugin is available"
@@ -91,6 +118,8 @@ jobs:
       matrix:
         java: [11, 17, 20]
     name: Build and Test MLCommons Plugin on Windows
+    if: github.repository == 'opensearch-project/ml-commons'
+    environment: ml-commons-cicd-env
     runs-on: windows-latest
 
     steps:
@@ -99,16 +128,33 @@ jobs:
         with:
           java-version: ${{ matrix.java }}
 
+      - uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.ML_ROLE }}
+          aws-region: us-west-2
+
       # ml-commons
       - name: Checkout MLCommons
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Build and Run Tests
+        shell: bash
         run: |
+          export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
+          export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
+          echo "::add-mask::$OPENAI_KEY"
+          echo "::add-mask::$COHERE_KEY"
           ./gradlew.bat build
       - name: Publish to Maven Local
         run: |
           ./gradlew publishToMavenLocal
 #      - name: Multi Nodes Integration Testing
+#        shell: bash
 #        run: |
+#          export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
+#          export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
+#          echo "::add-mask::$OPENAI_KEY"
+#          echo "::add-mask::$COHERE_KEY"
 #          ./gradlew integTest -PnumNodes=3

DEVELOPER_GUIDE.md

@@ -1,30 +1,81 @@
-- [Developer Guide](#developer-guide)
-    - [Forking and Cloning](#forking-and-cloning)
-    - [Install Prerequisites](#install-prerequisites)
-        - [JDK 14](#jdk-14)
-    - [Setup](#setup)
-    - [Build](#build)
-        - [Building from the command line](#building-from-the-command-line)
-        - [Building from the IDE](#building-from-the-ide)
-        - [Debugging](#debugging)
+<p center><img src="https://github.com/opensearch-project/opensearch-py/raw/main/OpenSearch.svg" height="64px" alt="Opensearch"/></p>
+<h1 center>ML-Commons Developer Guide</h1>
 
-## Developer Guide
+This guide applies to the development within the ML-Commons project
 
-### Forking and Cloning
+- [Getting started guide](#getting-started-guide)
+    - [Key technologies](#key-technologies)
+    - [Prerequisites](#prerequisites)
+    - [Fork and clone ML-Commons](#fork-and-clone-ml-commons)
+    - [Run OpenSearch](#run-opensearch)
+    - [Build](#Build)
+      - [Building from the command line](#Building-from-the-command-line)
+      - [Debugging](#Debugging)
+- [More documentation](#More-docs)
+- [Code guidelines](#code-guidelines)
 
-Fork this repository on GitHub, and clone locally with `git clone`.
 
-### Install Prerequisites
 
-#### JDK 14
 
-OpenSearch components build using Java 14 at a minimum. This means you must have a JDK 14 installed with the environment variable `JAVA_HOME` referencing the path to Java home for your JDK 14 installation, e.g. `JAVA_HOME=/usr/lib/jvm/jdk-14`.
+## Getting started guide
 
-### Setup
+This guide is for any developer who wants a running local development environment where you can make, see, and test changes. It's opinionated to get you running as quickly and easily as possible, but it's not the only way to set up a development environment.
 
-1. Clone the repository (see [Forking and Cloning](#forking-and-cloning))
-2. Make sure `JAVA_HOME` is pointing to a Java 14 JDK (see [Install Prerequisites](#install-prerequisites))
-3. Launch Intellij IDEA, Choose Import Project.
+If you're only interested in installing and using this plugin features, you can just install Opensearch and ml-commons plugin will be integrated with Opensearch.
+
+If you're planning to contribute code (features or fixes) to this repository, great! Make sure to also read the [contributing guide](CONTRIBUTING.md).
+
+
+### Key technologies
+
+ml-commons is primarily a Java based plugin for machine learning in opensearch. To effectively contribute you need to be familiar with Java.
+
+### Prerequisites
+
+To develop on ml-commons, you'll need:
+
+- A [GitHub account](https://docs.github.com/en/get-started/onboarding/getting-started-with-your-github-account)
+- [`git`](https://git-scm.com/) for version control
+- [`Java`](https://www.java.com/en/)
+- A code editor of your choice, configured for Java. If you don't have a favorite editor, we suggest [Intellij](https://www.jetbrains.com/idea/)
+
+If you already have these installed or have your own preferences for installing them, skip ahead to the [Fork and clone ml-commons](#fork-and-clone-ml-commons) section.
+
+
+#### Install `git`
+
+If you don't already have it installed (check with `git --version`) we recommend following [the `git` installation guide for your OS](https://git-scm.com/downloads).
+
+#### Install `Java`
+
+You can install any version of Java starting from 17. [`Jenv`](https://www.jenv.be/) is a good option to use so that you can have multiple versions of Java.
+
+
+### Fork and clone ml-commons
+
+All local development should be done in a [forked repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo).
+Fork ml-commons by clicking the "Fork" button at the top of the [GitHub repository](https://github.com/opensearch-project/ml-commons).
+
+Clone your forked version of ml-commons to your local machine (replace `opensearch-project` in the command below with your GitHub username):
+
+```bash
+$ git clone git@github.com:opensearch-project/ml-commons.git
+```
+
+### Run OpenSearch
+
+You can install Opensearch multiple ways:
+
+1. https://opensearch.org/downloads.html#docker-compose
+2. https://opensearch.org/docs/2.5/install-and-configure/install-opensearch/tar/
+
+#### Default setup for opensearch
+
+```yml
+opensearch.hosts: ["https://localhost:9200"]
+opensearch.username: "admin" # Default username
+opensearch.password: "admin" # Default password
+```
 
 ### Build
 
@@ -33,20 +84,16 @@ This package uses the [Gradle](https://docs.gradle.org/current/userguide/usergui
 #### Building from the command line
 
 1. `./gradlew build` builds and tests, `./gradlew build buildDeb buildRpm` build RPM and DEB.
-2. `./gradlew :run` launches a single node cluster with ml-commons plugin installed
-3. `./gradlew :integTest` launches a single node cluster with ml-commons plugin installed and runs all integration tests except security. Use `./gradlew integTest -PnumNodes=<number>` to launch multi-node cluster.
-4. ` ./gradlew :integTest --tests="<class path>.<test method>"` runs a single integration test class or method, for example `./gradlew integTest --tests="org.opensearch.ml.rest.RestMLTrainAndPredictIT.testTrainAndPredictKmeansWithEmptyParam"` or `./gradlew integTest --tests="org.opensearch.ml.rest.RestMLTrainAndPredictIT"`
+2. `./gradlew run` launches a single node cluster with ml-commons plugin installed
+3. `./gradlew integTest` launches a single node cluster with ml-commons plugin installed and runs all integration tests except security. Use `./gradlew integTest -PnumNodes=<number>` to launch multi-node cluster.
+4. ` ./gradlew integTest --tests="<class path>.<test method>"` runs a single integration test class or method, for example `./gradlew integTest --tests="org.opensearch.ml.rest.RestMLTrainAndPredictIT.testTrainAndPredictKmeansWithEmptyParam"` or `./gradlew integTest --tests="org.opensearch.ml.rest.RestMLTrainAndPredictIT"`
 5. `./gradlew integTest -Dtests.class="<class path>"` run specific integ test class, for example `./gradlew integTest -Dtests.class="org.opensearch.ml.rest.RestMLTrainAndPredictIT"`
 6. `./gradlew integTest -Dtests.method="<method name>"` run specific integ test method, for example `./gradlew integTest -Dtests.method="testTrainAndPredictKmeans"`
-7. `./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin` launches integration tests against a local cluster and run tests with security. Detail steps: (1)download OpenSearch tarball to local and install by running `opensearch-tar-install.sh`; (2)build ML plugin zip with your change and install ML plugin zip; (3)restart local test cluster; (4) run this gradle command to test.  
+7. `./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin` launches integration tests against a local cluster and run tests with security. Detail steps: (1)download OpenSearch tarball to local and install by running `opensearch-tar-install.sh`; (2)build ML plugin zip with your change and install ML plugin zip; (3)restart local test cluster; (4) run this gradle command to test.
 8. `./gradlew spotlessApply` formats code. And/or import formatting rules in `.eclipseformat.xml` with IDE.
 
 When launching a cluster using one of the above commands logs are placed in `/build/cluster/run node0/opensearch-<version>/logs`. Though the logs are teed to the console, in practices it's best to check the actual log file.
 
-#### Building from the IDE
-
-Currently, the only IDE we support is IntelliJ IDEA.  It's free, it's open source, it works. The gradle tasks above can also be launched from IntelliJ's Gradle toolbar and the extra parameters can be passed in via the Launch Configurations VM arguments.
-
 #### Debugging
 
 Sometimes it's useful to attach a debugger to either the OpenSearch cluster or the integ tests to see what's going on. When running unit tests you can just hit 'Debug' from the IDE's gutter to debug the tests.  To debug code running in an actual server run:
@@ -65,4 +112,36 @@ To debug code running in an integ test (which exercises the server from a separa
 ./gradlew -Dtest.debug :integTest 
 ```
 
-The test runner JVM will start suspended and wait for a debugger to attach to `localhost:5005` before running the tests.
+The test runner JVM will start suspended and wait for a debugger to attach to `localhost:5005` before running the tests.
+
+## More docs
+
+1. [Model serving framework](https://opensearch.org/docs/latest/ml-commons-plugin/model-serving-framework/)
+2. [Model Access Control](https://github.com/opensearch-project/ml-commons/blob/2.x/docs/model_access_control.md)
+3. [How to add a new function](https://github.com/opensearch-project/ml-commons/blob/2.x/docs/how-to-add-new-function.md)
+
+## Code guidelines
+
+#### Filenames
+
+All filenames should use `CamelCase`.
+
+**Right:** `ml-commons/common/src/main/java/org.opensearch/ml/common/MLModelGroup.java`
+
+**Wrong:** `ml-commons/common/src/main/java/org.opensearch/ml/common/ml_model_group.java`
+
+#### Do not comment out code
+
+We use a version management system. If a line of code is no longer needed,
+remove it, don't simply comment it out.
+
+#### Avoid global definitions
+
+Don't do this. Everything should be wrapped in a module that can be depended on
+by other modules. Even things as simple as a single value should be a module.
+
+#### Write small functions
+
+Keep your functions short. A good function fits on a slide that the people in
+the last row of a big room can comfortably read. So don't count on them having
+perfect vision and limit yourself to ~25 lines of code per function.

common/build.gradle

@@ -18,9 +18,9 @@ dependencies {
     compileOnly "org.opensearch:common-utils:${common_utils_version}"
     testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.4.0'
 
-    implementation group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
-    implementation group: 'com.google.code.gson', name: 'gson', version: '2.10.1'
-    implementation group: 'org.json', name: 'json', version: '20230227'
+    compileOnly group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
+    compileOnly group: 'com.google.code.gson', name: 'gson', version: '2.10.1'
+    compileOnly group: 'org.json', name: 'json', version: '20230227'
 }
 
 lombok {
@@ -40,11 +40,11 @@ jacocoTestCoverageVerification {
         rule {
             limit {
                 counter = 'LINE'
-                minimum = 0.6 //TODO: add more test to meet the coverage bar 0.9
+                minimum = 0.8 //TODO: add more test to meet the coverage bar 0.9
             }
             limit {
                 counter = 'BRANCH'
-                minimum = 0.6 //TODO: add more test to meet the coverage bar 0.9
+                minimum = 0.7 //TODO: add more test to meet the coverage bar 0.9
             }
         }
     }