Skip to content

Commit 96c8d4d

Browse files
peterzhuamazonpeterzhuamazon
authored
Onboarding Jenkins prod docker images to github ci check (opensearch-project#1538)
* Onboarding Jenkins prod docker images to github ci check Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Add suggested changes in the comments Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> --------- Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
1 parent 012cc6c commit 96c8d4d

File tree

1 file changed

+82
-31
lines changed

1 file changed

+82
-31
lines changed

‎.github/workflows/CI-workflow.yml

+82-31
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,29 @@ permissions:
1414
contents: read
1515

1616
jobs:
17-
Build-ml:
17+
Get-CI-Image-Tag:
18+
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
19+
with:
20+
product: opensearch
21+
22+
Build-ml-linux:
23+
needs: Get-CI-Image-Tag
1824
strategy:
1925
matrix:
2026
java: [11, 17, 20]
2127

22-
name: Build and Test MLCommons Plugin
28+
name: Build and Test MLCommons Plugin on linux
2329
if: github.repository == 'opensearch-project/ml-commons'
2430
environment: ml-commons-cicd-env
31+
outputs:
32+
build-test-linux: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
2533
runs-on: ubuntu-latest
34+
container:
35+
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
36+
# this image tag is subject to change as more dependencies and updates will arrive over time
37+
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
38+
# need to switch to root so that github actions can install runner binary on container without permission issues.
39+
options: --user root
2640

2741
steps:
2842
- name: Setup Java ${{ matrix.java }}
@@ -41,43 +55,86 @@ jobs:
4155
ref: ${{ github.event.pull_request.head.sha }}
4256

4357
- name: Build and Run Tests
58+
id: step-build-test-linux
4459
run: |
45-
export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
46-
export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
47-
echo "::add-mask::$OPENAI_KEY"
48-
echo "::add-mask::$COHERE_KEY"
49-
./gradlew build
50-
- name: Publish to Maven Local
51-
run: |
52-
./gradlew publishToMavenLocal
53-
- name: Multi Nodes Integration Testing
54-
run: |
55-
export OPENAI_KEY=$(aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text)
56-
export COHERE_KEY=$(aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text)
57-
echo "::add-mask::$OPENAI_KEY"
58-
echo "::add-mask::$COHERE_KEY"
59-
./gradlew integTest -PnumNodes=3
60+
chown -R 1000:1000 `pwd`
61+
su `id -un 1000` -c 'whoami && java -version &&
62+
export OPENAI_KEY=`aws secretsmanager get-secret-value --secret-id github_openai_key --query SecretString --output text` &&
63+
export COHERE_KEY=`aws secretsmanager get-secret-value --secret-id github_cohere_key --query SecretString --output text` &&
64+
echo "::add-mask::$OPENAI_KEY" &&
65+
echo "::add-mask::$COHERE_KEY" &&
66+
echo "build and run tests" && ./gradlew build &&
67+
echo "Publish to Maven Local" && ./gradlew publishToMavenLocal &&
68+
echo "Multi Nodes Integration Testing" && ./gradlew integTest -PnumNodes=3'
69+
plugin=`basename $(ls plugin/build/distributions/*.zip)`
70+
echo $plugin
71+
mv -v plugin/build/distributions/$plugin ./
72+
echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT
73+
74+
- name: Upload Coverage Report
75+
uses: codecov/codecov-action@v1
76+
with:
77+
flags: ml-commons
78+
token: ${{ secrets.CODECOV_TOKEN }}
79+
80+
- uses: actions/upload-artifact@v3
81+
with:
82+
name: ml-plugin-linux-${{ matrix.java }}
83+
path: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
84+
if-no-files-found: error
85+
86+
87+
Test-ml-linux-docker:
88+
needs: Build-ml-linux
89+
strategy:
90+
matrix:
91+
java: [11, 17, 20]
92+
93+
name: Test MLCommons Plugin on linux docker
94+
if: github.repository == 'opensearch-project/ml-commons'
95+
environment: ml-commons-cicd-env
96+
runs-on: ubuntu-latest
97+
98+
steps:
99+
- uses: aws-actions/configure-aws-credentials@v2
100+
with:
101+
role-to-assume: ${{ secrets.ML_ROLE }}
102+
aws-region: us-west-2
103+
104+
- name: Checkout MLCommons
105+
uses: actions/checkout@v3
106+
with:
107+
ref: ${{ github.event.pull_request.head.sha }}
108+
109+
- uses: actions/download-artifact@v3
110+
with:
111+
name: ml-plugin-linux-${{ matrix.java }}
60112

61113
- name: Pull and Run Docker
62114
run: |
63-
plugin=`basename $(ls plugin/build/distributions/*.zip)`
115+
plugin=${{ needs.Build-ml-linux.outputs.build-test-linux }}
64116
version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3`
65117
plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4`
66118
qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`
67-
candidate_version=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1`
68-
docker_version=$version-$qualifier
69119
70-
[[ -z $candidate_version ]] && candidate_version=$qualifier && qualifier=""
120+
if [ -n "$qualifier" ] && [ "$qualifier" != "SNAPSHOT" ]; then
121+
qualifier=-${qualifier}
122+
else
123+
qualifier=""
124+
fi
71125
72-
echo plugin version plugin_version qualifier candidate_version docker_version
73-
echo "($plugin) ($version) ($plugin_version) ($qualifier) ($candidate_version) ($docker_version)"
126+
docker_version=$version$qualifier
127+
128+
echo plugin version plugin_version qualifier docker_version
129+
echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
130+
131+
pwd && ls -l ./$plugin
74132
75-
cd ..
76133
if docker pull opensearchstaging/opensearch:$docker_version
77134
then
78135
echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
79136
echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-ml ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-ml; fi" >> Dockerfile
80-
echo "ADD ml-commons/plugin/build/distributions/$plugin /tmp/" >> Dockerfile
137+
echo "COPY $plugin /tmp/" >> Dockerfile
81138
echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
82139
docker build -t opensearch-ml:test .
83140
echo "imagePresent=true" >> $GITHUB_ENV
@@ -107,12 +164,6 @@ jobs:
107164
./gradlew integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster"
108165
fi
109166
110-
- name: Upload Coverage Report
111-
uses: codecov/codecov-action@v1
112-
with:
113-
flags: ml-commons
114-
token: ${{ secrets.CODECOV_TOKEN }}
115-
116167
Build-ml-windows:
117168
strategy:
118169
matrix:

0 commit comments

Comments
 (0)