Skip to content

Commit e70aa78

Browse files
peterzhuamazonpeterzhuamazon
authored
Onboard jenkins prod docker images to github actions (opensearch-project#507)
* Onboard jenkins prod docker images to github actions Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Add more Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Add more Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Add more Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> --------- Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
1 parent 568e17a commit e70aa78

File tree

3 files changed

+88
-52
lines changed

3 files changed

+88
-52
lines changed

‎.github/workflows/build.yml

+70-32
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,13 @@ on:
99
- "*"
1010

1111
jobs:
12+
Get-CI-Image-Tag:
13+
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
14+
with:
15+
product: opensearch
16+
1217
linux-build:
18+
needs: Get-CI-Image-Tag
1319
strategy:
1420
matrix:
1521
java:
@@ -18,7 +24,16 @@ jobs:
1824
# Job name
1925
name: Build Asynchronous Search
2026
# This job runs on Linux.
27+
outputs:
28+
build-test-linux: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
2129
runs-on: ubuntu-latest
30+
container:
31+
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
32+
# this image tag is subject to change as more dependencies and updates will arrive over time
33+
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
34+
# need to switch to root so that github actions can install runner binary on container without permission issues.
35+
options: --user root
36+
2237
steps:
2338
# This step uses the setup-java Github action: https://github.com/actions/setup-java
2439
- name: Set Up JDK ${{ matrix.java }}
@@ -29,33 +44,68 @@ jobs:
2944
- name: Checkout Branch
3045
uses: actions/checkout@v2
3146
- name: Build with Gradle
32-
run: ./gradlew build
47+
id: step-build-test-linux
48+
run: |
49+
chown -R 1000:1000 `pwd`
50+
su `id -un 1000` -c "./gradlew build"
51+
plugin=`basename $(ls build/distributions/*.zip)`
52+
echo plugin $plugin
53+
mv -v build/distributions/$plugin ./
54+
echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT
55+
- name: Uploads coverage
56+
uses: codecov/codecov-action@v1.2.1
57+
with:
58+
token: ${{ secrets.CODECOV_TOKEN }}
59+
- uses: actions/upload-artifact@v3
60+
with:
61+
name: async-plugin-linux-${{ matrix.java }}
62+
path: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
63+
if-no-files-found: error
64+
65+
linux-test-docker:
66+
needs: linux-build
67+
strategy:
68+
matrix:
69+
java:
70+
- 11
71+
- 17
72+
# Job name
73+
name: Test Asynchronous Search with opensearchstaging docker
74+
# This job runs on Linux.
75+
runs-on: ubuntu-latest
76+
steps:
77+
- name: Checkout Branch
78+
uses: actions/checkout@v2
79+
- uses: actions/download-artifact@v3
80+
with:
81+
name: async-plugin-linux-${{ matrix.java }}
3382
- name: Pull and Run Docker for security tests
3483
run: |
35-
version=`./gradlew properties -q | grep "opensearch_version:" | awk '{print $2}'`
36-
IFS='-' read -r -a version_array <<< "$version"
37-
plugin_version="${version_array[0]}.0"; for entry in ${version_array[@]:1}; do plugin_version+="-$entry"; done
38-
docker_version="${version_array[0]}-${version_array[1]}"
84+
plugin=${{ needs.linux-build.outputs.build-test-linux }}
85+
version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-3`
86+
plugin_version=`echo $plugin|awk -F- '{print $3}'| cut -d. -f 1-4`
87+
qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`
3988
40-
echo version $version
41-
echo plugin_version $plugin_version
42-
echo docker_version $docker_version
89+
if [ -n "$qualifier" ] && [ "$qualifier" != "SNAPSHOT" ]; then
90+
qualifier=-${qualifier}
91+
else
92+
qualifier=""
93+
fi
4394
44-
pwd=`pwd`
45-
echo $pwd
46-
cd ..
47-
pwd1=`pwd`
48-
echo $pwd1
49-
list_of_all_files=`ls asynchronous-search/build/distributions/`
50-
echo "listing distributions"
51-
echo $list_of_all_files
95+
docker_version=$version$qualifier
96+
97+
98+
echo plugin version plugin_version qualifier docker_version
99+
echo "($plugin) ($version) ($plugin_version) ($qualifier) ($docker_version)"
100+
101+
pwd && ls -l ./$plugin
52102
53103
if docker pull opensearchstaging/opensearch:$docker_version
54104
then
55105
echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
56106
echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-asynchronous-search ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-asynchronous-search; fi" >> Dockerfile
57-
echo "ADD asynchronous-search/build/distributions/opensearch-asynchronous-search-$plugin_version.zip /tmp/" >> Dockerfile
58-
echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/opensearch-asynchronous-search-$plugin_version.zip" >> Dockerfile
107+
echo "ADD $plugin /tmp/" >> Dockerfile
108+
echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
59109
docker build -t opensearch-asynchronous-search:test .
60110
echo "imagePresent=true" >> $GITHUB_ENV
61111
else
@@ -85,20 +135,7 @@ jobs:
85135
with:
86136
name: logs
87137
path: build/testclusters/integTest-*/logs/*
88-
- name: Create Artifact Path
89-
run: |
90-
mkdir -p asynchronous-search-artifacts
91-
cp ./build/distributions/*.zip asynchronous-search-artifacts
92-
- name: Uploads coverage
93-
uses: codecov/codecov-action@v1.2.1
94-
with:
95-
token: ${{ secrets.CODECOV_TOKEN }}
96-
# This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact
97-
- name: Upload Artifacts
98-
uses: actions/upload-artifact@v1
99-
with:
100-
name: asynchronous-search-plugin-ubuntu
101-
path: asynchronous-search-artifacts
138+
102139
windows-build:
103140
# Job name
104141
name: Build Asynchronous Search
@@ -127,6 +164,7 @@ jobs:
127164
with:
128165
name: asynchronous-search-plugin-windows
129166
path: asynchronous-search-artifacts
167+
130168
mac-os-build:
131169
# Job name
132170
name: Build Asynchronous Search

‎.github/workflows/dco-check.yml

-18
This file was deleted.

‎.github/workflows/multi-node-test-workflow.yml

+18-2
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,24 @@ on:
1212
- "*"
1313

1414
jobs:
15+
Get-CI-Image-Tag:
16+
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
17+
with:
18+
product: opensearch
19+
1520
build:
1621
# Job name
22+
needs: Get-CI-Image-Tag
1723
name: Build Asynchronous Search
1824
# This job runs on Linux
1925
runs-on: ubuntu-latest
26+
container:
27+
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
28+
# this image tag is subject to change as more dependencies and updates will arrive over time
29+
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
30+
# need to switch to root so that github actions can install runner binary on container without permission issues.
31+
options: --user root
32+
2033
steps:
2134
# This step uses the setup-java Github action: https://github.com/actions/setup-java
2235
- name: Set Up JDK 11
@@ -27,11 +40,14 @@ jobs:
2740
- name: Checkout Branch
2841
uses: actions/checkout@v2
2942
- name: Run integration tests with multi node config
30-
run: ./gradlew integTest -PnumNodes=5
43+
run: |
44+
chown -R 1000:1000 `pwd`
45+
su `id -un 1000` -c "./gradlew integTest -PnumNodes=5"
3146
- name: Run Backwards Compatibility Tests
3247
run: |
3348
echo "Running backwards compatibility tests ..."
34-
./gradlew bwcTestSuite -Dtests.security.manager=false
49+
chown -R 1000:1000 `pwd`
50+
su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false"
3551
- name: Upload failed logs
3652
uses: actions/upload-artifact@v2
3753
if: failure()

0 commit comments

Comments
 (0)