Skip to content

Commit 8b97cd5

Browse files
rishabh6788reta
authored and
kaushalmahi12
committed
Run performance benchmark on pull requests (opensearch-project#14760)
* add performance benchmark workflow for pull requests Signed-off-by: Rishabh Singh <sngri@amazon.com> * Update PERFORMANCE_BENCHMARKS.md Co-authored-by: Andriy Redko <drreta@gmail.com> Signed-off-by: Rishabh Singh <rishabhksingh@gmail.com> * Update PERFORMANCE_BENCHMARKS.md Co-authored-by: Andriy Redko <drreta@gmail.com> Signed-off-by: Rishabh Singh <rishabhksingh@gmail.com> * Update .github/workflows/benchmark-pull-request.yml Co-authored-by: Andriy Redko <drreta@gmail.com> Signed-off-by: Rishabh Singh <rishabhksingh@gmail.com> * Update .github/workflows/benchmark-pull-request.yml Co-authored-by: Andriy Redko <drreta@gmail.com> Signed-off-by: Rishabh Singh <rishabhksingh@gmail.com> * Update .github/workflows/benchmark-pull-request.yml Co-authored-by: Andriy Redko <drreta@gmail.com> Signed-off-by: Rishabh Singh <rishabhksingh@gmail.com> * Update .github/workflows/benchmark-pull-request.yml Co-authored-by: Andriy Redko <drreta@gmail.com> Signed-off-by: Rishabh Singh <rishabhksingh@gmail.com> --------- Signed-off-by: Rishabh Singh <sngri@amazon.com> Signed-off-by: Rishabh Singh <rishabhksingh@gmail.com> Co-authored-by: Andriy Redko <drreta@gmail.com> Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>
1 parent eff7595 commit 8b97cd5

4 files changed

+434
-0
lines changed

‎.github/benchmark-configs.json

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
{
2+
"name": "Cluster and opensearch-benchmark configurations",
3+
"id_1": {
4+
"description": "Indexing only configuration for NYC_TAXIS workload",
5+
"supported_major_versions": ["2", "3"],
6+
"cluster-benchmark-configs": {
7+
"SINGLE_NODE_CLUSTER": "true",
8+
"MIN_DISTRIBUTION": "true",
9+
"TEST_WORKLOAD": "nyc_taxis",
10+
"WORKLOAD_PARAMS": "{\"number_of_replicas\":\"0\",\"number_of_shards\":\"1\"}",
11+
"EXCLUDE_TASKS": "type:search",
12+
"CAPTURE_NODE_STAT": "true"
13+
},
14+
"cluster_configuration": {
15+
"size": "Single-Node",
16+
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
17+
}
18+
},
19+
"id_2": {
20+
"description": "Indexing only configuration for HTTP_LOGS workload",
21+
"supported_major_versions": ["2", "3"],
22+
"cluster-benchmark-configs": {
23+
"SINGLE_NODE_CLUSTER": "true",
24+
"MIN_DISTRIBUTION": "true",
25+
"TEST_WORKLOAD": "http_logs",
26+
"WORKLOAD_PARAMS": "{\"number_of_replicas\":\"0\",\"number_of_shards\":\"1\"}",
27+
"EXCLUDE_TASKS": "type:search",
28+
"CAPTURE_NODE_STAT": "true"
29+
},
30+
"cluster_configuration": {
31+
"size": "Single-Node",
32+
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
33+
}
34+
},
35+
"id_3": {
36+
"description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-3.0.0",
37+
"supported_major_versions": ["3"],
38+
"cluster-benchmark-configs": {
39+
"SINGLE_NODE_CLUSTER": "true",
40+
"MIN_DISTRIBUTION": "true",
41+
"TEST_WORKLOAD": "nyc_taxis",
42+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
43+
"CAPTURE_NODE_STAT": "true"
44+
},
45+
"cluster_configuration": {
46+
"size": "Single-Node",
47+
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
48+
}
49+
},
50+
"id_4": {
51+
"description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.0.0",
52+
"supported_major_versions": ["3"],
53+
"cluster-benchmark-configs": {
54+
"SINGLE_NODE_CLUSTER": "true",
55+
"MIN_DISTRIBUTION": "true",
56+
"TEST_WORKLOAD": "http_logs",
57+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"http_logs_1_shard\"}",
58+
"CAPTURE_NODE_STAT": "true"
59+
},
60+
"cluster_configuration": {
61+
"size": "Single-Node",
62+
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
63+
}
64+
},
65+
"id_5": {
66+
"description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.0.0",
67+
"supported_major_versions": ["3"],
68+
"cluster-benchmark-configs": {
69+
"SINGLE_NODE_CLUSTER": "true",
70+
"MIN_DISTRIBUTION": "true",
71+
"TEST_WORKLOAD": "big5",
72+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-300\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-300\",\"snapshot_name\":\"big5_1_shard\"}",
73+
"CAPTURE_NODE_STAT": "true"
74+
},
75+
"cluster_configuration": {
76+
"size": "Single-Node",
77+
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
78+
}
79+
},
80+
"id_6": {
81+
"description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-2.x",
82+
"supported_major_versions": ["2"],
83+
"cluster-benchmark-configs": {
84+
"SINGLE_NODE_CLUSTER": "true",
85+
"MIN_DISTRIBUTION": "true",
86+
"TEST_WORKLOAD": "nyc_taxis",
87+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
88+
"CAPTURE_NODE_STAT": "true"
89+
},
90+
"cluster_configuration": {
91+
"size": "Single-Node",
92+
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
93+
}
94+
},
95+
"id_7": {
96+
"description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-2.x",
97+
"supported_major_versions": ["2"],
98+
"cluster-benchmark-configs": {
99+
"SINGLE_NODE_CLUSTER": "true",
100+
"MIN_DISTRIBUTION": "true",
101+
"TEST_WORKLOAD": "http_logs",
102+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"http_logs_1_shard\"}",
103+
"CAPTURE_NODE_STAT": "true"
104+
},
105+
"cluster_configuration": {
106+
"size": "Single-Node",
107+
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
108+
}
109+
},
110+
"id_8": {
111+
"description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-2.x",
112+
"supported_major_versions": ["2"],
113+
"cluster-benchmark-configs": {
114+
"SINGLE_NODE_CLUSTER": "true",
115+
"MIN_DISTRIBUTION": "true",
116+
"TEST_WORKLOAD": "big5",
117+
"WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots\",\"snapshot_name\":\"big5_1_shard\"}",
118+
"CAPTURE_NODE_STAT": "true"
119+
},
120+
"cluster_configuration": {
121+
"size": "Single-Node",
122+
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
123+
}
124+
},
125+
"id_9": {
126+
"description": "Indexing and search configuration for pmc workload",
127+
"supported_major_versions": ["2", "3"],
128+
"cluster-benchmark-configs": {
129+
"SINGLE_NODE_CLUSTER": "true",
130+
"MIN_DISTRIBUTION": "true",
131+
"TEST_WORKLOAD": "pmc",
132+
"WORKLOAD_PARAMS": "{\"number_of_replicas\":\"0\",\"number_of_shards\":\"1\"}",
133+
"CAPTURE_NODE_STAT": "true"
134+
},
135+
"cluster_configuration": {
136+
"size": "Single-Node",
137+
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
138+
}
139+
},
140+
"id_10": {
141+
"description": "Indexing only configuration for stack-overflow workload",
142+
"supported_major_versions": ["2", "3"],
143+
"cluster-benchmark-configs": {
144+
"SINGLE_NODE_CLUSTER": "true",
145+
"MIN_DISTRIBUTION": "true",
146+
"TEST_WORKLOAD": "so",
147+
"WORKLOAD_PARAMS": "{\"number_of_replicas\":\"0\",\"number_of_shards\":\"1\"}",
148+
"CAPTURE_NODE_STAT": "true"
149+
},
150+
"cluster_configuration": {
151+
"size": "Single-Node",
152+
"data_instance_config": "4vCPU, 32G Mem, 16G Heap"
153+
}
154+
}
155+
}

‎.github/workflows/add-performance-comment.yml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: Performance Label Action
2+
3+
on:
4+
pull_request:
5+
types: [labeled]
6+
7+
jobs:
8+
add-comment:
9+
if: github.event.label.name == 'Performance'
10+
runs-on: ubuntu-latest
11+
permissions:
12+
pull-requests: write
13+
14+
steps:
15+
- name: Add comment to PR
16+
uses: actions/github-script@v6
17+
with:
18+
github-token: ${{secrets.GITHUB_TOKEN}}
19+
script: |
20+
github.rest.issues.createComment({
21+
issue_number: context.issue.number,
22+
owner: context.repo.owner,
23+
repo: context.repo.repo,
24+
body: "Hello!\nWe have added a performance benchmark workflow that runs by adding a comment on the PR.\n Please refer https://github.com/opensearch-project/OpenSearch/blob/main/PERFORMANCE_BENCHMARKS.md on how to run benchmarks on pull requests."
25+
})

‎.github/workflows/benchmark-pull-request.yml

+142
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
name: Run performance benchmark on pull request
2+
on:
3+
issue_comment:
4+
types: [created]
5+
jobs:
6+
run-performance-benchmark-on-pull-request:
7+
if: ${{ (github.event.issue.pull_request) && (contains(github.event.comment.body, '"run-benchmark-test"')) }}
8+
runs-on: ubuntu-latest
9+
permissions:
10+
id-token: write
11+
contents: read
12+
issues: write
13+
pull-requests: write
14+
steps:
15+
- name: Checkout Repository
16+
uses: actions/checkout@v3
17+
- name: Set up required env vars
18+
run: |
19+
echo "PR_NUMBER=${{ github.event.issue.number }}" >> $GITHUB_ENV
20+
echo "REPOSITORY=${{ github.event.repository.full_name }}" >> $GITHUB_ENV
21+
OPENSEARCH_VERSION=$(awk -F '=' '/^opensearch[[:space:]]*=/ {gsub(/[[:space:]]/, "", $2); print $2}' buildSrc/version.properties)
22+
echo "OPENSEARCH_VERSION=$OPENSEARCH_VERSION" >> $GITHUB_ENV
23+
major_version=$(echo $OPENSEARCH_VERSION | cut -d'.' -f1)
24+
echo "OPENSEARCH_MAJOR_VERSION=$major_version" >> $GITHUB_ENV
25+
echo "USER_TAGS=pull_request_number:${{ github.event.issue.number }},repository:OpenSearch" >> $GITHUB_ENV
26+
- name: Check comment format
27+
id: check_comment
28+
run: |
29+
comment='${{ github.event.comment.body }}'
30+
if echo "$comment" | jq -e 'has("run-benchmark-test")'; then
31+
echo "Valid comment format detected, check if valid config id is provided"
32+
config_id=$(echo $comment | jq -r '.["run-benchmark-test"]')
33+
benchmark_configs=$(cat .github/benchmark-configs.json)
34+
if echo $benchmark_configs | jq -e --arg id "$config_id" 'has($id)' && echo "$benchmark_configs" | jq -e --arg version "$OPENSEARCH_MAJOR_VERSION" --arg id "$config_id" '.[$id].supported_major_versions | index($version) != null' > /dev/null; then
35+
echo $benchmark_configs | jq -r --arg id "$config_id" '.[$id]."cluster-benchmark-configs" | to_entries[] | "\(.key)=\(.value)"' >> $GITHUB_ENV
36+
else
37+
echo "invalid=true" >> $GITHUB_OUTPUT
38+
fi
39+
else
40+
echo "invalid=true" >> $GITHUB_OUTPUT
41+
fi
42+
- name: Post invalid format comment
43+
if: steps.check_comment.outputs.invalid == 'true'
44+
uses: actions/github-script@v6
45+
with:
46+
github-token: ${{secrets.GITHUB_TOKEN}}
47+
script: |
48+
github.rest.issues.createComment({
49+
issue_number: context.issue.number,
50+
owner: context.repo.owner,
51+
repo: context.repo.repo,
52+
body: 'Invalid comment format or config id. Please refer to https://github.com/opensearch-project/OpenSearch/blob/main/PERFORMANCE_BENCHMARKS.md on how to run benchmarks on pull requests.'
53+
})
54+
- name: Fail workflow for invalid comment
55+
if: steps.check_comment.outputs.invalid == 'true'
56+
run: |
57+
echo "Invalid comment format detected. Failing the workflow."
58+
exit 1
59+
- id: get_approvers
60+
run: |
61+
echo "approvers=$(cat .github/CODEOWNERS | grep '^\*' | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')" >> $GITHUB_OUTPUT
62+
- uses: trstringer/manual-approval@v1
63+
if: (!contains(steps.get_approvers.outputs.approvers, github.event.comment.user.login))
64+
with:
65+
secret: ${{ github.TOKEN }}
66+
approvers: ${{ steps.get_approvers.outputs.approvers }}
67+
minimum-approvals: 1
68+
issue-title: 'Request to approve/deny benchmark run for PR #${{ env.PR_NUMBER }}'
69+
issue-body: "Please approve or deny the benchmark run for PR #${{ env.PR_NUMBER }}"
70+
exclude-workflow-initiator-as-approver: false
71+
- name: Get PR Details
72+
id: get_pr
73+
uses: actions/github-script@v7
74+
with:
75+
script: |
76+
const issue = context.payload.issue;
77+
const prNumber = issue.number;
78+
console.log(`Pull Request Number: ${prNumber}`);
79+
80+
const { data: pull_request } = await github.rest.pulls.get({
81+
owner: context.repo.owner,
82+
repo: context.repo.repo,
83+
pull_number: prNumber,
84+
});
85+
86+
return {
87+
"headRepoFullName": pull_request.head.repo.full_name,
88+
"headRef": pull_request.head.ref
89+
};
90+
- name: Set pr details env vars
91+
run: |
92+
echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRepoFullName'
93+
echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRef'
94+
headRepo=$(echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRepoFullName')
95+
headRef=$(echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRef')
96+
echo "prHeadRepo=$headRepo" >> $GITHUB_ENV
97+
echo "prheadRef=$headRef" >> $GITHUB_ENV
98+
- name: Checkout PR Repo
99+
uses: actions/checkout@v2
100+
with:
101+
repository: ${{ env.prHeadRepo }}
102+
ref: ${{ env.prHeadRef }}
103+
token: ${{ secrets.GITHUB_TOKEN }}
104+
- name: Setup Java
105+
uses: actions/setup-java@v1
106+
with:
107+
java-version: 21
108+
- name: Build and Assemble OpenSearch from PR
109+
run: |
110+
./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=false
111+
- name: Configure AWS credentials
112+
uses: aws-actions/configure-aws-credentials@v4
113+
with:
114+
role-to-assume: ${{ secrets.UPLOAD_ARCHIVE_ARTIFACT_ROLE }}
115+
role-session-name: publish-to-s3
116+
aws-region: us-west-2
117+
- name: Push to S3
118+
run: |
119+
aws s3 cp distribution/archives/linux-tar/build/distributions/opensearch-min-$OPENSEARCH_VERSION-linux-x64.tar.gz s3://${{ secrets.ARCHIVE_ARTIFACT_BUCKET_NAME }}/PR-$PR_NUMBER/
120+
echo "DISTRIBUTION_URL=${{ secrets.ARTIFACT_BUCKET_CLOUDFRONT_URL }}/PR-$PR_NUMBER/opensearch-min-$OPENSEARCH_VERSION-linux-x64.tar.gz" >> $GITHUB_ENV
121+
- name: Checkout opensearch-build repo
122+
uses: actions/checkout@v4
123+
with:
124+
repository: opensearch-project/opensearch-build
125+
ref: main
126+
path: opensearch-build
127+
- name: Trigger jenkins workflow to run gradle check
128+
run: |
129+
cat $GITHUB_ENV
130+
bash opensearch-build/scripts/benchmark/benchmark-pull-request.sh ${{ secrets.JENKINS_PR_BENCHMARK_GENERIC_WEBHOOK_TOKEN }}
131+
- name: Update PR with Job Url
132+
uses: actions/github-script@v6
133+
with:
134+
github-token: ${{ secrets.GITHUB_TOKEN }}
135+
script: |
136+
const workflowUrl = process.env.WORKFLOW_URL;
137+
github.rest.issues.createComment({
138+
issue_number: context.issue.number,
139+
owner: context.repo.owner,
140+
repo: context.repo.repo,
141+
body: `The Jenkins job url is ${workflowUrl} . Final results will be published once the job is completed.`
142+
})

0 commit comments

Comments
 (0)