Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add qualifier support to metrics queries #611

Merged
merged 2 commits into from
Mar 12, 2025
Merged

Add qualifier support to metrics queries #611

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c175b17
Add qualifier support to metrics queries
gaiksaya Mar 12, 2025
02e25bb
Refactor null check
gaiksaya Mar 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 23 additions & 5 deletions src/jenkins/ComponentBuildStatus.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,37 +18,40 @@
String awsSecretKey
String awsSessionToken
String indexName
String product
String product
String version
String qualifier
String distributionBuildNumber
String buildStartTimeFrom
String buildStartTimeTo
def script
OpenSearchMetricsQuery openSearchMetricsQuery

ComponentBuildStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, String distributionBuildNumber, String buildStartTimeFrom, String buildStartTimeTo, def script) {
ComponentBuildStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, String qualifier, String distributionBuildNumber, String buildStartTimeFrom, String buildStartTimeTo, def script) {
this.metricsUrl = metricsUrl
this.awsAccessKey = awsAccessKey
this.awsSecretKey = awsSecretKey
this.awsSessionToken = awsSessionToken
this.indexName = indexName
this.product = product
this.version = version
this.qualifier = qualifier
this.distributionBuildNumber = distributionBuildNumber
this.buildStartTimeFrom = buildStartTimeFrom
this.buildStartTimeTo = buildStartTimeTo
this.script = script
this.openSearchMetricsQuery = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script)
}

ComponentBuildStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, def script) {
ComponentBuildStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, String qualifier, def script) {
this.metricsUrl = metricsUrl
this.awsAccessKey = awsAccessKey
this.awsSecretKey = awsSecretKey
this.awsSessionToken = awsSessionToken
this.indexName = indexName
this.product = product
this.version = version
this.qualifier = qualifier
this.script = script
this.openSearchMetricsQuery = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script)
}
Expand All @@ -57,7 +60,7 @@
def queryMap = [
_source: [
"component",
],
],
query: [
bool: [
filter: [
Expand Down Expand Up @@ -93,6 +96,13 @@
]
]
]
if (this.qualifier != null && this.qualifier != "None") {
queryMap.query.bool.filter.add([
match_phrase: [
qualifier: "${this.qualifier}"
]
])
}
def query = JsonOutput.toJson(queryMap)
return query.replace('"', '\\"')
}
Expand All @@ -102,7 +112,7 @@
size : 1,
_source: [
"distribution_build_number",
],
],
query: [
bool: [
filter: [
Expand All @@ -127,6 +137,14 @@
]
]
]

if (this.qualifier != null && this.qualifier != "None") {
queryMap.query.bool.filter.add([

Check warning on line 142 in src/jenkins/ComponentBuildStatus.groovy

View check run for this annotation

Codecov / codecov/patch

src/jenkins/ComponentBuildStatus.groovy#L142 

Added line #L142 was not covered by tests
match_phrase: [
qualifier: "${this.qualifier}"
]
])
}
def query = JsonOutput.toJson(queryMap)
return query.replace('"', '\\"')
}
Expand Down
18 changes: 17 additions & 1 deletion src/jenkins/ComponentIntegTestStatus.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,20 @@
String indexName
String product
String version
String qualifier
String distributionBuildNumber
def script
OpenSearchMetricsQuery openSearchMetricsQuery

ComponentIntegTestStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, String distributionBuildNumber, def script) {
ComponentIntegTestStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String product, String version, String qualifier, String distributionBuildNumber, def script) {
this.metricsUrl = metricsUrl
this.awsAccessKey = awsAccessKey
this.awsSecretKey = awsSecretKey
this.awsSessionToken = awsSessionToken
this.indexName = indexName
this.product = product
this.version = version
this.qualifier = qualifier
this.distributionBuildNumber = distributionBuildNumber
this.script = script
this.openSearchMetricsQuery = new OpenSearchMetricsQuery(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, script)
Expand Down Expand Up @@ -70,6 +72,13 @@
]
]
]
if (this.qualifier != null && this.qualifier != "None") {
queryMap.query.bool.filter.add([

Check warning on line 76 in src/jenkins/ComponentIntegTestStatus.groovy

View check run for this annotation

Codecov / codecov/patch

src/jenkins/ComponentIntegTestStatus.groovy#L76 

Added line #L76 was not covered by tests
match_phrase: [
qualifier: "${this.qualifier}"
]
])
}
def query = JsonOutput.toJson(queryMap)
return query.replace('"', '\\"')
}
Expand Down Expand Up @@ -106,6 +115,13 @@
]
]
]
if (this.qualifier != null && this.qualifier != "None") {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Too many of this calls can we replace with:

opensearch-build-libraries/vars/abortStaleJenkinsJobs.groovy

Line 63 in a0e5039

boolean isNullOrEmpty(String str) { return (str == 'Null' || str == null || str.allWhitespace || str.isEmpty()) }

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We want to check for "None" as that is what is indexed by default for non-qualifiers while indexing data.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a private method to incorporate above logic.
Thanks!

queryMap.query.bool.filter.add([
match_phrase: [
qualifier: "${this.qualifier}"
]
])
}
def query = JsonOutput.toJson(queryMap)
return query.replace('"', '\\"')
}
Expand Down
57 changes: 21 additions & 36 deletions src/jenkins/ReleaseCandidateStatus.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,18 @@
String awsSessionToken
String indexName
String version
String qualifier
def script
def openSearchMetricsQuery

ReleaseCandidateStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String version, def script) {
ReleaseCandidateStatus(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, String indexName, String version, String qualifier, def script) {
this.metricsUrl = metricsUrl
this.awsAccessKey = awsAccessKey
this.awsSecretKey = awsSecretKey
this.awsSessionToken = awsSessionToken
this.indexName = indexName
this.version = version
this.qualifier = qualifier
this.script = script
this.openSearchMetricsQuery = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, indexName, script)
}
Expand Down Expand Up @@ -71,43 +73,19 @@
]

if (rcNumber != null) {
queryMap = [
_source: "distribution_build_number",
sort: [
[
distribution_build_number: [
order: "desc"
]
queryMap.sort[0].remove("rc_number")
queryMap.query.bool.filter.add([
match_phrase: [
rc_number: "${rcNumber}"
]
],
size: 1,
query: [
bool: [
filter: [
[
match_phrase: [
component: "${componentName}"
]
],
[
match_phrase: [
rc: "true"
]
],
[
match_phrase: [
version: "${this.version}"
]
],
[
match_phrase: [
rc_number: "${rcNumber}"
]
]
]
])
}
if (this.qualifier != null && this.qualifier != "None") {
queryMap.query.bool.filter.add([

Check warning on line 84 in src/jenkins/ReleaseCandidateStatus.groovy

View check run for this annotation

Codecov / codecov/patch

src/jenkins/ReleaseCandidateStatus.groovy#L84 

Added line #L84 was not covered by tests
match_phrase: [
qualifier: "${this.qualifier}"
]
]
]
])
}

def query = JsonOutput.toJson(queryMap)
Expand Down Expand Up @@ -150,6 +128,13 @@
]
]
]
if (this.qualifier != null && this.qualifier != "None") {
queryMap.query.bool.filter.add([
match_phrase: [
qualifier: "${this.qualifier}"
]
])
}
def query = JsonOutput.toJson(queryMap)
return query.replace('"', '\\"')
}
Expand Down
53 changes: 52 additions & 1 deletion tests/jenkins/TestComponentBuildStatus.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ class TestComponentBuildStatus {
private final String indexName = 'opensearch-distribution-build-results-*'
private final String product = "OpenSearch"
private final String version = "2.18.0"
private final String qualifier = "None"
private final String distributionBuildNumber = "4891"
private final String buildStartTimeFrom = "now-6h"
private final String buildStartTimeTo = "now"
Expand Down Expand Up @@ -66,7 +67,7 @@ class TestComponentBuildStatus {
}
return ""
}
componentBuildStatus = new ComponentBuildStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, distributionBuildNumber, buildStartTimeFrom, buildStartTimeTo, script)
componentBuildStatus = new ComponentBuildStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, qualifier, distributionBuildNumber, buildStartTimeFrom, buildStartTimeTo, script)
}

@Test
Expand Down Expand Up @@ -114,6 +115,56 @@ class TestComponentBuildStatus {
def result = componentBuildStatus.getQuery('failed')
assert result == expectedOutput
}
@Test
void testGetQueryReturnsExpectedQueryWithQualifier() {
def expectedOutput = JsonOutput.toJson([
_source: [
"component",
],
query: [
bool: [
filter: [
[
match_phrase: [
component_category: "OpenSearch"
]
],
[
match_phrase: [
component_build_result: "failed"
]
],
[
match_phrase: [
version: "2.18.0"
]
],
[
match_phrase : [
distribution_build_number : "4891"
]
],
[
range: [
build_start_time: [
from: "now-6h",
to: "now"
]
]
],
[
match_phrase : [
qualifier : "alpha1"
]
]
]
]
]
]).replace('"', '\\"')
def componentBuildStatusNew = new ComponentBuildStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, 'alpha1', distributionBuildNumber, buildStartTimeFrom, buildStartTimeTo, script)
def result = componentBuildStatusNew.getQuery('failed')
assert result == expectedOutput
}

@Test
void testComponentBuildStatusReturns() {
Expand Down
48 changes: 46 additions & 2 deletions tests/jenkins/TestComponentIntegTestStatus.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ class TestComponentIntegTestStatus {
private final String indexName = 'opensearch-integration-test-results'
private final String product = "OpenSearch"
private final String version = "2.18.0"
private final String qualifier = "None"
private final String distributionBuildNumber = "4891"
private def script

Expand Down Expand Up @@ -97,7 +98,7 @@ class TestComponentIntegTestStatus {
}
return ""
}
componentIntegTestStatus = new ComponentIntegTestStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, distributionBuildNumber, script)
componentIntegTestStatus = new ComponentIntegTestStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, qualifier, distributionBuildNumber, script)
}

@Test
Expand Down Expand Up @@ -176,6 +177,49 @@ class TestComponentIntegTestStatus {
assert result == expectedOutput
}

@Test
void testComponentIntegTestFailedDataQueryWithQualifier() {
def expectedOutput = JsonOutput.toJson([
_source : [
"platform",
"architecture",
"distribution",
"test_report_manifest_yml",
"integ_test_build_url",
"rc_number"
],
query: [
bool: [
filter: [
[
match_phrase: [
component: "k-NN"
]
],
[
match_phrase: [
version: "2.18.0"
]
],
[
match_phrase: [
distribution_build_number: "4891"
]
],
[
match_phrase: [
qualifier: "beta1"
]
]
]
]
]
]).replace('"', '\\"')
def componentIntegTestStatusNew = new ComponentIntegTestStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, 'beta1', distributionBuildNumber, script)
def result = componentIntegTestStatusNew.componentIntegTestFailedDataQuery('k-NN')
assert result == expectedOutput
}

@Test
void testGetComponents() {
def expectedOutput = ['cross-cluster-replication', 'k-NN', 'cross-cluster-replication', 'index-management', 'neural-search']
Expand Down Expand Up @@ -237,7 +281,7 @@ class TestComponentIntegTestStatus {
}
return ""
}
componentIntegTestStatus = new ComponentIntegTestStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, distributionBuildNumber, script)
componentIntegTestStatus = new ComponentIntegTestStatus(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, indexName, product, version, qualifier, distributionBuildNumber, script)
def componentData = '''
{
"took": 5,
Expand Down
Loading