Skip to content

Commit 8efef9a

Browse files
lukas-vlceklukas-vlcek
committed
Introduce BWC testing infrastructure
This commit introduces BWC tests. Right now BWC tests are not executed on Windows, thus Windows support is not guarantied. Closes: #315 Signed-off-by: Lukáš Vlček <lukas.vlcek@aiven.io>
1 parent 1410181 commit 8efef9a

File tree

7 files changed

+420
-0
lines changed

7 files changed

+420
-0
lines changed

‎.github/workflows/BWC.yml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Build and Run BWC Tests
2+
3+
on:
4+
push:
5+
branches:
6+
- "*"
7+
pull_request:
8+
branches:
9+
- "*"
10+
11+
jobs:
12+
build:
13+
strategy:
14+
matrix:
15+
os: [ubuntu-latest]
16+
java: [21]
17+
name: Build and Run BWC Tests
18+
runs-on: ${{ matrix.os }}
19+
permissions:
20+
contents: read
21+
22+
steps:
23+
- uses: actions/checkout@v4
24+
- name: Set up JDK ${{ matrix.java }}
25+
uses: actions/setup-java@v4
26+
with:
27+
distribution: "temurin"
28+
java-version: ${{ matrix.java }}
29+
30+
- name: Build and Run Tests
31+
run: |
32+
./gradlew --info bwcTestSuite -Dtests.security.manager=false

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
# https://discuss.elastic.co/t/leftovers-after-integtestrunner-in-root/152610
33
\.local*-integTestRunner-execution-times.log
44

5+
# This location is used by BWC test clusters to host older and actual versions of plugin ZIP files.
6+
src/test/resources/org/opensearch/prometheus-exporter/bwc/
7+
58
# intellij files
69
.idea/
710
*.iml

‎README.md

+15
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ The [Prometheus® exporter](https://prometheus.io/docs/instrumenting/writing_exp
1212
- [Usage](#usage)
1313
- [Build from Source](#build-from-source)
1414
- [Testing](#testing)
15+
- [BWC Testing](#bwc-testing)
1516
- [License](#license)
1617
- [Trademarks & Attributions](#trademarks--attributions)
1718

@@ -279,6 +280,20 @@ To run individual integration rest test file use:
279280
-Dtests.method="test {yaml=/20_11_index_level_metrics_disabled/Dynamically disable index level metrics}"
280281
```
281282

283+
### BWC Testing
284+
285+
Backward Compatibility (BWC) Testing is run manually using provided shell script:
286+
287+
```
288+
./bwctest.sh
289+
```
290+
291+
It is not part of `./gradlew [build|check]` task(s), but it is included in the CI workflow.
292+
293+
OpenSearch versions used during BWC tests use determined by properties located in `gradle.properties` file. Specifically `project.version` and `project.BWCversion`. Version of plugin deployed into `project.BWCversion` cluster is specified by `project.BWCPluginVersion` property.
294+
295+
In the beginning of BWC tests the actual version of plugin (`project.version`) is build using `bundlePlugin` gradle task and the `project.BWCPluginVersion` plugin is downloaded from GitHub releases. Both ZIP files are placed into `src/test/resource/org/opensearch/prometheus-exrpoter/bwc/prometheus-exporter` folder (this folder is ignored by git).
296+
282297
## License
283298

284299
Licensed under the Apache License, Version 2.0 (the "License");

‎build.gradle

+187
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import org.opensearch.gradle.test.RestIntegTestTask
22

3+
import java.util.concurrent.Callable
34
import java.util.regex.Matcher
45
import java.util.regex.Pattern
56

@@ -36,6 +37,12 @@ buildscript {
3637
"opensearch": opensearch_version,
3738
"prometheus": "0.16.0"
3839
]
40+
41+
bwcPluginDownloadLink = 'https://github.com/Aiven-Open/prometheus-exporter-plugin-for-opensearch/releases/download/' +
42+
project.BWCPluginVersion + '/prometheus-exporter-' + project.BWCPluginVersion + '.zip'
43+
baseName = "bwcCluster"
44+
bwcFilePath = "src/test/resources/org/opensearch/prometheus-exporter/bwc/"
45+
bwcPrometheusExporterPath = bwcFilePath + "prometheus-exporter/"
3946
}
4047

4148
repositories {
@@ -46,7 +53,12 @@ buildscript {
4653
maven { url "https://plugins.gradle.org/m2/" }
4754
}
4855

56+
configurations {
57+
zipArchive
58+
}
59+
4960
dependencies {
61+
zipArchive group: 'org.opensearch.plugin.prometheus', name:'prometheus-exporter', version: "${versions.opensearch}"
5062
classpath "org.opensearch.gradle:build-tools:${versions.opensearch}"
5163
}
5264
}
@@ -118,6 +130,9 @@ task integTest(type: RestIntegTestTask) {
118130
description = "Run tests against a cluster"
119131
testClassesDirs = sourceSets.test.output.classesDirs
120132
classpath = sourceSets.test.runtimeClasspath
133+
filter {
134+
excludeTestsMatching "org.opensearch.plugin.bwc.*IT"
135+
}
121136
}
122137
tasks.named("check").configure { dependsOn(integTest) }
123138

@@ -146,6 +161,178 @@ testClusters.integTest {
146161
plugin(project.tasks.bundlePlugin.archiveFile)
147162
}
148163

164+
task copyZIPBundle {
165+
dependsOn(bundlePlugin)
166+
167+
doLast {
168+
// By using ant.copy we can "hack" around gradle check for other task's resources modification.
169+
// It seems like a dirty hack but some official plugins seem to use this practice;
170+
// for instance see https://github.com/opensearch-project/anomaly-detection/
171+
ant.copy(todir: bwcPrometheusExporterPath + project.version) {
172+
ant.fileset(dir: 'build/distributions/', includes: 'prometheus-exporter-' + project.version + '.zip')
173+
}
174+
}
175+
}
176+
177+
// Clusters for BWC tests
178+
2.times { i ->
179+
testClusters {
180+
"${baseName}$i" {
181+
versions = [project.BWCversion, "2.17.1"]
182+
numberOfNodes = 3
183+
plugin(provider(new Callable<RegularFile>() {
184+
@Override
185+
RegularFile call() throws Exception {
186+
return new RegularFile() {
187+
@Override
188+
File getAsFile() {
189+
if (new File("$project.rootDir/$bwcFilePath/prometheus-exporter/$project.BWCPluginVersion").exists()) {
190+
project.delete(files("$project.rootDir/$bwcFilePath/prometheus-exporter/$project.BWCPluginVersion"))
191+
}
192+
project.mkdir bwcPrometheusExporterPath + project.BWCPluginVersion
193+
ant.get(src: bwcPluginDownloadLink,
194+
dest: bwcPrometheusExporterPath + project.BWCPluginVersion,
195+
httpusecaches: false)
196+
return fileTree(bwcPrometheusExporterPath + project.BWCPluginVersion).getSingleFile()
197+
}
198+
}
199+
}
200+
}))
201+
setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
202+
setting 'http.content_type.required', 'true'
203+
}
204+
}
205+
}
206+
207+
List<Provider<RegularFile>> plugins = [
208+
provider(new Callable<RegularFile>(){
209+
@Override
210+
RegularFile call() throws Exception {
211+
return new RegularFile() {
212+
@Override
213+
File getAsFile() {
214+
return fileTree(bwcPrometheusExporterPath + project.version).getSingleFile()
215+
}
216+
}
217+
}
218+
})
219+
]
220+
221+
// Creates 2 test clusters with 3 nodes of the old version.
222+
2.times {i ->
223+
task "${baseName}#oldVersionClusterTask$i"(type: RestIntegTestTask) {
224+
useCluster testClusters."${baseName}$i"
225+
filter {
226+
includeTestsMatching "org.opensearch.plugin.bwc.*IT"
227+
}
228+
systemProperty 'tests.rest.bwcsuite', 'old_cluster'
229+
systemProperty 'tests.rest.bwcsuite_round', 'old'
230+
systemProperty 'tests.plugin_bwc_version', project.BWCPluginVersion
231+
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
232+
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
233+
}
234+
}
235+
236+
// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version
237+
// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
238+
// This is also used as a one third upgraded cluster for a rolling upgrade.
239+
task "${baseName}#mixedClusterTask"(type: RestIntegTestTask) {
240+
dependsOn tasks.named("copyZIPBundle")
241+
useCluster testClusters."${baseName}0"
242+
dependsOn "${baseName}#oldVersionClusterTask0"
243+
doFirst {
244+
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
245+
}
246+
filter {
247+
includeTestsMatching "org.opensearch.plugin.bwc.*IT"
248+
}
249+
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
250+
systemProperty 'tests.rest.bwcsuite_round', 'first'
251+
systemProperty 'tests.plugin_bwc_version', project.BWCPluginVersion
252+
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
253+
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
254+
}
255+
256+
// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
257+
// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
258+
// This is used for rolling upgrade.
259+
task "${baseName}#twoThirdsUpgradedClusterTask"(type: RestIntegTestTask) {
260+
dependsOn tasks.named("copyZIPBundle")
261+
dependsOn "${baseName}#mixedClusterTask"
262+
useCluster testClusters."${baseName}0"
263+
doFirst {
264+
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
265+
}
266+
filter {
267+
includeTestsMatching "org.opensearch.plugin.bwc.*IT"
268+
}
269+
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
270+
systemProperty 'tests.rest.bwcsuite_round', 'second'
271+
systemProperty 'tests.plugin_bwc_version', project.BWCPluginVersion
272+
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
273+
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
274+
}
275+
276+
// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
277+
// This results in a fully upgraded cluster.
278+
// This is used for rolling upgrade.
279+
task "${baseName}#rollingUpgradeClusterTask"(type: RestIntegTestTask) {
280+
dependsOn tasks.named("copyZIPBundle")
281+
dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
282+
useCluster testClusters."${baseName}0"
283+
doFirst {
284+
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
285+
}
286+
filter {
287+
includeTestsMatching "org.opensearch.plugin.bwc.*IT"
288+
}
289+
mustRunAfter "${baseName}#mixedClusterTask"
290+
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
291+
systemProperty 'tests.rest.bwcsuite_round', 'third'
292+
systemProperty 'tests.plugin_bwc_version', project.BWCPluginVersion
293+
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
294+
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
295+
}
296+
297+
// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
298+
// at the same time resulting in a fully upgraded cluster.
299+
task "${baseName}#fullRestartClusterTask"(type: RestIntegTestTask) {
300+
dependsOn tasks.named("copyZIPBundle")
301+
dependsOn "${baseName}#oldVersionClusterTask1"
302+
useCluster testClusters."${baseName}1"
303+
doFirst {
304+
testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
305+
}
306+
filter {
307+
includeTestsMatching "org.opensearch.plugin.bwc.*IT"
308+
}
309+
systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
310+
systemProperty 'tests.plugin_bwc_version', project.BWCPluginVersion
311+
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
312+
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
313+
}
314+
315+
// A BWC test suite which runs all the bwc tasks combined.
316+
task bwcTestSuite(type: RestIntegTestTask) {
317+
318+
// Delete all downloaded and built plugin ZIP files.
319+
// Again – we are using ant task to workaround gradle resources modification alert.
320+
doFirst {
321+
ant.delete(includeEmptyDirs: true, verbose: true, removeNotFollowedSymlinks: true) {
322+
ant.fileset(
323+
dir: 'src/test/resources/org/opensearch/prometheus-exporter/bwc/prometheus-exporter'
324+
)
325+
}
326+
}
327+
328+
exclude '**/*Test*'
329+
exclude '**/*IT*'
330+
331+
dependsOn tasks.named("${baseName}#mixedClusterTask")
332+
dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
333+
dependsOn tasks.named("${baseName}#fullRestartClusterTask")
334+
}
335+
149336
run {
150337
useCluster testClusters.integTest
151338
}

‎bwctest.sh

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
function usage() {
6+
echo ""
7+
echo "This script is used to run Backwards Compatibility tests"
8+
echo "--------------------------------------------------------------------------"
9+
echo "Usage: $0 [args]"
10+
echo ""
11+
echo "Required arguments:"
12+
echo "None"
13+
echo ""
14+
echo -e "-h\tPrint this message."
15+
echo "--------------------------------------------------------------------------"
16+
}
17+
18+
while getopts ":h" arg; do
19+
case $arg in
20+
h)
21+
usage
22+
exit 1
23+
;;
24+
?)
25+
echo "Invalid option: -${OPTARG}"
26+
exit 1
27+
;;
28+
esac
29+
done
30+
31+
# Warning!
32+
# This should be done from gradle, see bwcTestSuite task, but that task is skipped.
33+
# TODO: Skipping task ':bwcTestSuite' as it has no source files and no previous output files.
34+
rm -rf src/test/resources/org/opensearch/prometheus-exporter/bwc/prometheus-exporter
35+
36+
./gradlew bwcTestSuite -Dtests.security.manager=false

‎gradle.properties

+7
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,14 @@
11
#group = org.opensearch.plugin.prometheus
22

3+
# An actual version of plugin
34
version = 2.17.1.0
45

6+
# A version of OpenSearch cluster to run BWC tests against
7+
BWCversion = 2.17.0
8+
9+
# A version of plugin to deploy to BWC clusters
10+
BWCPluginVersion = 2.17.0.0
11+
512
pluginName = prometheus-exporter
613
pluginClassname = org.opensearch.plugin.prometheus.PrometheusExporterPlugin
714
pluginDescription = Prometheus exporter plugin for OpenSearch

0 commit comments

Comments
 (0)