Skip to content

Commit 3253242

Browse files
DivyaasmDivyaasm
committed
add debian and zip and refactor code
Signed-off-by: Divya Madala <divyaasm@amazon.com>
1 parent 9ecf30a commit 3253242

17 files changed

+196
-87
lines changed

‎src/validation_workflow/deb/__init__.py

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Copyright OpenSearch Contributors
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# The OpenSearch Contributors require contributions made to
5+
# this file be licensed under the Apache-2.0 license or a
6+
# compatible open source license.
7+
#
8+
# This page intentionally left blank.

‎src/validation_workflow/deb/validation_deb.py

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Copyright OpenSearch Contributors
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# The OpenSearch Contributors require contributions made to
5+
# this file be licensed under the Apache-2.0 license or a
6+
# compatible open source license.
7+
8+
import logging
9+
import os
10+
import time
11+
12+
from system.execute import execute
13+
from test_workflow.integ_test.utils import get_password
14+
from validation_workflow.api_test_cases import ApiTestCases
15+
from validation_workflow.download_utils import DownloadUtils
16+
from validation_workflow.validation import Validation
17+
from validation_workflow.validation_args import ValidationArgs
18+
19+
20+
class ValidateDeb(Validation, DownloadUtils):
21+
def __init__(self, args: ValidationArgs) -> None:
22+
super().__init__(args)
23+
24+
def installation(self) -> bool:
25+
try:
26+
execute(f"sudo env OPENSEARCH_INITIAL_ADMIN_PASSWORD={get_password(str(self.args.version))}", ".", True, False)
27+
for project in self.args.projects:
28+
execute(f'sudo dpkg --purge {project}', ".")
29+
execute(f'sudo dpkg -i {os.path.basename(self.args.file_path.get(project))}', str(self.tmp_dir.path))
30+
time.sleep(80)
31+
32+
except:
33+
raise Exception("Failed to install Opensearch")
34+
return True
35+
36+
def start_cluster(self) -> bool:
37+
try:
38+
for project in self.args.projects:
39+
execute(f'sudo systemctl enable {project}', ".")
40+
execute(f'sudo systemctl start {project}', ".")
41+
time.sleep(20)
42+
execute(f'sudo systemctl status {project}', ".")
43+
except:
44+
raise Exception('Failed to Start Cluster')
45+
return True
46+
47+
def validation(self) -> bool:
48+
test_result, counter = ApiTestCases().test_apis(self.args.version, self.args.projects, self.check_for_security_plugin(os.path.join(os.sep, "usr", "share", "opensearch")) if not self.args.force_https else True) # noqa: E501
49+
if (test_result):
50+
logging.info(f'All tests Pass : {counter}')
51+
return True
52+
else:
53+
raise Exception(f'Not all tests Pass : {counter}')
54+
55+
def cleanup(self) -> bool:
56+
try:
57+
for project in self.args.projects:
58+
execute(f'sudo dpkg --purge {project}', ".")
59+
except Exception as e:
60+
raise Exception(f'Exception occurred either while attempting to stop cluster or removing OpenSearch/OpenSearch-Dashboards. {str(e)}')
61+
return True

‎src/validation_workflow/docker/validation_docker.py

-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
import requests
1717

18-
from system.temporary_directory import TemporaryDirectory
1918
from validation_workflow.api_request import ApiTest
2019
from validation_workflow.api_test_cases import ApiTestCases
2120
from validation_workflow.docker.inspect_docker_image import InspectDockerImage
@@ -241,7 +240,6 @@ def run_container(self, image_ids: dict, version: str) -> Any:
241240
'2': 'docker-compose-2.x.yml'
242241
}
243242

244-
self.tmp_dir = TemporaryDirectory()
245243
self.target_yml_file = os.path.join(self.tmp_dir.name, 'docker-compose.yml')
246244

247245
self.major_version_number = version[0]

‎src/validation_workflow/rpm/validation_rpm.py

-21
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
import time
1111

1212
from system.execute import execute
13-
from system.temporary_directory import TemporaryDirectory
1413
from test_workflow.integ_test.utils import get_password
1514
from validation_workflow.api_test_cases import ApiTestCases
1615
from validation_workflow.download_utils import DownloadUtils
@@ -22,26 +21,6 @@ class ValidateRpm(Validation, DownloadUtils):
2221

2322
def __init__(self, args: ValidationArgs) -> None:
2423
super().__init__(args)
25-
self.base_url_production = "https://artifacts.opensearch.org/releases/bundle/"
26-
self.base_url_staging = "https://ci.opensearch.org/ci/dbc/distribution-build-"
27-
self.tmp_dir = TemporaryDirectory()
28-
29-
def download_artifacts(self) -> bool:
30-
isFilePathEmpty = bool(self.args.file_path)
31-
for project in self.args.projects:
32-
if (isFilePathEmpty):
33-
if ("https:" not in self.args.file_path.get(project)):
34-
self.copy_artifact(self.args.file_path.get(project), str(self.tmp_dir.path))
35-
else:
36-
self.args.version = self.get_version(self.args.file_path.get(project))
37-
self.check_url(self.args.file_path.get(project))
38-
else:
39-
if (self.args.artifact_type == "staging"):
40-
self.args.file_path[project] = f"{self.base_url_staging}{project}/{self.args.version}/{self.args.build_number[project]}/linux/{self.args.arch}/{self.args.distribution}/dist/{project}/{project}-{self.args.version}-linux-{self.args.arch}.rpm" # noqa: E501
41-
else:
42-
self.args.file_path[project] = f"{self.base_url_production}{project}/{self.args.version}/{project}-{self.args.version}-linux-{self.args.arch}.rpm"
43-
self.check_url(self.args.file_path.get(project))
44-
return True
4524

4625
def installation(self) -> bool:
4726
try:

‎src/validation_workflow/tar/validation_tar.py

-21
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111

1212
from system.execute import execute
1313
from system.process import Process
14-
from system.temporary_directory import TemporaryDirectory
1514
from test_workflow.integ_test.utils import get_password
1615
from validation_workflow.api_test_cases import ApiTestCases
1716
from validation_workflow.download_utils import DownloadUtils
@@ -23,29 +22,9 @@ class ValidateTar(Validation, DownloadUtils):
2322

2423
def __init__(self, args: ValidationArgs) -> None:
2524
super().__init__(args)
26-
self.base_url_production = "https://artifacts.opensearch.org/releases/bundle/"
27-
self.base_url_staging = "https://ci.opensearch.org/ci/dbc/distribution-build-"
28-
self.tmp_dir = TemporaryDirectory()
2925
self.os_process = Process()
3026
self.osd_process = Process()
3127

32-
def download_artifacts(self) -> bool:
33-
isFilePathEmpty = bool(self.args.file_path)
34-
for project in self.args.projects:
35-
if (isFilePathEmpty):
36-
if ("https:" not in self.args.file_path.get(project)):
37-
self.copy_artifact(self.args.file_path.get(project), str(self.tmp_dir.path))
38-
else:
39-
self.args.version = self.get_version(self.args.file_path.get(project))
40-
self.check_url(self.args.file_path.get(project))
41-
else:
42-
if (self.args.artifact_type == "staging"):
43-
self.args.file_path[project] = f"{self.base_url_staging}{project}/{self.args.version}/{self.args.build_number[project]}/linux/{self.args.arch}/{self.args.distribution}/dist/{project}/{project}-{self.args.version}-linux-{self.args.arch}.tar.gz" # noqa: E501
44-
else:
45-
self.args.file_path[project] = f"{self.base_url_production}{project}/{self.args.version}/{project}-{self.args.version}-linux-{self.args.arch}.tar.gz"
46-
self.check_url(self.args.file_path.get(project))
47-
return True
48-
4928
def installation(self) -> bool:
5029
try:
5130
for project in self.args.projects:

‎src/validation_workflow/validation.py

+26-3
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from abc import ABC, abstractmethod
1414
from typing import Any
1515

16+
from system.temporary_directory import TemporaryDirectory
1617
from validation_workflow.download_utils import DownloadUtils
1718
from validation_workflow.validation_args import ValidationArgs
1819

@@ -23,8 +24,10 @@ class Validation(ABC):
2324
"""
2425

2526
def __init__(self, args: ValidationArgs) -> None:
26-
super().__init__()
2727
self.args = args
28+
self.base_url_production = "https://artifacts.opensearch.org/releases/bundle/"
29+
self.base_url_staging = "https://ci.opensearch.org/ci/dbc/distribution-build-"
30+
self.tmp_dir = TemporaryDirectory()
2831

2932
def check_url(self, url: str) -> bool:
3033
if DownloadUtils().download(url, self.tmp_dir) and DownloadUtils().is_url_valid(url): # type: ignore
@@ -53,9 +56,29 @@ def run(self) -> Any:
5356
except Exception as e:
5457
raise Exception(f'An error occurred while running the validation tests: {str(e)}')
5558

56-
@abstractmethod
5759
def download_artifacts(self) -> bool:
58-
pass
60+
isFilePathEmpty = bool(self.args.file_path)
61+
for project in self.args.projects:
62+
if (isFilePathEmpty):
63+
if ("https:" not in self.args.file_path.get(project)):
64+
self.copy_artifact(self.args.file_path.get(project), str(self.tmp_dir.path))
65+
else:
66+
self.args.version = self.get_version(self.args.file_path.get(project))
67+
self.check_url(self.args.file_path.get(project))
68+
else:
69+
self.args.file_path[project] = self.get_filepath(project)
70+
self.check_url(self.args.file_path.get(project))
71+
return True
72+
73+
def get_filepath(self, project: str) -> str:
74+
file_name_suffix = "tar.gz" if self.args.distribution == "tar" else self.args.distribution
75+
if self.args.artifact_type == "staging":
76+
if self.args.distribution == "yum":
77+
return f"{self.base_url_staging}{project}/{self.args.version}/{self.args.build_number[project]}/{self.args.platform}/{self.args.arch}/rpm/dist/{project}/{project}-{self.args.version}.staging.repo" # noqa: E501
78+
return f"{self.base_url_staging}{project}/{self.args.version}/{self.args.build_number[project]}/{self.args.platform}/{self.args.arch}/{self.args.distribution}/dist/{project}/{project}-{self.args.version}-{self.args.platform}-{self.args.arch}.{file_name_suffix}" # noqa: E501
79+
if self.args.distribution == "yum":
80+
return f"{self.base_url_production}{project}/{self.args.version[0:1]}.x/{project}-{self.args.version[0:1]}.x.repo"
81+
return f"{self.base_url_production}{project}/{self.args.version}/{project}-{self.args.version}-{self.args.platform}-{self.args.arch}.{file_name_suffix}"
5982

6083
@abstractmethod
6184
def installation(self) -> bool:

‎src/validation_workflow/validation_args.py

+8-4
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313

1414

1515
class ValidationArgs:
16-
SUPPORTED_PLATFORMS = ["linux"]
16+
SUPPORTED_PLATFORMS = ["linux", "windows"]
1717
DOCKER_SOURCE = ["dockerhub", "ecr"]
1818

1919
def __init__(self) -> None:
@@ -134,11 +134,11 @@ def __init__(self) -> None:
134134

135135
if (not (args.version or args.file_path)):
136136
raise Exception("Provide either version number or File Path")
137-
if(args.file_path):
137+
if (args.file_path):
138+
if 'opensearch' not in args.file_path.keys() or not set(args.file_path.keys()) <= {'opensearch', 'opensearch-dashboards'}:
139+
raise Exception("Missing OpenSearch artifact details! Please provide the valid product names among opensearch and opensearch-dashboards")
138140
args.distribution = self.get_distribution_type(args.file_path)
139141
args.projects = args.file_path.keys()
140-
if (('opensearch' not in args.projects)):
141-
raise Exception("Missing OpenSearch OpenSearch artifact details! Please provide the same along with OpenSearch-Dashboards to validate")
142142

143143
self.version = args.version
144144
self.file_path = args.file_path
@@ -165,6 +165,10 @@ def get_distribution_type(self, file_path: dict) -> str:
165165
return 'yum'
166166
elif (any("rpm" in value for value in file_path.values())):
167167
return 'rpm'
168+
elif (any("zip" in value for value in file_path.values())):
169+
return 'zip'
170+
elif (any("deb" in value for value in file_path.values())):
171+
return 'deb'
168172
else:
169173
raise Exception("Provided distribution is not supported")
170174

‎src/validation_workflow/validation_test_runner.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,24 @@
66
# compatible open source license.
77
# type: ignore
88

9+
from validation_workflow.deb.validation_deb import ValidateDeb
910
from validation_workflow.docker.validation_docker import ValidateDocker
1011
from validation_workflow.rpm.validation_rpm import ValidateRpm
1112
from validation_workflow.tar.validation_tar import ValidateTar
1213
from validation_workflow.validation import Validation
1314
from validation_workflow.validation_args import ValidationArgs
1415
from validation_workflow.yum.validation_yum import ValidateYum
16+
from validation_workflow.zip.validation_zip import ValidateZip
1517

1618

1719
class ValidationTestRunner:
1820
RUNNERS = {
1921
"docker": ValidateDocker,
2022
"tar": ValidateTar,
2123
"rpm": ValidateRpm,
22-
"yum": ValidateYum
24+
"yum": ValidateYum,
25+
"zip": ValidateZip,
26+
"deb": ValidateDeb
2327
}
2428

2529
@classmethod

‎src/validation_workflow/yum/validation_yum.py

-30
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,8 @@
1-
# Copyright OpenSearch Contributors
2-
# SPDX-License-Identifier: Apache-2.0
3-
#
4-
# The OpenSearch Contributors require contributions made to
5-
# this file be licensed under the Apache-2.0 license or a
6-
# compatible open source license.
7-
81
import logging
92
import os
103
import time
114

125
from system.execute import execute
13-
from system.temporary_directory import TemporaryDirectory
146
from test_workflow.integ_test.utils import get_password
157
from validation_workflow.api_test_cases import ApiTestCases
168
from validation_workflow.download_utils import DownloadUtils
@@ -22,28 +14,6 @@ class ValidateYum(Validation, DownloadUtils):
2214

2315
def __init__(self, args: ValidationArgs) -> None:
2416
super().__init__(args)
25-
self.base_url_production = "https://artifacts.opensearch.org/releases/bundle/"
26-
self.base_url_staging = "https://ci.opensearch.org/ci/dbc/distribution-build-"
27-
self.tmp_dir = TemporaryDirectory()
28-
29-
def download_artifacts(self) -> bool:
30-
isFilePathEmpty = bool(self.args.file_path)
31-
for project in self.args.projects:
32-
if (isFilePathEmpty):
33-
if ("https:" not in self.args.file_path.get(project)):
34-
self.copy_artifact(self.args.file_path.get(project), str(self.tmp_dir.path))
35-
else:
36-
self.args.version = self.get_version(self.args.file_path.get(project))
37-
self.check_url(self.args.file_path.get(project))
38-
39-
else:
40-
if (self.args.artifact_type == "staging"):
41-
self.args.file_path[project] = f"{self.base_url_staging}{project}/{self.args.version}/{self.args.build_number[project]}/linux/{self.args.arch}/rpm/dist/{project}/{project}-{self.args.version}.staging.repo" # noqa: E501
42-
else:
43-
self.args.file_path[project] = f"{self.base_url_production}{project}/{self.args.version[0:1]}.x/{project}-{self.args.version[0:1]}.x.repo"
44-
45-
self.check_url(self.args.file_path.get(project))
46-
return True
4717

4818
def installation(self) -> bool:
4919
try:

‎src/validation_workflow/zip/__init__.py

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Copyright OpenSearch Contributors
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# The OpenSearch Contributors require contributions made to
5+
# this file be licensed under the Apache-2.0 license or a
6+
# compatible open source license.
7+
#
8+
# This page intentionally left blank.

‎src/validation_workflow/zip/validation_zip.py

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Copyright OpenSearch Contributors
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# The OpenSearch Contributors require contributions made to
5+
# this file be licensed under the Apache-2.0 license or a
6+
# compatible open source license.
7+
8+
import logging
9+
import os
10+
import time
11+
12+
from system.execute import execute
13+
from system.process import Process
14+
from system.zip_file import ZipFile
15+
from test_workflow.integ_test.utils import get_password
16+
from validation_workflow.api_test_cases import ApiTestCases
17+
from validation_workflow.download_utils import DownloadUtils
18+
from validation_workflow.validation import Validation
19+
from validation_workflow.validation_args import ValidationArgs
20+
21+
22+
class ValidateZip(Validation, DownloadUtils):
23+
def __init__(self, args: ValidationArgs) -> None:
24+
super().__init__(args)
25+
self.os_process = Process()
26+
self.osd_process = Process()
27+
28+
def installation(self) -> bool:
29+
try:
30+
for project in self.args.projects:
31+
with ZipFile(os.path.join(self.tmp_dir.path, os.path.basename(self.args.file_path.get(project))), "r") as zip:
32+
zip.extractall(self.tmp_dir.path)
33+
except:
34+
raise Exception("Failed to install Opensearch")
35+
return True
36+
37+
def start_cluster(self) -> bool:
38+
try:
39+
execute(f"set OPENSEARCH_INITIAL_ADMIN_PASSWORD={get_password(str(self.args.version))}", ".", True)
40+
self.os_process.start(".\\opensearch-windows-install.bat", os.path.join(self.tmp_dir.path, f"opensearch-{self.args.version}"), False)
41+
time.sleep(85)
42+
if "opensearch-dashboards" in self.args.projects:
43+
self.osd_process.start(".\\bin\\opensearch-dashboards.bat", os.path.join(self.tmp_dir.path, f"opensearch-dashboards-{self.args.version}"), False)
44+
time.sleep(20)
45+
logging.info("Starting cluster")
46+
47+
except:
48+
raise Exception('Failed to Start Cluster')
49+
return True
50+
51+
def validation(self) -> bool:
52+
test_result, counter = ApiTestCases().test_apis(self.args.version, self.args.projects, self.check_for_security_plugin(os.path.join(self.tmp_dir.path, "opensearch")) if not self.args.force_https else True) # noqa: E501
53+
if test_result:
54+
logging.info(f'All tests Pass : {counter}')
55+
else:
56+
raise Exception(f'Not all tests Pass : {counter}')
57+
return True
58+
59+
def cleanup(self) -> bool:
60+
try:
61+
self.os_process.terminate()
62+
if ("opensearch-dashboards" in self.args.projects):
63+
self.osd_process.terminate()
64+
except:
65+
raise Exception('Failed to terminate the processes that started OpenSearch and OpenSearch-Dashboards')
66+
return True

0 commit comments

Comments
 (0)