Skip to content

Commit ae7fa5a

Browse files
DivyaasmDivyaasm
committed
add debian and zip and refactor code
Signed-off-by: Divya Madala <divyaasm@amazon.com>
1 parent 9ecf30a commit ae7fa5a

17 files changed

+192
-80
lines changed

‎src/validation_workflow/deb/__init__.py

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Copyright OpenSearch Contributors
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# The OpenSearch Contributors require contributions made to
5+
# this file be licensed under the Apache-2.0 license or a
6+
# compatible open source license.
7+
#
8+
# This page intentionally left blank.

‎src/validation_workflow/deb/validation_deb.py

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Copyright OpenSearch Contributors
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# The OpenSearch Contributors require contributions made to
5+
# this file be licensed under the Apache-2.0 license or a
6+
# compatible open source license.
7+
8+
import logging
9+
import os
10+
import time
11+
12+
from system.execute import execute
13+
from test_workflow.integ_test.utils import get_password
14+
from validation_workflow.api_test_cases import ApiTestCases
15+
from validation_workflow.download_utils import DownloadUtils
16+
from validation_workflow.validation import Validation
17+
from validation_workflow.validation_args import ValidationArgs
18+
19+
20+
class ValidateDeb(Validation, DownloadUtils):
21+
def __init__(self, args: ValidationArgs) -> None:
22+
super().__init__(args)
23+
24+
def installation(self) -> bool:
25+
try:
26+
for project in self.args.projects:
27+
set_password = f' env OPENSEARCH_INITIAL_ADMIN_PASSWORD={get_password(str(self.args.version))}' if project == "opensearch" else ""
28+
execute(f'sudo dpkg --purge {project}', ".")
29+
execute(f'sudo {set_password} dpkg -i {os.path.basename(self.args.file_path.get(project))}', str(self.tmp_dir.path))
30+
time.sleep(80)
31+
except:
32+
raise Exception("Failed to install Opensearch")
33+
return True
34+
35+
def start_cluster(self) -> bool:
36+
try:
37+
for project in self.args.projects:
38+
execute(f'sudo systemctl enable {project}', ".")
39+
execute(f'sudo systemctl start {project}', ".")
40+
time.sleep(20)
41+
execute(f'sudo systemctl status {project}', ".")
42+
except:
43+
raise Exception('Failed to Start Cluster')
44+
return True
45+
46+
def validation(self) -> bool:
47+
test_result, counter = ApiTestCases().test_apis(self.args.version, self.args.projects, self.check_for_security_plugin(os.path.join(os.sep, "usr", "share", "opensearch")) if not self.args.force_https else True) # noqa: E501
48+
if (test_result):
49+
logging.info(f'All tests Pass : {counter}')
50+
return True
51+
else:
52+
raise Exception(f'Not all tests Pass : {counter}')
53+
54+
def cleanup(self) -> bool:
55+
try:
56+
for project in self.args.projects:
57+
execute(f'sudo dpkg --purge {project}', ".")
58+
except Exception as e:
59+
raise Exception(f'Exception occurred either while attempting to stop cluster or removing OpenSearch/OpenSearch-Dashboards. {str(e)}')
60+
return True

‎src/validation_workflow/docker/validation_docker.py

-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
import requests
1717

18-
from system.temporary_directory import TemporaryDirectory
1918
from validation_workflow.api_request import ApiTest
2019
from validation_workflow.api_test_cases import ApiTestCases
2120
from validation_workflow.docker.inspect_docker_image import InspectDockerImage
@@ -241,7 +240,6 @@ def run_container(self, image_ids: dict, version: str) -> Any:
241240
'2': 'docker-compose-2.x.yml'
242241
}
243242

244-
self.tmp_dir = TemporaryDirectory()
245243
self.target_yml_file = os.path.join(self.tmp_dir.name, 'docker-compose.yml')
246244

247245
self.major_version_number = version[0]

‎src/validation_workflow/rpm/validation_rpm.py

-21
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
import time
1111

1212
from system.execute import execute
13-
from system.temporary_directory import TemporaryDirectory
1413
from test_workflow.integ_test.utils import get_password
1514
from validation_workflow.api_test_cases import ApiTestCases
1615
from validation_workflow.download_utils import DownloadUtils
@@ -22,26 +21,6 @@ class ValidateRpm(Validation, DownloadUtils):
2221

2322
def __init__(self, args: ValidationArgs) -> None:
2423
super().__init__(args)
25-
self.base_url_production = "https://artifacts.opensearch.org/releases/bundle/"
26-
self.base_url_staging = "https://ci.opensearch.org/ci/dbc/distribution-build-"
27-
self.tmp_dir = TemporaryDirectory()
28-
29-
def download_artifacts(self) -> bool:
30-
isFilePathEmpty = bool(self.args.file_path)
31-
for project in self.args.projects:
32-
if (isFilePathEmpty):
33-
if ("https:" not in self.args.file_path.get(project)):
34-
self.copy_artifact(self.args.file_path.get(project), str(self.tmp_dir.path))
35-
else:
36-
self.args.version = self.get_version(self.args.file_path.get(project))
37-
self.check_url(self.args.file_path.get(project))
38-
else:
39-
if (self.args.artifact_type == "staging"):
40-
self.args.file_path[project] = f"{self.base_url_staging}{project}/{self.args.version}/{self.args.build_number[project]}/linux/{self.args.arch}/{self.args.distribution}/dist/{project}/{project}-{self.args.version}-linux-{self.args.arch}.rpm" # noqa: E501
41-
else:
42-
self.args.file_path[project] = f"{self.base_url_production}{project}/{self.args.version}/{project}-{self.args.version}-linux-{self.args.arch}.rpm"
43-
self.check_url(self.args.file_path.get(project))
44-
return True
4524

4625
def installation(self) -> bool:
4726
try:

‎src/validation_workflow/tar/validation_tar.py

-21
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111

1212
from system.execute import execute
1313
from system.process import Process
14-
from system.temporary_directory import TemporaryDirectory
1514
from test_workflow.integ_test.utils import get_password
1615
from validation_workflow.api_test_cases import ApiTestCases
1716
from validation_workflow.download_utils import DownloadUtils
@@ -23,29 +22,9 @@ class ValidateTar(Validation, DownloadUtils):
2322

2423
def __init__(self, args: ValidationArgs) -> None:
2524
super().__init__(args)
26-
self.base_url_production = "https://artifacts.opensearch.org/releases/bundle/"
27-
self.base_url_staging = "https://ci.opensearch.org/ci/dbc/distribution-build-"
28-
self.tmp_dir = TemporaryDirectory()
2925
self.os_process = Process()
3026
self.osd_process = Process()
3127

32-
def download_artifacts(self) -> bool:
33-
isFilePathEmpty = bool(self.args.file_path)
34-
for project in self.args.projects:
35-
if (isFilePathEmpty):
36-
if ("https:" not in self.args.file_path.get(project)):
37-
self.copy_artifact(self.args.file_path.get(project), str(self.tmp_dir.path))
38-
else:
39-
self.args.version = self.get_version(self.args.file_path.get(project))
40-
self.check_url(self.args.file_path.get(project))
41-
else:
42-
if (self.args.artifact_type == "staging"):
43-
self.args.file_path[project] = f"{self.base_url_staging}{project}/{self.args.version}/{self.args.build_number[project]}/linux/{self.args.arch}/{self.args.distribution}/dist/{project}/{project}-{self.args.version}-linux-{self.args.arch}.tar.gz" # noqa: E501
44-
else:
45-
self.args.file_path[project] = f"{self.base_url_production}{project}/{self.args.version}/{project}-{self.args.version}-linux-{self.args.arch}.tar.gz"
46-
self.check_url(self.args.file_path.get(project))
47-
return True
48-
4928
def installation(self) -> bool:
5029
try:
5130
for project in self.args.projects:

‎src/validation_workflow/validation.py

+26-3
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from abc import ABC, abstractmethod
1414
from typing import Any
1515

16+
from system.temporary_directory import TemporaryDirectory
1617
from validation_workflow.download_utils import DownloadUtils
1718
from validation_workflow.validation_args import ValidationArgs
1819

@@ -23,8 +24,10 @@ class Validation(ABC):
2324
"""
2425

2526
def __init__(self, args: ValidationArgs) -> None:
26-
super().__init__()
2727
self.args = args
28+
self.base_url_production = "https://artifacts.opensearch.org/releases/bundle/"
29+
self.base_url_staging = "https://ci.opensearch.org/ci/dbc/distribution-build-"
30+
self.tmp_dir = TemporaryDirectory()
2831

2932
def check_url(self, url: str) -> bool:
3033
if DownloadUtils().download(url, self.tmp_dir) and DownloadUtils().is_url_valid(url): # type: ignore
@@ -53,9 +56,29 @@ def run(self) -> Any:
5356
except Exception as e:
5457
raise Exception(f'An error occurred while running the validation tests: {str(e)}')
5558

56-
@abstractmethod
5759
def download_artifacts(self) -> bool:
58-
pass
60+
isFilePathEmpty = bool(self.args.file_path)
61+
for project in self.args.projects:
62+
if (isFilePathEmpty):
63+
if ("https:" not in self.args.file_path.get(project)):
64+
self.copy_artifact(self.args.file_path.get(project), str(self.tmp_dir.path))
65+
else:
66+
self.args.version = self.get_version(self.args.file_path.get(project))
67+
self.check_url(self.args.file_path.get(project))
68+
else:
69+
self.args.file_path[project] = self.get_filepath(project)
70+
self.check_url(self.args.file_path.get(project))
71+
return True
72+
73+
def get_filepath(self, project: str) -> str:
74+
file_name_suffix = "tar.gz" if self.args.distribution == "tar" else self.args.distribution
75+
if self.args.artifact_type == "staging":
76+
if self.args.distribution == "yum":
77+
return f"{self.base_url_staging}{project}/{self.args.version}/{self.args.build_number[project]}/{self.args.platform}/{self.args.arch}/rpm/dist/{project}/{project}-{self.args.version}.staging.repo" # noqa: E501
78+
return f"{self.base_url_staging}{project}/{self.args.version}/{self.args.build_number[project]}/{self.args.platform}/{self.args.arch}/{self.args.distribution}/dist/{project}/{project}-{self.args.version}-{self.args.platform}-{self.args.arch}.{file_name_suffix}" # noqa: E501
79+
if self.args.distribution == "yum":
80+
return f"{self.base_url_production}{project}/{self.args.version[0:1]}.x/{project}-{self.args.version[0:1]}.x.repo"
81+
return f"{self.base_url_production}{project}/{self.args.version}/{project}-{self.args.version}-{self.args.platform}-{self.args.arch}.{file_name_suffix}"
5982

6083
@abstractmethod
6184
def installation(self) -> bool:

‎src/validation_workflow/validation_args.py

+8-4
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313

1414

1515
class ValidationArgs:
16-
SUPPORTED_PLATFORMS = ["linux"]
16+
SUPPORTED_PLATFORMS = ["linux", "windows"]
1717
DOCKER_SOURCE = ["dockerhub", "ecr"]
1818

1919
def __init__(self) -> None:
@@ -134,11 +134,11 @@ def __init__(self) -> None:
134134

135135
if (not (args.version or args.file_path)):
136136
raise Exception("Provide either version number or File Path")
137-
if(args.file_path):
137+
if (args.file_path):
138+
if 'opensearch' not in args.file_path.keys() or not set(args.file_path.keys()) <= {'opensearch', 'opensearch-dashboards'}:
139+
raise Exception("Missing OpenSearch artifact details! Please provide the valid product names among opensearch and opensearch-dashboards")
138140
args.distribution = self.get_distribution_type(args.file_path)
139141
args.projects = args.file_path.keys()
140-
if (('opensearch' not in args.projects)):
141-
raise Exception("Missing OpenSearch OpenSearch artifact details! Please provide the same along with OpenSearch-Dashboards to validate")
142142

143143
self.version = args.version
144144
self.file_path = args.file_path
@@ -165,6 +165,10 @@ def get_distribution_type(self, file_path: dict) -> str:
165165
return 'yum'
166166
elif (any("rpm" in value for value in file_path.values())):
167167
return 'rpm'
168+
elif (any("zip" in value for value in file_path.values())):
169+
return 'zip'
170+
elif (any("deb" in value for value in file_path.values())):
171+
return 'deb'
168172
else:
169173
raise Exception("Provided distribution is not supported")
170174

‎src/validation_workflow/validation_test_runner.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,24 @@
66
# compatible open source license.
77
# type: ignore
88

9+
from validation_workflow.deb.validation_deb import ValidateDeb
910
from validation_workflow.docker.validation_docker import ValidateDocker
1011
from validation_workflow.rpm.validation_rpm import ValidateRpm
1112
from validation_workflow.tar.validation_tar import ValidateTar
1213
from validation_workflow.validation import Validation
1314
from validation_workflow.validation_args import ValidationArgs
1415
from validation_workflow.yum.validation_yum import ValidateYum
16+
from validation_workflow.zip.validation_zip import ValidateZip
1517

1618

1719
class ValidationTestRunner:
1820
RUNNERS = {
1921
"docker": ValidateDocker,
2022
"tar": ValidateTar,
2123
"rpm": ValidateRpm,
22-
"yum": ValidateYum
24+
"yum": ValidateYum,
25+
"zip": ValidateZip,
26+
"deb": ValidateDeb
2327
}
2428

2529
@classmethod

‎src/validation_workflow/yum/validation_yum.py

-23
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
import time
1111

1212
from system.execute import execute
13-
from system.temporary_directory import TemporaryDirectory
1413
from test_workflow.integ_test.utils import get_password
1514
from validation_workflow.api_test_cases import ApiTestCases
1615
from validation_workflow.download_utils import DownloadUtils
@@ -22,28 +21,6 @@ class ValidateYum(Validation, DownloadUtils):
2221

2322
def __init__(self, args: ValidationArgs) -> None:
2423
super().__init__(args)
25-
self.base_url_production = "https://artifacts.opensearch.org/releases/bundle/"
26-
self.base_url_staging = "https://ci.opensearch.org/ci/dbc/distribution-build-"
27-
self.tmp_dir = TemporaryDirectory()
28-
29-
def download_artifacts(self) -> bool:
30-
isFilePathEmpty = bool(self.args.file_path)
31-
for project in self.args.projects:
32-
if (isFilePathEmpty):
33-
if ("https:" not in self.args.file_path.get(project)):
34-
self.copy_artifact(self.args.file_path.get(project), str(self.tmp_dir.path))
35-
else:
36-
self.args.version = self.get_version(self.args.file_path.get(project))
37-
self.check_url(self.args.file_path.get(project))
38-
39-
else:
40-
if (self.args.artifact_type == "staging"):
41-
self.args.file_path[project] = f"{self.base_url_staging}{project}/{self.args.version}/{self.args.build_number[project]}/linux/{self.args.arch}/rpm/dist/{project}/{project}-{self.args.version}.staging.repo" # noqa: E501
42-
else:
43-
self.args.file_path[project] = f"{self.base_url_production}{project}/{self.args.version[0:1]}.x/{project}-{self.args.version[0:1]}.x.repo"
44-
45-
self.check_url(self.args.file_path.get(project))
46-
return True
4724

4825
def installation(self) -> bool:
4926
try:

‎src/validation_workflow/zip/__init__.py

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Copyright OpenSearch Contributors
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# The OpenSearch Contributors require contributions made to
5+
# this file be licensed under the Apache-2.0 license or a
6+
# compatible open source license.
7+
#
8+
# This page intentionally left blank.

‎src/validation_workflow/zip/validation_zip.py

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Copyright OpenSearch Contributors
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
# The OpenSearch Contributors require contributions made to
5+
# this file be licensed under the Apache-2.0 license or a
6+
# compatible open source license.
7+
8+
import logging
9+
import os
10+
import time
11+
12+
from system.process import Process
13+
from system.zip_file import ZipFile
14+
from test_workflow.integ_test.utils import get_password
15+
from validation_workflow.api_test_cases import ApiTestCases
16+
from validation_workflow.download_utils import DownloadUtils
17+
from validation_workflow.validation import Validation
18+
from validation_workflow.validation_args import ValidationArgs
19+
20+
21+
class ValidateZip(Validation, DownloadUtils):
22+
def __init__(self, args: ValidationArgs) -> None:
23+
super().__init__(args)
24+
self.os_process = Process()
25+
self.osd_process = Process()
26+
27+
def installation(self) -> bool:
28+
try:
29+
for project in self.args.projects:
30+
with ZipFile(os.path.join(self.tmp_dir.path, os.path.basename(self.args.file_path.get(project))), "r") as zip:
31+
zip.extractall(self.tmp_dir.path)
32+
except:
33+
raise Exception("Failed to install Opensearch")
34+
return True
35+
36+
def start_cluster(self) -> bool:
37+
try:
38+
self.os_process.start(f"env OPENSEARCH_INITIAL_ADMIN_PASSWORD={get_password(str(self.args.version))} .\\opensearch-windows-install.bat", os.path.join(self.tmp_dir.path, f"opensearch-{self.args.version}"), False) # noqa: E501
39+
time.sleep(85)
40+
if "opensearch-dashboards" in self.args.projects:
41+
self.osd_process.start(".\\bin\\opensearch-dashboards.bat", os.path.join(self.tmp_dir.path, f"opensearch-dashboards-{self.args.version}"), False)
42+
time.sleep(20)
43+
logging.info("Starting cluster")
44+
except:
45+
raise Exception('Failed to Start Cluster')
46+
return True
47+
48+
def validation(self) -> bool:
49+
test_result, counter = ApiTestCases().test_apis(self.args.version, self.args.projects, self.check_for_security_plugin(os.path.join(self.tmp_dir.path, "opensearch")) if not self.args.force_https else True) # noqa: E501
50+
if test_result:
51+
logging.info(f'All tests Pass : {counter}')
52+
else:
53+
raise Exception(f'Not all tests Pass : {counter}')
54+
return True
55+
56+
def cleanup(self) -> bool:
57+
try:
58+
self.os_process.terminate()
59+
if ("opensearch-dashboards" in self.args.projects):
60+
self.osd_process.terminate()
61+
except:
62+
raise Exception('Failed to terminate the processes that started OpenSearch and OpenSearch-Dashboards')
63+
return True

‎tests/tests_validation_workflow/test_validation.py

-3
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,6 @@ def __init__(self, args: ValidationArgs) -> None:
1919
super().__init__(args)
2020
self.tmp_dir = TemporaryDirectory()
2121

22-
def download_artifacts(self) -> None:
23-
return None
24-
2522
def installation(self) -> None:
2623
return None
2724

0 commit comments

Comments
 (0)