Update the report workflow to accommodate reporting for smoke test results

src/report_workflow/test_report_runner.py

@@ -101,7 +101,11 @@ def component_entry(self, component_name: str, ci_group: int = None) -> Any:
 
         test_component = self.test_manifest.components[component_name]
 
-        config_names = [config for config in test_component.__to_dict__().get(self.test_type)["test-configs"]]
+        config_names = []
+        if "test-configs" in test_component.__to_dict__().get(self.test_type):
+            config_names = [config for config in test_component.__to_dict__().get(self.test_type)["test-configs"]]
+        elif "test-spec" in test_component.__to_dict__().get(self.test_type):
+            config_names = self.get_spec_path(self.test_report_data["version"], test_component.__to_dict__().get(self.test_type)["test-spec"])
         logging.info(f"Configs for {component_name} on {self.test_type} are {config_names}")
         for config in config_names:
             config_dict = {
@@ -164,6 +168,28 @@ def test_report_manifest_data_template(self, template_type: str) -> Any:
 
         return templates[template_type]
 
+    def get_spec_path(self, version: str, spec_file: str) -> list:
+        # loading spec yaml file
+        api_paths = []
+        spec_paths = [
+            os.path.join(os.getcwd(), "src", "test_workflow", "smoke_test", "smoke_tests_spec", f"{version.split('.')[0]}.x", spec_file),
+            os.path.join(os.getcwd(), "src", "test_workflow", "smoke_test", "smoke_tests_spec", "default", spec_file)
+        ]
+        logging.info(f"spec_paths is {spec_paths}")
+        for spec_file_path in spec_paths:
+            if os.path.exists(spec_file_path):
+                logging.info(f"Loading {spec_file} from {spec_file_path}")
+                with open(spec_file_path, 'r') as file:
+                    data = yaml.safe_load(file)
+                paths = data.get('paths', {})
+                for api_requests, api_details in paths.items():
+                    for method in api_details.keys():
+                        api_path = '_'.join([method, api_requests.replace("/", "_")])
+                        logging.info(f"api_path is {api_path}")
+                        api_paths.append(api_path)
+                break
+        return api_paths
+
 
 def generate_component_yml_ref(base_path: str, test_number: str, test_type: str, component_name: str,
                                config: str) -> str:
@@ -198,19 +224,22 @@ def get_os_cluster_logs(base_path: str, test_number: str, test_type: str, compon
     os_stdout: list = []
     os_stderr: list = []
     cluster_ids: list
-
-    if product_name == 'opensearch':
-        cluster_ids = ['id-0'] if config == 'with-security' else ['id-1']
-    else:
-        cluster_ids = ['id-0', 'id-1'] if config == 'with-security' else ['id-2', 'id-3']
-
-    for ids in cluster_ids:
-        if base_path.startswith("https://"):
-            os_stdout.append("/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stdout.txt"]))
-            os_stderr.append("/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stderr.txt"]))
+    if test_type == "integ-test":
+        if product_name == 'opensearch':
+            cluster_ids = ['id-0'] if config == 'with-security' else ['id-1']
         else:
-            os_stdout.append(os.path.join(base_path, "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stdout.txt"))
-            os_stderr.append(os.path.join(base_path, "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stderr.txt"))
+            cluster_ids = ['id-0', 'id-1'] if config == 'with-security' else ['id-2', 'id-3']
+
+        for ids in cluster_ids:
+            if base_path.startswith("https://"):
+                os_stdout.append("/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stdout.txt"]))
+                os_stderr.append("/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stderr.txt"]))
+            else:
+                os_stdout.append(os.path.join(base_path, "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stdout.txt"))
+                os_stderr.append(os.path.join(base_path, "test-results", test_number, test_type, component_name, config, "local-cluster-logs", ids, "stderr.txt"))
+    elif test_type == "smoke-test":
+        os_stdout.append(os.path.join(base_path, "test-results", test_number, test_type, "local-cluster-logs", "stdout.txt"))
+        os_stderr.append(os.path.join(base_path, "test-results", test_number, test_type, "local-cluster-logs", "stderr.txt"))
 
     return [os_stdout, os_stderr]
 

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/GET___plugins__ism_explain_ism-index-000001/index-management.yml

@@ -0,0 +1,8 @@
+component_name: index-management
+run_id: 12345
+test_config: GET___plugins__ism_explain_ism-index-000001
+test_result: PASS
+test_result_files:
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/GET___plugins__ism_explain_ism-index-000001/stdout.txt
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/GET___plugins__ism_explain_ism-index-000001/stderr.txt
+test_type: smoke-test

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/GET___plugins__ism_explain_ism-index-000001/stdout.txt

@@ -0,0 +1,7 @@
+Validating api request /_plugins/_ism/explain/ism-index-000001
+API request URL is https://localhost:9200/_plugins/_ism/explain/ism-index-000001
+Parameter is  and type is <class 'str'>
+header is {'Content-Type': 'application/json'}
+Response is 
+{"ism-index-000001":{"index.plugins.index_state_management.policy_id":"policy_1","index.opendistro.index_state_management.policy_id":"policy_1","index":"ism-index-000001","index_uuid":"tUf53qETQZW5hqoABOGajA","policy_id":"policy_1","enabled":true},"total_managed_indices":1}
+Response is validated.

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/GET___plugins__ism_policies/index-management.yml

@@ -0,0 +1,8 @@
+component_name: index-management
+run_id: 12345
+test_config: GET___plugins__ism_policies
+test_result: PASS
+test_result_files:
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/GET___plugins__ism_policies/stdout.txt
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/GET___plugins__ism_policies/stderr.txt
+test_type: smoke-test

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/GET___plugins__ism_policies/stdout.txt

@@ -0,0 +1,7 @@
+Validating api request /_plugins/_ism/policies
+API request URL is https://localhost:9200/_plugins/_ism/policies
+Parameter is  and type is <class 'str'>
+header is {'Content-Type': 'application/json'}
+Response is 
+{"policies":[{"_id":"policy_1","_seq_no":0,"_primary_term":1,"policy":{"policy_id":"policy_1","description":"ingesting logs","last_updated_time":1740092877535,"schema_version":21,"error_notification":null,"default_state":"ingest","states":[{"name":"ingest","actions":[{"retry":{"count":3,"backoff":"exponential","delay":"1m"},"rollover":{"min_doc_count":5,"copy_alias":false}}],"transitions":[{"state_name":"search"}]},{"name":"search","actions":[],"transitions":[{"state_name":"delete","conditions":{"min_index_age":"5m"}}]},{"name":"delete","actions":[{"retry":{"count":3,"backoff":"exponential","delay":"1m"},"delete":{}}],"transitions":[]}],"ism_template":null}}],"total_policies":1}
+Response is validated.

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/POST___plugins__ism_add_ism-index-000001/index-management.yml

@@ -0,0 +1,8 @@
+component_name: index-management
+run_id: 12345
+test_config: POST___plugins__ism_add_ism-index-000001
+test_result: PASS
+test_result_files:
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/POST___plugins__ism_add_ism-index-000001/stdout.txt
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/POST___plugins__ism_add_ism-index-000001/stderr.txt
+test_type: smoke-test

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/POST___plugins__ism_add_ism-index-000001/stdout.txt

@@ -0,0 +1,8 @@
+Validating api request /_plugins/_ism/add/ism-index-000001
+API request URL is https://localhost:9200/_plugins/_ism/add/ism-index-000001
+Parameter is {"policy_id": "policy_1"}
+ and type is <class 'str'>
+header is {'Content-Type': 'application/json'}
+Response is 
+{"updated_indices":1,"failures":false,"failed_indices":[]}
+Response is validated.

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/POST___plugins__ism_remove_ism-index-000001/index-management.yml

@@ -0,0 +1,8 @@
+component_name: index-management
+run_id: 12345
+test_config: POST___plugins__ism_remove_ism-index-000001
+test_result: PASS
+test_result_files:
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/POST___plugins__ism_remove_ism-index-000001/stdout.txt
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/POST___plugins__ism_remove_ism-index-000001/stderr.txt
+test_type: smoke-test

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/POST___plugins__ism_remove_ism-index-000001/stdout.txt

@@ -0,0 +1,7 @@
+Validating api request /_plugins/_ism/remove/ism-index-000001
+API request URL is https://localhost:9200/_plugins/_ism/remove/ism-index-000001
+Parameter is  and type is <class 'str'>
+header is {'Content-Type': 'application/json'}
+Response is 
+{"updated_indices":1,"failures":false,"failed_indices":[]}
+Response is validated.

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/POST__rollover-alias__rollover/index-management.yml

@@ -0,0 +1,8 @@
+component_name: index-management
+run_id: 12345
+test_config: POST__rollover-alias__rollover
+test_result: PASS
+test_result_files:
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/POST__rollover-alias__rollover/stdout.txt
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/POST__rollover-alias__rollover/stderr.txt
+test_type: smoke-test

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/POST__rollover-alias__rollover/stdout.txt

@@ -0,0 +1,7 @@
+Validating api request /rollover-alias/_rollover
+API request URL is https://localhost:9200/rollover-alias/_rollover
+Parameter is  and type is <class 'str'>
+header is {'Content-Type': 'application/json'}
+Response is 
+{"acknowledged":true,"shards_acknowledged":true,"old_index":"ism-index-000001","new_index":"ism-index-000002","rolled_over":true,"dry_run":false,"conditions":{}}
+Response is validated.

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/PUT___plugins__ism_policies_policy_1/index-management.yml

@@ -0,0 +1,8 @@
+component_name: index-management
+run_id: 12345
+test_config: PUT___plugins__ism_policies_policy_1
+test_result: PASS
+test_result_files:
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/PUT___plugins__ism_policies_policy_1/stdout.txt
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/PUT___plugins__ism_policies_policy_1/stderr.txt
+test_type: smoke-test

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/PUT___plugins__ism_policies_policy_1/stdout.txt

@@ -0,0 +1,8 @@
+Validating api request /_plugins/_ism/policies/policy_1
+API request URL is https://localhost:9200/_plugins/_ism/policies/policy_1
+Parameter is {"policy": {"description": "ingesting logs", "default_state": "ingest", "states": [{"name": "ingest", "actions": [{"rollover": {"min_doc_count": 5}}], "transitions": [{"state_name": "search"}]}, {"name": "search", "actions": [], "transitions": [{"state_name": "delete", "conditions": {"min_index_age": "5m"}}]}, {"name": "delete", "actions": [{"delete": {}}], "transitions": []}]}}
+ and type is <class 'str'>
+header is {'Content-Type': 'application/json'}
+Response is 
+{"_id":"policy_1","_version":1,"_primary_term":1,"_seq_no":0,"policy":{"policy":{"policy_id":"policy_1","description":"ingesting logs","last_updated_time":1740092877535,"schema_version":1,"error_notification":null,"default_state":"ingest","states":[{"name":"ingest","actions":[{"retry":{"count":3,"backoff":"exponential","delay":"1m"},"rollover":{"min_doc_count":5,"copy_alias":false}}],"transitions":[{"state_name":"search"}]},{"name":"search","actions":[],"transitions":[{"state_name":"delete","conditions":{"min_index_age":"5m"}}]},{"name":"delete","actions":[{"retry":{"count":3,"backoff":"exponential","delay":"1m"},"delete":{}}],"transitions":[]}],"ism_template":null}}}
+Response is validated.

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/PUT__ism-index-000001/index-management.yml

@@ -0,0 +1,8 @@
+component_name: index-management
+run_id: 12345
+test_config: PUT__ism-index-000001
+test_result: PASS
+test_result_files:
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/PUT__ism-index-000001/stdout.txt
+- /Users/zelinhao/workspace/fork/opensearch-build/test-results/12345/smoke-test/index-management/PUT__ism-index-000001/stderr.txt
+test_type: smoke-test

tests/tests_report_workflow/data/test-results/12345/smoke-test/index-management/PUT__ism-index-000001/stdout.txt

@@ -0,0 +1,8 @@
+Validating api request /ism-index-000001
+API request URL is https://localhost:9200/ism-index-000001
+Parameter is {"aliases": {"rollover-alias": {"is_write_index": true}}}
+ and type is <class 'str'>
+header is {'Content-Type': 'application/json'}
+Response is 
+{"acknowledged":true,"shards_acknowledged":true,"index":"ism-index-000001"}
+Response is validated.

tests/tests_report_workflow/data/test-results/12345/smoke-test/local-cluster-logs/stderr.txt

@@ -0,0 +1,11 @@
+WARNING: Using incubator modules: jdk.incubator.vector
+WARNING: A terminally deprecated method in java.lang.System has been called
+WARNING: System::setSecurityManager has been called by org.opensearch.bootstrap.OpenSearch (file:/private/var/folders/ft/7lyj4lg52l53_ppfk94gszph0000gq/T/tmphzkuouen/opensearch-2.19.0/lib/opensearch-2.19.0.jar)
+WARNING: Please consider reporting this to the maintainers of org.opensearch.bootstrap.OpenSearch
+WARNING: System::setSecurityManager will be removed in a future release
+Feb 20, 2025 3:07:00 PM sun.util.locale.provider.LocaleProviderAdapter <clinit>
+WARNING: COMPAT locale provider will be removed in a future release
+WARNING: A terminally deprecated method in java.lang.System has been called
+WARNING: System::setSecurityManager has been called by org.opensearch.bootstrap.Security (file:/private/var/folders/ft/7lyj4lg52l53_ppfk94gszph0000gq/T/tmphzkuouen/opensearch-2.19.0/lib/opensearch-2.19.0.jar)
+WARNING: Please consider reporting this to the maintainers of org.opensearch.bootstrap.Security
+WARNING: System::setSecurityManager will be removed in a future release