Adding reporting workflow to generate test_run manifest based on test workflow (opensearch-project#3636)

Signed-off-by: Zelin Hao <zelinhao@amazon.com>

Author: zelinh

report.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+set -e
+
+DIR="$(dirname "$0")"
+"$DIR/run.sh" "$DIR/src/run_test_report.py" $@

src/manifests/test_run_manifest.py

@@ -0,0 +1,155 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+from typing import Optional
+
+from manifests.component_manifest import Component, ComponentManifest, Components
+
+
+class TestRunManifest(ComponentManifest['TestRunManifest', 'TestComponents']):
+    """
+    TestRunManifest contains the aggregated test results for the components.
+
+    The format for schema version 1.0 is:
+        schema-version: '1.0'
+        name: name of the product e.g. OpenSearch
+        test-run:
+          Command: command to trigger the integ test
+          TestType: type of test this manifest reports. e.g. integ-test
+          TestManifest: location of the test manifest used
+          DistributionManifest: URL or local path of the bundle manifest.
+          TestID: test id
+        components:
+          - name: sql
+            command: command to trigger the integ test for only sql component
+            configs:
+              - name: with-security
+                status: the status of the test run with this config. e.g. pass/fail
+                yml: URL or local path to the component yml file
+    """
+
+    SCHEMA = {
+        "schema-version": {"required": True, "type": "string", "allowed": ["1.0"]},
+        "name": {"required": True, "type": "string", "allowed": ["OpenSearch", "OpenSearch Dashboards"]},
+        "test-run": {
+            "required": False,
+            "type": "dict",
+            "schema": {
+                "Command": {"required": False, "type": "string"},
+                "TestType": {"required": False, "type": "string"},
+                "TestManifest": {"required": False, "type": "string"},
+                "DistributionManifest": {"required": False, "type": "string"},
+                "TestID": {"required": False, "type": "string"}
+            },
+        },
+        "components": {
+            "type": "list",
+            "schema": {
+                "type": "dict",
+                "schema": {
+                    "name": {"required": True, "type": "string"},
+                    "command": {"type": "string"},
+                    "configs": {
+                        "type": "list",
+                        "schema": {
+                            "type": "dict",
+                            "schema": {
+                                "name": {"type": "string"},
+                                "status": {"type": "string"},
+                                "yml": {"type": "string"},
+                            }
+                        },
+                    },
+                },
+            },
+        },
+    }
+
+    def __init__(self, data: dict) -> None:
+        super().__init__(data)
+        self.name = str(data["name"])
+        self.test_run = self.TestRun(data.get("test-run", None))
+        self.components = TestComponents(data.get("components", []))  # type: ignore[assignment]
+
+    def __to_dict__(self) -> dict:
+        return {
+            "schema-version": "1.0",
+            "name": self.name,
+            "test-run": None if self.test_run is None else self.test_run.__to_dict__(),
+            "components": self.components.__to_dict__()
+        }
+
+    class TestRun:
+        def __init__(self, data: dict) -> None:
+            if data is None:
+                self.test_run = None
+            else:
+                self.command = data["Command"]
+                self.test_type = data["TestType"]
+                self.test_manifest = data["TestManifest"]
+                self.distribution_manifest = data["DistributionManifest"]
+                self.test_id = data["TestID"]
+
+        def __to_dict__(self) -> Optional[dict]:
+            if (self.command and self.test_type and self.test_manifest and self.distribution_manifest and self.test_id) is None:
+                return None
+            else:
+                return {
+                    "Command": self.command,
+                    "TestType": self.test_type,
+                    "TestManifest": self.test_manifest,
+                    "DistributionManifest": self.distribution_manifest,
+                    "TestID": self.test_id
+                }
+
+
+class TestComponents(Components['TestComponent']):
+    @classmethod
+    def __create__(self, data: dict) -> 'TestComponent':
+        return TestComponent(data)
+
+
+class TestComponent(Component):
+    def __init__(self, data: dict) -> None:
+        super().__init__(data)
+        self.command = data["command"]
+        self.configs = self.TestComponentConfigs(data.get("configs", None))
+
+    def __to_dict__(self) -> dict:
+        return {
+            "name": self.name,
+            "command": self.command,
+            "configs": self.configs.__to_list__()
+        }
+
+    class TestComponentConfigs:
+        def __init__(self, data: list) -> None:
+            self.configs = []
+            for config in data:
+                self.configs.append(self.TestComponentConfig(config).__to_dict__())
+
+        def __to_list__(self) -> list:
+            return self.configs
+
+        class TestComponentConfig:
+            def __init__(self, data: dict) -> None:
+                self.name = data["name"]
+                self.status = data["status"]
+                self.yml = data["yml"]
+
+            def __to_dict__(self) -> dict:
+                return {
+                    "name": self.name,
+                    "status": self.status,
+                    "yml": self.yml
+                }
+
+
+TestRunManifest.VERSIONS = {"1.0": TestRunManifest}
+
+TestComponent.__test__ = False  # type: ignore[attr-defined]
+TestRunManifest.__test__ = False  # type: ignore[attr-defined]

src/report_workflow/__init__.py

@@ -0,0 +1,8 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# This page intentionally left blank.

src/report_workflow/report_args.py

@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+import argparse
+import logging
+
+from test_workflow.test_kwargs import TestKwargs
+
+
+class ReportArgs:
+    test_run_id: str
+    keep: bool
+    test_manifest_path: str
+    artifact_paths: dict
+    test_type: str
+    logging_level: int
+
+    def __init__(self) -> None:
+        parser = argparse.ArgumentParser(description="Generate test report for given test data")
+        parser.add_argument("test_manifest_path", type=str, help="Specify a test manifest path.")
+        parser.add_argument("-p", "--artifact-paths", nargs='*', action=TestKwargs, default={},
+                            help="Specify aritfacts paths for OpenSearch and OpenSearch Dashboards.")
+        # e.g. --base-path https://ci.opensearch.org/ci/dbc/integ-test/2.7.0/7771/linux/x64/tar/test-results/1234<test-run-id>/integ-test use more to save arguments number
+        parser.add_argument("--base-path", type=str, default="",
+                            help="Specify base paths for the integration test logs.")
+        parser.add_argument("--test-type", type=str, default="integ-test", help="Specify test type of this.")
+        parser.add_argument("--output-path", type=str, help="Specify the path location for the test-run manifest.")
+        parser.add_argument("--test-run-id", type=int, help="The unique execution id for the test")
+        parser.add_argument("--component", type=str, dest="components", nargs='*', help="Test a specific component or components instead of the entire distribution.")
+        parser.add_argument(
+            "-v", "--verbose", help="Show more verbose output.", action="store_const", default=logging.INFO, const=logging.DEBUG, dest="logging_level"
+        )
+
+        args = parser.parse_args()
+        self.test_run_id = args.test_run_id
+        self.logging_level = args.logging_level
+        self.test_manifest_path = args.test_manifest_path
+        self.artifact_paths = args.artifact_paths
+        self.base_path = args.base_path
+        self.test_type = args.test_type
+        self.components = args.components
+        self.output_path = args.output_path

src/report_workflow/test_run_runner.py

@@ -0,0 +1,137 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+import os
+import urllib.request
+from typing import Any
+from urllib.error import HTTPError
+
+import validators
+import yaml
+
+from manifests.test_manifest import TestManifest
+from manifests.test_run_manifest import TestRunManifest
+from report_workflow.report_args import ReportArgs
+
+
+class TestRunRunner:
+    args: ReportArgs
+    test_manifest: TestManifest
+    tests_dir: str
+    test_run_manifest: TestRunManifest
+    test_run_data: dict
+
+    def __init__(self, args: ReportArgs, test_manifest: TestManifest) -> None:
+        self.args = args
+        self.base_path = args.base_path
+        self.test_manifest = test_manifest
+        self.test_run_data = self.test_run_manifest_data_template("manifest")
+        self.product_name = test_manifest.__to_dict__().get("name")
+        self.name = self.product_name.replace(" ", "-").lower()
+        self.components = self.args.components
+        self.test_run_id = args.test_run_id
+        self.test_type = self.args.test_type
+        self.test_manifest_path = self.args.test_manifest_path
+        self.artifact_paths = ""
+        for k, v in self.args.artifact_paths.items():
+            self.artifact_paths = " ".join([self.artifact_paths, k + "=" + v]).strip(" ")
+
+        self.dist_manifest = "/".join([self.args.artifact_paths[self.name], "dist", self.name, "manifest.yml"]) if self.args.artifact_paths[self.name].startswith("https://") \
+            else os.path.join(self.args.artifact_paths[self.name], "dist", self.name, "manifest.yml")
+        self.test_components = self.test_manifest.components
+
+    def update_data(self) -> dict:
+        self.test_run_data["name"] = self.product_name
+        self.test_run_data["test-run"] = self.update_test_run_data()
+        for component in self.test_components.select(focus=self.args.components):
+            self.test_run_data["components"].append(self.component_entry(component.name))
+        return self.test_run_data
+
+    def update_test_run_data(self) -> dict:
+        test_run_data = {
+            "Command": generate_test_command(self.test_type, self.test_manifest_path, self.artifact_paths),
+            "TestType": self.test_type,
+            "TestManifest": self.test_manifest_path,
+            "DistributionManifest": self.dist_manifest,
+            "TestID": str(self.test_run_id)
+        }
+        return test_run_data
+
+    def generate_report(self, data: dict, output_dir: str) -> Any:
+        test_run_manifest = TestRunManifest(data)
+        test_run_manifetest_run_manifest_file = os.path.join(output_dir, "test-run.yml")
+        logging.info(f"Generating test-run.yml in {output_dir}")
+        return test_run_manifest.to_file(test_run_manifetest_run_manifest_file)
+
+    def component_entry(self, component_name: str) -> Any:
+        component = self.test_run_manifest_data_template("component")
+        component["name"] = component_name
+        component["command"] = generate_test_command(self.test_type, self.test_manifest_path, self.artifact_paths, component_name)
+
+        test_component = self.test_manifest.components[component_name]
+
+        config_names = [config for config in test_component.__to_dict__().get(self.test_type)["test-configs"]]
+        logging.info(f"Configs for {component_name} on {self.test_type} are {config_names}")
+        for config in config_names:
+            config_dict = {
+                "name": config,
+            }
+
+            component_yml_ref = generate_component_yml_ref(self.base_path, str(self.test_run_id), self.test_type, component_name, config)
+            logging.info(f"Loading {component_yml_ref}")
+            try:
+                if validators.url(component_yml_ref):
+                    with urllib.request.urlopen(component_yml_ref) as f:
+                        component_yml = yaml.safe_load(f.read().decode("utf-8"))
+                        test_result = component_yml["test_result"]
+                else:
+                    with open(component_yml_ref, "r", encoding='utf8') as f:
+                        component_yml = yaml.safe_load(f)
+                        test_result = component_yml["test_result"]
+            except (FileNotFoundError, HTTPError):
+                logging.info(f"Component yml file for {component_name} for {config} is missing or the base path is incorrect.")
+                test_result = "Not Available"
+                component_yml_ref = "URL not available"
+            config_dict["yml"] = component_yml_ref
+            config_dict["status"] = test_result
+            component["configs"].append(config_dict)
+        return component
+
+    def test_run_manifest_data_template(self, template_type: str) -> Any:
+        templates = {
+            "manifest": {
+                "schema-version": "1.0",
+                "name": "",
+                "test-run": {},
+                "components": []
+            },
+            "component": {
+                "name": "",
+                "command": "",
+                "configs": []
+            }
+        }
+        return templates[template_type]
+
+
+def generate_component_yml_ref(base_path: str, test_number: str, test_type: str, component_name: str, config: str) -> str:
+    if base_path.startswith("https://"):
+        return "/".join([base_path.strip("/"), "test-results", test_number, test_type, component_name, config, f"{component_name}.yml"])
+    else:
+        return os.path.join(base_path, "test-results", test_number, test_type, component_name, config, f"{component_name}.yml")
+
+
+def generate_test_command(test_type: str, test_manifest_path: str, artifacts_path: str, component: str = "") -> str:
+    command = " ".join(["./test.sh", test_type, test_manifest_path, "--paths", artifacts_path])
+    if component:
+        command = " ".join([command, "--component", component])
+    logging.info(command)
+    return command
+
+
+TestRunRunner.__test__ = False  # type:ignore