Make input manifest ci checks accept commit id as refs (opensearch-project#2856)

* Make input manifest ci checks accept commit id as refs

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Resolve isort issues

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Use try catch to resolve issues with component without refs

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Resolve isort issues

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Change commit to sha1 var name in initial verification

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

Author: peterzhuamazon

src/ci_workflow/ci_check_list.py

@@ -5,6 +5,7 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
+import logging
 from abc import ABC, abstractmethod
 from typing import Any
 
@@ -14,6 +15,11 @@
 class CiCheckList(ABC):
     def __init__(self, component: Any, target: CiTarget) -> None:
         self.component = component
+        try:
+            self.component_ref_is_sha1 = True if len(self.component.ref) == 40 and int(self.component.ref, 16) else False
+            logging.debug("sha1 exists in ref")
+        except AttributeError:
+            logging.debug("sha1 not exist in ref")
         self.target = target
 
     @abstractmethod

src/ci_workflow/ci_check_list_source_ref.py

@@ -6,9 +6,11 @@
 # compatible open source license.
 
 import logging
+import os
 import subprocess
 
 from ci_workflow.ci_check_list import CiCheckList
+from git.git_repository import GitRepository
 
 
 class CiCheckListSourceRef(CiCheckList):
@@ -17,12 +19,31 @@ def __init__(self, url: str, ref: str) -> None:
             super().__init__(f"Missing {url}@{ref}.")
 
     def checkout(self, work_dir: str) -> None:
+
+        # If ref is commit id, checkout the repository instead, as ls-remote does not allow non-branch/non-tag to be checked
+        if self.component_ref_is_sha1:
+            self.git_repo = GitRepository(
+                self.component.repository, self.component.ref, os.path.join(work_dir, self.component.name), self.component.working_directory
+            )
+
         return super().checkout(work_dir)
 
     def check(self) -> None:
         logging.info(f"Checking {self.component.repository} at {self.component.ref}.")
-        results = subprocess.check_output(f"git ls-remote {self.component.repository} {self.component.ref}", shell=True).decode().strip().split("\t")
-        if len(results) != 2:
+
+        if self.component_ref_is_sha1:
+            logging.info(f"Detect {self.component.name} with ref {self.component.ref} in sha1 format, treat as commit.")
+            results = subprocess.check_output(f"git cat-file -t {self.component.ref}", shell=True, cwd=self.git_repo.working_directory).decode().strip().split("\t")
+            check_failure = False if len(results) == 1 and results[0] == 'commit' else True
+        else:
+            logging.info(f"Treat {self.component.name} with ref {self.component.ref} as branch/tag since it is not sha1 commit")
+            results = subprocess.check_output(f"git ls-remote {self.component.repository} {self.component.ref}", shell=True).decode().strip().split("\t")
+            check_failure = True if len(results) != 2 else False
+
+        if check_failure:
             raise CiCheckListSourceRef.MissingRefError(self.component.repository, self.component.ref)
         else:
-            logging.info(f"Found {self.component.repository} at {self.component.ref} ({results[0]}).")
+            if self.component_ref_is_sha1:
+                logging.info(f"Found {self.component.repository} at {self.component.ref}.")
+            else:
+                logging.info(f"Found {self.component.repository} at {self.component.ref} ({results[0]}).")

tests/tests_ci_workflow/test_ci_check_lists_source_ref.py

@@ -14,7 +14,7 @@
 
 class TestCiCheckListsSourceRef(unittest.TestCase):
     @patch("subprocess.check_output", return_value="invalid".encode())
-    def test_ref_does_not_exist(self, mock_check_output: MagicMock) -> None:
+    def test_ref_branch_tag_does_not_exist(self, mock_check_output: MagicMock) -> None:
         component = InputComponentFromSource({"name": "common-utils", "repository": "url", "ref": "ref"})
         with self.assertRaises(CiCheckListSourceRef.MissingRefError) as ctx:
             list = CiCheckListSourceRef(component, MagicMock())
@@ -23,8 +23,35 @@ def test_ref_does_not_exist(self, mock_check_output: MagicMock) -> None:
         mock_check_output.assert_called_with("git ls-remote url ref", shell=True)
 
     @patch("subprocess.check_output", return_value="valid\tref".encode())
-    def test_ref_exists(self, mock_check_output: MagicMock) -> None:
+    def test_ref_branch_tag_exists(self, mock_check_output: MagicMock) -> None:
         component = InputComponentFromSource({"name": "common-utils", "repository": "url", "ref": "ref"})
         list = CiCheckListSourceRef(component, MagicMock())
         list.check()
         mock_check_output.assert_called_with("git ls-remote url ref", shell=True)
+
+    @patch("ci_workflow.ci_check_list_source_ref.GitRepository")
+    def test_ref_commit_checkout(self, mock_git_repo: MagicMock) -> None:
+        component = InputComponentFromSource({"name": "common-utils", "repository": "url", "ref": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
+        list = CiCheckListSourceRef(component, MagicMock())
+        list.checkout("path")
+        mock_git_repo.assert_called()
+
+    @patch("ci_workflow.ci_check_list_source_ref.GitRepository")
+    @patch("subprocess.check_output", return_value="fatal".encode())
+    def test_ref_commit_does_not_exist(self, mock_check_output: MagicMock, mock_git_repo: MagicMock) -> None:
+        component = InputComponentFromSource({"name": "common-utils", "repository": "url", "ref": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
+        with self.assertRaises(CiCheckListSourceRef.MissingRefError) as ctx:
+            list = CiCheckListSourceRef(component, MagicMock())
+            list.checkout("path")
+            list.check()
+        self.assertEqual("Missing url@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.", str(ctx.exception))
+        mock_check_output.assert_called_with("git cat-file -t aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", shell=True, cwd=mock_git_repo().working_directory)
+
+    @patch("ci_workflow.ci_check_list_source_ref.GitRepository")
+    @patch("subprocess.check_output", return_value="commit".encode())
+    def test_ref_commit_exists(self, mock_check_output: MagicMock, mock_git_repo: MagicMock) -> None:
+        component = InputComponentFromSource({"name": "common-utils", "repository": "url", "ref": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
+        list = CiCheckListSourceRef(component, MagicMock())
+        list.checkout("path")
+        list.check()
+        mock_check_output.assert_called_with("git cat-file -t aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", shell=True, cwd=mock_git_repo().working_directory)