From c91333be0d2940d158704c980536d44666cc3f0a Mon Sep 17 00:00:00 2001 From: Justin Smith Date: Wed, 5 Feb 2025 08:43:36 -0500 Subject: [PATCH 1/5] GitHub CI job to verify tags are on expected branches --- .github/workflows/misc-tests.yaml | 12 ++++ util/git-tag-check/git-tag-check.py | 105 ++++++++++++++++++++++++++++ util/git-tag-check/tag-branch.json | 14 ++++ 3 files changed, 131 insertions(+) create mode 100644 util/git-tag-check/git-tag-check.py create mode 100644 util/git-tag-check/tag-branch.json diff --git a/.github/workflows/misc-tests.yaml b/.github/workflows/misc-tests.yaml index dab7960711..5d4ec33bf8 100644 --- a/.github/workflows/misc-tests.yaml +++ b/.github/workflows/misc-tests.yaml @@ -121,3 +121,15 @@ jobs: - if: ${{ matrix.os != 'windows-latest' }} name: Run tests run: cmake --build "path has spaces/build-fips" --target run_tests + git-tag-check: + if: github.repository_owner == 'aws' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-tags: true + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Check for git tag + run: python ./util/git-tag-check/git-tag-check.py diff --git a/util/git-tag-check/git-tag-check.py b/util/git-tag-check/git-tag-check.py new file mode 100644 index 0000000000..545285a385 --- /dev/null +++ b/util/git-tag-check/git-tag-check.py @@ -0,0 +1,105 @@ +import os +import subprocess +import json +import re +from typing import List + +REMOTE_NAME = "origin" +TAG_BRANCH_JSON = os.path.abspath(os.path.join(os.path.dirname(__file__), 'tag-branch.json')) +REPO_PATH = os.path.abspath(os.path.join(__file__, '..', '..')) + + +def get_git_tags(repo_path: str) -> List[str]: + try: + result = subprocess.run( + ['git', '-C', repo_path, 'tag', '--list'], + capture_output=True, + text=True, + check=True + ) + + tags = result.stdout.strip().split('\n') + + # Remove empty strings from list + return [tag for tag in tags if tag] + + except subprocess.CalledProcessError as e: + print(f"Error getting tags: {e}") + return [] + +def is_same_commit(tag: str, branch: str) -> bool: + tag_result = subprocess.run( + ['git', '-C', REPO_PATH, 'rev-parse', tag], + capture_output=True, + text=True, + check=True + ) + tag_sha = tag_result.stdout.strip() + + branch_result = subprocess.run( + ['git', '-C', REPO_PATH, 'rev-parse', branch], + capture_output=True, + text=True, + check=True + ) + branch_sha = branch_result.stdout.strip() + + #print(f"Comparing {tag_sha} and {branch_sha}") + return tag_sha == branch_sha + +def is_tag_reachable(tag: str, branch: str) -> bool: + # Sanity check - Verify the tag exists + subprocess.run( + ['git', '-C', REPO_PATH, 'rev-parse', '--verify', tag], + capture_output=True, + check=True + ) + + # Sanity check - Verify the branch exists + subprocess.run( + ['git', '-C', REPO_PATH, 'rev-parse', '--verify', branch], + capture_output=True, + check=True + ) + + result = subprocess.run( + ['git', '-C', REPO_PATH, 'merge-base', '--is-ancestor', tag, branch], + capture_output=True, + text=True + ) + + return result.returncode == 0 or is_same_commit(tag, branch) + +def main(): + with open(TAG_BRANCH_JSON, 'r') as file: + branch_tag_patterns = json.load(file) + + if len(branch_tag_patterns) == 0: + raise Exception("Empty JSON file?") + + result = subprocess.run( + ['git', '-C', REPO_PATH, 'fetch', '--tags', REMOTE_NAME], + capture_output=True, + text=True, + check=True + ) + if result.returncode != 0: + raise Exception("Error fetching tags") + + tags = get_git_tags(REPO_PATH) + if len(tags) == 0: + raise Exception("No tags found") + + for item in branch_tag_patterns: + branch = item['branch'] + tag_pattern = item['tag_pattern'] + print(f"Processing branch: '{branch}', pattern: '{tag_pattern}'") + for tag in tags: + if re.match(tag_pattern, tag): + if is_tag_reachable(tag, '/'.join([REMOTE_NAME, branch])): + print(f"Tag found: {tag} on branch: {branch}") + else: + raise Exception(f"Tag NOT found: {tag} on branch: {branch}") + +if __name__ == '__main__': + main() diff --git a/util/git-tag-check/tag-branch.json b/util/git-tag-check/tag-branch.json new file mode 100644 index 0000000000..4d1bcf0a0a --- /dev/null +++ b/util/git-tag-check/tag-branch.json @@ -0,0 +1,14 @@ +[ + { + "tag_pattern": "AWS-LC-FIPS-2\\..*", + "branch": "fips-2022-11-02" + }, + { + "tag_pattern": "AWS-LC-FIPS-3\\..*", + "branch": "fips-2024-09-27" + }, + { + "tag_pattern": "v1\\..*", + "branch": "main" + } +] From ce06b4e53e42abeacb88cb34e2306330a33deafa Mon Sep 17 00:00:00 2001 From: Justin Smith Date: Wed, 5 Feb 2025 14:23:42 -0500 Subject: [PATCH 2/5] Refactoring --- util/git-tag-check/git-tag-check.py | 157 +++++++++++++++------------- 1 file changed, 84 insertions(+), 73 deletions(-) diff --git a/util/git-tag-check/git-tag-check.py b/util/git-tag-check/git-tag-check.py index 545285a385..f4b7547224 100644 --- a/util/git-tag-check/git-tag-check.py +++ b/util/git-tag-check/git-tag-check.py @@ -2,104 +2,115 @@ import subprocess import json import re -from typing import List +from typing import List, Tuple +from pathlib import Path REMOTE_NAME = "origin" -TAG_BRANCH_JSON = os.path.abspath(os.path.join(os.path.dirname(__file__), 'tag-branch.json')) -REPO_PATH = os.path.abspath(os.path.join(__file__, '..', '..')) +TAG_BRANCH_JSON = Path(__file__).parent / 'tag-branch.json' +REPO_PATH = Path(__file__).parent.parent -def get_git_tags(repo_path: str) -> List[str]: +class GitError(Exception): + pass + + +def run_git_command(args: List[str], error_msg: str) -> subprocess.CompletedProcess: try: - result = subprocess.run( - ['git', '-C', repo_path, 'tag', '--list'], + return subprocess.run( + args, capture_output=True, text=True, check=True ) + except subprocess.CalledProcessError as e: + raise GitError(f"{error_msg}: {e.stderr.strip()}") - tags = result.stdout.strip().split('\n') - # Remove empty strings from list - return [tag for tag in tags if tag] +def get_git_tags() -> List[str]: + result = run_git_command( + ['git', '-C', str(REPO_PATH), 'tag', '--list'], + "Error getting tags" + ) + return [tag for tag in result.stdout.strip().split('\n') if tag] - except subprocess.CalledProcessError as e: - print(f"Error getting tags: {e}") - return [] - -def is_same_commit(tag: str, branch: str) -> bool: - tag_result = subprocess.run( - ['git', '-C', REPO_PATH, 'rev-parse', tag], - capture_output=True, - text=True, - check=True + +def get_commit_sha(ref: str) -> str: + result = run_git_command( + ['git', '-C', str(REPO_PATH), 'rev-parse', ref], + f"Error getting SHA for {ref}" ) - tag_sha = tag_result.stdout.strip() + return result.stdout.strip() + - branch_result = subprocess.run( - ['git', '-C', REPO_PATH, 'rev-parse', branch], - capture_output=True, - text=True, - check=True +def verify_ref_exists(ref: str) -> None: + run_git_command( + ['git', '-C', str(REPO_PATH), 'rev-parse', '--verify', ref], + f"Reference {ref} does not exist" ) - branch_sha = branch_result.stdout.strip() - #print(f"Comparing {tag_sha} and {branch_sha}") - return tag_sha == branch_sha def is_tag_reachable(tag: str, branch: str) -> bool: - # Sanity check - Verify the tag exists - subprocess.run( - ['git', '-C', REPO_PATH, 'rev-parse', '--verify', tag], - capture_output=True, - check=True - ) + # Verify both references exist + verify_ref_exists(tag) + verify_ref_exists(branch) - # Sanity check - Verify the branch exists - subprocess.run( - ['git', '-C', REPO_PATH, 'rev-parse', '--verify', branch], - capture_output=True, - check=True - ) + try: + run_git_command( + ['git', '-C', str(REPO_PATH), 'merge-base', '--is-ancestor', tag, branch], + f"Error checking if {tag} is ancestor of {branch}" + ) + return True + except GitError: + tag_sha = get_commit_sha(tag) + branch_sha = get_commit_sha(branch) + return tag_sha == branch_sha - result = subprocess.run( - ['git', '-C', REPO_PATH, 'merge-base', '--is-ancestor', tag, branch], - capture_output=True, - text=True - ) - return result.returncode == 0 or is_same_commit(tag, branch) +def load_branch_tag_patterns() -> List[dict]: + try: + with open(TAG_BRANCH_JSON) as file: + patterns = json.load(file) + if not patterns: + raise Exception("Empty JSON file") + return patterns + except json.JSONDecodeError as e: + raise Exception(f"Invalid JSON file: {e}") + except IOError as e: + raise Exception(f"Error reading JSON file: {e}") + def main(): - with open(TAG_BRANCH_JSON, 'r') as file: - branch_tag_patterns = json.load(file) + try: + branch_tag_patterns = load_branch_tag_patterns() + + run_git_command( + ['git', '-C', str(REPO_PATH), 'fetch', '--tags', REMOTE_NAME], + "Error fetching tags" + ) - if len(branch_tag_patterns) == 0: - raise Exception("Empty JSON file?") + # Get tags + tags = get_git_tags() + if not tags: + raise GitError("No tags found") + + # Process patterns + for item in branch_tag_patterns: + branch = item['branch'] + tag_pattern = item['tag_pattern'] + print(f"Processing branch: '{branch}', pattern: '{tag_pattern}'") + + for tag in tags: + if re.match(tag_pattern, tag): + full_branch = f"{REMOTE_NAME}/{branch}" + if is_tag_reachable(tag, full_branch): + print(f"Tag found: {tag} on branch: {branch}") + else: + raise GitError(f"Tag NOT found: {tag} on branch: {branch}") + + except GitError as e: + print(f"Error: {e}") + exit(1) - result = subprocess.run( - ['git', '-C', REPO_PATH, 'fetch', '--tags', REMOTE_NAME], - capture_output=True, - text=True, - check=True - ) - if result.returncode != 0: - raise Exception("Error fetching tags") - - tags = get_git_tags(REPO_PATH) - if len(tags) == 0: - raise Exception("No tags found") - - for item in branch_tag_patterns: - branch = item['branch'] - tag_pattern = item['tag_pattern'] - print(f"Processing branch: '{branch}', pattern: '{tag_pattern}'") - for tag in tags: - if re.match(tag_pattern, tag): - if is_tag_reachable(tag, '/'.join([REMOTE_NAME, branch])): - print(f"Tag found: {tag} on branch: {branch}") - else: - raise Exception(f"Tag NOT found: {tag} on branch: {branch}") if __name__ == '__main__': main() From 025f609e7e694d6c343d6d89a8ccd8829b1f8a41 Mon Sep 17 00:00:00 2001 From: Justin Smith Date: Wed, 5 Feb 2025 14:31:00 -0500 Subject: [PATCH 3/5] Remove unnecessary exception handling --- util/git-tag-check/git-tag-check.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/util/git-tag-check/git-tag-check.py b/util/git-tag-check/git-tag-check.py index f4b7547224..2efabb36a0 100644 --- a/util/git-tag-check/git-tag-check.py +++ b/util/git-tag-check/git-tag-check.py @@ -54,16 +54,17 @@ def is_tag_reachable(tag: str, branch: str) -> bool: verify_ref_exists(tag) verify_ref_exists(branch) - try: - run_git_command( + tag_sha = get_commit_sha(tag) + branch_sha = get_commit_sha(branch) + if tag_sha != branch_sha: + result = run_git_command( ['git', '-C', str(REPO_PATH), 'merge-base', '--is-ancestor', tag, branch], f"Error checking if {tag} is ancestor of {branch}" ) - return True - except GitError: - tag_sha = get_commit_sha(tag) - branch_sha = get_commit_sha(branch) - return tag_sha == branch_sha + # This return should always be true as an exception will occur if the return code != 0 + return result.returncode == 0 + return True + def load_branch_tag_patterns() -> List[dict]: From 3c71e51f2155989cac78334ad96cbe9201066fe4 Mon Sep 17 00:00:00 2001 From: Justin W Smith <103147162+justsmth@users.noreply.github.com> Date: Wed, 5 Feb 2025 14:46:33 -0500 Subject: [PATCH 4/5] More cleanup --- util/git-tag-check/git-tag-check.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/git-tag-check/git-tag-check.py b/util/git-tag-check/git-tag-check.py index 2efabb36a0..d729cdf5e5 100644 --- a/util/git-tag-check/git-tag-check.py +++ b/util/git-tag-check/git-tag-check.py @@ -57,12 +57,12 @@ def is_tag_reachable(tag: str, branch: str) -> bool: tag_sha = get_commit_sha(tag) branch_sha = get_commit_sha(branch) if tag_sha != branch_sha: - result = run_git_command( + # An exception will occur if the return code != 0 + run_git_command( ['git', '-C', str(REPO_PATH), 'merge-base', '--is-ancestor', tag, branch], f"Error checking if {tag} is ancestor of {branch}" ) - # This return should always be true as an exception will occur if the return code != 0 - return result.returncode == 0 + return True From 345269233eb975e20a16732f967b69e6cf3903b1 Mon Sep 17 00:00:00 2001 From: Justin Smith Date: Tue, 18 Feb 2025 12:34:50 -0500 Subject: [PATCH 5/5] Cleanup --- util/git-tag-check/git-tag-check.py | 2 ++ util/git-tag-check/tag-branch.json | 4 ++++ 2 files changed, 6 insertions(+) mode change 100644 => 100755 util/git-tag-check/git-tag-check.py diff --git a/util/git-tag-check/git-tag-check.py b/util/git-tag-check/git-tag-check.py old mode 100644 new mode 100755 index d729cdf5e5..3f5b142616 --- a/util/git-tag-check/git-tag-check.py +++ b/util/git-tag-check/git-tag-check.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import os import subprocess import json diff --git a/util/git-tag-check/tag-branch.json b/util/git-tag-check/tag-branch.json index 4d1bcf0a0a..b33a1ca16c 100644 --- a/util/git-tag-check/tag-branch.json +++ b/util/git-tag-check/tag-branch.json @@ -1,4 +1,8 @@ [ + { + "tag_pattern": "AWS-LC-FIPS-1\\..*", + "branch": "fips-2021-10-20" + }, { "tag_pattern": "AWS-LC-FIPS-2\\..*", "branch": "fips-2022-11-02"