From 9997d32f3ceb4edf4ab0ae7e9e45df5f553e29f0 Mon Sep 17 00:00:00 2001 From: Lucas Guillermou Date: Wed, 30 Oct 2024 17:39:19 +0100 Subject: [PATCH] Migrate GitRepositoryAdd to prefect --- backend/infrahub/git/models.py | 12 ++++ backend/infrahub/git/tasks.py | 27 ++++++++ .../infrahub/graphql/mutations/repository.py | 10 ++- .../infrahub/message_bus/messages/__init__.py | 2 - .../messages/git_repository_add.py | 17 ----- .../message_bus/operations/__init__.py | 1 - .../message_bus/operations/git/repository.py | 31 --------- backend/infrahub/workflows/catalogue.py | 10 +++ backend/tests/unit/git/test_git_rpc.py | 66 ++++++++++--------- docs/docs/reference/message-bus-events.mdx | 41 ------------ 10 files changed, 90 insertions(+), 127 deletions(-) delete mode 100644 backend/infrahub/message_bus/messages/git_repository_add.py diff --git a/backend/infrahub/git/models.py b/backend/infrahub/git/models.py index 93bc3ab95b..7504357926 100644 --- a/backend/infrahub/git/models.py +++ b/backend/infrahub/git/models.py @@ -55,3 +55,15 @@ class GitRepositoryMerge(BaseModel): source_branch: str = Field(..., description="The source branch") destination_branch: str = Field(..., description="The source branch") default_branch: str = Field(..., description="The default branch in Git") + + +class GitRepositoryAdd(BaseModel): + """Clone and sync an external repository after creation.""" + + location: str = Field(..., description="The external URL of the repository") + repository_id: str = Field(..., description="The unique ID of the Repository") + repository_name: str = Field(..., description="The name of the repository") + created_by: Optional[str] = Field(default=None, description="The user ID of the user that created the repository") + default_branch_name: Optional[str] = Field(None, description="Default branch for this repository") + infrahub_branch_name: str = Field(..., description="Infrahub branch on which to sync the remote repository") + internal_status: str = Field(..., description="Administrative status of the repository") diff --git a/backend/infrahub/git/tasks.py b/backend/infrahub/git/tasks.py index 95863e9dab..be4443c341 100644 --- a/backend/infrahub/git/tasks.py +++ b/backend/infrahub/git/tasks.py @@ -13,6 +13,7 @@ from ..workflows.catalogue import REQUEST_ARTIFACT_DEFINITION_GENERATE, REQUEST_ARTIFACT_GENERATE from ..workflows.utils import add_branch_tag from .models import ( + GitRepositoryAdd, GitRepositoryMerge, GitRepositoryPullReadOnly, RequestArtifactDefinitionGenerate, @@ -23,6 +24,32 @@ log = get_logger() +@flow(name="git_repository_add_read_write") +async def add_git_repository(model: GitRepositoryAdd) -> None: + service = services.service + async with service.git_report( + related_node=model.repository_id, + title=f"Initial import of the repository in branch: {model.infrahub_branch_name}", + created_by=model.created_by, + ) as git_report: + async with lock.registry.get(name=model.repository_name, namespace="repository"): + repo = await InfrahubRepository.new( + id=model.repository_id, + name=model.repository_name, + location=model.location, + client=service.client, + task_report=git_report, + infrahub_branch_name=model.infrahub_branch_name, + internal_status=model.internal_status, + default_branch_name=model.default_branch_name, + ) + await repo.import_objects_from_files( + infrahub_branch_name=model.infrahub_branch_name, git_branch_name=model.default_branch_name + ) + if model.internal_status == RepositoryInternalStatus.ACTIVE.value: + await repo.sync() + + @flow(name="git_repositories_create_branch") async def create_branch(branch: str, branch_id: str) -> None: """Request to the creation of git branches in available repositories.""" diff --git a/backend/infrahub/graphql/mutations/repository.py b/backend/infrahub/graphql/mutations/repository.py index bd2263c4d9..88218e3c3c 100644 --- a/backend/infrahub/graphql/mutations/repository.py +++ b/backend/infrahub/graphql/mutations/repository.py @@ -17,6 +17,8 @@ from infrahub.message_bus.messages.git_repository_connectivity import GitRepositoryConnectivityResponse from infrahub.workflows.catalogue import GIT_REPOSITORIES_PULL_READ_ONLY +from ...git.models import GitRepositoryAdd +from ...workflows.catalogue import GIT_REPOSITORY_ADD from .main import InfrahubMutationMixin, InfrahubMutationOptions if TYPE_CHECKING: @@ -101,7 +103,7 @@ async def mutate_create( ) else: obj = cast(CoreRepository, obj) - message = messages.GitRepositoryAdd( + git_repo_add_model = GitRepositoryAdd( repository_id=obj.id, repository_name=obj.name.value, location=obj.location.value, @@ -111,8 +113,10 @@ async def mutate_create( created_by=authenticated_user, ) - if context.service: - await context.service.send(message=message) + if context.service: + context.service.workflow.submit_workflow( + workflow=GIT_REPOSITORY_ADD, parameters={"model": git_repo_add_model} + ) # TODO Validate that the creation of the repository went as expected diff --git a/backend/infrahub/message_bus/messages/__init__.py b/backend/infrahub/message_bus/messages/__init__.py index 7a5954c096..b99c6d2b23 100644 --- a/backend/infrahub/message_bus/messages/__init__.py +++ b/backend/infrahub/message_bus/messages/__init__.py @@ -15,7 +15,6 @@ from .finalize_validator_execution import FinalizeValidatorExecution from .git_diff_namesonly import GitDiffNamesOnly, GitDiffNamesOnlyResponse from .git_file_get import GitFileGet, GitFileGetResponse -from .git_repository_add import GitRepositoryAdd from .git_repository_connectivity import GitRepositoryConnectivity from .git_repository_importobjects import GitRepositoryImportObjects from .git_repository_read_only_add import GitRepositoryAddReadOnly @@ -56,7 +55,6 @@ "finalize.validator.execution": FinalizeValidatorExecution, "git.diff.names_only": GitDiffNamesOnly, "git.file.get": GitFileGet, - "git.repository.add": GitRepositoryAdd, "git.repository.connectivity": GitRepositoryConnectivity, "git.repository.add_read_only": GitRepositoryAddReadOnly, "git.repository.import_objects": GitRepositoryImportObjects, diff --git a/backend/infrahub/message_bus/messages/git_repository_add.py b/backend/infrahub/message_bus/messages/git_repository_add.py deleted file mode 100644 index 231f851753..0000000000 --- a/backend/infrahub/message_bus/messages/git_repository_add.py +++ /dev/null @@ -1,17 +0,0 @@ -from typing import Optional - -from pydantic import Field - -from infrahub.message_bus import InfrahubMessage - - -class GitRepositoryAdd(InfrahubMessage): - """Clone and sync an external repository after creation.""" - - location: str = Field(..., description="The external URL of the repository") - repository_id: str = Field(..., description="The unique ID of the Repository") - repository_name: str = Field(..., description="The name of the repository") - created_by: Optional[str] = Field(default=None, description="The user ID of the user that created the repository") - default_branch_name: Optional[str] = Field(None, description="Default branch for this repository") - infrahub_branch_name: str = Field(..., description="Infrahub branch on which to sync the remote repository") - internal_status: str = Field(..., description="Administrative status of the repository") diff --git a/backend/infrahub/message_bus/operations/__init__.py b/backend/infrahub/message_bus/operations/__init__.py index 482e50a3d7..d8f50e4b7c 100644 --- a/backend/infrahub/message_bus/operations/__init__.py +++ b/backend/infrahub/message_bus/operations/__init__.py @@ -34,7 +34,6 @@ "finalize.validator.execution": finalize.validator.execution, "git.diff.names_only": git.diff.names_only, "git.file.get": git.file.get, - "git.repository.add": git.repository.add, "git.repository.add_read_only": git.repository.add_read_only, "git.repository.connectivity": git.repository.connectivity, "git.repository.import_objects": git.repository.import_objects, diff --git a/backend/infrahub/message_bus/operations/git/repository.py b/backend/infrahub/message_bus/operations/git/repository.py index de1f328a62..d89d3173f1 100644 --- a/backend/infrahub/message_bus/operations/git/repository.py +++ b/backend/infrahub/message_bus/operations/git/repository.py @@ -15,37 +15,6 @@ log = get_logger() -@flow(name="git-repository-add-read-write") -async def add(message: messages.GitRepositoryAdd, service: InfrahubServices) -> None: - log.info( - "Cloning and importing repository", - repository=message.repository_name, - location=message.location, - internal_status=message.internal_status, - ) - async with service.git_report( - related_node=message.repository_id, - title=f"Initial import of the repository in branch: {message.infrahub_branch_name}", - created_by=message.created_by, - ) as git_report: - async with lock.registry.get(name=message.repository_name, namespace="repository"): - repo = await InfrahubRepository.new( - id=message.repository_id, - name=message.repository_name, - location=message.location, - client=service.client, - task_report=git_report, - infrahub_branch_name=message.infrahub_branch_name, - internal_status=message.internal_status, - default_branch_name=message.default_branch_name, - ) - await repo.import_objects_from_files( - infrahub_branch_name=message.infrahub_branch_name, git_branch_name=message.default_branch_name - ) - if message.internal_status == RepositoryInternalStatus.ACTIVE.value: - await repo.sync() - - @flow(name="git-repository-add-read-only") async def add_read_only(message: messages.GitRepositoryAddReadOnly, service: InfrahubServices) -> None: log.info( diff --git a/backend/infrahub/workflows/catalogue.py b/backend/infrahub/workflows/catalogue.py index bb5ed92327..79afd30580 100644 --- a/backend/infrahub/workflows/catalogue.py +++ b/backend/infrahub/workflows/catalogue.py @@ -146,6 +146,15 @@ tags=[WorkflowTag.DATABASE_CHANGE], ) +GIT_REPOSITORY_ADD = WorkflowDefinition( + name="git-repository-add-read-write", + type=WorkflowType.INTERNAL, + module="infrahub.git.tasks", + function="add_git_repository", + branch_support=BranchSupportType.AWARE, + tags=[WorkflowTag.DATABASE_CHANGE], +) + BRANCH_REBASE = WorkflowDefinition( name="branch-rebase", type=WorkflowType.INTERNAL, @@ -202,6 +211,7 @@ REQUEST_DIFF_REFRESH, GIT_REPOSITORIES_PULL_READ_ONLY, GIT_REPOSITORIES_MERGE, + GIT_REPOSITORY_ADD, TRIGGER_GENERATOR_DEFINITION_RUN, BRANCH_CANCEL_PROPOSED_CHANGES, ] diff --git a/backend/tests/unit/git/test_git_rpc.py b/backend/tests/unit/git/test_git_rpc.py index 816f98f72c..59aab7da74 100644 --- a/backend/tests/unit/git/test_git_rpc.py +++ b/backend/tests/unit/git/test_git_rpc.py @@ -10,9 +10,9 @@ from infrahub.core.constants import InfrahubKind, RepositoryInternalStatus from infrahub.exceptions import RepositoryError from infrahub.git import InfrahubRepository -from infrahub.git.models import GitRepositoryMerge, GitRepositoryPullReadOnly +from infrahub.git.models import GitRepositoryAdd, GitRepositoryMerge, GitRepositoryPullReadOnly from infrahub.git.repository import InfrahubReadOnlyRepository -from infrahub.git.tasks import pull_read_only +from infrahub.git.tasks import add_git_repository, pull_read_only from infrahub.lock import InfrahubLockRegistry from infrahub.message_bus import Meta, messages from infrahub.message_bus.operations import git @@ -53,28 +53,22 @@ def setup_method(self): self.default_branch_name = "default-branch" self.client = AsyncMock(spec=InfrahubClient) self.git_report = AsyncContextManagerMock() + self.original_services = services.service + services.service = InfrahubServices(client=self.client) + services.service.git_report = self.git_report - self.services = InfrahubServices(client=self.client) - self.services.git_report = self.git_report - lock_patcher = patch("infrahub.message_bus.operations.git.repository.lock") - self.mock_infra_lock = lock_patcher.start() - self.mock_infra_lock.registry = AsyncMock(spec=InfrahubLockRegistry) - repo_class_patcher = patch( - "infrahub.message_bus.operations.git.repository.InfrahubRepository", spec=InfrahubRepository - ) - self.mock_repo_class = repo_class_patcher.start() self.mock_repo = AsyncMock(spec=InfrahubRepository) self.mock_repo.default_branch = self.default_branch_name self.mock_repo.infrahub_branch_name = self.default_branch_name self.mock_repo.internal_status = "active" - self.mock_repo_class.new.return_value = self.mock_repo def teardown_method(self): patch.stopall() + services.service = self.original_services async def test_git_rpc_create_successful(self, git_upstream_repo_01: dict[str, str]): repo_id = str(UUIDT()) - message = messages.GitRepositoryAdd( + model = GitRepositoryAdd( repository_id=repo_id, repository_name=git_upstream_repo_01["name"], location=git_upstream_repo_01["path"], @@ -83,25 +77,33 @@ async def test_git_rpc_create_successful(self, git_upstream_repo_01: dict[str, s internal_status="active", ) - await git.repository.add(message=message, service=self.services) - - self.mock_infra_lock.registry.get.assert_called_once_with( - name=git_upstream_repo_01["name"], namespace="repository" - ) - self.mock_repo_class.new.assert_awaited_once_with( - id=repo_id, - name=git_upstream_repo_01["name"], - location=git_upstream_repo_01["path"], - client=self.client, - task_report=self.git_report, - infrahub_branch_name=self.default_branch_name, - internal_status="active", - default_branch_name=self.default_branch_name, - ) - self.mock_repo.import_objects_from_files.assert_awaited_once_with( - infrahub_branch_name=self.default_branch_name, git_branch_name=self.default_branch_name - ) - self.mock_repo.sync.assert_awaited_once_with() + with ( + patch("infrahub.git.tasks.lock") as mock_infra_lock, + patch("infrahub.git.tasks.InfrahubRepository", spec=InfrahubRepository) as mock_repo_class, + ): + mock_infra_lock.registry = AsyncMock(spec=InfrahubLockRegistry) + mock_repo_class.new.return_value = self.mock_repo + + await add_git_repository(model=model) + + mock_infra_lock.registry.get.assert_called_once_with( + name=git_upstream_repo_01["name"], namespace="repository" + ) + + mock_repo_class.new.assert_awaited_once_with( + id=repo_id, + name=git_upstream_repo_01["name"], + location=git_upstream_repo_01["path"], + client=self.client, + task_report=self.git_report, + infrahub_branch_name=self.default_branch_name, + internal_status="active", + default_branch_name=self.default_branch_name, + ) + self.mock_repo.import_objects_from_files.assert_awaited_once_with( + infrahub_branch_name=self.default_branch_name, git_branch_name=self.default_branch_name + ) + self.mock_repo.sync.assert_awaited_once_with() async def test_git_rpc_merge( diff --git a/docs/docs/reference/message-bus-events.mdx b/docs/docs/reference/message-bus-events.mdx index d3629f0b22..da12d36e11 100644 --- a/docs/docs/reference/message-bus-events.mdx +++ b/docs/docs/reference/message-bus-events.mdx @@ -356,26 +356,6 @@ For more detailed explanations on how to use these events within Infrahub, see t ### Git Repository - -#### Event git.repository.add - - -**Description**: Clone and sync an external repository after creation. - -**Priority**: 3 - - -| Key | Description | Type | Default Value | -|-----|-------------|------|---------------| -| **meta** | Meta properties for the message | N/A | None | -| **location** | The external URL of the repository | string | None | -| **repository_id** | The unique ID of the Repository | string | None | -| **repository_name** | The name of the repository | string | None | -| **created_by** | The user ID of the user that created the repository | N/A | None | -| **default_branch_name** | Default branch for this repository | N/A | None | -| **infrahub_branch_name** | Infrahub branch on which to sync the remote repository | string | None | -| **internal_status** | Administrative status of the repository | string | None | - #### Event git.repository.connectivity @@ -1188,27 +1168,6 @@ For more detailed explanations on how to use these events within Infrahub, see t ### Git Repository - -#### Event git.repository.add - - -**Description**: Clone and sync an external repository after creation. - -**Priority**: 3 - - - -| Key | Description | Type | Default Value | -|-----|-------------|------|---------------| -| **meta** | Meta properties for the message | N/A | None | -| **location** | The external URL of the repository | string | None | -| **repository_id** | The unique ID of the Repository | string | None | -| **repository_name** | The name of the repository | string | None | -| **created_by** | The user ID of the user that created the repository | N/A | None | -| **default_branch_name** | Default branch for this repository | N/A | None | -| **infrahub_branch_name** | Infrahub branch on which to sync the remote repository | string | None | -| **internal_status** | Administrative status of the repository | string | None | - #### Event git.repository.connectivity