diff --git a/backend/infrahub/core/diff/calculator.py b/backend/infrahub/core/diff/calculator.py index 947dc67a24..7a91f31f84 100644 --- a/backend/infrahub/core/diff/calculator.py +++ b/backend/infrahub/core/diff/calculator.py @@ -5,7 +5,7 @@ from infrahub.core.timestamp import Timestamp from infrahub.database import InfrahubDatabase -from .model.path import CalculatedDiffs +from .model.path import CalculatedDiffs, NodeFieldSpecifier class DiffCalculator: @@ -13,24 +13,56 @@ def __init__(self, db: InfrahubDatabase) -> None: self.db = db async def calculate_diff( - self, base_branch: Branch, diff_branch: Branch, from_time: Timestamp, to_time: Timestamp + self, + base_branch: Branch, + diff_branch: Branch, + from_time: Timestamp, + to_time: Timestamp, + previous_node_specifiers: set[NodeFieldSpecifier] | None = None, ) -> CalculatedDiffs: - diff_query = await DiffAllPathsQuery.init( + if diff_branch.name == registry.default_branch: + diff_branch_create_time = from_time + else: + diff_branch_create_time = Timestamp(diff_branch.get_created_at()) + diff_parser = DiffQueryParser( + base_branch=base_branch, + diff_branch=diff_branch, + schema_manager=registry.schema, + from_time=from_time, + to_time=to_time, + ) + branch_diff_query = await DiffAllPathsQuery.init( db=self.db, branch=diff_branch, base_branch=base_branch, + diff_branch_create_time=diff_branch_create_time, diff_from=from_time, diff_to=to_time, ) - await diff_query.execute(db=self.db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=base_branch.name, - diff_branch_name=diff_branch.name, - schema_manager=registry.schema, - from_time=from_time, - to_time=to_time, - ) + await branch_diff_query.execute(db=self.db) + for query_result in branch_diff_query.get_results(): + diff_parser.read_result(query_result=query_result) + + if base_branch.name != diff_branch.name: + branch_node_specifiers = diff_parser.get_node_field_specifiers_for_branch(branch_name=diff_branch.name) + new_node_field_specifiers = branch_node_specifiers - (previous_node_specifiers or set()) + current_node_field_specifiers = (previous_node_specifiers or set()) - new_node_field_specifiers + base_diff_query = await DiffAllPathsQuery.init( + db=self.db, + branch=base_branch, + base_branch=base_branch, + diff_branch_create_time=diff_branch_create_time, + diff_from=from_time, + diff_to=to_time, + current_node_field_specifiers=[ + (nfs.node_uuid, nfs.field_name) for nfs in current_node_field_specifiers + ], + new_node_field_specifiers=[(nfs.node_uuid, nfs.field_name) for nfs in new_node_field_specifiers], + ) + await base_diff_query.execute(db=self.db) + for query_result in base_diff_query.get_results(): + diff_parser.read_result(query_result=query_result) + diff_parser.parse() return CalculatedDiffs( base_branch_name=base_branch.name, diff --git a/backend/infrahub/core/diff/combiner.py b/backend/infrahub/core/diff/combiner.py index 3a959c307a..67ecc7bb14 100644 --- a/backend/infrahub/core/diff/combiner.py +++ b/backend/infrahub/core/diff/combiner.py @@ -13,6 +13,7 @@ EnrichedDiffProperty, EnrichedDiffRelationship, EnrichedDiffRoot, + EnrichedDiffs, EnrichedDiffSingleRelationship, ) @@ -356,16 +357,33 @@ def _link_child_nodes(self, nodes: Iterable[EnrichedDiffNode]) -> None: parent_rel = child_node.get_relationship(name=parent_rel_name) parent_rel.nodes.add(parent_node) - async def combine(self, earlier_diff: EnrichedDiffRoot, later_diff: EnrichedDiffRoot) -> EnrichedDiffRoot: - self._initialize(earlier_diff=earlier_diff, later_diff=later_diff) - filtered_node_pairs = self._filter_nodes_to_keep(earlier_diff=earlier_diff, later_diff=later_diff) - combined_nodes = self._combine_nodes(node_pairs=filtered_node_pairs) - self._link_child_nodes(nodes=combined_nodes) - return EnrichedDiffRoot( - uuid=str(uuid4()), - base_branch_name=later_diff.base_branch_name, - diff_branch_name=later_diff.diff_branch_name, - from_time=earlier_diff.from_time, - to_time=later_diff.to_time, - nodes=combined_nodes, + async def combine(self, earlier_diffs: EnrichedDiffs, later_diffs: EnrichedDiffs) -> EnrichedDiffs: + combined_diffs: list[EnrichedDiffRoot] = [] + for earlier, later in ( + (earlier_diffs.base_branch_diff, later_diffs.base_branch_diff), + (earlier_diffs.diff_branch_diff, later_diffs.diff_branch_diff), + ): + self._initialize(earlier_diff=earlier, later_diff=later) + filtered_node_pairs = self._filter_nodes_to_keep(earlier_diff=earlier, later_diff=later) + combined_nodes = self._combine_nodes(node_pairs=filtered_node_pairs) + self._link_child_nodes(nodes=combined_nodes) + combined_diffs.append( + EnrichedDiffRoot( + uuid=str(uuid4()), + partner_uuid=later.partner_uuid, + base_branch_name=later.base_branch_name, + diff_branch_name=later.diff_branch_name, + from_time=earlier.from_time, + to_time=later.to_time, + nodes=combined_nodes, + ) + ) + base_branch_diff, diff_branch_diff = combined_diffs # pylint: disable=unbalanced-tuple-unpacking + base_branch_diff.partner_uuid = diff_branch_diff.uuid + diff_branch_diff.partner_uuid = base_branch_diff.uuid + return EnrichedDiffs( + base_branch_name=later_diffs.base_branch_name, + diff_branch_name=later_diffs.diff_branch_name, + base_branch_diff=base_branch_diff, + diff_branch_diff=diff_branch_diff, ) diff --git a/backend/infrahub/core/diff/conflicts_enricher.py b/backend/infrahub/core/diff/conflicts_enricher.py index 1973b1956e..1502e79433 100644 --- a/backend/infrahub/core/diff/conflicts_enricher.py +++ b/backend/infrahub/core/diff/conflicts_enricher.py @@ -2,7 +2,6 @@ from infrahub.core.constants import DiffAction, RelationshipCardinality from infrahub.core.constants.database import DatabaseEdgeType -from infrahub.database import InfrahubDatabase from .model.path import ( EnrichedDiffAttribute, @@ -16,10 +15,9 @@ class ConflictsEnricher: - def __init__(self, db: InfrahubDatabase) -> None: + def __init__(self) -> None: self._base_branch_name: str | None = None self._diff_branch_name: str | None = None - self.schema_manager = db.schema @property def base_branch_name(self) -> str: @@ -66,7 +64,6 @@ def _add_node_conflicts(self, base_node: EnrichedDiffNode, branch_node: Enriched base_relationship = base_relationship_map[relationship_name] branch_relationship = branch_relationship_map[relationship_name] self._add_relationship_conflicts( - branch_node=branch_node, base_relationship=base_relationship, branch_relationship=branch_relationship, ) @@ -100,7 +97,11 @@ def _add_attribute_conflicts( for property_type in common_property_types: base_property = base_property_map[property_type] branch_property = branch_property_map[property_type] - if base_property.new_value != branch_property.new_value: + same_value = base_property.new_value == branch_property.new_value or ( + base_property.action is DiffAction.UNCHANGED + and base_property.previous_value == branch_property.previous_value + ) + if not same_value: self._add_property_conflict( base_property=base_property, branch_property=branch_property, @@ -110,15 +111,10 @@ def _add_attribute_conflicts( def _add_relationship_conflicts( self, - branch_node: EnrichedDiffNode, base_relationship: EnrichedDiffRelationship, branch_relationship: EnrichedDiffRelationship, ) -> None: - node_schema = self.schema_manager.get_node_schema( - name=branch_node.kind, branch=self.diff_branch_name, duplicate=False - ) - relationship_schema = node_schema.get_relationship(name=branch_relationship.name) - is_cardinality_one = relationship_schema.cardinality is RelationshipCardinality.ONE + is_cardinality_one = branch_relationship.cardinality is RelationshipCardinality.ONE if is_cardinality_one: if not base_relationship.relationships or not branch_relationship.relationships: return diff --git a/backend/infrahub/core/diff/coordinator.py b/backend/infrahub/core/diff/coordinator.py index 80d8d775e5..59166f0e3e 100644 --- a/backend/infrahub/core/diff/coordinator.py +++ b/backend/infrahub/core/diff/coordinator.py @@ -1,13 +1,22 @@ from __future__ import annotations -from dataclasses import dataclass, replace +from dataclasses import dataclass, field from typing import TYPE_CHECKING from infrahub import lock +from infrahub.core import registry from infrahub.core.timestamp import Timestamp from infrahub.log import get_logger -from .model.path import BranchTrackingId, EnrichedDiffRoot, NameTrackingId, TimeRange, TrackingId +from .model.path import ( + BranchTrackingId, + EnrichedDiffRoot, + EnrichedDiffs, + NameTrackingId, + NodeFieldSpecifier, + TimeRange, + TrackingId, +) if TYPE_CHECKING: from infrahub.core.branch import Branch @@ -33,10 +42,7 @@ class EnrichedDiffRequest: diff_branch: Branch from_time: Timestamp to_time: Timestamp - - def __hash__(self) -> int: - hash_keys = [self.base_branch.name, self.diff_branch.name, self.from_time.to_string(), self.to_time.to_string()] - return hash("-".join(hash_keys)) + node_field_specifiers: set[NodeFieldSpecifier] = field(default_factory=set) class DiffCoordinator: @@ -64,7 +70,6 @@ def __init__( self.data_check_synchronizer = data_check_synchronizer self.conflict_transferer = conflict_transferer self.lock_registry = lock.registry - self._enriched_diff_cache: dict[EnrichedDiffRequest, EnrichedDiffRoot] = {} async def run_update( self, @@ -126,20 +131,19 @@ async def update_branch_diff(self, base_branch: Branch, diff_branch: Branch) -> self.lock_registry.get(name=incremental_lock_name, namespace=self.lock_namespace), ): log.debug(f"Acquired lock to run branch diff update for {base_branch.name} - {diff_branch.name}") - base_enriched_diff, branch_enriched_diff = await self._update_diffs( + enriched_diffs = await self._update_diffs( base_branch=base_branch, diff_branch=diff_branch, from_time=from_time, to_time=to_time, tracking_id=tracking_id, ) - await self.summary_counts_enricher.enrich(enriched_diff_root=base_enriched_diff) - await self.diff_repo.save(enriched_diff=base_enriched_diff) - await self.summary_counts_enricher.enrich(enriched_diff_root=branch_enriched_diff) - await self.diff_repo.save(enriched_diff=branch_enriched_diff) - await self._update_core_data_checks(enriched_diff=branch_enriched_diff) + await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.base_branch_diff) + await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.diff_branch_diff) + await self.diff_repo.save(enriched_diffs=enriched_diffs) + await self._update_core_data_checks(enriched_diff=enriched_diffs.diff_branch_diff) log.debug(f"Branch diff update complete for {base_branch.name} - {diff_branch.name}") - return branch_enriched_diff + return enriched_diffs.diff_branch_diff async def create_or_update_arbitrary_timeframe_diff( self, @@ -157,20 +161,19 @@ async def create_or_update_arbitrary_timeframe_diff( ) async with self.lock_registry.get(name=general_lock_name, namespace=self.lock_namespace): log.debug(f"Acquired lock to run arbitrary diff update for {base_branch.name} - {diff_branch.name}") - base_enriched_diff, branch_enriched_diff = await self._update_diffs( + enriched_diffs = await self._update_diffs( base_branch=base_branch, diff_branch=diff_branch, from_time=from_time, to_time=to_time, tracking_id=tracking_id, ) - await self.summary_counts_enricher.enrich(enriched_diff_root=base_enriched_diff) - await self.diff_repo.save(enriched_diff=base_enriched_diff) - await self.summary_counts_enricher.enrich(enriched_diff_root=branch_enriched_diff) - await self.diff_repo.save(enriched_diff=branch_enriched_diff) - await self._update_core_data_checks(enriched_diff=branch_enriched_diff) + await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.base_branch_diff) + await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.diff_branch_diff) + await self.diff_repo.save(enriched_diffs=enriched_diffs) + await self._update_core_data_checks(enriched_diff=enriched_diffs.diff_branch_diff) log.debug(f"Arbitrary diff update complete for {base_branch.name} - {diff_branch.name}") - return branch_enriched_diff + return enriched_diffs.diff_branch_diff async def recalculate( self, @@ -189,7 +192,7 @@ async def recalculate( else: to_time = current_branch_diff.to_time await self.diff_repo.delete_diff_roots(diff_root_uuids=[current_branch_diff.uuid]) - base_branch_diff, fresh_branch_diff = await self._update_diffs( + enriched_diffs = await self._update_diffs( base_branch=base_branch, diff_branch=diff_branch, from_time=current_branch_diff.from_time, @@ -199,15 +202,16 @@ async def recalculate( ) if current_branch_diff: - await self.conflict_transferer.transfer(earlier=current_branch_diff, later=fresh_branch_diff) + await self.conflict_transferer.transfer( + earlier=current_branch_diff, later=enriched_diffs.diff_branch_diff + ) - await self.summary_counts_enricher.enrich(enriched_diff_root=base_branch_diff) - await self.diff_repo.save(enriched_diff=base_branch_diff) - await self.summary_counts_enricher.enrich(enriched_diff_root=fresh_branch_diff) - await self.diff_repo.save(enriched_diff=fresh_branch_diff) - await self._update_core_data_checks(enriched_diff=fresh_branch_diff) + await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.base_branch_diff) + await self.summary_counts_enricher.enrich(enriched_diff_root=enriched_diffs.diff_branch_diff) + await self.diff_repo.save(enriched_diffs=enriched_diffs) + await self._update_core_data_checks(enriched_diff=enriched_diffs.diff_branch_diff) log.debug(f"Diff recalculation complete for {base_branch.name} - {diff_branch.name}") - return fresh_branch_diff + return enriched_diffs.diff_branch_diff async def _update_diffs( self, @@ -217,87 +221,100 @@ async def _update_diffs( to_time: Timestamp, tracking_id: TrackingId | None = None, force_branch_refresh: bool = False, - ) -> tuple[EnrichedDiffRoot, EnrichedDiffRoot]: - requested_diff_branches = {base_branch.name: base_branch, diff_branch.name: diff_branch} - aggregated_diffs_by_branch_name: dict[str, EnrichedDiffRoot] = {} + ) -> EnrichedDiffs: diff_uuids_to_delete = [] - for branch in requested_diff_branches.values(): - retrieved_enriched_diffs = await self.diff_repo.get( - base_branch_name=base_branch.name, - diff_branch_names=[branch.name], + retrieved_enriched_diffs = await self.diff_repo.get_pairs( + base_branch_name=base_branch.name, + diff_branch_name=diff_branch.name, + from_time=from_time, + to_time=to_time, + ) + for enriched_diffs in retrieved_enriched_diffs: + if tracking_id: + if enriched_diffs.base_branch_diff.tracking_id: + diff_uuids_to_delete.append(enriched_diffs.base_branch_diff.uuid) + if enriched_diffs.diff_branch_diff.tracking_id: + diff_uuids_to_delete.append(enriched_diffs.diff_branch_diff.uuid) + aggregated_enriched_diffs = await self._get_aggregated_enriched_diffs( + diff_request=EnrichedDiffRequest( + base_branch=base_branch, + diff_branch=diff_branch, from_time=from_time, to_time=to_time, - tracking_id=tracking_id, - include_empty=True, - ) - if tracking_id: - diff_uuids_to_delete += [ - diff.uuid for diff in retrieved_enriched_diffs if diff.tracking_id == tracking_id - ] - if branch is diff_branch and force_branch_refresh: - covered_time_ranges = [] + ), + partial_enriched_diffs=retrieved_enriched_diffs if not force_branch_refresh else [], + ) + + await self.conflicts_enricher.add_conflicts_to_branch_diff( + base_diff_root=aggregated_enriched_diffs.base_branch_diff, + branch_diff_root=aggregated_enriched_diffs.diff_branch_diff, + ) + await self.labels_enricher.enrich( + enriched_diff_root=aggregated_enriched_diffs.diff_branch_diff, conflicts_only=True + ) + + if tracking_id: + aggregated_enriched_diffs.base_branch_diff.tracking_id = tracking_id + aggregated_enriched_diffs.diff_branch_diff.tracking_id = tracking_id + if diff_uuids_to_delete: + await self.diff_repo.delete_diff_roots(diff_root_uuids=diff_uuids_to_delete) + return aggregated_enriched_diffs + + async def _get_aggregated_enriched_diffs( + self, diff_request: EnrichedDiffRequest, partial_enriched_diffs: list[EnrichedDiffs] + ) -> EnrichedDiffs: + if not partial_enriched_diffs: + return await self._get_enriched_diff(diff_request=diff_request) + + remaining_diffs = sorted(partial_enriched_diffs, key=lambda d: d.diff_branch_diff.from_time) + current_time = diff_request.from_time + previous_diffs: EnrichedDiffs | None = None + while current_time < diff_request.to_time: + if remaining_diffs and remaining_diffs[0].diff_branch_diff.from_time == current_time: + current_diffs = remaining_diffs.pop(0) else: - covered_time_ranges = [ - TimeRange(from_time=enriched_diff.from_time, to_time=enriched_diff.to_time) - for enriched_diff in retrieved_enriched_diffs - ] - missing_time_ranges = self._get_missing_time_ranges( - time_ranges=covered_time_ranges, from_time=from_time, to_time=to_time - ) - all_enriched_diffs = list(retrieved_enriched_diffs) - for missing_time_range in missing_time_ranges: + if remaining_diffs: + end_time = remaining_diffs[0].diff_branch_diff.from_time + else: + end_time = diff_request.to_time + if previous_diffs is None: + node_field_specifiers = set() + else: + node_field_specifiers = self._get_node_field_specifiers( + enriched_diff=previous_diffs.diff_branch_diff + ) diff_request = EnrichedDiffRequest( - base_branch=base_branch, - diff_branch=branch, - from_time=missing_time_range.from_time, - to_time=missing_time_range.to_time, + base_branch=diff_request.base_branch, + diff_branch=diff_request.diff_branch, + from_time=current_time, + to_time=end_time, + node_field_specifiers=node_field_specifiers, ) - enriched_diff = await self._get_enriched_diff(diff_request=diff_request) - all_enriched_diffs.append(enriched_diff) - all_enriched_diffs.sort(key=lambda e_diff: e_diff.from_time) - combined_diff = all_enriched_diffs[0] - for next_diff in all_enriched_diffs[1:]: - combined_diff = await self.diff_combiner.combine(earlier_diff=combined_diff, later_diff=next_diff) - aggregated_diffs_by_branch_name[branch.name] = combined_diff - - if len(aggregated_diffs_by_branch_name) > 1: - await self.conflicts_enricher.add_conflicts_to_branch_diff( - base_diff_root=aggregated_diffs_by_branch_name[base_branch.name], - branch_diff_root=aggregated_diffs_by_branch_name[diff_branch.name], - ) - await self.labels_enricher.enrich( - enriched_diff_root=aggregated_diffs_by_branch_name[diff_branch.name], conflicts_only=True - ) + current_diffs = await self._get_enriched_diff(diff_request=diff_request) - if tracking_id: - for enriched_diff in aggregated_diffs_by_branch_name.values(): - enriched_diff.tracking_id = tracking_id - if diff_uuids_to_delete: - await self.diff_repo.delete_diff_roots(diff_root_uuids=diff_uuids_to_delete) - - return ( - aggregated_diffs_by_branch_name[base_branch.name], - aggregated_diffs_by_branch_name[diff_branch.name], - ) + if previous_diffs: + current_diffs = await self.diff_combiner.combine( + earlier_diffs=previous_diffs, later_diffs=current_diffs + ) + + previous_diffs = current_diffs + current_time = current_diffs.diff_branch_diff.to_time + + return current_diffs async def _update_core_data_checks(self, enriched_diff: EnrichedDiffRoot) -> list[Node]: return await self.data_check_synchronizer.synchronize(enriched_diff=enriched_diff) - async def _get_enriched_diff(self, diff_request: EnrichedDiffRequest) -> EnrichedDiffRoot: - if diff_request in self._enriched_diff_cache: - return self._enriched_diff_cache[diff_request] + async def _get_enriched_diff(self, diff_request: EnrichedDiffRequest) -> EnrichedDiffs: calculated_diff_pair = await self.diff_calculator.calculate_diff( base_branch=diff_request.base_branch, diff_branch=diff_request.diff_branch, from_time=diff_request.from_time, to_time=diff_request.to_time, + previous_node_specifiers=diff_request.node_field_specifiers, ) enriched_diff_pair = await self.diff_enricher.enrich(calculated_diffs=calculated_diff_pair) - self._enriched_diff_cache[diff_request] = enriched_diff_pair.diff_branch_diff - if diff_request.base_branch.name != diff_request.diff_branch.name: - base_diff_request = replace(diff_request, diff_branch=diff_request.base_branch) - self._enriched_diff_cache[base_diff_request] = enriched_diff_pair.base_branch_diff - return enriched_diff_pair.diff_branch_diff + return enriched_diff_pair def _get_missing_time_ranges( self, time_ranges: list[TimeRange], from_time: Timestamp, to_time: Timestamp @@ -318,3 +335,17 @@ def _get_missing_time_ranges( if sorted_time_ranges[-1].to_time < to_time: missing_time_ranges.append(TimeRange(from_time=sorted_time_ranges[-1].to_time, to_time=to_time)) return missing_time_ranges + + def _get_node_field_specifiers(self, enriched_diff: EnrichedDiffRoot) -> set[NodeFieldSpecifier]: + specifiers = set() + schema_branch = registry.schema.get_schema_branch(name=enriched_diff.diff_branch_name) + for node in enriched_diff.nodes: + for attribute in node.attributes: + specifiers.add(NodeFieldSpecifier(node_uuid=node.uuid, field_name=attribute.name)) + if not node.relationships: + continue + node_schema = schema_branch.get_node(name=node.kind, duplicate=False) + for relationship in node.relationships: + relationship_schema = node_schema.get_relationship(name=relationship.name) + specifiers.add(NodeFieldSpecifier(node_uuid=node.uuid, field_name=relationship_schema.get_identifier())) + return specifiers diff --git a/backend/infrahub/core/diff/model/path.py b/backend/infrahub/core/diff/model/path.py index 98d746363e..d3a1bd580f 100644 --- a/backend/infrahub/core/diff/model/path.py +++ b/backend/infrahub/core/diff/model/path.py @@ -3,7 +3,6 @@ from dataclasses import dataclass, field, replace from enum import Enum from typing import TYPE_CHECKING, Any, Optional -from uuid import uuid4 from infrahub.core.constants import DiffAction, RelationshipCardinality, RelationshipDirection, RelationshipStatus from infrahub.core.constants.database import DatabaseEdgeType @@ -75,6 +74,15 @@ def deserialize_tracking_id(tracking_id_str: str) -> TrackingId: raise ValueError(f"{tracking_id_str} is not a valid TrackingId") +@dataclass +class NodeFieldSpecifier: + node_uuid: str + field_name: str + + def __hash__(self) -> int: + return hash(f"{self.node_uuid}:{self.field_name}") + + @dataclass class BaseSummary: num_added: int = field(default=0, kw_only=True) @@ -366,6 +374,7 @@ class EnrichedDiffRoot(BaseSummary): from_time: Timestamp to_time: Timestamp uuid: str + partner_uuid: str tracking_id: TrackingId | None = field(default=None, kw_only=True) nodes: set[EnrichedDiffNode] = field(default_factory=set) @@ -399,13 +408,16 @@ def get_all_conflicts(self) -> list[EnrichedDiffConflict]: return all_conflicts @classmethod - def from_calculated_diff(cls, calculated_diff: DiffRoot, base_branch_name: str) -> EnrichedDiffRoot: + def from_calculated_diff( + cls, calculated_diff: DiffRoot, base_branch_name: str, partner_uuid: str + ) -> EnrichedDiffRoot: return EnrichedDiffRoot( base_branch_name=base_branch_name, diff_branch_name=calculated_diff.branch, from_time=calculated_diff.from_time, to_time=calculated_diff.to_time, - uuid=str(uuid4()), + uuid=calculated_diff.uuid, + partner_uuid=partner_uuid, nodes={EnrichedDiffNode.from_calculated_node(calculated_node=n) for n in calculated_diff.nodes}, ) @@ -461,10 +473,14 @@ class EnrichedDiffs: @classmethod def from_calculated_diffs(cls, calculated_diffs: CalculatedDiffs) -> EnrichedDiffs: base_branch_diff = EnrichedDiffRoot.from_calculated_diff( - calculated_diff=calculated_diffs.base_branch_diff, base_branch_name=calculated_diffs.base_branch_name + calculated_diff=calculated_diffs.base_branch_diff, + base_branch_name=calculated_diffs.base_branch_name, + partner_uuid=calculated_diffs.diff_branch_diff.uuid, ) diff_branch_diff = EnrichedDiffRoot.from_calculated_diff( - calculated_diff=calculated_diffs.diff_branch_diff, base_branch_name=calculated_diffs.base_branch_name + calculated_diff=calculated_diffs.diff_branch_diff, + base_branch_name=calculated_diffs.base_branch_name, + partner_uuid=calculated_diffs.base_branch_diff.uuid, ) return EnrichedDiffs( base_branch_name=calculated_diffs.base_branch_name, diff --git a/backend/infrahub/core/diff/query/diff_get.py b/backend/infrahub/core/diff/query/diff_get.py index b9e1411f26..61465aa288 100644 --- a/backend/infrahub/core/diff/query/diff_get.py +++ b/backend/infrahub/core/diff/query/diff_get.py @@ -1,5 +1,6 @@ from typing import Any +from infrahub import config from infrahub.core.query import Query, QueryType from infrahub.core.timestamp import Timestamp from infrahub.database import InfrahubDatabase @@ -15,20 +16,9 @@ AND diff_root.from_time >= $from_time AND diff_root.to_time <= $to_time AND ($tracking_id IS NULL OR diff_root.tracking_id = $tracking_id) - AND ($diff_id IS NULL OR diff_root.uuid = $diff_id) + AND ($diff_ids IS NULL OR diff_root.uuid IN $diff_ids) WITH diff_root ORDER BY diff_root.base_branch, diff_root.diff_branch, diff_root.from_time, diff_root.to_time - WITH diff_root.base_branch AS bb, diff_root.diff_branch AS db, collect(diff_root) AS same_branch_diff_roots - WITH reduce( - non_overlapping = [], dr in same_branch_diff_roots | - CASE - WHEN size(non_overlapping) = 0 THEN [dr] - WHEN dr.from_time >= (non_overlapping[-1]).from_time AND dr.to_time <= (non_overlapping[-1]).to_time THEN non_overlapping - WHEN (non_overlapping[-1]).from_time >= dr.from_time AND (non_overlapping[-1]).to_time <= dr.to_time THEN non_overlapping[..-1] + [dr] - ELSE non_overlapping + [dr] - END - ) AS non_overlapping_diff_roots - UNWIND non_overlapping_diff_roots AS diff_root // get all the nodes attached to the diffs OPTIONAL MATCH (diff_root)-[:DIFF_HAS_NODE]->(diff_node:DiffNode) """ @@ -45,12 +35,12 @@ def __init__( self, base_branch_name: str, diff_branch_names: list[str], - filters: EnrichedDiffQueryFilters, max_depth: int, + filters: EnrichedDiffQueryFilters | None = None, from_time: Timestamp | None = None, to_time: Timestamp | None = None, tracking_id: TrackingId | None = None, - diff_id: str | None = None, + diff_ids: list[str] | None = None, **kwargs: Any, ) -> None: super().__init__(**kwargs) @@ -60,7 +50,7 @@ def __init__( self.to_time: Timestamp = to_time or Timestamp() self.max_depth = max_depth self.tracking_id = tracking_id - self.diff_id = diff_id + self.diff_ids = diff_ids self.filters = filters or EnrichedDiffQueryFilters() async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None: @@ -70,8 +60,8 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None: "from_time": self.from_time.to_string(), "to_time": self.to_time.to_string(), "tracking_id": self.tracking_id.serialize() if self.tracking_id else None, - "diff_id": self.diff_id, - "limit": self.limit, + "diff_ids": self.diff_ids, + "limit": self.limit or config.SETTINGS.database.query_size_limit, "offset": self.offset, } # ruff: noqa: E501 diff --git a/backend/infrahub/core/diff/query/diff_summary.py b/backend/infrahub/core/diff/query/diff_summary.py index ced27f6f14..06b3924cb2 100644 --- a/backend/infrahub/core/diff/query/diff_summary.py +++ b/backend/infrahub/core/diff/query/diff_summary.py @@ -68,7 +68,7 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None: "from_time": self.from_time.to_string(), "to_time": self.to_time.to_string(), "tracking_id": self.tracking_id.serialize() if self.tracking_id else None, - "diff_id": None, + "diff_ids": None, } # ruff: noqa: E501 diff --git a/backend/infrahub/core/diff/query/save_query.py b/backend/infrahub/core/diff/query/save_query.py index cd35611fed..1b6b9439fb 100644 --- a/backend/infrahub/core/diff/query/save_query.py +++ b/backend/infrahub/core/diff/query/save_query.py @@ -9,7 +9,7 @@ EnrichedDiffNode, EnrichedDiffProperty, EnrichedDiffRelationship, - EnrichedDiffRoot, + EnrichedDiffs, EnrichedDiffSingleRelationship, ) @@ -17,99 +17,124 @@ class EnrichedDiffSaveQuery(Query): name = "enriched_diff_save" type = QueryType.WRITE + insert_return = False - def __init__(self, enriched_diff_root: EnrichedDiffRoot, **kwargs: Any) -> None: + def __init__(self, enriched_diffs: EnrichedDiffs, **kwargs: Any) -> None: super().__init__(**kwargs) - self.enriched_diff_root = enriched_diff_root + self.enriched_diffs = enriched_diffs async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None: - self.params = self._build_diff_root_params(enriched_diff=self.enriched_diff_root) + self.params = self._build_diff_root_params(enriched_diffs=self.enriched_diffs) # ruff: noqa: E501 query = """ - MERGE (diff_root:DiffRoot { - base_branch: $diff_root_props.base_branch, - diff_branch: $diff_root_props.diff_branch, - from_time: $diff_root_props.from_time, - to_time: $diff_root_props.to_time, - uuid: $diff_root_props.uuid, - num_added: $diff_root_props.num_added, - num_updated: $diff_root_props.num_updated, - num_removed: $diff_root_props.num_removed, - num_conflicts: $diff_root_props.num_conflicts, - contains_conflict: $diff_root_props.contains_conflict - }) - SET diff_root.tracking_id = $diff_root_props.tracking_id - WITH diff_root - UNWIND $node_maps AS node_map - CREATE (diff_root)-[:DIFF_HAS_NODE]->(diff_node:DiffNode) - SET diff_node = node_map.node_properties - // node conflict - FOREACH (i in CASE WHEN node_map.conflict_params IS NOT NULL THEN [1] ELSE [] END | - CREATE (diff_node)-[:DIFF_HAS_CONFLICT]->(diff_node_conflict:DiffConflict) - SET diff_node_conflict = node_map.conflict_params - ) - - // attributes - WITH diff_root, diff_node, node_map + UNWIND [$base_branch_diff, $diff_branch_diff] AS diff_root_map + WITH diff_root_map CALL { - WITH diff_node, node_map - UNWIND node_map.attributes AS node_attribute - CREATE (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(diff_attribute:DiffAttribute) - SET diff_attribute = node_attribute.node_properties + WITH diff_root_map + MERGE (diff_root:DiffRoot { + base_branch: diff_root_map.diff_root_props.base_branch, + diff_branch: diff_root_map.diff_root_props.diff_branch, + from_time: diff_root_map.diff_root_props.from_time, + to_time: diff_root_map.diff_root_props.to_time, + uuid: diff_root_map.diff_root_props.uuid, + num_added: diff_root_map.diff_root_props.num_added, + num_updated: diff_root_map.diff_root_props.num_updated, + num_removed: diff_root_map.diff_root_props.num_removed, + num_conflicts: diff_root_map.diff_root_props.num_conflicts, + contains_conflict: diff_root_map.diff_root_props.contains_conflict + }) - // node attribute properties - WITH diff_attribute, node_attribute - UNWIND node_attribute.properties AS attr_property - CREATE (diff_attribute)-[:DIFF_HAS_PROPERTY]->(diff_attr_prop:DiffProperty) - SET diff_attr_prop = attr_property.node_properties - // attribute property conflict - FOREACH (i in CASE WHEN attr_property.conflict_params IS NOT NULL THEN [1] ELSE [] END | - CREATE (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(diff_attribute_property_conflict:DiffConflict) - SET diff_attribute_property_conflict = attr_property.conflict_params - ) - } - // relationships - WITH diff_root, diff_node, node_map - CALL { - WITH diff_node, node_map - UNWIND node_map.relationships as node_relationship - CREATE (diff_node)-[:DIFF_HAS_RELATIONSHIP]->(diff_relationship:DiffRelationship) - SET diff_relationship = node_relationship.node_properties + SET diff_root.tracking_id = diff_root_map.diff_root_props.tracking_id + WITH diff_root, diff_root_map + CALL { + WITH diff_root, diff_root_map + UNWIND diff_root_map.node_maps AS node_map + CREATE (diff_root)-[:DIFF_HAS_NODE]->(diff_node:DiffNode) + SET diff_node = node_map.node_properties + // node conflict + FOREACH (i in CASE WHEN node_map.conflict_params IS NOT NULL THEN [1] ELSE [] END | + CREATE (diff_node)-[:DIFF_HAS_CONFLICT]->(diff_node_conflict:DiffConflict) + SET diff_node_conflict = node_map.conflict_params + ) + + // attributes + WITH diff_root, diff_root_map, diff_node, node_map + CALL { + WITH diff_node, node_map + UNWIND node_map.attributes AS node_attribute + CREATE (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(diff_attribute:DiffAttribute) + SET diff_attribute = node_attribute.node_properties - // node single relationships - WITH diff_relationship, node_relationship - UNWIND node_relationship.relationships as node_single_relationship - CREATE (diff_relationship)-[:DIFF_HAS_ELEMENT]->(diff_relationship_element:DiffRelationshipElement) - SET diff_relationship_element = node_single_relationship.node_properties - // single relationship conflict - FOREACH (i in CASE WHEN node_single_relationship.conflict_params IS NOT NULL THEN [1] ELSE [] END | - CREATE (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(diff_relationship_conflict:DiffConflict) - SET diff_relationship_conflict = node_single_relationship.conflict_params - ) + // node attribute properties + WITH diff_attribute, node_attribute + UNWIND node_attribute.properties AS attr_property + CREATE (diff_attribute)-[:DIFF_HAS_PROPERTY]->(diff_attr_prop:DiffProperty) + SET diff_attr_prop = attr_property.node_properties + // attribute property conflict + FOREACH (i in CASE WHEN attr_property.conflict_params IS NOT NULL THEN [1] ELSE [] END | + CREATE (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(diff_attribute_property_conflict:DiffConflict) + SET diff_attribute_property_conflict = attr_property.conflict_params + ) + } - // node relationship properties - WITH diff_relationship_element, node_single_relationship - UNWIND node_single_relationship.properties as node_relationship_property - CREATE (diff_relationship_element)-[:DIFF_HAS_PROPERTY]->(diff_relationship_property:DiffProperty) - SET diff_relationship_property = node_relationship_property.node_properties - // relationship property conflict - FOREACH (i in CASE WHEN node_relationship_property.conflict_params IS NOT NULL THEN [1] ELSE [] END | - CREATE (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(diff_relationship_property_conflict:DiffConflict) - SET diff_relationship_property_conflict = node_relationship_property.conflict_params - ) + // relationships + WITH diff_root, diff_root_map, diff_node, node_map + CALL { + WITH diff_node, node_map + UNWIND node_map.relationships as node_relationship + CREATE (diff_node)-[:DIFF_HAS_RELATIONSHIP]->(diff_relationship:DiffRelationship) + SET diff_relationship = node_relationship.node_properties + + // node single relationships + WITH diff_relationship, node_relationship + UNWIND node_relationship.relationships as node_single_relationship + CREATE (diff_relationship)-[:DIFF_HAS_ELEMENT]->(diff_relationship_element:DiffRelationshipElement) + SET diff_relationship_element = node_single_relationship.node_properties + // single relationship conflict + FOREACH (i in CASE WHEN node_single_relationship.conflict_params IS NOT NULL THEN [1] ELSE [] END | + CREATE (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(diff_relationship_conflict:DiffConflict) + SET diff_relationship_conflict = node_single_relationship.conflict_params + ) + + // node relationship properties + WITH diff_relationship_element, node_single_relationship + UNWIND node_single_relationship.properties as node_relationship_property + CREATE (diff_relationship_element)-[:DIFF_HAS_PROPERTY]->(diff_relationship_property:DiffProperty) + SET diff_relationship_property = node_relationship_property.node_properties + // relationship property conflict + FOREACH (i in CASE WHEN node_relationship_property.conflict_params IS NOT NULL THEN [1] ELSE [] END | + CREATE (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(diff_relationship_property_conflict:DiffConflict) + SET diff_relationship_property_conflict = node_relationship_property.conflict_params + ) + } + } + + WITH diff_root, diff_root_map + CALL { + WITH diff_root, diff_root_map + UNWIND diff_root_map.node_parent_links AS node_parent_link + CALL { + WITH diff_root, node_parent_link + MATCH (diff_root)-[:DIFF_HAS_NODE]->(parent_node:DiffNode {uuid: node_parent_link.parent_uuid})-[:DIFF_HAS_RELATIONSHIP]->(diff_rel_group:DiffRelationship {name: node_parent_link.relationship_name}) + MATCH (diff_root)-[:DIFF_HAS_NODE]->(child_node:DiffNode {uuid: node_parent_link.child_uuid}) + MERGE (diff_rel_group)-[:DIFF_HAS_NODE]->(child_node) + } + } + RETURN diff_root } - WITH diff_root - UNWIND $node_parent_links AS node_parent_link + + WITH DISTINCT diff_root AS diff_root + WITH collect(diff_root) AS diff_roots CALL { - WITH diff_root, node_parent_link - MATCH (diff_root)-[:DIFF_HAS_NODE]->(parent_node:DiffNode {uuid: node_parent_link.parent_uuid})-[:DIFF_HAS_RELATIONSHIP]->(diff_rel_group:DiffRelationship {name: node_parent_link.relationship_name}) - MATCH (diff_root)-[:DIFF_HAS_NODE]->(child_node:DiffNode {uuid: node_parent_link.child_uuid}) - MERGE (diff_rel_group)-[:DIFF_HAS_NODE]->(child_node) + WITH diff_roots + WITH diff_roots[0] AS base_diff_node, diff_roots[1] AS branch_diff_node + MERGE (base_diff_node)-[:DIFF_HAS_PARTNER]-(branch_diff_node) + SET (base_diff_node).partner_uuid = (branch_diff_node).uuid + SET (branch_diff_node).partner_uuid = (base_diff_node).uuid } """ self.add_to_query(query=query) - self.return_labels = ["diff_root.uuid"] def _build_conflict_params(self, enriched_conflict: EnrichedDiffConflict) -> dict[str, Any]: return { @@ -261,26 +286,32 @@ def _build_node_parent_links(self, enriched_node: EnrichedDiffNode) -> list[dict parent_links.extend(self._build_node_parent_links(enriched_node=child_node)) return parent_links - def _build_diff_root_params(self, enriched_diff: EnrichedDiffRoot) -> dict[str, Any]: + def _build_diff_root_params(self, enriched_diffs: EnrichedDiffs) -> dict[str, Any]: params: dict[str, Any] = {} - params["diff_root_props"] = { - "base_branch": enriched_diff.base_branch_name, - "diff_branch": enriched_diff.diff_branch_name, - "from_time": enriched_diff.from_time.to_string(), - "to_time": enriched_diff.to_time.to_string(), - "uuid": enriched_diff.uuid, - "tracking_id": enriched_diff.tracking_id.serialize() if enriched_diff.tracking_id else None, - "num_added": enriched_diff.num_added, - "num_updated": enriched_diff.num_updated, - "num_removed": enriched_diff.num_removed, - "num_conflicts": enriched_diff.num_conflicts, - "contains_conflict": enriched_diff.contains_conflict, - } - node_maps = [] - node_parent_links = [] - for node in enriched_diff.nodes: - node_maps.append(self._build_diff_node_params(enriched_node=node)) - node_parent_links.extend(self._build_node_parent_links(enriched_node=node)) - params["node_maps"] = node_maps - params["node_parent_links"] = node_parent_links + for enriched_diff, param_key in ( + (enriched_diffs.base_branch_diff, "base_branch_diff"), + (enriched_diffs.diff_branch_diff, "diff_branch_diff"), + ): + diff_params: dict[str, Any] = {} + diff_params["diff_root_props"] = { + "base_branch": enriched_diff.base_branch_name, + "diff_branch": enriched_diff.diff_branch_name, + "from_time": enriched_diff.from_time.to_string(), + "to_time": enriched_diff.to_time.to_string(), + "uuid": enriched_diff.uuid, + "tracking_id": enriched_diff.tracking_id.serialize() if enriched_diff.tracking_id else None, + "num_added": enriched_diff.num_added, + "num_updated": enriched_diff.num_updated, + "num_removed": enriched_diff.num_removed, + "num_conflicts": enriched_diff.num_conflicts, + "contains_conflict": enriched_diff.contains_conflict, + } + node_maps = [] + node_parent_links = [] + for node in enriched_diff.nodes: + node_maps.append(self._build_diff_node_params(enriched_node=node)) + node_parent_links.extend(self._build_node_parent_links(enriched_node=node)) + diff_params["node_maps"] = node_maps + diff_params["node_parent_links"] = node_parent_links + params[param_key] = diff_params return params diff --git a/backend/infrahub/core/diff/query_parser.py b/backend/infrahub/core/diff/query_parser.py index 12a6c63a2a..0ae45e291d 100644 --- a/backend/infrahub/core/diff/query_parser.py +++ b/backend/infrahub/core/diff/query_parser.py @@ -16,10 +16,12 @@ DiffRelationship, DiffRoot, DiffSingleRelationship, + NodeFieldSpecifier, ) if TYPE_CHECKING: - from infrahub.core.query.diff import DiffAllPathsQuery + from infrahub.core.branch import Branch + from infrahub.core.query import QueryResult from infrahub.core.schema.relationship_schema import RelationshipSchema from infrahub.core.schema_manager import SchemaManager @@ -290,6 +292,7 @@ def get_final_single_relationship(self, from_time: Timestamp) -> DiffSingleRelat @dataclass class DiffRelationshipIntermediate: name: str + identifier: str cardinality: RelationshipCardinality properties_by_db_id: dict[str, set[DiffRelationshipPropertyIntermediate]] = field(default_factory=dict) _single_relationship_list: list[DiffSingleRelationshipIntermediate] = field(default_factory=list) @@ -389,19 +392,21 @@ def to_diff_root(self, from_time: Timestamp, to_time: Timestamp) -> DiffRoot: class DiffQueryParser: def __init__( self, - diff_query: DiffAllPathsQuery, - base_branch_name: str, - diff_branch_name: str, + base_branch: Branch, + diff_branch: Branch, schema_manager: SchemaManager, from_time: Timestamp, to_time: Optional[Timestamp] = None, ) -> None: - self.diff_query = diff_query - self.base_branch_name = base_branch_name - self.diff_branch_name = diff_branch_name + self.base_branch_name = base_branch.name + self.diff_branch_name = diff_branch.name self.schema_manager = schema_manager self.from_time = from_time self.to_time = to_time or Timestamp() + if diff_branch.name == base_branch.name: + self.diff_branch_create_time = from_time + else: + self.diff_branch_create_time = Timestamp(diff_branch.get_created_at()) self._diff_root_by_branch: dict[str, DiffRootIntermediate] = {} self._final_diff_root_by_branch: dict[str, DiffRoot] = {} @@ -415,17 +420,29 @@ def get_diff_root_for_branch(self, branch: str) -> DiffRoot: return self._final_diff_root_by_branch[branch] return DiffRoot(from_time=self.from_time, to_time=self.to_time, uuid=str(uuid4()), branch=branch, nodes=[]) + def get_node_field_specifiers_for_branch(self, branch_name: str) -> set[NodeFieldSpecifier]: + if branch_name not in self._diff_root_by_branch: + return set() + node_field_specifiers = set() + diff_root = self._diff_root_by_branch[branch_name] + for node in diff_root.nodes_by_id.values(): + for attribute_name in node.attributes_by_name: + node_field_specifiers.add(NodeFieldSpecifier(node_uuid=node.uuid, field_name=attribute_name)) + for relationship_diff in node.relationships_by_name.values(): + node_field_specifiers.add( + NodeFieldSpecifier(node_uuid=node.uuid, field_name=relationship_diff.identifier) + ) + return node_field_specifiers + + def read_result(self, query_result: QueryResult) -> None: + path = query_result.get_path(label="diff_path") + database_path = DatabasePath.from_cypher_path(cypher_path=path) + self._parse_path(database_path=database_path) + def parse(self) -> None: - if not self.diff_query.has_been_executed: - raise RuntimeError("query must be executed before indexing") - - for query_result in self.diff_query.get_results(): - paths = query_result.get_paths(label="full_diff_paths") - for path in paths: - database_path = DatabasePath.from_cypher_path(cypher_path=path) - self._parse_path(database_path=database_path) - self._apply_base_branch_previous_values() - self._remove_empty_base_diff_root() + if len(self._diff_root_by_branch) > 1: + self._apply_base_branch_previous_values() + self._remove_empty_base_diff_root() self._finalize() def _parse_path(self, database_path: DatabasePath) -> None: @@ -436,7 +453,7 @@ def _parse_path(self, database_path: DatabasePath) -> None: def _get_diff_root(self, database_path: DatabasePath) -> DiffRootIntermediate: branch = database_path.deepest_branch if branch not in self._diff_root_by_branch: - self._diff_root_by_branch[branch] = DiffRootIntermediate(uuid=database_path.root_id, branch=branch) + self._diff_root_by_branch[branch] = DiffRootIntermediate(uuid=str(uuid4()), branch=branch) return self._diff_root_by_branch[branch] def _get_diff_node(self, database_path: DatabasePath, diff_root: DiffRootIntermediate) -> DiffNodeIntermediate: @@ -500,7 +517,9 @@ def _get_diff_relationship( diff_relationship = diff_node.relationships_by_name.get(relationship_schema.name) if not diff_relationship: diff_relationship = DiffRelationshipIntermediate( - name=relationship_schema.name, cardinality=relationship_schema.cardinality + name=relationship_schema.name, + cardinality=relationship_schema.cardinality, + identifier=relationship_schema.get_identifier(), ) diff_node.relationships_by_name[relationship_schema.name] = diff_relationship return diff_relationship @@ -591,17 +610,21 @@ def _remove_empty_base_diff_root(self) -> None: ordered_diff_values = property_diff.get_ordered_values_asc() if not ordered_diff_values: continue - if ordered_diff_values[-1].changed_at >= self.from_time: + if ordered_diff_values[-1].changed_at >= self.diff_branch_create_time: return for relationship_diff in node_diff.relationships_by_name.values(): for diff_relationship_property_list in relationship_diff.properties_by_db_id.values(): for diff_relationship_property in diff_relationship_property_list: - if diff_relationship_property.changed_at >= self.from_time: + if diff_relationship_property.changed_at >= self.diff_branch_create_time: return del self._diff_root_by_branch[self.base_branch_name] def _finalize(self) -> None: for branch, diff_root_intermediate in self._diff_root_by_branch.items(): + if branch == self.base_branch_name: + from_time = self.diff_branch_create_time + else: + from_time = self.from_time self._final_diff_root_by_branch[branch] = diff_root_intermediate.to_diff_root( - from_time=self.from_time, to_time=self.to_time + from_time=from_time, to_time=self.to_time ) diff --git a/backend/infrahub/core/diff/repository/deserializer.py b/backend/infrahub/core/diff/repository/deserializer.py index 4af9c39556..207c69150c 100644 --- a/backend/infrahub/core/diff/repository/deserializer.py +++ b/backend/infrahub/core/diff/repository/deserializer.py @@ -167,6 +167,7 @@ def build_diff_root(cls, root_node: Neo4jNode) -> EnrichedDiffRoot: from_time=from_time, to_time=to_time, uuid=str(root_node.get("uuid")), + partner_uuid=str(root_node.get("partner_uuid")), tracking_id=tracking_id, num_added=int(root_node.get("num_added")), num_updated=int(root_node.get("num_updated")), diff --git a/backend/infrahub/core/diff/repository/repository.py b/backend/infrahub/core/diff/repository/repository.py index 1647192156..057c1bfc4a 100644 --- a/backend/infrahub/core/diff/repository/repository.py +++ b/backend/infrahub/core/diff/repository/repository.py @@ -4,7 +4,7 @@ from infrahub.database import InfrahubDatabase from infrahub.exceptions import ResourceNotFoundError -from ..model.path import ConflictSelection, EnrichedDiffConflict, EnrichedDiffRoot, TimeRange, TrackingId +from ..model.path import ConflictSelection, EnrichedDiffConflict, EnrichedDiffRoot, EnrichedDiffs, TimeRange, TrackingId from ..query.delete_query import EnrichedDiffDeleteQuery from ..query.diff_get import EnrichedDiffGetQuery from ..query.diff_summary import DiffSummaryCounters, DiffSummaryQuery @@ -33,7 +33,7 @@ async def get( limit: int | None = None, offset: int | None = None, tracking_id: TrackingId | None = None, - diff_id: str | None = None, + diff_ids: list[str] | None = None, include_empty: bool = False, ) -> list[EnrichedDiffRoot]: final_max_depth = config.SETTINGS.database.max_depth_search_hierarchy @@ -49,7 +49,7 @@ async def get( limit=final_limit, offset=offset, tracking_id=tracking_id, - diff_id=diff_id, + diff_ids=diff_ids, ) await query.execute(db=self.db) diff_roots = await self.deserializer.deserialize( @@ -59,6 +59,49 @@ async def get( diff_roots = [dr for dr in diff_roots if len(dr.nodes) > 0] return diff_roots + async def get_pairs( + self, + base_branch_name: str, + diff_branch_name: str, + from_time: Timestamp, + to_time: Timestamp, + ) -> list[EnrichedDiffs]: + max_depth = config.SETTINGS.database.max_depth_search_hierarchy + query = await EnrichedDiffGetQuery.init( + db=self.db, + base_branch_name=base_branch_name, + diff_branch_names=[diff_branch_name], + from_time=from_time, + to_time=to_time, + max_depth=max_depth, + ) + await query.execute(db=self.db) + diff_branch_roots = await self.deserializer.deserialize( + database_results=query.get_results(), include_parents=True + ) + diffs_by_uuid = {dbr.uuid: dbr for dbr in diff_branch_roots} + base_partner_query = await EnrichedDiffGetQuery.init( + db=self.db, + base_branch_name=base_branch_name, + diff_branch_names=[base_branch_name], + max_depth=max_depth, + diff_ids=[d.partner_uuid for d in diffs_by_uuid.values()], + ) + await base_partner_query.execute(db=self.db) + base_branch_roots = await self.deserializer.deserialize( + database_results=base_partner_query.get_results(), include_parents=True + ) + diffs_by_uuid.update({bbr.uuid: bbr for bbr in base_branch_roots}) + return [ + EnrichedDiffs( + base_branch_name=base_branch_name, + diff_branch_name=diff_branch_name, + base_branch_diff=diffs_by_uuid[dbr.partner_uuid], + diff_branch_diff=dbr, + ) + for dbr in diff_branch_roots + ] + async def get_one( self, diff_branch_name: str, @@ -71,7 +114,7 @@ async def get_one( base_branch_name=registry.default_branch, diff_branch_names=[diff_branch_name], tracking_id=tracking_id, - diff_id=diff_id, + diff_ids=[diff_id] if diff_id else None, filters=filters, include_parents=include_parents, include_empty=True, @@ -87,8 +130,8 @@ async def get_one( raise ResourceNotFoundError(f"Multiple diffs for {error_str}") return enriched_diffs[0] - async def save(self, enriched_diff: EnrichedDiffRoot) -> None: - query = await EnrichedDiffSaveQuery.init(db=self.db, enriched_diff_root=enriched_diff) + async def save(self, enriched_diffs: EnrichedDiffs) -> None: + query = await EnrichedDiffSaveQuery.init(db=self.db, enriched_diffs=enriched_diffs) await query.execute(db=self.db) async def summary( diff --git a/backend/infrahub/core/graph/__init__.py b/backend/infrahub/core/graph/__init__.py index bb23cc8351..cdfe3d1b83 100644 --- a/backend/infrahub/core/graph/__init__.py +++ b/backend/infrahub/core/graph/__init__.py @@ -1 +1 @@ -GRAPH_VERSION = 14 +GRAPH_VERSION = 15 diff --git a/backend/infrahub/core/migrations/graph/__init__.py b/backend/infrahub/core/migrations/graph/__init__.py index c87fda7514..5b6daa4b78 100644 --- a/backend/infrahub/core/migrations/graph/__init__.py +++ b/backend/infrahub/core/migrations/graph/__init__.py @@ -16,13 +16,14 @@ from .m012_convert_account_generic import Migration012 from .m013_convert_git_password_credential import Migration013 from .m014_remove_index_attr_value import Migration014 +from .m015_diff_format_update import Migration015 if TYPE_CHECKING: from infrahub.core.root import Root - from ..shared import GraphMigration, InternalSchemaMigration + from ..shared import ArbitraryMigration, GraphMigration, InternalSchemaMigration -MIGRATIONS: list[type[Union[GraphMigration, InternalSchemaMigration]]] = [ +MIGRATIONS: list[type[Union[GraphMigration, InternalSchemaMigration, ArbitraryMigration]]] = [ Migration001, Migration002, Migration003, @@ -37,10 +38,13 @@ Migration012, Migration013, Migration014, + Migration015, ] -async def get_graph_migrations(root: Root) -> Sequence[Union[GraphMigration, InternalSchemaMigration]]: +async def get_graph_migrations( + root: Root, +) -> Sequence[Union[GraphMigration, InternalSchemaMigration, ArbitraryMigration]]: applicable_migrations = [] for migration_class in MIGRATIONS: migration = migration_class.init() diff --git a/backend/infrahub/core/migrations/graph/m015_diff_format_update.py b/backend/infrahub/core/migrations/graph/m015_diff_format_update.py new file mode 100644 index 0000000000..2fa2e772ea --- /dev/null +++ b/backend/infrahub/core/migrations/graph/m015_diff_format_update.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from infrahub.core import registry +from infrahub.core.diff.repository.repository import DiffRepository +from infrahub.core.migrations.shared import MigrationResult +from infrahub.dependencies.registry import build_component_registry, get_component_registry +from infrahub.log import get_logger + +from ..shared import ArbitraryMigration + +if TYPE_CHECKING: + from infrahub.database import InfrahubDatabase + +log = get_logger() + + +class Migration015(ArbitraryMigration): + name: str = "015_diff_format_update" + minimum_version: int = 14 + + async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult: + result = MigrationResult() + + return result + + async def execute(self, db: InfrahubDatabase) -> MigrationResult: + default_branch = registry.get_branch_from_registry() + build_component_registry() + component_registry = get_component_registry() + diff_repo = await component_registry.get_component(DiffRepository, db=db, branch=default_branch) + + diff_roots = await diff_repo.get_empty_roots() + await diff_repo.delete_diff_roots(diff_root_uuids=[d.uuid for d in diff_roots]) + return MigrationResult() diff --git a/backend/infrahub/core/migrations/shared.py b/backend/infrahub/core/migrations/shared.py index 2dfe31b871..d0c721937e 100644 --- a/backend/infrahub/core/migrations/shared.py +++ b/backend/infrahub/core/migrations/shared.py @@ -172,3 +172,18 @@ async def execute(self, db: InfrahubDatabase) -> MigrationResult: return result return result + + +class ArbitraryMigration(BaseModel): + name: str = Field(..., description="Name of the migration") + minimum_version: int = Field(..., description="Minimum version of the graph to execute this migration") + + @classmethod + def init(cls, **kwargs: dict[str, Any]) -> Self: + return cls(**kwargs) # type: ignore[arg-type] + + async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult: + raise NotImplementedError() + + async def execute(self, db: InfrahubDatabase) -> MigrationResult: + raise NotImplementedError() diff --git a/backend/infrahub/core/query/__init__.py b/backend/infrahub/core/query/__init__.py index c2d4318398..516cba5951 100644 --- a/backend/infrahub/core/query/__init__.py +++ b/backend/infrahub/core/query/__init__.py @@ -285,8 +285,13 @@ def get_nodes(self) -> Generator[Neo4jNode, None, None]: if isinstance(item, Neo4jNode): yield item + def get_path(self, label: str) -> Neo4jPath: + path = self.get(label=label) + if isinstance(path, Neo4jPath): + return path + raise ValueError(f"{label} is not a Path") + def get_paths(self, label: str) -> Generator[Neo4jPath, None, None]: - """Return all nodes.""" for path in self.get(label=label): if isinstance(path, Neo4jPath): yield path diff --git a/backend/infrahub/core/query/diff.py b/backend/infrahub/core/query/diff.py index 8048479407..2c6e871e02 100644 --- a/backend/infrahub/core/query/diff.py +++ b/backend/infrahub/core/query/diff.py @@ -2,7 +2,6 @@ from typing import TYPE_CHECKING, Optional, Union -from infrahub.core import registry from infrahub.core.constants import BranchSupportType from infrahub.core.query import Query, QueryResult, QueryType, sort_results_by_time from infrahub.core.timestamp import Timestamp @@ -508,232 +507,317 @@ class DiffAllPathsQuery(DiffQuery): def __init__( self, base_branch: Branch, - namespaces_include: Optional[list[str]] = None, - namespaces_exclude: Optional[list[str]] = None, - kinds_include: Optional[list[str]] = None, - kinds_exclude: Optional[list[str]] = None, - branch_support: Optional[list[BranchSupportType]] = None, + diff_branch_create_time: Timestamp, + branch_support: list[BranchSupportType] | None = None, + current_node_field_specifiers: list[tuple[str, str]] | None = None, + new_node_field_specifiers: list[tuple[str, str]] | None = None, *args, **kwargs, ): self.base_branch = base_branch - self.namespaces_include = namespaces_include - self.namespaces_exclude = namespaces_exclude - self.kinds_include = kinds_include - self.kinds_exclude = kinds_exclude + self.diff_branch_create_time = diff_branch_create_time self.branch_support = branch_support or [BranchSupportType.AWARE] + self.current_node_field_specifiers = current_node_field_specifiers + self.new_node_field_specifiers = new_node_field_specifiers super().__init__(*args, **kwargs) - def _get_node_where_clause(self, node_variable_name: str) -> str: - where_clause_parts = [] - where_clause_parts.append( - f"($namespaces_include IS NULL OR {node_variable_name}.namespace IN $namespaces_include)" - ) - where_clause_parts.append( - f"($namespaces_exclude IS NULL OR NOT({node_variable_name}.namespace IN $namespaces_exclude))" - ) - where_clause_parts.append(f"($kinds_include IS NULL OR {node_variable_name}.kind IN $kinds_include)") - where_clause_parts.append(f"($kinds_exclude IS NULL OR NOT({node_variable_name}.kind IN $kinds_exclude))") - where_clause = " AND ".join(where_clause_parts) - return f"({where_clause})" - async def query_init(self, db: InfrahubDatabase, **kwargs): + from_str = self.diff_from.to_string() self.params.update( { - "namespaces_include": self.namespaces_include, - "namespaces_exclude": self.namespaces_exclude, - "kinds_include": self.kinds_include, - "kinds_exclude": self.kinds_exclude, "base_branch_name": self.base_branch.name, "branch_name": self.branch.name, - "branch_names": [registry.default_branch, self.branch.name], - "from_time": self.diff_from.to_string(), + "branch_create_time": self.diff_branch_create_time.to_string(), + "from_time": from_str, "to_time": self.diff_to.to_string(), + "branch_support": [item.value for item in self.branch_support], + "new_node_field_specifiers": self.new_node_field_specifiers, + "current_node_field_specifiers": self.current_node_field_specifiers, } ) - p_node_where = self._get_node_where_clause(node_variable_name="p") - n_node_where = self._get_node_where_clause(node_variable_name="n") - - diff_rel_filter_parts, br_params = self.branch.get_query_filter_range( - rel_label="diff_rel", - start_time=self.diff_from, - end_time=self.diff_to, - ) - diff_rel_filter = " AND ".join(diff_rel_filter_parts) - - self.params.update(br_params) - self.params["branch_support"] = [item.value for item in self.branch_support] - - # ruff: noqa: E501 query = """ - // all updated edges for our branches and time frame - MATCH (p)-[diff_rel]-(q) - WHERE any(l in labels(p) WHERE l in ["Node", "Attribute", "Relationship"]) - AND %(diff_rel_filter)s +WITH CASE + WHEN $new_node_field_specifiers IS NULL AND $current_node_field_specifiers IS NULL THEN [[NULL, $from_time]] + WHEN $new_node_field_specifiers IS NULL OR size($new_node_field_specifiers) = 0 THEN [[$current_node_field_specifiers, $from_time]] + WHEN $current_node_field_specifiers IS NULL OR size($current_node_field_specifiers) = 0 THEN [[$new_node_field_specifiers, $branch_create_time]] + ELSE [[$new_node_field_specifiers, $branch_create_time], [$current_node_field_specifiers, $from_time]] +END AS diff_filter_params_list +UNWIND diff_filter_params_list AS diff_filter_params +CALL { + WITH diff_filter_params + WITH diff_filter_params[0] AS node_field_specifiers_list, diff_filter_params[1] AS from_time + CALL { + WITH node_field_specifiers_list, from_time + WITH reduce(node_ids = [], nfs IN node_field_specifiers_list | node_ids + [nfs[0]]) AS node_ids_list, from_time + // ------------------------------------- + // Identify nodes added/removed on branch + // ------------------------------------- + MATCH (q:Root)<-[diff_rel:IS_PART_OF {branch: $branch_name}]-(p:Node) + WHERE (node_ids_list IS NULL OR p.uuid IN node_ids_list) + AND (from_time <= diff_rel.from < $to_time) + AND (diff_rel.to IS NULL OR (from_time <= diff_rel.to < $to_time)) + AND (p.branch_support IN $branch_support OR q.branch_support IN $branch_support) + WITH p, q, diff_rel + // ------------------------------------- + // Get every path on this branch under each node + // ------------------------------------- + CALL { + WITH p, q, diff_rel + OPTIONAL MATCH path = ( + (q)<-[top_diff_rel:IS_PART_OF]-(p)-[r_node]-(node)-[r_prop]-(prop) + ) + WHERE %(id_func)s(diff_rel) = %(id_func)s(top_diff_rel) + AND type(r_node) IN ["HAS_ATTRIBUTE", "IS_RELATED"] + AND any(l in labels(node) WHERE l in ["Attribute", "Relationship"]) + AND type(r_prop) IN ["IS_VISIBLE", "IS_PROTECTED", "HAS_SOURCE", "HAS_OWNER", "HAS_VALUE", "IS_RELATED"] + AND any(l in labels(prop) WHERE l in ["Boolean", "Node", "AttributeValue"]) + AND ALL( + r in [r_node, r_prop] + WHERE r.from <= $to_time AND r.branch = top_diff_rel.branch + ) + AND top_diff_rel.from <= r_node.from + AND (top_diff_rel.to IS NULL OR top_diff_rel.to >= r_node.from) + AND r_node.from <= r_prop.from + AND (r_node.to IS NULL OR r_node.to >= r_prop.from) + AND [%(id_func)s(p), type(r_node)] <> [%(id_func)s(prop), type(r_prop)] + AND top_diff_rel.status = r_node.status + AND top_diff_rel.status = r_prop.status + WITH path, node, prop, r_prop, r_node + ORDER BY + %(id_func)s(node), + %(id_func)s(prop), + r_prop.from DESC, + r_node.from DESC + WITH node, prop, type(r_prop) AS r_prop_type, type(r_node) AS r_node_type, head(collect(path)) AS top_diff_path + RETURN top_diff_path + } + RETURN top_diff_path AS diff_path + } + RETURN diff_path + UNION + WITH diff_filter_params + WITH diff_filter_params[0] AS node_field_specifiers_list, diff_filter_params[1] AS from_time + CALL { + WITH node_field_specifiers_list, from_time + // ------------------------------------- + // Identify attributes/relationships added/removed on branch + // ------------------------------------- + CALL { + WITH node_field_specifiers_list, from_time + MATCH (root:Root)<-[r_root:IS_PART_OF]-(p:Node)-[diff_rel:HAS_ATTRIBUTE {branch: $branch_name}]->(q:Attribute) + // exclude attributes and relationships under added/removed nodes b/c they are covered above + WHERE (node_field_specifiers_list IS NULL OR [p.uuid, q.name] IN node_field_specifiers_list) + AND r_root.branch IN [$branch_name, $base_branch_name] + AND r_root.from < from_time + AND r_root.status = "active" + // get attributes and relationships added on the branch during the timeframe + AND (from_time <= diff_rel.from < $to_time) + AND (diff_rel.to IS NULL OR (from_time <= diff_rel.to < $to_time)) + AND r_root.from <= diff_rel.from + AND (r_root.to IS NULL OR r_root.to >= diff_rel.from) AND (p.branch_support IN $branch_support OR q.branch_support IN $branch_support) - AND %(p_node_where)s - // subqueries to get full paths associated with the above update edges - WITH p, diff_rel, q - // -- DEEPEST EDGE SUBQUERY -- - // get full path for every HAS_VALUE, HAS_SOURCE/OWNER, IS_VISIBLE/PROTECTED - // can be multiple paths in the case of HAS_SOURCE/OWNER, IS_VISIBLE/PROTECTED to include - // both peers in the relationship - CALL { - WITH p, q, diff_rel - OPTIONAL MATCH path = ( - (:Root)<-[r_root:IS_PART_OF]-(n:Node)-[r_node]-(inner_p)-[inner_diff_rel]->(inner_q) - ) - WHERE %(id_func)s(inner_p) = %(id_func)s(p) AND %(id_func)s(inner_diff_rel) = %(id_func)s(diff_rel) AND %(id_func)s(inner_q) = %(id_func)s(q) - AND any(l in labels(inner_p) WHERE l in ["Attribute", "Relationship"]) - AND type(inner_diff_rel) IN ["IS_VISIBLE", "IS_PROTECTED", "HAS_SOURCE", "HAS_OWNER", "HAS_VALUE"] - AND any(l in labels(inner_q) WHERE l in ["Boolean", "Node", "AttributeValue"]) - AND type(r_node) IN ["HAS_ATTRIBUTE", "IS_RELATED"] - AND %(n_node_where)s - AND [%(id_func)s(n), type(r_node)] <> [%(id_func)s(inner_q), type(inner_diff_rel)] - AND ALL( - r in [r_root, r_node] - WHERE r.from <= $to_time AND r.branch IN $branch_names - ) - // exclude paths where an active edge is below a deleted edge - AND (inner_diff_rel.status = "deleted" OR r_node.status = "active") - AND (r_node.status = "deleted" OR r_root.status = "active") - WITH path AS diff_rel_path, diff_rel, r_root, n, r_node, p - ORDER BY - %(id_func)s(n) DESC, - %(id_func)s(p) DESC, - r_node.branch = diff_rel.branch DESC, - r_root.branch = diff_rel.branch DESC, - r_node.from DESC, - r_root.from DESC - WITH p, n, head(collect(diff_rel_path)) AS deepest_diff_path - RETURN deepest_diff_path - } - WITH p, diff_rel, q, deepest_diff_path - // explicitly add in base branch path, if it exists to capture previous value - // explicitly add in far-side of any relationship to get peer_id for rel properties + RETURN root, r_root, p, diff_rel, q + UNION ALL + WITH node_field_specifiers_list, from_time + MATCH (root:Root)<-[r_root:IS_PART_OF]-(p:Node)-[diff_rel:IS_RELATED {branch: $branch_name}]-(q:Relationship) + // exclude attributes and relationships under added/removed nodes b/c they are covered above + WHERE (node_field_specifiers_list IS NULL OR [p.uuid, q.name] IN node_field_specifiers_list) + AND r_root.branch IN [$branch_name, $base_branch_name] + AND r_root.from < from_time + AND r_root.status = "active" + // get attributes and relationships added on the branch during the timeframe + AND (from_time <= diff_rel.from < $to_time) + AND (diff_rel.to IS NULL OR (from_time <= diff_rel.to < $to_time)) + AND r_root.from <= diff_rel.from + AND (r_root.to IS NULL OR r_root.to >= diff_rel.from) + AND (p.branch_support IN $branch_support OR q.branch_support IN $branch_support) + RETURN root, r_root, p, diff_rel, q + } + WITH root, r_root, p, diff_rel, q, from_time + // ------------------------------------- + // Exclude attributes/relationship under nodes deleted on this branch in the timeframe + // because those were all handled above at the node level + // ------------------------------------- + CALL { + WITH root, p, from_time + OPTIONAL MATCH (root)<-[r_root_deleted:IS_PART_OF {branch: $branch_name}]-(p) + WHERE from_time <= r_root_deleted.from <= $to_time + WITH r_root_deleted + ORDER BY r_root_deleted.status DESC + LIMIT 1 + RETURN COALESCE(r_root_deleted.status = "deleted", FALSE) AS node_deleted + } + WITH root, r_root, p, diff_rel, q, node_deleted + WHERE node_deleted = FALSE + // ------------------------------------- + // Get every path on this branch under each attribute/relationship + // ------------------------------------- + CALL { + WITH root, r_root, p, diff_rel, q + OPTIONAL MATCH path = ( + (root:Root)<-[mid_r_root:IS_PART_OF]-(p)-[mid_diff_rel]-(q)-[r_prop]-(prop) + ) + WHERE %(id_func)s(mid_r_root) = %(id_func)s(r_root) + AND %(id_func)s(mid_diff_rel) = %(id_func)s(diff_rel) + AND type(r_prop) IN ["IS_VISIBLE", "IS_PROTECTED", "HAS_SOURCE", "HAS_OWNER", "HAS_VALUE", "IS_RELATED"] + AND any(l in labels(prop) WHERE l in ["Boolean", "Node", "AttributeValue"]) + AND r_prop.from <= $to_time AND r_prop.branch = mid_diff_rel.branch + AND mid_diff_rel.from <= r_prop.from + AND (mid_diff_rel.to IS NULL OR mid_diff_rel.to >= r_prop.from) + AND [%(id_func)s(p), type(mid_diff_rel)] <> [%(id_func)s(prop), type(r_prop)] + // exclude paths where an active edge is below a deleted edge + AND (mid_diff_rel.status = "active" OR r_prop.status = "deleted") + WITH path, prop, r_prop, mid_r_root + ORDER BY + type(r_prop), + mid_r_root.branch = mid_diff_rel.branch DESC, + r_prop.from DESC, + mid_r_root.from DESC + WITH prop, type(r_prop) AS type_r_prop, head(collect(path)) AS latest_prop_path + RETURN latest_prop_path + } + RETURN latest_prop_path AS mid_diff_path + } + RETURN mid_diff_path AS diff_path + UNION + WITH diff_filter_params + WITH diff_filter_params[0] AS node_field_specifiers_list, diff_filter_params[1] AS from_time + CALL { + WITH node_field_specifiers_list, from_time + // ------------------------------------- + // Identify properties added/removed on branch + // ------------------------------------- + MATCH diff_rel_path = (root:Root)<-[r_root:IS_PART_OF]-(n:Node)-[r_node]-(p)-[diff_rel {branch: $branch_name}]->(q) + WHERE (node_field_specifiers_list IS NULL OR [n.uuid, p.name] IN node_field_specifiers_list) + AND (from_time <= diff_rel.from < $to_time) + AND (diff_rel.to IS NULL OR (from_time <= diff_rel.to < $to_time)) + // exclude attributes and relationships under added/removed nodes, attrs, and rels b/c they are covered above + AND ALL( + r in [r_root, r_node] + WHERE r.from <= from_time AND r.branch IN [$branch_name, $base_branch_name] + ) + AND (p.branch_support IN $branch_support OR q.branch_support IN $branch_support) + AND any(l in labels(p) WHERE l in ["Attribute", "Relationship"]) + AND type(diff_rel) IN ["IS_VISIBLE", "IS_PROTECTED", "HAS_SOURCE", "HAS_OWNER", "HAS_VALUE"] + AND any(l in labels(q) WHERE l in ["Boolean", "Node", "AttributeValue"]) + AND type(r_node) IN ["HAS_ATTRIBUTE", "IS_RELATED"] + AND ALL( + r_pair IN [[r_root, r_node], [r_node, diff_rel]] + // filter out paths where a base branch edge follows a branch edge + WHERE ((r_pair[0]).branch = $base_branch_name OR (r_pair[1]).branch = $branch_name) + // filter out paths where an active edge follows a deleted edge + AND ((r_pair[0]).status = "active" OR (r_pair[1]).status = "deleted") + // filter out paths where an earlier from time follows a later from time + AND (r_pair[0]).from <= (r_pair[1]).from + // require adjacent edge pairs to have overlapping times, but only if on the same branch + AND ( + (r_pair[0]).branch <> (r_pair[1]).branch + OR (r_pair[0]).to IS NULL + OR (r_pair[0]).to >= (r_pair[1]).from + ) + ) + AND [%(id_func)s(n), type(r_node)] <> [%(id_func)s(q), type(diff_rel)] + WITH diff_rel_path, r_root, n, r_node, p, diff_rel, from_time + ORDER BY + %(id_func)s(n) DESC, + %(id_func)s(p) DESC, + type(diff_rel), + r_node.branch = diff_rel.branch DESC, + r_root.branch = diff_rel.branch DESC, + diff_rel.from DESC, + r_node.from DESC, + r_root.from DESC + // ------------------------------------- + // Add base branch paths, if they exist, to capture previous values + // Add peer-side of any relationships to get the peer's ID + // ------------------------------------- + WITH n, p, from_time, diff_rel, diff_rel_path + CALL { + // ------------------------------------- + // Exclude properties under nodes and attributes/relationship deleted + // on this branch in the timeframe because those were all handled above + // ------------------------------------- + WITH n, p, from_time CALL { - WITH p, diff_rel, deepest_diff_path - WITH p, diff_rel, deepest_diff_path AS diff_rel_path, nodes(deepest_diff_path) AS drp_nodes, relationships(deepest_diff_path) AS drp_relationships - WITH p, diff_rel, diff_rel_path, drp_relationships[0] AS r_root, drp_nodes[1] AS n, drp_relationships[1] AS r_node - // get base branch version of the diff path, if it exists - WITH diff_rel_path, diff_rel, r_root, n, r_node, p - OPTIONAL MATCH latest_base_path = (:Root)<-[r_root2]-(n2)-[r_node2]-(inner_p2)-[base_diff_rel]->(base_prop) - WHERE %(id_func)s(r_root2) = %(id_func)s(r_root) AND %(id_func)s(n2) = %(id_func)s(n) AND %(id_func)s(r_node2) = %(id_func)s(r_node) AND %(id_func)s(inner_p2) = %(id_func)s(p) - AND any(r in relationships(diff_rel_path) WHERE r.branch = $branch_name) - AND %(id_func)s(n2) <> %(id_func)s(base_prop) - AND type(base_diff_rel) = type(diff_rel) - AND all( - r in relationships(latest_base_path) - WHERE r.branch = $base_branch_name AND r.from <= $from_time - ) - // exclude paths where an active edge is below a deleted edge - AND (base_diff_rel.status = "deleted" OR r_node2.status = "active") - AND (r_node2.status = "deleted" OR r_root2.status = "active") - WITH diff_rel_path, latest_base_path, diff_rel, r_root, n, r_node, p - ORDER BY base_diff_rel.from DESC, r_node.from DESC, r_root.from DESC + WITH n, from_time + OPTIONAL MATCH (root:Root)<-[r_root_deleted:IS_PART_OF {branch: $branch_name}]-(n) + WHERE from_time <= r_root_deleted.from <= $to_time + WITH r_root_deleted + ORDER BY r_root_deleted.status DESC LIMIT 1 - // get peer node for updated relationship properties - WITH diff_rel_path, latest_base_path, diff_rel, r_root, n, r_node, p - OPTIONAL MATCH base_peer_path = ( - (:Root)<-[r_root3]-(n3)-[r_node3]-(inner_p3:Relationship)-[base_r_peer:IS_RELATED]-(base_peer:Node) - ) - WHERE %(id_func)s(r_root3) = %(id_func)s(r_root) AND %(id_func)s(n3) = %(id_func)s(n) AND %(id_func)s(r_node3) = %(id_func)s(r_node) AND %(id_func)s(inner_p3) = %(id_func)s(p) - AND type(diff_rel) <> "IS_RELATED" - AND [%(id_func)s(n3), type(r_node3)] <> [%(id_func)s(base_peer), type(base_r_peer)] - AND base_r_peer.from <= $to_time - AND base_r_peer.branch IN $branch_names - // exclude paths where an active edge is below a deleted edge - AND (base_r_peer.status = "deleted" OR r_node3.status = "active") - AND (r_node3.status = "deleted" OR r_root3.status = "active") - WITH diff_rel_path, latest_base_path, base_peer_path, base_r_peer, diff_rel - ORDER BY base_r_peer.branch = diff_rel.branch DESC, base_r_peer.from DESC - LIMIT 1 - RETURN reduce( - diff_rel_paths = [], item IN [diff_rel_path, latest_base_path, base_peer_path] | - CASE WHEN item IS NULL THEN diff_rel_paths ELSE diff_rel_paths + [item] END - ) AS diff_rel_paths - } - // -- MIDDLE EDGE SUBQUERY -- - // get full paths for every HAS_ATTRIBUTE, IS_RELATED edge - // this includes at least one path for every property under the middle edge in question - WITH p, q, diff_rel, diff_rel_paths AS full_diff_paths - CALL { - WITH p, q, diff_rel - OPTIONAL MATCH path = ( - (:Root)<-[r_root:IS_PART_OF]-(inner_p)-[inner_diff_rel]-(inner_q)-[r_prop]-(prop) - ) - WHERE %(id_func)s(inner_p) = %(id_func)s(p) AND %(id_func)s(inner_diff_rel) = %(id_func)s(diff_rel) AND %(id_func)s(inner_q) = %(id_func)s(q) - AND "Node" IN labels(inner_p) - AND type(inner_diff_rel) IN ["HAS_ATTRIBUTE", "IS_RELATED"] - AND any(l in labels(inner_q) WHERE l in ["Attribute", "Relationship"]) - AND type(r_prop) IN ["IS_VISIBLE", "IS_PROTECTED", "HAS_SOURCE", "HAS_OWNER", "HAS_VALUE", "IS_RELATED"] - AND any(l in labels(prop) WHERE l in ["Boolean", "Node", "AttributeValue"]) - AND ALL( - r in [r_root, r_prop] - WHERE r.from <= $to_time AND r.branch IN $branch_names - ) - AND [%(id_func)s(inner_p), type(inner_diff_rel)] <> [%(id_func)s(prop), type(r_prop)] - // exclude paths where an active edge is below a deleted edge - AND (inner_diff_rel.status = "active" OR (r_prop.status = "deleted" AND inner_diff_rel.branch = r_prop.branch)) - AND (inner_diff_rel.status = "deleted" OR r_root.status = "active") - WITH path, prop, r_prop, r_root - ORDER BY - %(id_func)s(prop), - r_prop.branch = diff_rel.branch DESC, - r_root.branch = diff_rel.branch DESC, - r_prop.from DESC, - r_root.from DESC - WITH prop, head(collect(path)) AS latest_prop_path - RETURN collect(latest_prop_path) AS latest_paths + RETURN COALESCE(r_root_deleted.status = "deleted", FALSE) AS node_deleted } - WITH p, q, diff_rel, full_diff_paths + latest_paths AS full_diff_paths - // -- TOP EDGE SUBQUERY -- - // get full paths for every IS_PART_OF edge - // this edge indicates a whole node was added or deleted - // we need to get every attribute and relationship on the node to capture the new and previous values + WITH n, p, from_time, node_deleted CALL { - WITH p, q, diff_rel - OPTIONAL MATCH path = ( - (inner_q:Root)<-[inner_diff_rel:IS_PART_OF]-(inner_p:Node)-[r_node]-(node)-[r_prop]-(prop) - ) - WHERE %(id_func)s(inner_p) = %(id_func)s(p) AND %(id_func)s(inner_diff_rel) = %(id_func)s(diff_rel) AND %(id_func)s(inner_q) = %(id_func)s(q) - AND type(r_node) IN ["HAS_ATTRIBUTE", "IS_RELATED"] - AND any(l in labels(node) WHERE l in ["Attribute", "Relationship"]) - AND type(r_prop) IN ["IS_VISIBLE", "IS_PROTECTED", "HAS_SOURCE", "HAS_OWNER", "HAS_VALUE", "IS_RELATED"] - AND any(l in labels(prop) WHERE l in ["Boolean", "Node", "AttributeValue"]) - AND ALL( - r in [r_node, r_prop] - WHERE r.from <= $to_time AND r.branch IN $branch_names - ) - AND [%(id_func)s(inner_p), type(r_node)] <> [%(id_func)s(prop), type(r_prop)] - // exclude paths where an active edge is below a deleted edge - AND (inner_diff_rel.status = "active" OR - ( - r_node.status = "deleted" AND inner_diff_rel.branch = r_node.branch - AND r_prop.status = "deleted" AND inner_diff_rel.branch = r_prop.branch - ) - ) - AND (r_prop.status = "deleted" OR r_node.status = "active") - WITH path, node, prop, r_prop, r_node - ORDER BY - %(id_func)s(node), - %(id_func)s(prop), - r_prop.branch = diff_rel.branch DESC, - r_node.branch = diff_rel.branch DESC, - r_prop.from DESC, - r_node.from DESC - WITH node, prop, type(r_prop) AS r_prop_type, type(r_node) AS r_node_type, head(collect(path)) AS latest_path - RETURN latest_path + WITH n, p, from_time + OPTIONAL MATCH (n)-[r_node_deleted {branch: $branch_name}]-(p) + WHERE from_time <= r_node_deleted.from <= $to_time + AND type(r_node_deleted) IN ["HAS_ATTRIBUTE", "IS_RELATED"] + WITH r_node_deleted + ORDER BY r_node_deleted.status DESC + LIMIT 1 + RETURN COALESCE(r_node_deleted.status = "deleted", FALSE) AS field_deleted } - WITH p, q, diff_rel, full_diff_paths, collect(latest_path) AS latest_paths - WITH p, q, diff_rel, full_diff_paths + latest_paths AS full_diff_paths - """ % { - "diff_rel_filter": diff_rel_filter, - "id_func": db.get_id_function_name(), - "p_node_where": p_node_where, - "n_node_where": n_node_where, + RETURN node_deleted OR field_deleted AS node_or_field_deleted } - + WITH n, p, diff_rel, diff_rel_path, node_or_field_deleted + WHERE node_or_field_deleted = FALSE + WITH n, p, type(diff_rel) AS drt, head(collect(diff_rel_path)) AS deepest_diff_path + CALL { + WITH n, p, deepest_diff_path + WITH n, p, deepest_diff_path AS diff_rel_path, relationships(deepest_diff_path) AS drp_relationships + WITH n, p, diff_rel_path, drp_relationships[2] as diff_rel, drp_relationships[0] AS r_root, drp_relationships[1] AS r_node + // get base branch version of the diff path, if it exists + WITH diff_rel_path, diff_rel, r_root, n, r_node, p + OPTIONAL MATCH latest_base_path = (:Root)<-[deep_r_root]-(n)-[deep_r_node]-(p)-[base_diff_rel]->(base_prop) + WHERE %(id_func)s(deep_r_node) = %(id_func)s(r_node) + AND %(id_func)s(deep_r_root) = %(id_func)s(r_root) + AND %(id_func)s(n) <> %(id_func)s(base_prop) + AND type(base_diff_rel) = type(diff_rel) + AND all( + r in relationships(latest_base_path) + WHERE r.branch = $base_branch_name AND r.from <= $branch_create_time + ) + WITH diff_rel_path, latest_base_path, diff_rel, r_root, n, r_node, p + ORDER BY base_diff_rel.from DESC, r_node.from DESC, r_root.from DESC + LIMIT 1 + // get peer node for updated relationship properties + WITH diff_rel_path, latest_base_path, diff_rel, r_root, n, r_node, p + OPTIONAL MATCH base_peer_path = ( + (:Root)<-[base_r_root]-(n)-[base_r_node]-(p:Relationship)-[base_r_peer:IS_RELATED]-(base_peer:Node) + ) + WHERE type(diff_rel) <> "IS_RELATED" + AND %(id_func)s(base_r_root) = %(id_func)s(r_root) + AND %(id_func)s(base_r_node) = %(id_func)s(r_node) + AND [%(id_func)s(n), type(base_r_node)] <> [%(id_func)s(base_peer), type(base_r_peer)] + AND base_r_peer.from <= $to_time + // filter out paths where an earlier from time follows a later from time + AND base_r_node.from <= base_r_peer.from + // filter out paths where a base branch edge follows a branch edge + AND (base_r_node.branch = $base_branch_name OR base_r_peer.branch = $branch_name) + // filter out paths where an active edge follows a deleted edge + AND (base_r_node.status = "active" OR base_r_peer.status = "deleted") + // require adjacent edge pairs to have overlapping times, but only if on the same branch + AND ( + base_r_node.branch <> base_r_peer.branch + OR base_r_node.to IS NULL + OR base_r_node.to >= base_r_peer.from + ) + WITH diff_rel_path, latest_base_path, base_peer_path, base_r_peer, diff_rel + ORDER BY base_r_peer.branch = diff_rel.branch DESC, base_r_peer.from DESC + LIMIT 1 + RETURN latest_base_path, base_peer_path + } + WITH reduce( + diff_rel_paths = [], item IN [deepest_diff_path, latest_base_path, base_peer_path] | + CASE WHEN item IS NULL THEN diff_rel_paths ELSE diff_rel_paths + [item] END + ) AS diff_rel_paths + UNWIND diff_rel_paths AS bottom_diff_path + RETURN bottom_diff_path + } + RETURN bottom_diff_path AS diff_path +} + """ % {"id_func": db.get_id_function_name()} self.add_to_query(query) - self.return_labels = ["diff_rel", "full_diff_paths"] + self.return_labels = ["DISTINCT diff_path AS diff_path"] diff --git a/backend/infrahub/dependencies/builder/diff/conflicts_enricher.py b/backend/infrahub/dependencies/builder/diff/conflicts_enricher.py index 49dd39ffaf..9c8e8e514b 100644 --- a/backend/infrahub/dependencies/builder/diff/conflicts_enricher.py +++ b/backend/infrahub/dependencies/builder/diff/conflicts_enricher.py @@ -5,4 +5,4 @@ class DiffConflictsEnricherDependency(DependencyBuilder[ConflictsEnricher]): @classmethod def build(cls, context: DependencyBuilderContext) -> ConflictsEnricher: - return ConflictsEnricher(db=context.db) + return ConflictsEnricher() diff --git a/backend/infrahub/dependencies/registry.py b/backend/infrahub/dependencies/registry.py index 9a00d4a466..d9acc6de5c 100644 --- a/backend/infrahub/dependencies/registry.py +++ b/backend/infrahub/dependencies/registry.py @@ -11,6 +11,7 @@ from .builder.constraint.schema.uniqueness import SchemaUniquenessConstraintDependency from .builder.diff.calculator import DiffCalculatorDependency from .builder.diff.combiner import DiffCombinerDependency +from .builder.diff.conflict_transferer import DiffConflictTransfererDependency from .builder.diff.coordinator import DiffCoordinatorDependency from .builder.diff.data_check_synchronizer import DiffDataCheckSynchronizerDependency from .builder.diff.enricher.aggregated import DiffAggregatedEnricherDependency @@ -41,6 +42,7 @@ def build_component_registry() -> ComponentDependencyRegistry: component_registry.track_dependency(DiffCalculatorDependency) component_registry.track_dependency(DiffCombinerDependency) component_registry.track_dependency(DiffRepositoryDependency) + component_registry.track_dependency(DiffConflictTransfererDependency) component_registry.track_dependency(DiffCoordinatorDependency) component_registry.track_dependency(DiffDataCheckSynchronizerDependency) return component_registry diff --git a/backend/tests/unit/core/diff/test_conflicts_enricher.py b/backend/tests/unit/core/diff/test_conflicts_enricher.py index 8674c6a24f..5eadf9f67b 100644 --- a/backend/tests/unit/core/diff/test_conflicts_enricher.py +++ b/backend/tests/unit/core/diff/test_conflicts_enricher.py @@ -2,7 +2,7 @@ from uuid import uuid4 from infrahub.core.branch import Branch -from infrahub.core.constants import DiffAction +from infrahub.core.constants import DiffAction, RelationshipCardinality from infrahub.core.constants.database import DatabaseEdgeType from infrahub.core.diff.conflicts_enricher import ConflictsEnricher from infrahub.core.diff.model.path import EnrichedDiffConflict, EnrichedDiffRoot @@ -40,7 +40,7 @@ def _set_conflicts_to_none(self, enriched_diff: EnrichedDiffRoot): element_prop.conflict = None async def __call_system_under_test(self, db: InfrahubDatabase, base_enriched_diff, branch_enriched_diff) -> None: - conflicts_enricher = ConflictsEnricher(db=db) + conflicts_enricher = ConflictsEnricher() return await conflicts_enricher.add_conflicts_to_branch_diff( base_diff_root=base_enriched_diff, branch_diff_root=branch_enriched_diff ) @@ -208,6 +208,7 @@ async def test_cardinality_one_conflicts(self, db: InfrahubDatabase, car_person_ peer_id=new_base_peer_id, properties=base_properties, action=DiffAction.UPDATED ) }, + cardinality=RelationshipCardinality.ONE, ) } base_nodes = { @@ -235,6 +236,7 @@ async def test_cardinality_one_conflicts(self, db: InfrahubDatabase, car_person_ peer_id=previous_peer_id, properties=branch_properties, action=DiffAction.REMOVED ) }, + cardinality=RelationshipCardinality.ONE, ) } branch_nodes = { @@ -318,6 +320,7 @@ async def test_cardinality_many_conflicts(self, db: InfrahubDatabase, car_person EnrichedRelationshipElementFactory.build(peer_id=peer_id_1, properties=base_properties_1), EnrichedRelationshipElementFactory.build(peer_id=peer_id_2, properties=base_properties_2), }, + cardinality=RelationshipCardinality.MANY, ) } base_nodes = { @@ -357,6 +360,7 @@ async def test_cardinality_many_conflicts(self, db: InfrahubDatabase, car_person EnrichedRelationshipElementFactory.build(peer_id=peer_id_1, properties=branch_properties_1), EnrichedRelationshipElementFactory.build(peer_id=peer_id_2, properties=branch_properties_2), }, + cardinality=RelationshipCardinality.MANY, ) } branch_nodes = { diff --git a/backend/tests/unit/core/diff/test_coordinator_lock.py b/backend/tests/unit/core/diff/test_coordinator_lock.py index bcb830a26e..a83e10c405 100644 --- a/backend/tests/unit/core/diff/test_coordinator_lock.py +++ b/backend/tests/unit/core/diff/test_coordinator_lock.py @@ -4,7 +4,7 @@ import pytest -from infrahub import lock +from infrahub import config, lock from infrahub.core.branch import Branch from infrahub.core.diff.coordinator import DiffCoordinator from infrahub.core.diff.data_check_synchronizer import DiffDataCheckSynchronizer @@ -32,6 +32,7 @@ async def branch_with_data(self, db: InfrahubDatabase, default_branch: Branch, c return branch_1 async def get_diff_coordinator(self, db: InfrahubDatabase, diff_branch: Branch) -> DiffCoordinator: + config.SETTINGS.database.max_depth_search_hierarchy = 10 component_registry = get_component_registry() diff_coordinator = await component_registry.get_component(DiffCoordinator, db=db, branch=diff_branch) mock_synchronizer = AsyncMock(spec=DiffDataCheckSynchronizer) @@ -54,8 +55,7 @@ async def test_incremental_diff_locks_do_not_queue_up( ) assert len(results) == 2 assert results[0] == results[1] - # called once to calculate diff on main and once to calculate diff on the branch - assert len(diff_coordinator.diff_calculator.calculate_diff.call_args_list) == 2 + assert len(diff_coordinator.diff_calculator.calculate_diff.call_args_list) == 1 # called instead of calculating the diff again diff_coordinator.diff_repo.get_one.assert_awaited_once() @@ -82,11 +82,13 @@ async def test_arbitrary_diff_locks_queue_up( assert len(results) == 2 assert results[0].to_time != results[1].to_time assert results[0].uuid != results[1].uuid + assert results[0].partner_uuid != results[1].partner_uuid results[0].to_time = results[1].to_time results[0].uuid = results[1].uuid + results[0].partner_uuid = results[1].partner_uuid assert results[0] == results[1] - # called once to calculate diff on main and once to calculate diff on the branch for each request - assert len(diff_coordinator.diff_calculator.calculate_diff.call_args_list) == 4 + # called once to calculate diff on main and once to calculate diff on the branch + assert len(diff_coordinator.diff_calculator.calculate_diff.call_args_list) == 2 # not called because diffs are calculated both times diff_coordinator.diff_repo.get_one.assert_not_awaited() @@ -108,13 +110,15 @@ async def test_arbitrary_diff_blocks_incremental_diff( assert len(results) == 2 assert results[0].to_time != results[1].to_time assert results[0].uuid != results[1].uuid + assert results[0].partner_uuid != results[1].partner_uuid assert results[0].tracking_id != results[1].tracking_id results[0].to_time = results[1].to_time results[0].uuid = results[1].uuid + results[0].partner_uuid = results[1].partner_uuid results[0].tracking_id = results[1].tracking_id assert results[0] == results[1] - # called once to calculate diff on main and once to calculate diff on the branch for each request - assert len(diff_coordinator.diff_calculator.calculate_diff.call_args_list) == 4 + # called once to calculate diff on main and once to calculate diff on the branch + assert len(diff_coordinator.diff_calculator.calculate_diff.call_args_list) == 2 # not called because diffs are calculated both times diff_coordinator.diff_repo.get_one.assert_not_awaited() @@ -136,12 +140,14 @@ async def test_incremental_diff_blocks_arbitrary_diff( assert len(results) == 2 assert results[0].to_time != results[1].to_time assert results[0].uuid != results[1].uuid + assert results[0].partner_uuid != results[1].partner_uuid assert results[0].tracking_id != results[1].tracking_id results[0].to_time = results[1].to_time results[0].uuid = results[1].uuid + results[0].partner_uuid = results[1].partner_uuid results[0].tracking_id = results[1].tracking_id assert results[0] == results[1] - # called once to calculate diff on main and once to calculate diff on the branch for each request - assert len(diff_coordinator.diff_calculator.calculate_diff.call_args_list) == 4 + # called once to calculate diff on main and once to calculate diff on the branch + assert len(diff_coordinator.diff_calculator.calculate_diff.call_args_list) == 2 # not called because diffs are calculated both times diff_coordinator.diff_repo.get_one.assert_not_awaited() diff --git a/backend/tests/unit/core/diff/test_diff_query_parser.py b/backend/tests/unit/core/diff/test_diff_calculator.py similarity index 66% rename from backend/tests/unit/core/diff/test_diff_query_parser.py rename to backend/tests/unit/core/diff/test_diff_calculator.py index 43db8f36b5..3c25fc2de9 100644 --- a/backend/tests/unit/core/diff/test_diff_query_parser.py +++ b/backend/tests/unit/core/diff/test_diff_calculator.py @@ -1,14 +1,14 @@ import pytest -from infrahub.core import registry from infrahub.core.branch import Branch -from infrahub.core.constants import DiffAction +from infrahub.core.constants import DiffAction, RelationshipCardinality from infrahub.core.constants.database import DatabaseEdgeType -from infrahub.core.diff.query_parser import DiffQueryParser +from infrahub.core.diff.calculator import DiffCalculator +from infrahub.core.diff.model.path import NodeFieldSpecifier from infrahub.core.initialization import create_branch from infrahub.core.manager import NodeManager from infrahub.core.node import Node -from infrahub.core.query.diff import DiffAllPathsQuery +from infrahub.core.schema_manager import SchemaBranch from infrahub.core.timestamp import Timestamp from infrahub.database import InfrahubDatabase @@ -29,23 +29,12 @@ async def test_diff_attribute_branch_update( await alfred_branch.save(db=db) branch_after_change = Timestamp() - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=default_branch, - ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - diff_parser.parse() - assert diff_parser.get_branches() == {default_branch.name, branch.name} - main_root_path = diff_parser.get_diff_root_for_branch(branch=default_branch.name) + main_root_path = calculated_diffs.base_branch_diff assert main_root_path.branch == default_branch.name assert len(main_root_path.nodes) == 1 node_diff = main_root_path.nodes[0] @@ -63,7 +52,7 @@ async def test_diff_attribute_branch_update( assert property_diff.new_value == "Big Alfred" assert property_diff.action is DiffAction.UPDATED assert main_before_change < property_diff.changed_at < main_after_change - branch_root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) + branch_root_path = calculated_diffs.diff_branch_diff assert branch_root_path.branch == branch.name assert len(branch_root_path.nodes) == 1 node_diff = branch_root_path.nodes[0] @@ -94,24 +83,14 @@ async def test_attribute_property_main_update( await alfred_main.save(db=db) after_change = Timestamp() - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=default_branch, - base_branch=default_branch, - diff_from=from_time, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=default_branch, from_time=from_time, to_time=Timestamp() ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=default_branch.name, - schema_manager=registry.schema, - from_time=from_time, - ) - diff_parser.parse() - assert diff_parser.get_branches() == {default_branch.name} - main_root_path = diff_parser.get_diff_root_for_branch(branch=default_branch.name) + base_root_path = calculated_diffs.base_branch_diff + main_root_path = calculated_diffs.diff_branch_diff + assert base_root_path == main_root_path assert main_root_path.branch == default_branch.name assert len(main_root_path.nodes) == 1 node_diff = main_root_path.nodes[0] @@ -147,23 +126,14 @@ async def test_attribute_branch_set_null(db: InfrahubDatabase, default_branch: B await car_branch.save(db=db) after_change = Timestamp() - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=default_branch, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, - ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name} - branch_root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.nodes == [] + branch_root_path = calculated_diffs.diff_branch_diff assert branch_root_path.branch == branch.name assert len(branch_root_path.nodes) == 1 node_diff = branch_root_path.nodes[0] @@ -193,24 +163,17 @@ async def test_node_delete(db: InfrahubDatabase, default_branch: Branch, car_acc car_branch = await NodeManager.get_one(db=db, branch=branch, id=car_accord_main.id) await car_branch.delete(db=db) - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=default_branch, - diff_from=from_time, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, - ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name} - branch_root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) + base_root_path = calculated_diffs.base_branch_diff + branch_root_path = calculated_diffs.diff_branch_diff + if branch is default_branch: + assert base_root_path == branch_root_path + else: + assert base_root_path.nodes == [] assert branch_root_path.branch == branch.name assert len(branch_root_path.nodes) == 2 node_diffs_by_id = {n.uuid: n for n in branch_root_path.nodes} @@ -266,24 +229,19 @@ async def test_node_base_delete_branch_update( car_branch.nbr_seats.value = 10 await car_branch.save(db=db) - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=default_branch, - diff_from=from_time, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, - ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name, default_branch.name} - branch_root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) + base_root_path = calculated_diffs.base_branch_diff + assert len(base_root_path.nodes) == 1 + node_diffs_by_id = {n.uuid: n for n in base_root_path.nodes} + node_diff = node_diffs_by_id[car_accord_main.id] + assert node_diff.uuid == car_accord_main.id + assert node_diff.kind == "TestCar" + assert node_diff.action is DiffAction.REMOVED + branch_root_path = calculated_diffs.diff_branch_diff assert branch_root_path.branch == branch.name assert len(branch_root_path.nodes) == 1 node_diffs_by_id = {n.uuid: n for n in branch_root_path.nodes} @@ -314,23 +272,14 @@ async def test_node_branch_add(db: InfrahubDatabase, default_branch: Branch, car await new_person.save(db=db) after_change = Timestamp() - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=default_branch, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, - ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name} - branch_root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.nodes == [] + branch_root_path = calculated_diffs.diff_branch_diff assert branch_root_path.branch == branch.name assert len(branch_root_path.nodes) == 1 node_diff = branch_root_path.nodes[0] @@ -365,23 +314,14 @@ async def test_attribute_property_multiple_branch_updates( await alfred_branch.save(db=db) after_last_change = Timestamp() - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=default_branch, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, - ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name} - root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.nodes == [] + root_path = calculated_diffs.diff_branch_diff assert root_path.branch == branch.name assert len(root_path.nodes) == 1 node_diff = root_path.nodes[0] @@ -421,28 +361,15 @@ async def test_relationship_one_peer_branch_and_main_update( await car_branch.save(db=db) after_branch_change = Timestamp() - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=default_branch, - diff_from=from_time, - ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - diff_parser.parse() - - assert diff_parser.get_branches() == {branch.name, default_branch.name} # check branch - root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) - assert root_path.branch == branch.name - nodes_by_id = {n.uuid: n for n in root_path.nodes} + branch_root_path = calculated_diffs.diff_branch_diff + assert branch_root_path.branch == branch.name + nodes_by_id = {n.uuid: n for n in branch_root_path.nodes} assert set(nodes_by_id.keys()) == {car_accord_main.id, person_john_main.id, person_alfred_main.id} # check relationship on car node on branch car_node = nodes_by_id[car_main.get_id()] @@ -545,10 +472,10 @@ async def test_relationship_one_peer_branch_and_main_update( for prop_diff in single_relationship.properties: assert before_branch_change < prop_diff.changed_at < after_branch_change # check main - root_path = diff_parser.get_diff_root_for_branch(branch=default_branch.name) - assert root_path.branch == default_branch.name - nodes_by_id = {n.uuid: n for n in root_path.nodes} - assert set(nodes_by_id.keys()) == {car_accord_main.id, person_john_main.id, person_jane_main.id} + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.branch == default_branch.name + nodes_by_id = {n.uuid: n for n in base_root_path.nodes} + assert set(nodes_by_id.keys()) == {car_accord_main.id, person_john_main.id} # check relationship on car node on main car_node = nodes_by_id[car_main.get_id()] assert car_node.uuid == car_accord_main.id @@ -622,34 +549,6 @@ async def test_relationship_one_peer_branch_and_main_update( } for prop_diff in single_relationship.properties: assert before_main_change < prop_diff.changed_at < after_main_change - # check relationship on added peer on main - jane_node = nodes_by_id[person_jane_main.get_id()] - assert jane_node.uuid == person_jane_main.get_id() - assert jane_node.kind == "TestPerson" - assert jane_node.action is DiffAction.UPDATED - assert len(jane_node.attributes) == 0 - assert len(jane_node.relationships) == 1 - relationship_diff = jane_node.relationships[0] - assert relationship_diff.name == "cars" - assert relationship_diff.action is DiffAction.UPDATED - elements_by_peer_id = {e.peer_id: e for e in relationship_diff.relationships} - assert set(elements_by_peer_id.keys()) == {car_accord_main.get_id()} - single_relationship = relationship_diff.relationships[0] - assert single_relationship.peer_id == car_accord_main.id - assert single_relationship.action is DiffAction.ADDED - properties_by_type = {p.property_type: p for p in single_relationship.properties} - assert set(properties_by_type.keys()) == { - DatabaseEdgeType.IS_RELATED, - DatabaseEdgeType.IS_PROTECTED, - DatabaseEdgeType.IS_VISIBLE, - } - assert {(p.property_type, p.action, p.previous_value, p.new_value) for p in single_relationship.properties} == { - (DatabaseEdgeType.IS_RELATED, DiffAction.ADDED, None, car_accord_main.id), - (DatabaseEdgeType.IS_VISIBLE, DiffAction.ADDED, None, True), - (DatabaseEdgeType.IS_PROTECTED, DiffAction.ADDED, None, False), - } - for prop_diff in single_relationship.properties: - assert before_main_change < prop_diff.changed_at < after_main_change async def test_relationship_one_property_branch_update( @@ -673,26 +572,14 @@ async def test_relationship_one_property_branch_update( await car_branch.save(db=db) after_branch_change = Timestamp() - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=default_branch, - diff_from=from_time, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, - ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name, default_branch.name} - root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) - assert root_path.branch == branch.name - nodes_by_id = {n.uuid: n for n in root_path.nodes} + branch_root_path = calculated_diffs.diff_branch_diff + assert branch_root_path.branch == branch.name + nodes_by_id = {n.uuid: n for n in branch_root_path.nodes} assert set(nodes_by_id.keys()) == {car_accord_main.id, person_john_main.id} # check relationship property on car node on branch car_node = nodes_by_id[car_main.get_id()] @@ -749,23 +636,10 @@ async def test_relationship_one_property_branch_update( assert property_diff.action is DiffAction.UNCHANGED assert property_diff.changed_at < before_branch_change # check relationship peer on new peer on main - root_main_path = diff_parser.get_diff_root_for_branch(branch=default_branch.name) + root_main_path = calculated_diffs.base_branch_diff assert root_main_path.branch == default_branch.name - assert len(root_main_path.nodes) == 3 diff_nodes_by_id = {n.uuid: n for n in root_main_path.nodes} - node_diff = diff_nodes_by_id[person_jane_main.id] - assert node_diff.uuid == person_jane_main.id - assert node_diff.kind == "TestPerson" - assert node_diff.action is DiffAction.UPDATED - assert len(node_diff.attributes) == 0 - assert len(node_diff.relationships) == 1 - relationship_diff = node_diff.relationships[0] - assert relationship_diff.name == "cars" - assert relationship_diff.action is DiffAction.UPDATED - assert len(relationship_diff.relationships) == 1 - single_relationship_diff = relationship_diff.relationships[0] - assert single_relationship_diff.peer_id == car_accord_main.id - assert single_relationship_diff.action is DiffAction.ADDED + assert set(diff_nodes_by_id.keys()) == {person_john_main.id, car_accord_main.id} # check relationship peer on old peer on main node_diff = diff_nodes_by_id[person_john_main.id] assert node_diff.uuid == person_john_main.id @@ -856,23 +730,14 @@ async def test_add_node_branch( await new_car.new(db=db, name="Batmobile", color="#000000", owner=person_jane_main) await new_car.save(db=db) - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=default_branch, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, - ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name} - root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.nodes == [] + root_path = calculated_diffs.diff_branch_diff assert root_path.branch == branch.name assert len(root_path.nodes) == 2 diff_nodes_by_id = {n.uuid: n for n in root_path.nodes} @@ -947,26 +812,15 @@ async def test_many_relationship_property_update( await branch_car.owner.update(db=db, data={"id": person_john_main.id, "_relation__source": person_jane_main.id}) await branch_car.save(db=db) - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=branch, - ) - await diff_query.execute(db=db) - - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name} - root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.nodes == [] + root_path = calculated_diffs.diff_branch_diff assert root_path.branch == branch.name - assert len(root_path.nodes) == 2 nodes_by_id = {n.uuid: n for n in root_path.nodes} assert set(nodes_by_id.keys()) == {person_john_main.get_id(), car_accord_main.get_id()} john_node = nodes_by_id[person_john_main.get_id()] @@ -1032,28 +886,16 @@ async def test_cardinality_one_peer_conflicting_updates( await main_car.save(db=db) main_update_done = Timestamp() - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=branch, - diff_from=from_time, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, - ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name, default_branch.name} # check branch - root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) - assert root_path.branch == branch.name - assert len(root_path.nodes) == 3 - nodes_by_id = {n.uuid: n for n in root_path.nodes} + branch_root_path = calculated_diffs.diff_branch_diff + assert branch_root_path.branch == branch.name + assert len(branch_root_path.nodes) == 3 + nodes_by_id = {n.uuid: n for n in branch_root_path.nodes} assert set(nodes_by_id.keys()) == {car_accord_main.get_id(), person_john_main.get_id(), person_albert_main.get_id()} # check car node on branch car_node = nodes_by_id[car_accord_main.id] @@ -1166,11 +1008,11 @@ async def test_cardinality_one_peer_conflicting_updates( assert diff_prop.new_value == new_value assert from_time < diff_prop.changed_at < branch_update_done # check main - root_path = diff_parser.get_diff_root_for_branch(branch=default_branch.name) - assert root_path.branch == default_branch.name - assert len(root_path.nodes) == 3 - nodes_by_id = {n.uuid: n for n in root_path.nodes} - assert set(nodes_by_id.keys()) == {car_accord_main.get_id(), person_john_main.get_id(), person_jane_main.get_id()} + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.branch == default_branch.name + assert len(base_root_path.nodes) == 2 + nodes_by_id = {n.uuid: n for n in base_root_path.nodes} + assert set(nodes_by_id.keys()) == {car_accord_main.get_id(), person_john_main.get_id()} # check car node on main car_node = nodes_by_id[car_accord_main.id] assert car_node.action is DiffAction.UPDATED @@ -1251,36 +1093,6 @@ async def test_cardinality_one_peer_conflicting_updates( assert diff_prop.previous_value == previous_value assert diff_prop.new_value is None assert branch_update_done < diff_prop.changed_at < main_update_done - # check jane node on main - jane_node = nodes_by_id[person_jane_main.id] - assert jane_node.action is DiffAction.UPDATED - assert jane_node.attributes == [] - assert len(jane_node.relationships) == 1 - assert jane_node.changed_at < from_time - cars_rel = jane_node.relationships.pop() - assert cars_rel.name == "cars" - assert cars_rel.action is DiffAction.UPDATED - assert branch_update_done < cars_rel.changed_at < main_update_done - assert len(cars_rel.relationships) == 1 - cars_element = cars_rel.relationships.pop() - assert cars_element.peer_id == car_accord_main.id - assert cars_element.action is DiffAction.ADDED - assert branch_update_done < cars_element.changed_at < main_update_done - properties_by_type = {p.property_type: p for p in cars_element.properties} - assert set(properties_by_type.keys()) == { - DatabaseEdgeType.IS_RELATED, - DatabaseEdgeType.IS_VISIBLE, - DatabaseEdgeType.IS_PROTECTED, - } - for prop_type, new_value in [ - (DatabaseEdgeType.IS_RELATED, car_accord_main.id), - (DatabaseEdgeType.IS_VISIBLE, True), - (DatabaseEdgeType.IS_PROTECTED, False), - ]: - diff_prop = properties_by_type[prop_type] - assert diff_prop.previous_value is None - assert diff_prop.new_value == new_value - assert branch_update_done < diff_prop.changed_at < main_update_done async def test_relationship_property_owner_conflicting_updates( @@ -1298,28 +1110,16 @@ async def test_relationship_property_owner_conflicting_updates( await branch_john.cars.update(db=db, data={"id": car_accord_main.id, "_relation__owner": car_accord_main.id}) await branch_john.save(db=db) - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=branch, - diff_from=from_time, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - await diff_query.execute(db=db) - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, - ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name, default_branch.name} # check branch - root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) - assert root_path.branch == branch.name - assert len(root_path.nodes) == 2 - nodes_by_id = {n.uuid: n for n in root_path.nodes} + branch_root_path = calculated_diffs.diff_branch_diff + assert branch_root_path.branch == branch.name + assert len(branch_root_path.nodes) == 2 + nodes_by_id = {n.uuid: n for n in branch_root_path.nodes} assert set(nodes_by_id.keys()) == {person_john_main.get_id(), car_accord_main.get_id()} # john node on branch john_node = nodes_by_id[person_john_main.get_id()] @@ -1366,10 +1166,10 @@ async def test_relationship_property_owner_conflicting_updates( assert owner_rel.previous_value is None assert owner_rel.new_value == car_accord_main.get_id() # check main - root_path = diff_parser.get_diff_root_for_branch(branch=default_branch.name) - assert root_path.branch == default_branch.name - assert len(root_path.nodes) == 2 - nodes_by_id = {n.uuid: n for n in root_path.nodes} + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.branch == default_branch.name + assert len(base_root_path.nodes) == 2 + nodes_by_id = {n.uuid: n for n in base_root_path.nodes} assert set(nodes_by_id.keys()) == {person_john_main.get_id(), car_accord_main.get_id()} # john node on main john_node = nodes_by_id[person_john_main.get_id()] @@ -1434,27 +1234,17 @@ async def test_agnostic_source_relationship_update( await branch_car.owner.update(db=db, data={"id": person_1.id, "_relation__source": person_1.id}) await branch_car.save(db=db) - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=default_branch, - ) - await diff_query.execute(db=db) - - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name} - root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) - assert root_path.branch == branch.name - assert len(root_path.nodes) == 1 - diff_node = root_path.nodes.pop() + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.nodes == [] + branch_root_path = calculated_diffs.diff_branch_diff + assert branch_root_path.branch == branch.name + assert len(branch_root_path.nodes) == 1 + diff_node = branch_root_path.nodes.pop() assert diff_node.uuid == new_car.get_id() assert diff_node.action is DiffAction.UPDATED assert diff_node.attributes == [] @@ -1493,26 +1283,16 @@ async def test_agnostic_owner_relationship_added( await new_car.owner.update(db=db, data={"id": person_1.id, "_relation__owner": person_1.id}) await new_car.save(db=db) - diff_query = await DiffAllPathsQuery.init( - db=db, - branch=branch, - base_branch=default_branch, - ) - await diff_query.execute(db=db) - - diff_parser = DiffQueryParser( - diff_query=diff_query, - base_branch_name=default_branch.name, - diff_branch_name=branch.name, - schema_manager=registry.schema, - from_time=from_time, + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() ) - diff_parser.parse() - assert diff_parser.get_branches() == {branch.name} - root_path = diff_parser.get_diff_root_for_branch(branch=branch.name) - assert root_path.branch == branch.name - diff_nodes_by_id = {n.uuid: n for n in root_path.nodes} + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.nodes == [] + branch_root_path = calculated_diffs.diff_branch_diff + assert branch_root_path.branch == branch.name + diff_nodes_by_id = {n.uuid: n for n in branch_root_path.nodes} assert set(diff_nodes_by_id.keys()) == {new_car.get_id()} diff_node_car = diff_nodes_by_id[new_car.get_id()] assert diff_node_car.action is DiffAction.ADDED @@ -1546,3 +1326,522 @@ async def test_agnostic_owner_relationship_added( (DatabaseEdgeType.IS_PROTECTED, DiffAction.ADDED, None, False), (DatabaseEdgeType.IS_VISIBLE, DiffAction.ADDED, None, True), } + + +async def test_diff_attribute_branch_update_with_previous_base_update_ignored( + db: InfrahubDatabase, default_branch: Branch, person_alfred_main, person_john_main, car_accord_main +): + branch = await create_branch(db=db, branch_name="branch") + # change that will be ignored + car_main = await NodeManager.get_one(db=db, branch=default_branch, id=car_accord_main.id) + car_main.color.value = "BLURPLE" + await car_main.save(db=db) + alfred_main = await NodeManager.get_one(db=db, branch=default_branch, id=person_alfred_main.id) + alfred_main.name.value = "Big Alfred" + await alfred_main.save(db=db) + from_time = Timestamp() + alfred_branch = await NodeManager.get_one(db=db, branch=branch, id=person_alfred_main.id) + alfred_branch.name.value = "Little Alfred" + branch_before_change = Timestamp() + await alfred_branch.save(db=db) + branch_after_change = Timestamp() + + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, + diff_branch=branch, + from_time=from_time, + to_time=Timestamp(), + previous_node_specifiers={NodeFieldSpecifier(node_uuid=alfred_main.id, field_name="name")}, + ) + + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.branch == default_branch.name + assert len(base_root_path.nodes) == 0 + branch_root_path = calculated_diffs.diff_branch_diff + assert branch_root_path.branch == branch.name + assert len(branch_root_path.nodes) == 1 + node_diff = branch_root_path.nodes[0] + assert node_diff.uuid == person_alfred_main.id + assert node_diff.kind == "TestPerson" + assert node_diff.action is DiffAction.UPDATED + assert len(node_diff.attributes) == 1 + attribute_diff = node_diff.attributes[0] + assert attribute_diff.name == "name" + assert attribute_diff.action is DiffAction.UPDATED + assert len(attribute_diff.properties) == 1 + property_diff = attribute_diff.properties[0] + assert property_diff.property_type == DatabaseEdgeType.HAS_VALUE + assert property_diff.previous_value == "Alfred" + assert property_diff.new_value == "Little Alfred" + assert property_diff.action is DiffAction.UPDATED + assert branch_before_change < property_diff.changed_at < branch_after_change + + +async def test_diff_attribute_branch_update_with_concurrent_base_update_captured( + db: InfrahubDatabase, default_branch: Branch, person_alfred_main, person_john_main, car_accord_main +): + branch = await create_branch(db=db, branch_name="branch") + from_time = Timestamp() + # change that will be ignored + car_main = await NodeManager.get_one(db=db, branch=default_branch, id=car_accord_main.id) + car_main.color.value = "BLURPLE" + await car_main.save(db=db) + alfred_main = await NodeManager.get_one(db=db, branch=default_branch, id=person_alfred_main.id) + alfred_main.name.value = "Big Alfred" + base_before_change = Timestamp() + await alfred_main.save(db=db) + base_after_change = Timestamp() + alfred_branch = await NodeManager.get_one(db=db, branch=branch, id=person_alfred_main.id) + alfred_branch.name.value = "Little Alfred" + branch_before_change = Timestamp() + await alfred_branch.save(db=db) + branch_after_change = Timestamp() + + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, + diff_branch=branch, + from_time=from_time, + to_time=Timestamp(), + previous_node_specifiers={NodeFieldSpecifier(node_uuid=alfred_main.id, field_name="name")}, + ) + + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.branch == default_branch.name + assert len(base_root_path.nodes) == 1 + node_diff = base_root_path.nodes[0] + assert node_diff.uuid == person_alfred_main.id + assert node_diff.kind == "TestPerson" + assert node_diff.action is DiffAction.UPDATED + assert len(node_diff.attributes) == 1 + attribute_diff = node_diff.attributes[0] + assert attribute_diff.name == "name" + assert attribute_diff.action is DiffAction.UPDATED + assert len(attribute_diff.properties) == 1 + property_diff = attribute_diff.properties[0] + assert property_diff.property_type == DatabaseEdgeType.HAS_VALUE + assert property_diff.previous_value == "Alfred" + assert property_diff.new_value == "Big Alfred" + assert property_diff.action is DiffAction.UPDATED + assert base_before_change < property_diff.changed_at < base_after_change + branch_root_path = calculated_diffs.diff_branch_diff + assert branch_root_path.branch == branch.name + assert len(branch_root_path.nodes) == 1 + node_diff = branch_root_path.nodes[0] + assert node_diff.uuid == person_alfred_main.id + assert node_diff.kind == "TestPerson" + assert node_diff.action is DiffAction.UPDATED + assert len(node_diff.attributes) == 1 + attribute_diff = node_diff.attributes[0] + assert attribute_diff.name == "name" + assert attribute_diff.action is DiffAction.UPDATED + assert len(attribute_diff.properties) == 1 + property_diff = attribute_diff.properties[0] + assert property_diff.property_type == DatabaseEdgeType.HAS_VALUE + assert property_diff.previous_value == "Alfred" + assert property_diff.new_value == "Little Alfred" + assert property_diff.action is DiffAction.UPDATED + assert branch_before_change < property_diff.changed_at < branch_after_change + + +async def test_diff_attribute_branch_update_with_previous_base_update_captured( + db: InfrahubDatabase, default_branch: Branch, person_alfred_main, person_john_main, car_accord_main +): + branch = await create_branch(db=db, branch_name="branch") + # change that will be ignored + car_main = await NodeManager.get_one(db=db, branch=default_branch, id=car_accord_main.id) + car_main.color.value = "BLURPLE" + await car_main.save(db=db) + alfred_main = await NodeManager.get_one(db=db, branch=default_branch, id=person_alfred_main.id) + alfred_main.name.value = "Big Alfred" + base_before_change = Timestamp() + await alfred_main.save(db=db) + base_after_change = Timestamp() + from_time = Timestamp() + alfred_branch = await NodeManager.get_one(db=db, branch=branch, id=person_alfred_main.id) + alfred_branch.name.value = "Little Alfred" + branch_before_change = Timestamp() + await alfred_branch.save(db=db) + branch_after_change = Timestamp() + + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, + diff_branch=branch, + from_time=from_time, + to_time=Timestamp(), + ) + + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.branch == default_branch.name + assert len(base_root_path.nodes) == 1 + node_diff = base_root_path.nodes[0] + assert node_diff.uuid == person_alfred_main.id + assert node_diff.kind == "TestPerson" + assert node_diff.action is DiffAction.UPDATED + assert len(node_diff.attributes) == 1 + attribute_diff = node_diff.attributes[0] + assert attribute_diff.name == "name" + assert attribute_diff.action is DiffAction.UPDATED + assert len(attribute_diff.properties) == 1 + property_diff = attribute_diff.properties[0] + assert property_diff.property_type == DatabaseEdgeType.HAS_VALUE + assert property_diff.previous_value == "Alfred" + assert property_diff.new_value == "Big Alfred" + assert property_diff.action is DiffAction.UPDATED + assert base_before_change < property_diff.changed_at < base_after_change + branch_root_path = calculated_diffs.diff_branch_diff + assert branch_root_path.branch == branch.name + assert len(branch_root_path.nodes) == 1 + node_diff = branch_root_path.nodes[0] + assert node_diff.uuid == person_alfred_main.id + assert node_diff.kind == "TestPerson" + assert node_diff.action is DiffAction.UPDATED + assert len(node_diff.attributes) == 1 + attribute_diff = node_diff.attributes[0] + assert attribute_diff.name == "name" + assert attribute_diff.action is DiffAction.UPDATED + assert len(attribute_diff.properties) == 1 + property_diff = attribute_diff.properties[0] + assert property_diff.property_type == DatabaseEdgeType.HAS_VALUE + assert property_diff.previous_value == "Alfred" + assert property_diff.new_value == "Little Alfred" + assert property_diff.action is DiffAction.UPDATED + assert branch_before_change < property_diff.changed_at < branch_after_change + + +async def test_diff_attribute_branch_update_with_separate_previous_base_update_captured( + db: InfrahubDatabase, default_branch: Branch, person_alfred_main, person_john_main, car_accord_main +): + branch = await create_branch(db=db, branch_name="branch") + alfred_main = await NodeManager.get_one(db=db, branch=default_branch, id=person_alfred_main.id) + alfred_main.name.value = "Big Alfred" + await alfred_main.save(db=db) + from_time = Timestamp() + car_main = await NodeManager.get_one(db=db, branch=default_branch, id=car_accord_main.id) + car_main.color.value = "BLURPLE" + base_before_change = Timestamp() + await car_main.save(db=db) + base_after_change = Timestamp() + alfred_branch = await NodeManager.get_one(db=db, branch=branch, id=person_alfred_main.id) + alfred_branch.name.value = "Little Alfred" + branch_before_change = Timestamp() + await alfred_branch.save(db=db) + branch_after_change = Timestamp() + + diff_calculator = DiffCalculator(db=db) + + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, + diff_branch=branch, + from_time=from_time, + to_time=Timestamp(), + previous_node_specifiers={ + NodeFieldSpecifier(node_uuid=car_accord_main.id, field_name="color"), + NodeFieldSpecifier(node_uuid=person_alfred_main.id, field_name="name"), + }, + ) + + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.branch == default_branch.name + nodes_by_id = {n.uuid: n for n in base_root_path.nodes} + node_diff = nodes_by_id[car_accord_main.id] + assert node_diff.uuid == car_accord_main.id + assert node_diff.kind == "TestCar" + assert node_diff.action is DiffAction.UPDATED + assert len(node_diff.attributes) == 1 + attribute_diff = node_diff.attributes[0] + assert attribute_diff.name == "color" + assert attribute_diff.action is DiffAction.UPDATED + assert len(attribute_diff.properties) == 1 + property_diff = attribute_diff.properties[0] + assert property_diff.property_type == DatabaseEdgeType.HAS_VALUE + assert property_diff.previous_value == "#444444" + assert property_diff.new_value == "BLURPLE" + assert property_diff.action is DiffAction.UPDATED + assert base_before_change < property_diff.changed_at < base_after_change + branch_root_path = calculated_diffs.diff_branch_diff + assert branch_root_path.branch == branch.name + assert len(branch_root_path.nodes) == 1 + node_diff = branch_root_path.nodes[0] + assert node_diff.uuid == person_alfred_main.id + assert node_diff.kind == "TestPerson" + assert node_diff.action is DiffAction.UPDATED + assert len(node_diff.attributes) == 1 + attribute_diff = node_diff.attributes[0] + assert attribute_diff.name == "name" + assert attribute_diff.action is DiffAction.UPDATED + assert len(attribute_diff.properties) == 1 + property_diff = attribute_diff.properties[0] + assert property_diff.property_type == DatabaseEdgeType.HAS_VALUE + assert property_diff.previous_value == "Alfred" + assert property_diff.new_value == "Little Alfred" + assert property_diff.action is DiffAction.UPDATED + assert branch_before_change < property_diff.changed_at < branch_after_change + + +async def test_branch_node_delete_with_base_updates( + db: InfrahubDatabase, default_branch: Branch, car_accord_main, person_john_main, person_jane_main +): + branch = await create_branch(db=db, branch_name="branch") + from_time = Timestamp() + car_branch = await NodeManager.get_one(db=db, branch=branch, id=car_accord_main.id) + await car_branch.delete(db=db) + + car_main = await NodeManager.get_one(db=db, id=car_accord_main.id) + car_main.color.value = "blurple" + await car_main.owner.update(db=db, data={"id": person_jane_main.id}) + await car_main.save(db=db) + + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() + ) + + base_root_path = calculated_diffs.base_branch_diff + assert base_root_path.branch == default_branch.name + node_diffs_by_id = {n.uuid: n for n in base_root_path.nodes} + assert set(node_diffs_by_id.keys()) == {car_accord_main.id, person_john_main.id} + node_diff = node_diffs_by_id[car_accord_main.id] + assert node_diff.kind == "TestCar" + assert node_diff.action is DiffAction.UPDATED + assert len(node_diff.relationships) == 1 + rel_diffs_by_name = {r.name: r for r in node_diff.relationships} + rel_diff = rel_diffs_by_name["owner"] + assert rel_diff.action is DiffAction.UPDATED + elements_by_peer_id = {e.peer_id: e for e in rel_diff.relationships} + assert set(elements_by_peer_id.keys()) == {person_john_main.id, person_jane_main.id} + added_element = elements_by_peer_id[person_jane_main.id] + assert added_element.action is DiffAction.ADDED + properties_by_type = {p.property_type: p for p in added_element.properties} + assert set(properties_by_type.keys()) == { + DatabaseEdgeType.IS_PROTECTED, + DatabaseEdgeType.IS_RELATED, + DatabaseEdgeType.IS_VISIBLE, + } + for prop_type, new_value in ( + (DatabaseEdgeType.IS_PROTECTED, False), + (DatabaseEdgeType.IS_RELATED, person_jane_main.id), + (DatabaseEdgeType.IS_VISIBLE, True), + ): + diff_prop = properties_by_type[prop_type] + assert diff_prop.action is DiffAction.ADDED + assert diff_prop.previous_value is None + assert diff_prop.new_value == new_value + removed_element = elements_by_peer_id[person_john_main.id] + assert removed_element.action is DiffAction.REMOVED + properties_by_type = {p.property_type: p for p in removed_element.properties} + assert set(properties_by_type.keys()) == { + DatabaseEdgeType.IS_PROTECTED, + DatabaseEdgeType.IS_RELATED, + DatabaseEdgeType.IS_VISIBLE, + } + for prop_type, previous_value in ( + (DatabaseEdgeType.IS_PROTECTED, False), + (DatabaseEdgeType.IS_RELATED, person_john_main.id), + (DatabaseEdgeType.IS_VISIBLE, True), + ): + diff_prop = properties_by_type[prop_type] + assert diff_prop.action is DiffAction.REMOVED + assert diff_prop.previous_value == previous_value + assert diff_prop.new_value is None + peer_node_diff = node_diffs_by_id[person_john_main.id] + assert peer_node_diff.action is DiffAction.UPDATED + assert len(peer_node_diff.attributes) == 0 + assert len(peer_node_diff.relationships) == 1 + rel_diffs_by_name = {r.name: r for r in peer_node_diff.relationships} + rel_diff = rel_diffs_by_name["cars"] + assert rel_diff.action is DiffAction.UPDATED + elements_by_peer_id = {e.peer_id: e for e in rel_diff.relationships} + assert len(elements_by_peer_id) == 1 + removed_element = elements_by_peer_id[car_accord_main.id] + assert removed_element.action is DiffAction.REMOVED + properties_by_type = {p.property_type: p for p in removed_element.properties} + assert set(properties_by_type.keys()) == { + DatabaseEdgeType.IS_PROTECTED, + DatabaseEdgeType.IS_RELATED, + DatabaseEdgeType.IS_VISIBLE, + } + for prop_type, previous_value in ( + (DatabaseEdgeType.IS_PROTECTED, False), + (DatabaseEdgeType.IS_RELATED, car_accord_main.id), + (DatabaseEdgeType.IS_VISIBLE, True), + ): + diff_prop = properties_by_type[prop_type] + assert diff_prop.action is DiffAction.REMOVED + assert diff_prop.previous_value == previous_value + assert diff_prop.new_value is None + attributes_by_name = {attr.name: attr for attr in node_diff.attributes} + assert len(attributes_by_name) == 1 + attribute_diff = attributes_by_name["color"] + assert attribute_diff.action is DiffAction.UPDATED + properties_by_type = {prop.property_type: prop for prop in attribute_diff.properties} + assert len(properties_by_type) == 1 + diff_property = properties_by_type[DatabaseEdgeType.HAS_VALUE] + assert diff_property.action is DiffAction.UPDATED + assert diff_property.previous_value == "#444444" + assert diff_property.new_value == "blurple" + + branch_root_path = calculated_diffs.diff_branch_diff + assert branch_root_path.branch == branch.name + assert len(branch_root_path.nodes) == 2 + node_diffs_by_id = {n.uuid: n for n in branch_root_path.nodes} + assert set(node_diffs_by_id.keys()) == {car_accord_main.id, person_john_main.id} + node_diff = node_diffs_by_id[car_accord_main.id] + assert node_diff.uuid == car_accord_main.id + assert node_diff.kind == "TestCar" + assert node_diff.action is DiffAction.REMOVED + assert len(node_diff.attributes) == 5 + assert len(node_diff.relationships) == 1 + relationship_diff = node_diff.relationships[0] + attributes_by_name = {attr.name: attr for attr in node_diff.attributes} + assert set(attributes_by_name.keys()) == {"name", "nbr_seats", "color", "is_electric", "transmission"} + for attribute_diff in attributes_by_name.values(): + assert attribute_diff.action is DiffAction.REMOVED + properties_by_type = {prop.property_type: prop for prop in attribute_diff.properties} + diff_property = properties_by_type[DatabaseEdgeType.HAS_VALUE] + assert diff_property.action is DiffAction.REMOVED + assert diff_property.new_value in (None, "NULL") + assert len(node_diff.relationships) == 1 + relationship_diff = node_diff.relationships[0] + assert relationship_diff.name == "owner" + assert relationship_diff.action is DiffAction.REMOVED + assert len(relationship_diff.relationships) == 1 + single_relationship_diff = relationship_diff.relationships[0] + assert single_relationship_diff.peer_id == person_john_main.id + assert single_relationship_diff.action is DiffAction.REMOVED + node_diff = node_diffs_by_id[person_john_main.id] + assert node_diff.uuid == person_john_main.id + assert node_diff.kind == "TestPerson" + assert node_diff.action is DiffAction.UPDATED + assert len(node_diff.attributes) == 0 + assert len(node_diff.relationships) == 1 + relationship_diff = node_diff.relationships[0] + assert relationship_diff.name == "cars" + assert relationship_diff.action is DiffAction.UPDATED + assert len(relationship_diff.relationships) == 1 + single_relationship_diff = relationship_diff.relationships[0] + assert single_relationship_diff.peer_id == car_branch.id + assert single_relationship_diff.action is DiffAction.REMOVED + assert len(single_relationship_diff.properties) == 3 + for diff_property in single_relationship_diff.properties: + assert diff_property.action is DiffAction.REMOVED + + +async def test_branch_relationship_delete_with_property_update( + db: InfrahubDatabase, default_branch: Branch, animal_person_schema: SchemaBranch +): + person_schema = animal_person_schema.get(name="TestPerson") + dog_schema = animal_person_schema.get(name="TestDog") + persons = [] + for i in range(3): + person = await Node.init(db=db, schema=person_schema, branch=default_branch) + await person.new(db=db, name=f"Person{i}") + await person.save(db=db) + persons.append(person) + dogs = [] + for i in range(3): + dog = await Node.init(db=db, schema=dog_schema, branch=default_branch) + await dog.new(db=db, name=f"Dog{i}", breed=f"Breed{i}", owner=persons[i], best_friend=persons[i]) + await dog.save(db=db) + dogs.append(dog) + branch = await create_branch(db=db, branch_name="branch") + from_time = Timestamp() + dog_branch = await NodeManager.get_one(db=db, branch=branch, id=dogs[0].id) + before_branch_change = Timestamp() + await dog_branch.best_friend.update(db=db, data=[None]) + await dog_branch.save(db=db) + after_branch_change = Timestamp() + + dog_main = await NodeManager.get_one(db=db, id=dogs[0].id) + before_main_change = Timestamp() + await dog_main.best_friend.update(db=db, data={"id": persons[0].id, "_relation__is_visible": False}) + await dog_main.save(db=db) + after_main_change = Timestamp() + + diff_calculator = DiffCalculator(db=db) + calculated_diffs = await diff_calculator.calculate_diff( + base_branch=default_branch, diff_branch=branch, from_time=from_time, to_time=Timestamp() + ) + + base_diff = calculated_diffs.base_branch_diff + assert base_diff.branch == default_branch.name + node_diffs_by_id = {n.uuid: n for n in base_diff.nodes} + node_diff = node_diffs_by_id[dog_main.id] + assert node_diff.uuid == dog_main.id + assert node_diff.kind == "TestDog" + assert node_diff.action is DiffAction.UPDATED + assert len(node_diff.attributes) == 0 + assert len(node_diff.relationships) == 1 + rel_diffs_by_name = {r.name: r for r in node_diff.relationships} + rel_diff = rel_diffs_by_name["best_friend"] + assert rel_diff.cardinality is RelationshipCardinality.ONE + assert rel_diff.action is DiffAction.UPDATED + assert len(rel_diff.relationships) == 1 + rel_elements_by_peer_id = {e.peer_id: e for e in rel_diff.relationships} + rel_element_diff = rel_elements_by_peer_id[persons[0].id] + assert rel_element_diff.action is DiffAction.UPDATED + prop_diff_by_type = {p.property_type: p for p in rel_element_diff.properties} + prop_diff = prop_diff_by_type[DatabaseEdgeType.IS_VISIBLE] + assert prop_diff.action is DiffAction.UPDATED + assert prop_diff.new_value is False + assert prop_diff.previous_value is True + assert before_main_change < prop_diff.changed_at < after_main_change + for prop_diff in prop_diff_by_type.values(): + if prop_diff.property_type is DatabaseEdgeType.IS_VISIBLE: + continue + assert prop_diff.action is DiffAction.UNCHANGED + + branch_diff = calculated_diffs.diff_branch_diff + assert branch_diff.branch == branch.name + node_diffs_by_id = {n.uuid: n for n in branch_diff.nodes} + assert set(node_diffs_by_id.keys()) == {dog_branch.id, persons[0].id} + dog_node = node_diffs_by_id[dog_branch.id] + assert dog_node.action is DiffAction.UPDATED + assert len(dog_node.attributes) == 0 + assert len(dog_node.relationships) == 1 + rel_diffs_by_name = {r.name: r for r in dog_node.relationships} + rel_diff = rel_diffs_by_name["best_friend"] + assert rel_diff.cardinality is RelationshipCardinality.ONE + assert rel_diff.action is DiffAction.REMOVED + assert len(rel_diff.relationships) == 1 + rel_elements_by_peer_id = {e.peer_id: e for e in rel_diff.relationships} + rel_element_diff = rel_elements_by_peer_id[persons[0].id] + assert rel_element_diff.action is DiffAction.REMOVED + prop_diff_by_type = {p.property_type: p for p in rel_element_diff.properties} + assert len(prop_diff_by_type) == 3 + for property_type, previous_value in [ + (DatabaseEdgeType.IS_RELATED, persons[0].id), + (DatabaseEdgeType.IS_VISIBLE, True), + (DatabaseEdgeType.IS_PROTECTED, False), + ]: + prop_diff = prop_diff_by_type[property_type] + assert prop_diff.action is DiffAction.REMOVED + assert prop_diff.new_value is None + assert prop_diff.previous_value == previous_value + assert before_branch_change < prop_diff.changed_at < after_branch_change + person_node = node_diffs_by_id[persons[0].id] + assert person_node.action is DiffAction.UPDATED + assert len(person_node.attributes) == 0 + assert len(person_node.relationships) == 1 + rel_diffs_by_name = {r.name: r for r in person_node.relationships} + rel_diff = rel_diffs_by_name["best_friends"] + assert rel_diff.cardinality is RelationshipCardinality.MANY + assert rel_diff.action is DiffAction.UPDATED + assert len(rel_diff.relationships) == 1 + rel_elements_by_peer_id = {e.peer_id: e for e in rel_diff.relationships} + rel_element_diff = rel_elements_by_peer_id[dog_branch.id] + assert rel_element_diff.action is DiffAction.REMOVED + prop_diff_by_type = {p.property_type: p for p in rel_element_diff.properties} + assert len(prop_diff_by_type) == 3 + for property_type, previous_value in [ + (DatabaseEdgeType.IS_RELATED, dog_branch.id), + (DatabaseEdgeType.IS_VISIBLE, True), + (DatabaseEdgeType.IS_PROTECTED, False), + ]: + prop_diff = prop_diff_by_type[property_type] + assert prop_diff.action is DiffAction.REMOVED + assert prop_diff.new_value is None + assert prop_diff.previous_value == previous_value + assert before_branch_change < prop_diff.changed_at < after_branch_change diff --git a/backend/tests/unit/core/diff/test_diff_combiner.py b/backend/tests/unit/core/diff/test_diff_combiner.py index e2d2f11b09..70906cdb5b 100644 --- a/backend/tests/unit/core/diff/test_diff_combiner.py +++ b/backend/tests/unit/core/diff/test_diff_combiner.py @@ -16,6 +16,7 @@ EnrichedDiffNode, EnrichedDiffProperty, EnrichedDiffRelationship, + EnrichedDiffs, EnrichedDiffSingleRelationship, ) from infrahub.core.schema.node_schema import NodeSchema @@ -87,7 +88,20 @@ def mock_get_node_schema(name, *args, **kwargs): self.schema_manager.get_node_schema.side_effect = mock_get_node_schema async def __call_system_under_test(self, diff_1, diff_2): - return await self.combiner.combine(earlier_diff=diff_1, later_diff=diff_2) + enriched_diffs_1 = EnrichedDiffs( + base_branch_name=self.base_branch, + diff_branch_name=self.diff_branch, + base_branch_diff=EnrichedRootFactory.build(nodes=set()), + diff_branch_diff=diff_1, + ) + enriched_diffs_2 = EnrichedDiffs( + base_branch_name=self.base_branch, + diff_branch_name=self.diff_branch, + base_branch_diff=EnrichedRootFactory.build(nodes=set()), + diff_branch_diff=diff_2, + ) + combined_diffs = await self.combiner.combine(earlier_diffs=enriched_diffs_1, later_diffs=enriched_diffs_2) + return combined_diffs.diff_branch_diff @pytest.mark.parametrize( "action_1,action_2", @@ -109,6 +123,7 @@ async def test_add_and_remove_node_cancel_one_another(self, action_1, action_2): combined = await self.__call_system_under_test(self.diff_root_1, self.diff_root_2) self.expected_combined.uuid = combined.uuid + self.expected_combined.partner_uuid = combined.partner_uuid assert combined == self.expected_combined @pytest.mark.parametrize( @@ -144,6 +159,7 @@ async def test_node_action_addition(self, action_1, action_2, expected_action): combined = await self.__call_system_under_test(self.diff_root_1, self.diff_root_2) self.expected_combined.uuid = combined.uuid + self.expected_combined.partner_uuid = combined.partner_uuid self.expected_combined.nodes = { EnrichedDiffNode( uuid=diff_node_2.uuid, @@ -191,6 +207,7 @@ async def test_stale_parent_node_removed(self): combined = await self.__call_system_under_test(self.diff_root_1, self.diff_root_2) self.expected_combined.uuid = combined.uuid + self.expected_combined.partner_uuid = combined.partner_uuid expected_parent_node = replace(parent_node_2) expected_rel = replace(relationship_2, nodes={expected_parent_node}) expected_child_node = replace( @@ -335,6 +352,7 @@ async def test_attributes_combined(self): combined = await self.__call_system_under_test(self.diff_root_1, self.diff_root_2) self.expected_combined.uuid = combined.uuid + self.expected_combined.partner_uuid = combined.partner_uuid assert combined == self.expected_combined async def test_relationship_one_combined(self, with_schema_manager): @@ -448,7 +466,7 @@ async def test_relationship_one_combined(self, with_schema_manager): combined = await self.__call_system_under_test(self.diff_root_1, self.diff_root_2) self.expected_combined.uuid = combined.uuid - + self.expected_combined.partner_uuid = combined.partner_uuid assert combined == self.expected_combined async def test_relationship_many_combined(self, with_schema_manager): @@ -613,7 +631,7 @@ async def test_relationship_many_combined(self, with_schema_manager): combined = await self.__call_system_under_test(self.diff_root_1, self.diff_root_2) self.expected_combined.uuid = combined.uuid - + self.expected_combined.partner_uuid = combined.partner_uuid assert combined == self.expected_combined async def test_relationship_with_only_nodes(self, with_schema_manager): @@ -677,7 +695,7 @@ async def test_relationship_with_only_nodes(self, with_schema_manager): combined = await self.__call_system_under_test(self.diff_root_1, self.diff_root_2) self.expected_combined.uuid = combined.uuid - + self.expected_combined.partner_uuid = combined.partner_uuid assert combined == self.expected_combined async def test_early_conflict_removed(self): @@ -838,6 +856,7 @@ async def test_unchanged_parents_correctly_updated(self): relationships={expected_relationship}, ) self.expected_combined.uuid = combined.uuid + self.expected_combined.partner_uuid = combined.partner_uuid self.expected_combined.nodes = {expected_parent_node, expected_child_node} assert combined == self.expected_combined @@ -907,5 +926,6 @@ async def test_updated_parents_correctly_updated(self): relationships={expected_child_rel}, ) self.expected_combined.uuid = combined.uuid + self.expected_combined.partner_uuid = combined.partner_uuid self.expected_combined.nodes = {expected_parent_1, expected_parent_2, expected_child_node} assert combined == self.expected_combined diff --git a/backend/tests/unit/core/diff/test_diff_repository.py b/backend/tests/unit/core/diff/test_diff_repository.py index 733269f24f..2d87e20184 100644 --- a/backend/tests/unit/core/diff/test_diff_repository.py +++ b/backend/tests/unit/core/diff/test_diff_repository.py @@ -13,6 +13,7 @@ BranchTrackingId, EnrichedDiffNode, EnrichedDiffRoot, + EnrichedDiffs, NameTrackingId, ) from infrahub.core.diff.repository.deserializer import EnrichedDiffDeserializer @@ -98,6 +99,28 @@ def _build_nodes(self, num_nodes: int, num_sub_fields: int) -> set[EnrichedDiffN nodes_to_check.append(child_node) return all_nodes + async def _save_single_diff( + self, diff_repository: DiffRepository, enriched_diff: EnrichedDiffRoot + ) -> EnrichedDiffs: + base_diff = EnrichedRootFactory.build( + base_branch_name=enriched_diff.base_branch_name, + diff_branch_name=enriched_diff.base_branch_name, + from_time=enriched_diff.from_time, + to_time=enriched_diff.to_time, + nodes=self._build_nodes(num_nodes=2, num_sub_fields=1), + tracking_id=enriched_diff.tracking_id, + partner_uuid=enriched_diff.uuid, + ) + enriched_diff.partner_uuid = base_diff.uuid + enriched_diffs = EnrichedDiffs( + base_branch_name=self.base_branch_name, + diff_branch_name=self.diff_branch_name, + diff_branch_diff=enriched_diff, + base_branch_diff=base_diff, + ) + await diff_repository.save(enriched_diffs=enriched_diffs) + return enriched_diffs + async def test_get_non_existent_diff(self, diff_repository: DiffRepository, reset_database): right_now = Timestamp() enriched_diffs = await diff_repository.get( @@ -118,7 +141,7 @@ async def test_save_and_retrieve(self, diff_repository: DiffRepository, reset_da tracking_id=NameTrackingId(name="the-best-diff"), ) - await diff_repository.save(enriched_diff=enriched_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=enriched_diff) retrieved = await diff_repository.get( base_branch_name=self.base_branch_name, @@ -141,7 +164,7 @@ async def test_base_branch_name_filter(self, diff_repository: DiffRepository, re uuid=root_uuid, nodes={EnrichedNodeFactory.build(relationships={})}, ) - await diff_repository.save(enriched_diff=enriched_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=enriched_diff) retrieved = await diff_repository.get( base_branch_name=self.base_branch_name, @@ -171,7 +194,7 @@ async def test_diff_branch_name_filter(self, diff_repository: DiffRepository, re uuid=root_uuid, nodes={EnrichedNodeFactory.build(relationships={})}, ) - await diff_repository.save(enriched_diff=enriched_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=enriched_diff) start_time = DateTime.create(2024, 6, 15, 18, 35, 20, tz=UTC) end_time = start_time.add(months=1) @@ -205,7 +228,7 @@ async def test_filter_time_ranges(self, diff_repository: DiffRepository, reset_d uuid=root_uuid, nodes={EnrichedNodeFactory.build(relationships={})}, ) - await diff_repository.save(enriched_diff=enriched_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=enriched_diff) # both before retrieved = await diff_repository.get( @@ -269,7 +292,7 @@ async def test_filter_root_node_uuids(self, diff_repository: DiffRepository, res nodes=nodes, ) enriched_diffs.append(enriched_diff) - await diff_repository.save(enriched_diff=enriched_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=enriched_diff) parent_node = EnrichedNodeFactory.build() middle_parent_rel = EnrichedRelationshipGroupFactory.build(nodes={parent_node}) @@ -286,7 +309,7 @@ async def test_filter_root_node_uuids(self, diff_repository: DiffRepository, res to_time=Timestamp(self.diff_to_time), nodes=other_nodes | {parent_node, middle_node, leaf_node}, ) - await diff_repository.save(enriched_diff=this_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=this_diff) diff_branch_names = [e.diff_branch_name for e in enriched_diffs] + ["diff"] # get parent node @@ -566,7 +589,7 @@ async def test_save_and_retrieve_many_diffs(self, diff_repository: DiffRepositor to_time=Timestamp(start_time.add(minutes=(i * 30) + 29)), nodes=nodes, ) - await diff_repository.save(enriched_diff=enriched_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=enriched_diff) diffs_to_retrieve.append(enriched_diff) for i in range(5): nodes = self._build_nodes(num_nodes=3, num_sub_fields=2) @@ -577,7 +600,7 @@ async def test_save_and_retrieve_many_diffs(self, diff_repository: DiffRepositor to_time=Timestamp(start_time.add(days=3, minutes=(i * 30) + 29)), nodes=nodes, ) - await diff_repository.save(enriched_diff=enriched_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=enriched_diff) retrieved = await diff_repository.get( base_branch_name=self.base_branch_name, @@ -588,38 +611,6 @@ async def test_save_and_retrieve_many_diffs(self, diff_repository: DiffRepositor assert len(retrieved) == 5 assert set(retrieved) == set(diffs_to_retrieve) - async def test_retrieve_overlapping_diffs_excludes_duplicates( - self, diff_repository: DiffRepository, reset_database - ): - for i in range(5): - nodes = self._build_nodes(num_nodes=3, num_sub_fields=2) - incremental_enriched_diff = EnrichedRootFactory.build( - base_branch_name=self.base_branch_name, - diff_branch_name=self.diff_branch_name, - from_time=Timestamp(self.diff_from_time.add(minutes=i * 30)), - to_time=Timestamp(self.diff_from_time.add(minutes=(i * 30) + 29)), - nodes=nodes, - ) - await diff_repository.save(enriched_diff=incremental_enriched_diff) - nodes = self._build_nodes(num_nodes=3, num_sub_fields=2) - super_enriched_diff = EnrichedRootFactory.build( - base_branch_name=self.base_branch_name, - diff_branch_name=self.diff_branch_name, - from_time=Timestamp(self.diff_from_time), - to_time=Timestamp(self.diff_from_time.add(minutes=(4 * 30) + 29)), - nodes=nodes, - ) - await diff_repository.save(enriched_diff=super_enriched_diff) - - retrieved = await diff_repository.get( - base_branch_name=self.base_branch_name, - diff_branch_names=[self.diff_branch_name], - from_time=Timestamp(self.diff_from_time), - to_time=Timestamp(self.diff_from_time.add(minutes=(4 * 30) + 29)), - ) - assert len(retrieved) == 1 - assert retrieved[0] == super_enriched_diff - async def test_delete_diff_by_uuid(self, diff_repository: DiffRepository, reset_database): diffs: list[EnrichedDiffRoot] = [] start_time = self.diff_from_time.add(seconds=1) @@ -632,7 +623,7 @@ async def test_delete_diff_by_uuid(self, diff_repository: DiffRepository, reset_ to_time=Timestamp(start_time.add(minutes=(i * 30) + 29)), nodes=nodes, ) - await diff_repository.save(enriched_diff=enriched_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=enriched_diff) diffs.append(enriched_diff) diff_to_delete = diffs.pop() @@ -662,7 +653,7 @@ async def test_get_by_tracking_id(self, diff_repository: DiffRepository, reset_d to_time=Timestamp(end_time.add(minutes=(i * 30) + 29)), nodes=nodes, ) - await diff_repository.save(enriched_diff=enriched_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=enriched_diff) nodes = self._build_nodes(num_nodes=2, num_sub_fields=2) branch_tracked_diff = EnrichedRootFactory.build( base_branch_name=self.base_branch_name, @@ -672,7 +663,7 @@ async def test_get_by_tracking_id(self, diff_repository: DiffRepository, reset_d nodes=nodes, tracking_id=branch_tracking_id, ) - await diff_repository.save(enriched_diff=branch_tracked_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=branch_tracked_diff) name_tracked_diff = EnrichedRootFactory.build( base_branch_name=self.base_branch_name, diff_branch_name=self.diff_branch_name, @@ -681,7 +672,7 @@ async def test_get_by_tracking_id(self, diff_repository: DiffRepository, reset_d nodes=nodes, tracking_id=name_tracking_id, ) - await diff_repository.save(enriched_diff=name_tracked_diff) + await self._save_single_diff(diff_repository=diff_repository, enriched_diff=name_tracked_diff) retrieved_branch_diff = await diff_repository.get_one( tracking_id=branch_tracking_id, diff --git a/backend/tests/unit/message_bus/operations/event/test_branch.py b/backend/tests/unit/message_bus/operations/event/test_branch.py index 674e0bea88..e29a13bd54 100644 --- a/backend/tests/unit/message_bus/operations/event/test_branch.py +++ b/backend/tests/unit/message_bus/operations/event/test_branch.py @@ -49,6 +49,7 @@ async def test_merged(default_branch: Branch): from_time=right_now, to_time=right_now, uuid=str(uuid4()), + partner_uuid=str(uuid4()), tracking_id=BranchTrackingId(name=str(uuid4())), ) for _ in range(2) @@ -60,6 +61,7 @@ async def test_merged(default_branch: Branch): from_time=right_now, to_time=right_now, uuid=str(uuid4()), + partner_uuid=str(uuid4()), ) for _ in range(2) ] @@ -99,6 +101,7 @@ async def test_rebased(default_branch: Branch): from_time=right_now, to_time=right_now, uuid=str(uuid4()), + partner_uuid=str(uuid4()), ) for _ in range(2) ] diff --git a/changelog/4438.fixed.md b/changelog/4438.fixed.md new file mode 100644 index 0000000000..fc954f7f6e --- /dev/null +++ b/changelog/4438.fixed.md @@ -0,0 +1,4 @@ +Updates internal logic to improve performance when generating a diff. + +BREAKING CHANGE: Diff data, including conflict selections, will be deleted. We recommend merging +any outstanding proposed changes before upgrading to this version. \ No newline at end of file