diff --git a/backend/infrahub/core/diff/enricher/cardinality_one.py b/backend/infrahub/core/diff/enricher/cardinality_one.py index a80b76473e..3d1917cf67 100644 --- a/backend/infrahub/core/diff/enricher/cardinality_one.py +++ b/backend/infrahub/core/diff/enricher/cardinality_one.py @@ -3,6 +3,7 @@ from infrahub.core.constants import NULL_VALUE, DiffAction, RelationshipCardinality from infrahub.core.constants.database import DatabaseEdgeType from infrahub.database import InfrahubDatabase +from infrahub.log import get_logger from ..model.path import ( CalculatedDiffs, @@ -16,6 +17,8 @@ if TYPE_CHECKING: from infrahub.core.schema import MainSchemaTypes +log = get_logger() + class DiffCardinalityOneEnricher(DiffEnricherInterface): """Clean up diffs for cardinality=one relationships to make them cleaner and more intuitive @@ -34,6 +37,7 @@ def __init__(self, db: InfrahubDatabase): async def enrich(self, enriched_diff_root: EnrichedDiffRoot, calculated_diffs: CalculatedDiffs) -> None: self._node_schema_map = {} + log.info("Beginning cardinality-one diff enrichment...") for diff_node in enriched_diff_root.nodes: for relationship_group in diff_node.relationships: if ( @@ -41,6 +45,7 @@ async def enrich(self, enriched_diff_root: EnrichedDiffRoot, calculated_diffs: C and len(relationship_group.relationships) > 0 ): self.consolidate_cardinality_one_diff_elements(diff_relationship=relationship_group) + log.info("Cardinality-one diff enrichment complete.") def _determine_action(self, previous_value: Any, new_value: Any) -> DiffAction: if previous_value == new_value: diff --git a/backend/infrahub/core/diff/enricher/hierarchy.py b/backend/infrahub/core/diff/enricher/hierarchy.py index a75ba7f46a..7af0ecd22c 100644 --- a/backend/infrahub/core/diff/enricher/hierarchy.py +++ b/backend/infrahub/core/diff/enricher/hierarchy.py @@ -7,19 +7,24 @@ from infrahub.core.query.relationship import RelationshipGetPeerQuery, RelationshipPeerData from infrahub.core.schema import ProfileSchema from infrahub.database import InfrahubDatabase +from infrahub.log import get_logger from ..model.path import ( CalculatedDiffs, EnrichedDiffRoot, ) +from ..parent_node_adder import DiffParentNodeAdder, ParentNodeAddRequest from .interface import DiffEnricherInterface +log = get_logger() + class DiffHierarchyEnricher(DiffEnricherInterface): """Add hierarchy and parent/component nodes to diff even if the higher-level nodes are unchanged""" - def __init__(self, db: InfrahubDatabase): + def __init__(self, db: InfrahubDatabase, parent_adder: DiffParentNodeAdder): self.db = db + self.parent_adder = parent_adder async def enrich( self, enriched_diff_root: EnrichedDiffRoot, calculated_diffs: CalculatedDiffs | None = None @@ -28,6 +33,8 @@ async def enrich( # - A node has a relationship of kind parent # - A node is part of a hierarchy + log.info("Beginning hierarchical diff enrichment...") + self.parent_adder.initialize(enriched_diff_root=enriched_diff_root) node_rel_parent_map: dict[str, list[str]] = defaultdict(list) node_hierarchy_map: dict[str, list[str]] = defaultdict(list) @@ -53,6 +60,7 @@ async def enrich( await self._enrich_nodes_with_parent(enriched_diff_root=enriched_diff_root, node_map=node_rel_parent_map) await self._enrich_hierarchical_nodes(enriched_diff_root=enriched_diff_root, node_map=node_hierarchy_map) + log.info("Hierarchical diff enrichment complete.") async def _enrich_hierarchical_nodes( self, @@ -63,6 +71,7 @@ async def _enrich_hierarchical_nodes( # Retrieve the ID of all ancestors for kind, node_ids in node_map.items(): + log.info(f"Beginning hierarchy enrichment for {kind} node, num_nodes={len(node_ids)}...") hierarchy_schema = self.db.schema.get( name=kind, branch=enriched_diff_root.diff_branch_name, duplicate=False ) @@ -87,7 +96,7 @@ async def _enrich_hierarchical_nodes( current_node = node for ancestor in ancestors: - parent = enriched_diff_root.add_parent( + parent_request = ParentNodeAddRequest( node_id=current_node.uuid, parent_id=str(ancestor.uuid), parent_kind=ancestor.kind, @@ -97,6 +106,7 @@ async def _enrich_hierarchical_nodes( parent_rel_cardinality=parent_rel.cardinality, parent_rel_label=parent_rel.label or "", ) + parent = self.parent_adder.add_parent(parent_request=parent_request) current_node = parent @@ -114,6 +124,7 @@ async def _enrich_nodes_with_parent( # Query the UUID of the parent for kind, ids in node_map.items(): + log.info(f"Beginning parent enrichment for {kind} node, num_nodes={len(ids)}...") schema_node = self.db.schema.get(name=kind, branch=enriched_diff_root.diff_branch_name, duplicate=False) parent_rel = [rel for rel in schema_node.relationships if rel.kind == RelationshipKind.PARENT][0] @@ -138,15 +149,16 @@ async def _enrich_nodes_with_parent( # Check if the parent are already present # If parent is already in the list of node we need to add a relationship # If parent is not in the list of node, we need to add it + diff_node_map = enriched_diff_root.get_node_map(node_uuids=set(parent_peers.keys())) for node_id, peer_parent in parent_peers.items(): # TODO check if we can optimize this part to avoid querying this multiple times - node = enriched_diff_root.get_node(node_uuid=node_id) + node = diff_node_map[node_id] schema_node = self.db.schema.get( name=node.kind, branch=enriched_diff_root.diff_branch_name, duplicate=False ) parent_rel = [rel for rel in schema_node.relationships if rel.kind == RelationshipKind.PARENT][0] - enriched_diff_root.add_parent( + parent_request = ParentNodeAddRequest( node_id=node.uuid, parent_id=str(peer_parent.peer_id), parent_kind=peer_parent.peer_kind, @@ -156,6 +168,7 @@ async def _enrich_nodes_with_parent( parent_rel_cardinality=parent_rel.cardinality, parent_rel_label=parent_rel.label or "", ) + self.parent_adder.add_parent(parent_request=parent_request) if node_parent_with_parent_map: await self._enrich_nodes_with_parent( diff --git a/backend/infrahub/core/diff/enricher/labels.py b/backend/infrahub/core/diff/enricher/labels.py index bf39bb9c16..b281a5df08 100644 --- a/backend/infrahub/core/diff/enricher/labels.py +++ b/backend/infrahub/core/diff/enricher/labels.py @@ -6,6 +6,7 @@ from infrahub.core.constants.database import DatabaseEdgeType from infrahub.core.query.node import NodeGetKindQuery from infrahub.database import InfrahubDatabase +from infrahub.log import get_logger from ..model.path import ( CalculatedDiffs, @@ -17,6 +18,8 @@ from ..payload_builder import get_display_labels from .interface import DiffEnricherInterface +log = get_logger() + PROPERTY_TYPES_WITH_LABELS = {DatabaseEdgeType.IS_RELATED, DatabaseEdgeType.HAS_OWNER, DatabaseEdgeType.HAS_SOURCE} @@ -194,6 +197,7 @@ async def enrich( calculated_diffs: CalculatedDiffs | None = None, conflicts_only: bool = False, ) -> None: + log.info("Beginning display labels diff enrichment...") self._base_branch_name = enriched_diff_root.base_branch_name self._diff_branch_name = enriched_diff_root.diff_branch_name self._conflicts_only = conflicts_only @@ -214,3 +218,4 @@ async def enrich( ... self._update_relationship_labels(enriched_diff=enriched_diff_root) + log.info("Display labels diff enrichment complete.") diff --git a/backend/infrahub/core/diff/enricher/path_identifier.py b/backend/infrahub/core/diff/enricher/path_identifier.py index 2d21b77522..b1591570e3 100644 --- a/backend/infrahub/core/diff/enricher/path_identifier.py +++ b/backend/infrahub/core/diff/enricher/path_identifier.py @@ -1,10 +1,13 @@ from infrahub.core.constants import PathType from infrahub.core.path import DataPath from infrahub.database import InfrahubDatabase +from infrahub.log import get_logger from ..model.path import CalculatedDiffs, EnrichedDiffRoot from .interface import DiffEnricherInterface +log = get_logger() + class DiffPathIdentifierEnricher(DiffEnricherInterface): """Add path identifiers to every element in the diff""" @@ -20,6 +23,7 @@ def diff_branch_name(self) -> str: return self._diff_branch_name async def enrich(self, enriched_diff_root: EnrichedDiffRoot, calculated_diffs: CalculatedDiffs) -> None: + log.info("Beginning path identifier diff enrichment...") self._diff_branch_name = enriched_diff_root.diff_branch_name for node in enriched_diff_root.nodes: node_path = DataPath( @@ -62,3 +66,4 @@ async def enrich(self, enriched_diff_root: EnrichedDiffRoot, calculated_diffs: C relationship_property_path = relationship_element_path.model_copy() relationship_property_path.property_name = relationship_property.property_type.value relationship_property.path_identifier = relationship_property_path.get_path(with_peer=False) + log.info("Path identifier diff enrichment complete.") diff --git a/backend/infrahub/core/diff/model/path.py b/backend/infrahub/core/diff/model/path.py index 9aa59628c9..3188785222 100644 --- a/backend/infrahub/core/diff/model/path.py +++ b/backend/infrahub/core/diff/model/path.py @@ -486,6 +486,13 @@ def has_node(self, node_uuid: str) -> bool: except ValueError: return False + def get_node_map(self, node_uuids: set[str] | None = None) -> dict[str, EnrichedDiffNode]: + node_map = {} + for node in self.nodes: + if node_uuids is None or node.uuid in node_uuids: + node_map[node.uuid] = node + return node_map + def get_all_conflicts(self) -> dict[str, EnrichedDiffConflict]: all_conflicts: dict[str, EnrichedDiffConflict] = {} for node in self.nodes: diff --git a/backend/infrahub/core/diff/parent_node_adder.py b/backend/infrahub/core/diff/parent_node_adder.py new file mode 100644 index 0000000000..32e9be838f --- /dev/null +++ b/backend/infrahub/core/diff/parent_node_adder.py @@ -0,0 +1,78 @@ +from dataclasses import dataclass, field + +from infrahub.core.constants import DiffAction, RelationshipCardinality + +from .model.path import EnrichedDiffNode, EnrichedDiffRelationship, EnrichedDiffRoot + + +@dataclass +class ParentNodeAddRequest: + node_id: str + parent_id: str + parent_kind: str + parent_label: str + parent_rel_name: str + parent_rel_identifier: str + parent_rel_cardinality: RelationshipCardinality + parent_rel_label: str = field(default="") + + +class DiffParentNodeAdder: + def __init__(self) -> None: + self._diff_root: EnrichedDiffRoot | None = None + self._node_map: dict[str, EnrichedDiffNode] = {} + + def initialize(self, enriched_diff_root: EnrichedDiffRoot) -> None: + self._diff_root = enriched_diff_root + self._node_map = enriched_diff_root.get_node_map() + + def get_root(self) -> EnrichedDiffRoot: + if not self._diff_root: + raise RuntimeError("Must call initialize before using") + return self._diff_root + + def get_node(self, node_uuid: str) -> EnrichedDiffNode: + return self._node_map[node_uuid] + + def has_node(self, node_uuid: str) -> bool: + return node_uuid in self._node_map + + def add_node(self, node: EnrichedDiffNode) -> None: + if node.uuid in self._node_map: + return + self._node_map[node.uuid] = node + self.get_root().nodes.add(node) + + def add_parent(self, parent_request: ParentNodeAddRequest) -> EnrichedDiffNode: + if not self._diff_root: + raise RuntimeError("Must call initialize before using") + node = self.get_node(node_uuid=parent_request.node_id) + if not self.has_node(node_uuid=parent_request.parent_id): + parent = EnrichedDiffNode( + uuid=parent_request.parent_id, + kind=parent_request.parent_kind, + label=parent_request.parent_label, + action=DiffAction.UNCHANGED, + changed_at=None, + ) + self.add_node(parent) + else: + parent = self.get_node(node_uuid=parent_request.parent_id) + + try: + rel = node.get_relationship(name=parent_request.parent_rel_name) + rel.nodes.add(parent) + except ValueError: + node.relationships.add( + EnrichedDiffRelationship( + name=parent_request.parent_rel_name, + identifier=parent_request.parent_rel_identifier, + label=parent_request.parent_rel_label, + cardinality=parent_request.parent_rel_cardinality, + changed_at=None, + action=DiffAction.UNCHANGED, + nodes={parent}, + ) + ) + + return parent diff --git a/backend/infrahub/core/diff/payload_builder.py b/backend/infrahub/core/diff/payload_builder.py index 4b918a264a..cdd7bf8ce5 100644 --- a/backend/infrahub/core/diff/payload_builder.py +++ b/backend/infrahub/core/diff/payload_builder.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING +from infrahub import config from infrahub.core.manager import NodeManager from infrahub.core.registry import registry from infrahub.exceptions import SchemaNotFoundError @@ -26,8 +27,18 @@ async def get_display_labels_per_kind( if skip_missing_schema: return {} raise - nodes = await NodeManager.get_many(ids=ids, fields=fields, db=db, branch=branch) - return {node_id: await node.render_display_label(db=db) for node_id, node in nodes.items()} + display_label_map: dict[str, str] = {} + offset = 0 + limit = config.SETTINGS.database.query_size_limit + while True: + limited_ids = ids[offset : offset + limit] + if not limited_ids: + break + node_map = await NodeManager.get_many(ids=limited_ids, fields=fields, db=db, branch=branch) + for node_id, node in node_map.items(): + display_label_map[node_id] = await node.render_display_label(db=db) + offset += limit + return display_label_map async def get_display_labels(nodes: dict[str, dict[str, list[str]]], db: InfrahubDatabase) -> dict[str, dict[str, str]]: diff --git a/backend/infrahub/core/diff/query/save.py b/backend/infrahub/core/diff/query/save.py index 19780b5e8b..742a7c11f8 100644 --- a/backend/infrahub/core/diff/query/save.py +++ b/backend/infrahub/core/diff/query/save.py @@ -1,4 +1,4 @@ -from typing import Any +from typing import Any, Iterable from infrahub.core.query import Query, QueryType from infrahub.database import InfrahubDatabase @@ -82,9 +82,12 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None: self.params = self._build_node_batch_params() query = """ UNWIND $node_details_list AS node_details -WITH node_details.root_uuid AS root_uuid, node_details.node_map AS node_map -MATCH (diff_root:DiffRoot {uuid: root_uuid}) -MERGE (diff_root)-[:DIFF_HAS_NODE]->(diff_node:DiffNode {uuid: node_map.node_properties.uuid}) +WITH + node_details.root_uuid AS root_uuid, + node_details.node_map AS node_map, + toString(node_details.node_map.node_properties.uuid) AS node_uuid +MERGE (diff_root:DiffRoot {uuid: root_uuid}) +MERGE (diff_root)-[:DIFF_HAS_NODE]->(diff_node:DiffNode {uuid: node_uuid}) WITH root_uuid, node_map, diff_node, (node_map.conflict_params IS NOT NULL) AS has_node_conflict SET diff_node.kind = node_map.node_properties.kind, @@ -148,138 +151,131 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None: // ------------------------- // add attributes for this node // ------------------------- - CALL { - WITH diff_node, node_map - UNWIND node_map.attributes AS node_attribute - MERGE (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(diff_attribute:DiffAttribute {name: node_attribute.node_properties.name}) - SET diff_attribute = node_attribute.node_properties - // ------------------------- - // add properties for this attribute - // ------------------------- - WITH diff_attribute, node_attribute - // ------------------------- - // remove stale properties for this attribute - // ------------------------- - CALL { - WITH diff_attribute, node_attribute - WITH diff_attribute, %(attr_props_list_comp)s AS prop_types - OPTIONAL MATCH (diff_attribute)-[:DIFF_HAS_PROPERTY]->(prop_to_delete:DiffProperty) - WHERE NOT (prop_to_delete.property_type IN prop_types) - OPTIONAL MATCH (prop_to_delete)-[*..4]->(next_to_delete) - DETACH DELETE next_to_delete - DETACH DELETE prop_to_delete - } - UNWIND node_attribute.properties AS attr_property - MERGE (diff_attribute)-[:DIFF_HAS_PROPERTY]->(diff_attr_prop:DiffProperty {property_type: attr_property.node_properties.property_type}) - SET diff_attr_prop = attr_property.node_properties - // ------------------------- - // add/remove conflict for this property - // ------------------------- - WITH diff_attr_prop, attr_property - OPTIONAL MATCH (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(current_attr_prop_conflict:DiffConflict) - WITH diff_attr_prop, attr_property, current_attr_prop_conflict, (attr_property.conflict_params IS NOT NULL) AS has_prop_conflict - FOREACH (i in CASE WHEN has_prop_conflict = FALSE THEN [1] ELSE [] END | - DETACH DELETE current_attr_prop_conflict - ) - FOREACH (i in CASE WHEN has_prop_conflict = TRUE THEN [1] ELSE [] END | - MERGE (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(diff_attr_prop_conflict:DiffConflict) - SET diff_attr_prop_conflict = attr_property.conflict_params - ) - } + UNWIND node_map.attributes AS node_attribute + MERGE (diff_node)-[:DIFF_HAS_ATTRIBUTE]->(diff_attribute:DiffAttribute {name: node_attribute.node_properties.name}) + SET diff_attribute = node_attribute.node_properties // ------------------------- - // remove stale relationships for this node + // add properties for this attribute + // ------------------------- + WITH diff_attribute, node_attribute + // ------------------------- + // remove stale properties for this attribute // ------------------------- - WITH diff_node, node_map CALL { - WITH diff_node, node_map - WITH diff_node, %(rel_name_list_comp)s AS rel_names - OPTIONAL MATCH (diff_node)-[:DIFF_HAS_RELATIONSHIP]->(rel_to_delete:DiffRelationship) - WHERE NOT (rel_to_delete.name IN rel_names) - OPTIONAL MATCH (rel_to_delete)-[*..8]->(next_to_delete) + WITH diff_attribute, node_attribute + WITH diff_attribute, %(attr_props_list_comp)s AS prop_types + OPTIONAL MATCH (diff_attribute)-[:DIFF_HAS_PROPERTY]->(prop_to_delete:DiffProperty) + WHERE NOT (prop_to_delete.property_type IN prop_types) + OPTIONAL MATCH (prop_to_delete)-[*..4]->(next_to_delete) DETACH DELETE next_to_delete - DETACH DELETE rel_to_delete + DETACH DELETE prop_to_delete } + UNWIND node_attribute.properties AS attr_property + MERGE (diff_attribute)-[:DIFF_HAS_PROPERTY]->(diff_attr_prop:DiffProperty {property_type: attr_property.node_properties.property_type}) + SET diff_attr_prop = attr_property.node_properties // ------------------------- - // add relationships for this node + // add/remove conflict for this property // ------------------------- + WITH diff_attr_prop, attr_property + OPTIONAL MATCH (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(current_attr_prop_conflict:DiffConflict) + WITH diff_attr_prop, attr_property, current_attr_prop_conflict, (attr_property.conflict_params IS NOT NULL) AS has_prop_conflict + FOREACH (i in CASE WHEN has_prop_conflict = FALSE THEN [1] ELSE [] END | + DETACH DELETE current_attr_prop_conflict + ) + FOREACH (i in CASE WHEN has_prop_conflict = TRUE THEN [1] ELSE [] END | + MERGE (diff_attr_prop)-[:DIFF_HAS_CONFLICT]->(diff_attr_prop_conflict:DiffConflict) + SET diff_attr_prop_conflict = attr_property.conflict_params + ) +} +// ------------------------- +// remove stale relationships for this node +// ------------------------- +CALL { WITH diff_node, node_map - CALL { - WITH diff_node, node_map - UNWIND node_map.relationships as node_relationship - MERGE (diff_node)-[:DIFF_HAS_RELATIONSHIP]->(diff_relationship:DiffRelationship {name: node_relationship.node_properties.name}) - SET diff_relationship = node_relationship.node_properties - // ------------------------- - // remove stale elements for this relationship group - // ------------------------- - WITH diff_relationship, node_relationship - CALL { - WITH diff_relationship, node_relationship - WITH diff_relationship, %(rel_peers_list_comp)s AS rel_peers - OPTIONAL MATCH (diff_relationship)-[:DIFF_HAS_ELEMENT]->(element_to_delete:DiffRelationshipElement) - WHERE NOT (element_to_delete.peer_id IN rel_peers) - OPTIONAL MATCH (element_to_delete)-[*..6]->(next_to_delete) - DETACH DELETE next_to_delete - DETACH DELETE element_to_delete - } - // ------------------------- - // add elements for this relationship group - // ------------------------- - WITH diff_relationship, node_relationship - UNWIND node_relationship.relationships as node_single_relationship - MERGE (diff_relationship)-[:DIFF_HAS_ELEMENT] - ->(diff_relationship_element:DiffRelationshipElement {peer_id: node_single_relationship.node_properties.peer_id}) - SET diff_relationship_element = node_single_relationship.node_properties - // ------------------------- - // add/remove conflict for this relationship element - // ------------------------- - WITH diff_relationship_element, node_single_relationship - OPTIONAL MATCH (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(current_element_conflict:DiffConflict) - WITH diff_relationship_element, node_single_relationship, current_element_conflict, - (node_single_relationship.conflict_params IS NOT NULL) AS has_element_conflict - FOREACH (i in CASE WHEN has_element_conflict = FALSE THEN [1] ELSE [] END | - DETACH DELETE current_element_conflict - ) - FOREACH (i in CASE WHEN has_element_conflict = TRUE THEN [1] ELSE [] END | - MERGE (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(element_conflict:DiffConflict) - SET element_conflict = node_single_relationship.conflict_params - ) - // ------------------------- - // remove stale properties for this relationship element - // ------------------------- - WITH diff_relationship_element, node_single_relationship - CALL { - WITH diff_relationship_element, node_single_relationship - WITH diff_relationship_element, %(element_props_list_comp)s AS element_props - OPTIONAL MATCH (diff_relationship_element)-[:DIFF_HAS_PROPERTY]->(property_to_delete:DiffProperty) - WHERE NOT (property_to_delete.property_type IN element_props) - OPTIONAL MATCH (property_to_delete)-[*..4]->(next_to_delete) - DETACH DELETE next_to_delete - DETACH DELETE property_to_delete - } - // ------------------------- - // add properties for this relationship element - // ------------------------- - WITH diff_relationship_element, node_single_relationship - UNWIND node_single_relationship.properties as node_relationship_property - MERGE (diff_relationship_element)-[:DIFF_HAS_PROPERTY] - ->(diff_relationship_property:DiffProperty {property_type: node_relationship_property.node_properties.property_type}) - SET diff_relationship_property = node_relationship_property.node_properties - // ------------------------- - // add conflict for this relationship element - // ------------------------- - WITH diff_relationship_property, node_relationship_property - OPTIONAL MATCH (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(diff_relationship_property_conflict:DiffConflict) - WITH diff_relationship_property, node_relationship_property, diff_relationship_property_conflict, - (node_relationship_property.conflict_params IS NOT NULL) AS has_property_conflict - FOREACH (i in CASE WHEN has_property_conflict = FALSE THEN [1] ELSE [] END | - DETACH DELETE diff_relationship_property_conflict - ) - FOREACH (i in CASE WHEN has_property_conflict = TRUE THEN [1] ELSE [] END | - MERGE (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(property_conflict:DiffConflict) - SET property_conflict = node_relationship_property.conflict_params - ) - } + WITH diff_node, %(rel_name_list_comp)s AS rel_names + OPTIONAL MATCH (diff_node)-[:DIFF_HAS_RELATIONSHIP]->(rel_to_delete:DiffRelationship) + WHERE NOT (rel_to_delete.name IN rel_names) + OPTIONAL MATCH (rel_to_delete)-[*..8]->(next_to_delete) + DETACH DELETE next_to_delete + DETACH DELETE rel_to_delete +} +// ------------------------- +// add relationships for this node +// ------------------------- +WITH diff_node, node_map +UNWIND node_map.relationships as node_relationship +MERGE (diff_node)-[:DIFF_HAS_RELATIONSHIP]->(diff_relationship:DiffRelationship {name: node_relationship.node_properties.name}) +SET diff_relationship = node_relationship.node_properties +// ------------------------- +// remove stale elements for this relationship group +// ------------------------- +WITH diff_relationship, node_relationship +CALL { + WITH diff_relationship, node_relationship + WITH diff_relationship, %(rel_peers_list_comp)s AS rel_peers + OPTIONAL MATCH (diff_relationship)-[:DIFF_HAS_ELEMENT]->(element_to_delete:DiffRelationshipElement) + WHERE NOT (element_to_delete.peer_id IN rel_peers) + OPTIONAL MATCH (element_to_delete)-[*..6]->(next_to_delete) + DETACH DELETE next_to_delete + DETACH DELETE element_to_delete +} +// ------------------------- +// add elements for this relationship group +// ------------------------- +WITH diff_relationship, node_relationship +UNWIND node_relationship.relationships as node_single_relationship +MERGE (diff_relationship)-[:DIFF_HAS_ELEMENT] + ->(diff_relationship_element:DiffRelationshipElement {peer_id: node_single_relationship.node_properties.peer_id}) +SET diff_relationship_element = node_single_relationship.node_properties +// ------------------------- +// add/remove conflict for this relationship element +// ------------------------- +WITH diff_relationship_element, node_single_relationship +OPTIONAL MATCH (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(current_element_conflict:DiffConflict) +WITH diff_relationship_element, node_single_relationship, current_element_conflict, + (node_single_relationship.conflict_params IS NOT NULL) AS has_element_conflict +FOREACH (i in CASE WHEN has_element_conflict = FALSE THEN [1] ELSE [] END | + DETACH DELETE current_element_conflict +) +FOREACH (i in CASE WHEN has_element_conflict = TRUE THEN [1] ELSE [] END | + MERGE (diff_relationship_element)-[:DIFF_HAS_CONFLICT]->(element_conflict:DiffConflict) + SET element_conflict = node_single_relationship.conflict_params +) +// ------------------------- +// remove stale properties for this relationship element +// ------------------------- +WITH diff_relationship_element, node_single_relationship +CALL { + WITH diff_relationship_element, node_single_relationship + WITH diff_relationship_element, %(element_props_list_comp)s AS element_props + OPTIONAL MATCH (diff_relationship_element)-[:DIFF_HAS_PROPERTY]->(property_to_delete:DiffProperty) + WHERE NOT (property_to_delete.property_type IN element_props) + OPTIONAL MATCH (property_to_delete)-[*..4]->(next_to_delete) + DETACH DELETE next_to_delete + DETACH DELETE property_to_delete } +// ------------------------- +// add properties for this relationship element +// ------------------------- +WITH diff_relationship_element, node_single_relationship +UNWIND node_single_relationship.properties as node_relationship_property +MERGE (diff_relationship_element)-[:DIFF_HAS_PROPERTY] + ->(diff_relationship_property:DiffProperty {property_type: node_relationship_property.node_properties.property_type}) +SET diff_relationship_property = node_relationship_property.node_properties +// ------------------------- +// add conflict for this relationship element +// ------------------------- +WITH diff_relationship_property, node_relationship_property +OPTIONAL MATCH (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(diff_relationship_property_conflict:DiffConflict) +WITH diff_relationship_property, node_relationship_property, diff_relationship_property_conflict, + (node_relationship_property.conflict_params IS NOT NULL) AS has_property_conflict +FOREACH (i in CASE WHEN has_property_conflict = FALSE THEN [1] ELSE [] END | + DETACH DELETE diff_relationship_property_conflict +) +FOREACH (i in CASE WHEN has_property_conflict = TRUE THEN [1] ELSE [] END | + MERGE (diff_relationship_property)-[:DIFF_HAS_CONFLICT]->(property_conflict:DiffConflict) + SET property_conflict = node_relationship_property.conflict_params +) """ % { "attr_name_list_comp": db.render_list_comprehension( items="node_map.attributes", item_name="node_properties.name" @@ -432,47 +428,34 @@ class EnrichedNodesLinkQuery(Query): type = QueryType.WRITE insert_return = False - def __init__(self, enriched_diffs: EnrichedDiffs, **kwargs: Any) -> None: + def __init__(self, diff_root_uuid: str, diff_nodes: Iterable[EnrichedDiffNode], **kwargs: Any) -> None: super().__init__(**kwargs) - self.enriched_diffs = enriched_diffs + self.diff_root_uuid = diff_root_uuid + self.diff_nodes = diff_nodes async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None: - parent_links_list = [] - for diff_root in (self.enriched_diffs.base_branch_diff, self.enriched_diffs.diff_branch_diff): - for node in diff_root.nodes: - parent_links_list.extend(self._build_node_parent_links(enriched_node=node, root_uuid=diff_root.uuid)) - self.params = {"node_links_list": parent_links_list} + parent_node_map: dict[str, dict[str, str]] = {} + for diff_node in self.diff_nodes: + if diff_node.uuid not in parent_node_map: + parent_node_map[diff_node.uuid] = {} + for relationship in diff_node.relationships: + for parent_node in relationship.nodes: + parent_node_map[diff_node.uuid][relationship.name] = parent_node.uuid + self.params = {"root_uuid": self.diff_root_uuid, "parent_node_map": parent_node_map} query = """ -UNWIND $node_links_list AS node_link_details -WITH - node_link_details.root_uuid AS root_uuid, - node_link_details.parent_uuid AS parent_uuid, - node_link_details.child_uuid AS child_uuid, - node_link_details.child_relationship_name AS relationship_name +WITH keys($parent_node_map) AS child_node_uuids +MATCH (diff_root:DiffRoot {uuid: $root_uuid}) +MATCH (diff_root)-[:DIFF_HAS_NODE]->(child_node:DiffNode) +WHERE child_node.uuid IN child_node_uuids CALL { - WITH root_uuid, parent_uuid, child_uuid, relationship_name - MATCH (diff_root {uuid: root_uuid}) - MATCH (diff_root)-[:DIFF_HAS_NODE]->(child_node:DiffNode {uuid: child_uuid}) - -[:DIFF_HAS_RELATIONSHIP]->(diff_rel_group:DiffRelationship {name: relationship_name}) + WITH diff_root, child_node + WITH diff_root, child_node, $parent_node_map[child_node.uuid] AS sub_map + WITH diff_root, child_node, sub_map, keys(sub_map) AS relationship_names + MATCH (child_node)-[:DIFF_HAS_RELATIONSHIP]->(diff_rel_group:DiffRelationship) + WHERE diff_rel_group.name IN relationship_names + WITH diff_root, diff_rel_group, toString(sub_map[diff_rel_group.name]) AS parent_uuid MATCH (diff_root)-[:DIFF_HAS_NODE]->(parent_node:DiffNode {uuid: parent_uuid}) MERGE (diff_rel_group)-[:DIFF_HAS_NODE]->(parent_node) } """ self.add_to_query(query) - - def _build_node_parent_links(self, enriched_node: EnrichedDiffNode, root_uuid: str) -> list[dict[str, str]]: - if not enriched_node.relationships: - return [] - parent_links = [] - for relationship in enriched_node.relationships: - for parent_node in relationship.nodes: - parent_links.append( - { - "child_uuid": enriched_node.uuid, - "child_relationship_name": relationship.name, - "parent_uuid": parent_node.uuid, - "root_uuid": root_uuid, - } - ) - parent_links.extend(self._build_node_parent_links(enriched_node=parent_node, root_uuid=root_uuid)) - return parent_links diff --git a/backend/infrahub/core/diff/query/summary_counts_enricher.py b/backend/infrahub/core/diff/query/summary_counts_enricher.py index 864fc5c292..209c93b9fa 100644 --- a/backend/infrahub/core/diff/query/summary_counts_enricher.py +++ b/backend/infrahub/core/diff/query/summary_counts_enricher.py @@ -6,10 +6,10 @@ from ..model.path import TrackingId -class DiffSummaryCountsEnricherQuery(Query): - """Update summary counters for a given diff""" +class DiffFieldsSummaryCountsEnricherQuery(Query): + """Update summary counters for the attributes and relationshipsin in a diff""" - name = "diff_summary_count_enricher" + name = "diff_fields_summary_count_enricher" type = QueryType.WRITE insert_return = False @@ -23,7 +23,9 @@ def __init__( ) -> None: super().__init__(**kwargs) if (diff_id is None and tracking_id is None) or (diff_id and tracking_id): - raise ValueError("EnrichedDiffAllConflictsQuery requires one and only one of `tracking_id` or `diff_id`") + raise ValueError( + "DiffFieldsSummaryCountsEnricherQuery requires one and only one of `tracking_id` or `diff_id`" + ) self.diff_branch_name = diff_branch_name self.tracking_id = tracking_id self.diff_id = diff_id @@ -138,6 +140,51 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None: SET dr.num_removed = num_removed } } + """ + self.add_to_query(query) + + +class DiffNodesSummaryCountsEnricherQuery(Query): + """Update summary counters for the nodes and root in a diff""" + + name = "diff_nodes_summary_count_enricher" + type = QueryType.WRITE + insert_return = False + + def __init__( + self, + diff_branch_name: str, + tracking_id: TrackingId | None = None, + diff_id: str | None = None, + node_uuids: list[str] | None = None, + **kwargs: Any, + ) -> None: + super().__init__(**kwargs) + if (diff_id is None and tracking_id is None) or (diff_id and tracking_id): + raise ValueError( + "DiffNodesSummaryCountsEnricherQuery requires one and only one of `tracking_id` or `diff_id`" + ) + self.diff_branch_name = diff_branch_name + self.tracking_id = tracking_id + self.diff_id = diff_id + if self.tracking_id is None and self.diff_id is None: + raise RuntimeError("tracking_id or diff_id is required") + self.node_uuids = node_uuids + + async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None: + self.params = { + "diff_branch_name": self.diff_branch_name, + "diff_id": self.diff_id, + "tracking_id": self.tracking_id.serialize() if self.tracking_id else None, + "node_uuids": self.node_uuids, + } + + query = """ +MATCH (root:DiffRoot) +WHERE ($diff_id IS NOT NULL AND root.uuid = $diff_id) +OR ($tracking_id IS NOT NULL AND root.tracking_id = $tracking_id AND root.diff_branch = $diff_branch_name) +MATCH (root)-[:DIFF_HAS_NODE]->(dn:DiffNode) +WHERE $node_uuids IS NULL OR dn.uuid IN $node_uuids // ---------------------- // handle node count updates // ---------------------- diff --git a/backend/infrahub/core/diff/repository/deserializer.py b/backend/infrahub/core/diff/repository/deserializer.py index 50a3a3e6a8..52515f832e 100644 --- a/backend/infrahub/core/diff/repository/deserializer.py +++ b/backend/infrahub/core/diff/repository/deserializer.py @@ -18,10 +18,12 @@ EnrichedDiffSingleRelationship, deserialize_tracking_id, ) +from ..parent_node_adder import DiffParentNodeAdder, ParentNodeAddRequest class EnrichedDiffDeserializer: - def __init__(self) -> None: + def __init__(self, parent_adder: DiffParentNodeAdder) -> None: + self.parent_adder = parent_adder self._diff_root_map: dict[str, EnrichedDiffRoot] = {} self._diff_node_map: dict[tuple[str, str], EnrichedDiffNode] = {} self._diff_node_attr_map: dict[tuple[str, str, str], EnrichedDiffAttribute] = {} @@ -127,6 +129,7 @@ def _deserialize_relationships( def _deserialize_parents(self) -> None: for enriched_root, node_path_tuples in self._parents_path_map.items(): + self.parent_adder.initialize(enriched_diff_root=enriched_root) for node_uuid, parents_path in node_path_tuples: # Remove the node itself from the path parents_path_slice = parents_path.nodes[1:] @@ -134,7 +137,7 @@ def _deserialize_parents(self) -> None: # TODO Ensure the list is even current_node_uuid = node_uuid for rel, parent in zip(parents_path_slice[::2], parents_path_slice[1::2]): - enriched_root.add_parent( + parent_request = ParentNodeAddRequest( node_id=current_node_uuid, parent_id=parent.get("uuid"), parent_kind=parent.get("kind"), @@ -144,6 +147,7 @@ def _deserialize_parents(self) -> None: parent_rel_cardinality=RelationshipCardinality(rel.get("cardinality")), parent_rel_label=rel.get("label"), ) + self.parent_adder.add_parent(parent_request=parent_request) current_node_uuid = parent.get("uuid") @classmethod diff --git a/backend/infrahub/core/diff/repository/repository.py b/backend/infrahub/core/diff/repository/repository.py index 05dd747ea2..e4a3fb783a 100644 --- a/backend/infrahub/core/diff/repository/repository.py +++ b/backend/infrahub/core/diff/repository/repository.py @@ -6,7 +6,10 @@ from infrahub import config from infrahub.core import registry from infrahub.core.diff.query.field_summary import EnrichedDiffNodeFieldSummaryQuery -from infrahub.core.diff.query.summary_counts_enricher import DiffSummaryCountsEnricherQuery +from infrahub.core.diff.query.summary_counts_enricher import ( + DiffFieldsSummaryCountsEnricherQuery, + DiffNodesSummaryCountsEnricherQuery, +) from infrahub.core.query.diff import DiffCountChanges from infrahub.core.timestamp import Timestamp from infrahub.database import InfrahubDatabase, retry_db_transaction @@ -16,6 +19,7 @@ from ..model.path import ( ConflictSelection, EnrichedDiffConflict, + EnrichedDiffNode, EnrichedDiffRoot, EnrichedDiffRootMetadata, EnrichedDiffs, @@ -262,35 +266,80 @@ async def _save_node_batch(self, node_create_batch: list[EnrichedNodeCreateReque raise log.exception("Database memory error during save. Trying smaller transactions") for node_request in node_create_batch: + log.info( + f"Updating node {node_request.node.uuid}, num_properties={node_request.node.num_properties}..." + ) single_node_query = await EnrichedNodeBatchCreateQuery.init( db=self.db, node_create_batch=[node_request] ) await single_node_query.execute(db=self.db) + @retry_db_transaction(name="enriched_diff_hierarchy_update") + async def _run_hierarchy_links_update_query(self, diff_root_uuid: str, diff_nodes: list[EnrichedDiffNode]) -> None: + log.info(f"Updating diff hierarchy links, num_nodes={len(diff_nodes)}") + link_query = await EnrichedNodesLinkQuery.init(db=self.db, diff_root_uuid=diff_root_uuid, diff_nodes=diff_nodes) + await link_query.execute(db=self.db) + + async def _update_hierarchy_links(self, enriched_diffs: EnrichedDiffs) -> None: + for diff_root in (enriched_diffs.base_branch_diff, enriched_diffs.diff_branch_diff): + nodes_to_update = [] + for node in diff_root.nodes: + if any(r.nodes for r in node.relationships): + nodes_to_update.append(node) + if len(nodes_to_update) >= config.SETTINGS.database.query_size_limit: + await self._run_hierarchy_links_update_query( + diff_root_uuid=diff_root.uuid, diff_nodes=nodes_to_update + ) + nodes_to_update = [] + if nodes_to_update: + await self._run_hierarchy_links_update_query(diff_root_uuid=diff_root.uuid, diff_nodes=nodes_to_update) + + async def _update_summary_counts(self, diff_root: EnrichedDiffRoot) -> None: + max_nodes_limit = config.SETTINGS.database.query_size_limit + num_nodes = len(diff_root.nodes) + if diff_root.exists_on_database and num_nodes < max_nodes_limit: + await self.add_summary_counts( + diff_branch_name=diff_root.diff_branch_name, + diff_id=diff_root.uuid, + node_uuids=None, + ) + return + node_uuids: list[str] = [] + for diff_node in diff_root.nodes: + node_uuids.append(diff_node.uuid) + if len(node_uuids) >= max_nodes_limit: + await self.add_summary_counts( + diff_branch_name=diff_root.diff_branch_name, + diff_id=diff_root.uuid, + node_uuids=node_uuids, + ) + node_uuids = [] + if node_uuids: + await self.add_summary_counts( + diff_branch_name=diff_root.diff_branch_name, + diff_id=diff_root.uuid, + node_uuids=node_uuids, + ) + async def save(self, enriched_diffs: EnrichedDiffs | EnrichedDiffsMetadata, do_summary_counts: bool = True) -> None: - await self._save_root_metadata(enriched_diffs=enriched_diffs) + # metadata-only update if not isinstance(enriched_diffs, EnrichedDiffs): + await self._save_root_metadata(enriched_diffs=enriched_diffs) return - num_nodes = len(enriched_diffs.base_branch_diff.nodes) + len(enriched_diffs.diff_branch_diff.nodes) - log.info(f"Saving diff (num_nodes={num_nodes})...") + + count_nodes_remaining = len(enriched_diffs.base_branch_diff.nodes) + len(enriched_diffs.diff_branch_diff.nodes) + log.info(f"Saving diff (num_nodes={count_nodes_remaining})...") for batch_num, node_create_batch in enumerate( self._get_node_create_request_batch(enriched_diffs=enriched_diffs) ): log.info(f"Saving node batch #{batch_num}...") await self._save_node_batch(node_create_batch=node_create_batch) - log.info(f"Batch saved. num_nodes={len(node_create_batch)}") - link_query = await EnrichedNodesLinkQuery.init(db=self.db, enriched_diffs=enriched_diffs) - await link_query.execute(db=self.db) - log.info("Diff saved.") + count_nodes_remaining -= len(node_create_batch) + log.info(f"Batch saved. {count_nodes_remaining=}") + await self._update_hierarchy_links(enriched_diffs=enriched_diffs) if do_summary_counts: - node_uuids: list[str] | None = None - if enriched_diffs.diff_branch_diff.exists_on_database: - node_uuids = list(enriched_diffs.branch_node_uuids) - await self.add_summary_counts( - diff_branch_name=enriched_diffs.diff_branch_name, - diff_id=enriched_diffs.diff_branch_diff.uuid, - node_uuids=node_uuids, - ) + await self._update_summary_counts(diff_root=enriched_diffs.diff_branch_diff) + await self._save_root_metadata(enriched_diffs=enriched_diffs) async def summary( self, @@ -468,7 +517,6 @@ async def get_node_field_specifiers(self, diff_id: str) -> dict[str, set[str]]: offset += limit return specifiers - @retry_db_transaction(name="enriched_diff_summary_counts") async def add_summary_counts( self, diff_branch_name: str, @@ -476,8 +524,48 @@ async def add_summary_counts( diff_id: str | None = None, node_uuids: list[str] | None = None, ) -> None: - log.info("Updating summary counts...") - query = await DiffSummaryCountsEnricherQuery.init( + await self._add_field_summary_counts( + diff_branch_name=diff_branch_name, + tracking_id=tracking_id, + diff_id=diff_id, + node_uuids=node_uuids, + ) + await self._add_node_summary_counts( + diff_branch_name=diff_branch_name, + tracking_id=tracking_id, + diff_id=diff_id, + node_uuids=node_uuids, + ) + + @retry_db_transaction(name="enriched_diff_field_summary_counts") + async def _add_field_summary_counts( + self, + diff_branch_name: str, + tracking_id: TrackingId | None = None, + diff_id: str | None = None, + node_uuids: list[str] | None = None, + ) -> None: + log.info("Updating field summary counts...") + query = await DiffFieldsSummaryCountsEnricherQuery.init( + db=self.db, + diff_branch_name=diff_branch_name, + tracking_id=tracking_id, + diff_id=diff_id, + node_uuids=node_uuids, + ) + await query.execute(db=self.db) + log.info("Field summary counts updated.") + + @retry_db_transaction(name="enriched_diff_node_summary_counts") + async def _add_node_summary_counts( + self, + diff_branch_name: str, + tracking_id: TrackingId | None = None, + diff_id: str | None = None, + node_uuids: list[str] | None = None, + ) -> None: + log.info("Updating node summary counts...") + query = await DiffNodesSummaryCountsEnricherQuery.init( db=self.db, diff_branch_name=diff_branch_name, tracking_id=tracking_id, @@ -485,4 +573,4 @@ async def add_summary_counts( node_uuids=node_uuids, ) await query.execute(db=self.db) - log.info("Summary counts updated.") + log.info("node summary counts updated.") diff --git a/backend/infrahub/core/graph/index.py b/backend/infrahub/core/graph/index.py index ef041cf982..bdedd6d0f8 100644 --- a/backend/infrahub/core/graph/index.py +++ b/backend/infrahub/core/graph/index.py @@ -12,6 +12,9 @@ IndexItem(name="attr_iphost_bin", label="AttributeIPHost", properties=["binary_address"], type=IndexType.RANGE), IndexItem(name="rel_uuid", label="Relationship", properties=["uuid"], type=IndexType.RANGE), IndexItem(name="rel_identifier", label="Relationship", properties=["name"], type=IndexType.RANGE), + # diff indices + IndexItem(name="diff_uuid", label="DiffRoot", properties=["uuid"], type=IndexType.TEXT), + IndexItem(name="diff_node_uuid", label="DiffNode", properties=["uuid"], type=IndexType.TEXT), ] rel_indexes: list[IndexItem] = [ diff --git a/backend/infrahub/dependencies/builder/diff/deserializer.py b/backend/infrahub/dependencies/builder/diff/deserializer.py index 34ca083acc..ae771553da 100644 --- a/backend/infrahub/dependencies/builder/diff/deserializer.py +++ b/backend/infrahub/dependencies/builder/diff/deserializer.py @@ -1,8 +1,10 @@ from infrahub.core.diff.repository.deserializer import EnrichedDiffDeserializer from infrahub.dependencies.interface import DependencyBuilder, DependencyBuilderContext +from .parent_node_adder import DiffParentNodeAdderDependency + class DiffDeserializerDependency(DependencyBuilder[EnrichedDiffDeserializer]): @classmethod def build(cls, context: DependencyBuilderContext) -> EnrichedDiffDeserializer: - return EnrichedDiffDeserializer() + return EnrichedDiffDeserializer(parent_adder=DiffParentNodeAdderDependency.build(context=context)) diff --git a/backend/infrahub/dependencies/builder/diff/enricher/hierarchy.py b/backend/infrahub/dependencies/builder/diff/enricher/hierarchy.py index ed4afd220b..3f4b533e7a 100644 --- a/backend/infrahub/dependencies/builder/diff/enricher/hierarchy.py +++ b/backend/infrahub/dependencies/builder/diff/enricher/hierarchy.py @@ -1,8 +1,10 @@ from infrahub.core.diff.enricher.hierarchy import DiffHierarchyEnricher from infrahub.dependencies.interface import DependencyBuilder, DependencyBuilderContext +from ..parent_node_adder import DiffParentNodeAdderDependency + class DiffHierarchyEnricherDependency(DependencyBuilder[DiffHierarchyEnricher]): @classmethod def build(cls, context: DependencyBuilderContext) -> DiffHierarchyEnricher: - return DiffHierarchyEnricher(db=context.db) + return DiffHierarchyEnricher(db=context.db, parent_adder=DiffParentNodeAdderDependency.build(context=context)) diff --git a/backend/infrahub/dependencies/builder/diff/parent_node_adder.py b/backend/infrahub/dependencies/builder/diff/parent_node_adder.py new file mode 100644 index 0000000000..db0859d0be --- /dev/null +++ b/backend/infrahub/dependencies/builder/diff/parent_node_adder.py @@ -0,0 +1,8 @@ +from infrahub.core.diff.parent_node_adder import DiffParentNodeAdder +from infrahub.dependencies.interface import DependencyBuilder, DependencyBuilderContext + + +class DiffParentNodeAdderDependency(DependencyBuilder[DiffParentNodeAdder]): + @classmethod + def build(cls, context: DependencyBuilderContext) -> DiffParentNodeAdder: + return DiffParentNodeAdder() diff --git a/backend/tests/unit/core/diff/query/test_read.py b/backend/tests/unit/core/diff/query/test_read.py index ecae630716..ee584dd828 100644 --- a/backend/tests/unit/core/diff/query/test_read.py +++ b/backend/tests/unit/core/diff/query/test_read.py @@ -7,6 +7,7 @@ from infrahub.core.branch import Branch from infrahub.core.diff.coordinator import DiffCoordinator from infrahub.core.diff.data_check_synchronizer import DiffDataCheckSynchronizer +from infrahub.core.diff.parent_node_adder import DiffParentNodeAdder from infrahub.core.diff.query.diff_summary import DiffSummaryCounters, DiffSummaryQuery, EnrichedDiffQueryFilters from infrahub.core.diff.repository.deserializer import EnrichedDiffDeserializer from infrahub.core.diff.repository.repository import DiffRepository @@ -236,7 +237,7 @@ async def test_summary_no_filter(self, db: InfrahubDatabase, default_branch: Bra assert summary == counters async def test_get_without_parent(self, db: InfrahubDatabase, default_branch: Branch, load_data): - repository = DiffRepository(db=db, deserializer=EnrichedDiffDeserializer()) + repository = DiffRepository(db=db, deserializer=EnrichedDiffDeserializer(DiffParentNodeAdder())) diffs_without = await repository.get( base_branch_name=default_branch.name, diff_branch_names=[load_data["diff_branch"].name], diff --git a/backend/tests/unit/core/diff/repository/test_diff_repository.py b/backend/tests/unit/core/diff/repository/test_diff_repository.py index dd3de4980b..632e702f7e 100644 --- a/backend/tests/unit/core/diff/repository/test_diff_repository.py +++ b/backend/tests/unit/core/diff/repository/test_diff_repository.py @@ -18,6 +18,7 @@ NameTrackingId, NodeDiffFieldSummary, ) +from infrahub.core.diff.parent_node_adder import DiffParentNodeAdder from infrahub.core.diff.repository.deserializer import EnrichedDiffDeserializer from infrahub.core.diff.repository.repository import DiffRepository from infrahub.core.timestamp import Timestamp @@ -48,7 +49,9 @@ def diff_repository(self, db: InfrahubDatabase) -> Generator[DiffRepository, Non original_size = config.SETTINGS.database.query_size_limit config.SETTINGS.database.max_depth_search_hierarchy = 10 config.SETTINGS.database.query_size_limit = 50 - diff_repository = DiffRepository(db=db, deserializer=EnrichedDiffDeserializer(), max_save_batch_size=30) + diff_repository = DiffRepository( + db=db, deserializer=EnrichedDiffDeserializer(DiffParentNodeAdder()), max_save_batch_size=30 + ) yield diff_repository config.SETTINGS.database.max_depth_search_hierarchy = original_depth config.SETTINGS.database.query_size_limit = original_size diff --git a/backend/tests/unit/core/diff/repository/test_diff_summary_counts.py b/backend/tests/unit/core/diff/repository/test_diff_summary_counts.py index 130f12a503..b0ddd6c686 100644 --- a/backend/tests/unit/core/diff/repository/test_diff_summary_counts.py +++ b/backend/tests/unit/core/diff/repository/test_diff_summary_counts.py @@ -8,6 +8,7 @@ EnrichedDiffNode, EnrichedDiffRoot, ) +from infrahub.core.diff.parent_node_adder import DiffParentNodeAdder from infrahub.core.diff.repository.deserializer import EnrichedDiffDeserializer from infrahub.core.diff.repository.repository import DiffRepository from infrahub.database import InfrahubDatabase @@ -22,7 +23,7 @@ class TestDiffSummaryCountsQuery(DiffRepositoryTestBase): @pytest.fixture def diff_repository(self, db: InfrahubDatabase) -> DiffRepository: - return DiffRepository(db=db, deserializer=EnrichedDiffDeserializer()) + return DiffRepository(db=db, deserializer=EnrichedDiffDeserializer(DiffParentNodeAdder())) async def __save_and_update_diff( self, diff_repository: DiffRepository, enriched_diff: EnrichedDiffRoot diff --git a/backend/tests/unit/core/diff/test_diff_get_all_conflicts.py b/backend/tests/unit/core/diff/test_diff_get_all_conflicts.py index c261711cf6..87a97a3315 100644 --- a/backend/tests/unit/core/diff/test_diff_get_all_conflicts.py +++ b/backend/tests/unit/core/diff/test_diff_get_all_conflicts.py @@ -4,6 +4,7 @@ BranchTrackingId, EnrichedDiffs, ) +from infrahub.core.diff.parent_node_adder import DiffParentNodeAdder from infrahub.core.diff.repository.deserializer import EnrichedDiffDeserializer from infrahub.core.diff.repository.repository import DiffRepository from infrahub.database import InfrahubDatabase @@ -22,7 +23,7 @@ class TestDiffGetAllConflicts: @pytest.fixture def diff_repository(self, db: InfrahubDatabase) -> DiffRepository: - return DiffRepository(db=db, deserializer=EnrichedDiffDeserializer()) + return DiffRepository(db=db, deserializer=EnrichedDiffDeserializer(DiffParentNodeAdder())) def _get_enriched_diffs(self) -> EnrichedDiffs: branch_diff_root = EnrichedRootFactory.build( diff --git a/backend/tests/unit/core/diff/test_diff_has_conflicts_query.py b/backend/tests/unit/core/diff/test_diff_has_conflicts_query.py index ecbf514943..e515ea6e85 100644 --- a/backend/tests/unit/core/diff/test_diff_has_conflicts_query.py +++ b/backend/tests/unit/core/diff/test_diff_has_conflicts_query.py @@ -4,6 +4,7 @@ BranchTrackingId, EnrichedDiffs, ) +from infrahub.core.diff.parent_node_adder import DiffParentNodeAdder from infrahub.core.diff.repository.deserializer import EnrichedDiffDeserializer from infrahub.core.diff.repository.repository import DiffRepository from infrahub.database import InfrahubDatabase @@ -22,7 +23,7 @@ class TestDiffHasConflictsCheck: @pytest.fixture def diff_repository(self, db: InfrahubDatabase) -> DiffRepository: - return DiffRepository(db=db, deserializer=EnrichedDiffDeserializer()) + return DiffRepository(db=db, deserializer=EnrichedDiffDeserializer(DiffParentNodeAdder())) def _get_enriched_diffs(self) -> EnrichedDiffs: branch_diff_root = EnrichedRootFactory.build( diff --git a/backend/tests/unit/core/diff/test_diff_hierarchy_enricher.py b/backend/tests/unit/core/diff/test_diff_hierarchy_enricher.py index 0bd14a1a10..1a6b4f79a5 100644 --- a/backend/tests/unit/core/diff/test_diff_hierarchy_enricher.py +++ b/backend/tests/unit/core/diff/test_diff_hierarchy_enricher.py @@ -1,5 +1,6 @@ from infrahub.core.diff.enricher.hierarchy import DiffHierarchyEnricher from infrahub.core.diff.model.path import DiffAction +from infrahub.core.diff.parent_node_adder import DiffParentNodeAdder from infrahub.core.initialization import create_branch from infrahub.core.node import Node from infrahub.database import InfrahubDatabase @@ -15,7 +16,7 @@ async def test_node_no_parent_no_rel(db: InfrahubDatabase, default_branch, perso diff_root = EnrichedRootFactory.build( base_branch_name=default_branch.name, diff_branch_name=branch.name, nodes={diff_node} ) - enricher = DiffHierarchyEnricher(db=db) + enricher = DiffHierarchyEnricher(db=db, parent_adder=DiffParentNodeAdder()) await enricher.enrich(enriched_diff_root=diff_root, calculated_diffs=None) assert len(diff_root.nodes) == 2 @@ -42,7 +43,7 @@ async def test_node_no_parent_rel(db: InfrahubDatabase, default_branch, person_j diff_root = EnrichedRootFactory.build( base_branch_name=default_branch.name, diff_branch_name=branch.name, nodes={diff_node} ) - enricher = DiffHierarchyEnricher(db=db) + enricher = DiffHierarchyEnricher(db=db, parent_adder=DiffParentNodeAdder()) await enricher.enrich(enriched_diff_root=diff_root, calculated_diffs=None) assert len(diff_root.nodes) == 2 @@ -88,7 +89,7 @@ async def test_node_hierarchy(db: InfrahubDatabase, default_branch, hierarchical diff_root = EnrichedRootFactory.build( base_branch_name=default_branch.name, diff_branch_name=branch.name, nodes={diff_node1, diff_node2} ) - enricher = DiffHierarchyEnricher(db=db) + enricher = DiffHierarchyEnricher(db=db, parent_adder=DiffParentNodeAdder()) await enricher.enrich(enriched_diff_root=diff_root, calculated_diffs=None) assert len(diff_root.nodes) == 6 diff --git a/backend/tests/unit/graphql/diff/test_diff_update_mutation.py b/backend/tests/unit/graphql/diff/test_diff_update_mutation.py index 5d5cd6e652..7623c4078b 100644 --- a/backend/tests/unit/graphql/diff/test_diff_update_mutation.py +++ b/backend/tests/unit/graphql/diff/test_diff_update_mutation.py @@ -2,6 +2,7 @@ from infrahub.core.branch import Branch from infrahub.core.diff.model.path import EnrichedDiffRootMetadata, NameTrackingId +from infrahub.core.diff.parent_node_adder import DiffParentNodeAdder from infrahub.core.diff.repository.deserializer import EnrichedDiffDeserializer from infrahub.core.diff.repository.repository import DiffRepository from infrahub.core.initialization import create_branch @@ -76,7 +77,7 @@ async def named_diff( assert result.errors is None assert result.data["DiffUpdate"]["ok"] is True - diff_repo = DiffRepository(db=db, deserializer=EnrichedDiffDeserializer()) + diff_repo = DiffRepository(db=db, deserializer=EnrichedDiffDeserializer(DiffParentNodeAdder())) return ( await diff_repo.get_roots_metadata( diff_branch_names=[diff_branch.name],