From ae2b3626afd12ff0089c8b15d00c3f009bb9a457 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Wed, 24 Jan 2024 13:33:05 -0500 Subject: [PATCH 1/5] first pass: move Macro to dbt/artifacts --- core/dbt/artifacts/resources/__init__.py | 2 ++ core/dbt/artifacts/resources/v1/docs.py | 10 ++++++ core/dbt/artifacts/resources/v1/macro.py | 40 ++++++++++++++++++++++++ core/dbt/contracts/graph/nodes.py | 39 ++++++----------------- core/dbt/contracts/graph/unparsed.py | 14 +-------- 5 files changed, 63 insertions(+), 42 deletions(-) create mode 100644 core/dbt/artifacts/resources/v1/docs.py create mode 100644 core/dbt/artifacts/resources/v1/macro.py diff --git a/core/dbt/artifacts/resources/__init__.py b/core/dbt/artifacts/resources/__init__.py index e7c97147b06..5b4a42ac25b 100644 --- a/core/dbt/artifacts/resources/__init__.py +++ b/core/dbt/artifacts/resources/__init__.py @@ -2,3 +2,5 @@ # alias to latest resource definitions from dbt.artifacts.resources.v1.documentation import Documentation +from dbt.artifacts.resources.v1.macro import Macro, MacroDependsOn, MacroArgument +from dbt.artifacts.resources.v1.docs import Docs diff --git a/core/dbt/artifacts/resources/v1/docs.py b/core/dbt/artifacts/resources/v1/docs.py new file mode 100644 index 00000000000..5dca7a88421 --- /dev/null +++ b/core/dbt/artifacts/resources/v1/docs.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass +from dbt_common.dataclass_schema import dbtClassMixin +from dbt_common.contracts.util import Replaceable +from typing import Optional + + +@dataclass +class Docs(dbtClassMixin, Replaceable): + show: bool = True + node_color: Optional[str] = None diff --git a/core/dbt/artifacts/resources/v1/macro.py b/core/dbt/artifacts/resources/v1/macro.py new file mode 100644 index 00000000000..26f37719643 --- /dev/null +++ b/core/dbt/artifacts/resources/v1/macro.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass, field +import time +from typing import Literal, List, Dict, Optional, Any + +from dbt_common.contracts.util import Replaceable +from dbt_common.dataclass_schema import dbtClassMixin +from dbt.artifacts.resources.base import BaseArtifactNode +from dbt.artifacts.resources.types import NodeType, ModelLanguage +from dbt.artifacts.resources.v1.docs import Docs + + +@dataclass +class MacroArgument(dbtClassMixin): + name: str + type: Optional[str] = None + description: str = "" + + +@dataclass +class MacroDependsOn(dbtClassMixin, Replaceable): + macros: List[str] = field(default_factory=list) + + # 'in' on lists is O(n) so this is O(n^2) for # of macros + def add_macro(self, value: str): + if value not in self.macros: + self.macros.append(value) + + +@dataclass +class Macro(BaseArtifactNode): + macro_sql: str + resource_type: Literal[NodeType.Macro] + depends_on: MacroDependsOn = field(default_factory=MacroDependsOn) + description: str = "" + meta: Dict[str, Any] = field(default_factory=dict) + docs: Docs = field(default_factory=Docs) + patch_path: Optional[str] = None + arguments: List[MacroArgument] = field(default_factory=list) + created_at: float = field(default_factory=lambda: time.time()) + supported_languages: Optional[List[ModelLanguage]] = None diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 9b93e995cff..a335bd59d9b 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -27,12 +27,10 @@ ) from dbt.contracts.graph.unparsed import ( ConstantPropertyInput, - Docs, ExposureType, ExternalTable, FreshnessThreshold, HasYamlMetadata, - MacroArgument, MaturityType, Owner, Quoting, @@ -61,7 +59,6 @@ from dbt_common.events.contextvars import set_log_contextvars from dbt.flags import get_flags from dbt.node_types import ( - ModelLanguage, NodeType, AccessType, REFABLE_NODE_TYPES, @@ -97,7 +94,14 @@ SavedQueryConfig, ) -from dbt.artifacts.resources import BaseArtifactNode, Documentation as DocumentationContract +from dbt.artifacts.resources import ( + BaseArtifactNode, + Docs, + MacroDependsOn, + MacroArgument, + Documentation as DocumentationResource, + Macro as MacroResource, +) # ===================================================================== @@ -240,18 +244,6 @@ def quoting_dict(self) -> Dict[str, bool]: return {} -@dataclass -class MacroDependsOn(dbtClassMixin, Replaceable): - """Used only in the Macro class""" - - macros: List[str] = field(default_factory=list) - - # 'in' on lists is O(n) so this is O(n^2) for # of macros - def add_macro(self, value: str): - if value not in self.macros: - self.macros.append(value) - - @dataclass class DeferRelation(HasRelationMetadata): alias: str @@ -1149,18 +1141,7 @@ class SnapshotNode(CompiledNode): @dataclass -class Macro(BaseNode): - macro_sql: str - resource_type: Literal[NodeType.Macro] - depends_on: MacroDependsOn = field(default_factory=MacroDependsOn) - description: str = "" - meta: Dict[str, Any] = field(default_factory=dict) - docs: Docs = field(default_factory=Docs) - patch_path: Optional[str] = None - arguments: List[MacroArgument] = field(default_factory=list) - created_at: float = field(default_factory=lambda: time.time()) - supported_languages: Optional[List[ModelLanguage]] = None - +class Macro(MacroResource, BaseNode): def same_contents(self, other: Optional["Macro"]) -> bool: if other is None: return False @@ -1179,7 +1160,7 @@ def depends_on_macros(self): @dataclass -class Documentation(DocumentationContract, BaseNode): +class Documentation(DocumentationResource, BaseNode): @property def search_name(self): return self.name diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 5f355635872..b17a2912432 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -29,6 +29,7 @@ from dbt.exceptions import ParsingError from dbt_semantic_interfaces.type_enums import ConversionCalculationType +from dbt.artifacts.resources import Docs, MacroArgument from dataclasses import dataclass, field from datetime import timedelta @@ -83,12 +84,6 @@ class UnparsedRunHook(UnparsedNode): index: Optional[int] = None -@dataclass -class Docs(dbtClassMixin, Replaceable): - show: bool = True - node_color: Optional[str] = None - - @dataclass class HasColumnProps(AdditionalPropertiesMixin, ExtensibleDbtClassMixin, Replaceable): name: str @@ -266,13 +261,6 @@ def get_tests_for_version(self, version: NodeVersion) -> List[TestDef]: ) -@dataclass -class MacroArgument(dbtClassMixin): - name: str - type: Optional[str] = None - description: str = "" - - @dataclass class UnparsedMacroUpdate(HasConfig, HasColumnProps, HasYamlMetadata): arguments: List[MacroArgument] = field(default_factory=list) From a6aaba675de62340287125394762eaa54a547bfe Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 25 Jan 2024 09:54:59 -0500 Subject: [PATCH 2/5] changelog entry --- .changes/unreleased/Under the Hood-20240125-095453.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20240125-095453.yaml diff --git a/.changes/unreleased/Under the Hood-20240125-095453.yaml b/.changes/unreleased/Under the Hood-20240125-095453.yaml new file mode 100644 index 00000000000..47efee0c27e --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240125-095453.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Define Macro resource in dbt/artifacts +time: 2024-01-25T09:54:53.974332-05:00 +custom: + Author: michelleark + Issue: "9382" From 4fbc2b8a1780c3bdeaba8aaff3a6a5fd56a49a17 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 25 Jan 2024 13:41:59 -0500 Subject: [PATCH 3/5] move Group resource to dbt/artifacts and rename BaseArtifactNode to BaseResource --- .../unreleased/Under the Hood-20240125-095453.yaml | 4 ++-- core/dbt/artifacts/resources/__init__.py | 4 +++- core/dbt/artifacts/resources/base.py | 2 +- core/dbt/artifacts/resources/v1/documentation.py | 4 ++-- core/dbt/artifacts/resources/v1/group.py | 13 +++++++++++++ core/dbt/artifacts/resources/v1/macro.py | 4 ++-- core/dbt/artifacts/resources/v1/owner.py | 11 +++++++++++ core/dbt/contracts/graph/nodes.py | 11 +++++------ core/dbt/contracts/graph/unparsed.py | 8 +------- 9 files changed, 40 insertions(+), 21 deletions(-) create mode 100644 core/dbt/artifacts/resources/v1/group.py create mode 100644 core/dbt/artifacts/resources/v1/owner.py diff --git a/.changes/unreleased/Under the Hood-20240125-095453.yaml b/.changes/unreleased/Under the Hood-20240125-095453.yaml index 47efee0c27e..2edb1eb487b 100644 --- a/.changes/unreleased/Under the Hood-20240125-095453.yaml +++ b/.changes/unreleased/Under the Hood-20240125-095453.yaml @@ -1,6 +1,6 @@ kind: Under the Hood -body: Define Macro resource in dbt/artifacts +body: Define Macro and Group resources in dbt/artifacts time: 2024-01-25T09:54:53.974332-05:00 custom: Author: michelleark - Issue: "9382" + Issue: 9381 9382 diff --git a/core/dbt/artifacts/resources/__init__.py b/core/dbt/artifacts/resources/__init__.py index 5b4a42ac25b..2b94afe3681 100644 --- a/core/dbt/artifacts/resources/__init__.py +++ b/core/dbt/artifacts/resources/__init__.py @@ -1,6 +1,8 @@ -from dbt.artifacts.resources.base import BaseArtifactNode +from dbt.artifacts.resources.base import BaseResource # alias to latest resource definitions from dbt.artifacts.resources.v1.documentation import Documentation from dbt.artifacts.resources.v1.macro import Macro, MacroDependsOn, MacroArgument from dbt.artifacts.resources.v1.docs import Docs +from dbt.artifacts.resources.v1.group import Group +from dbt.artifacts.resources.v1.owner import Owner diff --git a/core/dbt/artifacts/resources/base.py b/core/dbt/artifacts/resources/base.py index 72ce677e1ce..f82f072bbcf 100644 --- a/core/dbt/artifacts/resources/base.py +++ b/core/dbt/artifacts/resources/base.py @@ -6,7 +6,7 @@ @dataclass -class BaseArtifactNode(dbtClassMixin, Replaceable): +class BaseResource(dbtClassMixin, Replaceable): name: str resource_type: NodeType package_name: str diff --git a/core/dbt/artifacts/resources/v1/documentation.py b/core/dbt/artifacts/resources/v1/documentation.py index b62846512d0..59c19e4ee6d 100644 --- a/core/dbt/artifacts/resources/v1/documentation.py +++ b/core/dbt/artifacts/resources/v1/documentation.py @@ -1,11 +1,11 @@ from dataclasses import dataclass from typing import Literal -from dbt.artifacts.resources.base import BaseArtifactNode +from dbt.artifacts.resources.base import BaseResource from dbt.artifacts.resources.types import NodeType @dataclass -class Documentation(BaseArtifactNode): +class Documentation(BaseResource): resource_type: Literal[NodeType.Documentation] block_contents: str diff --git a/core/dbt/artifacts/resources/v1/group.py b/core/dbt/artifacts/resources/v1/group.py new file mode 100644 index 00000000000..c5351268eaa --- /dev/null +++ b/core/dbt/artifacts/resources/v1/group.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass +from typing import Literal + +from dbt.artifacts.resources.base import BaseResource +from dbt.artifacts.resources.types import NodeType +from dbt.artifacts.resources.v1.owner import Owner + + +@dataclass +class Group(BaseResource): + name: str + owner: Owner + resource_type: Literal[NodeType.Group] diff --git a/core/dbt/artifacts/resources/v1/macro.py b/core/dbt/artifacts/resources/v1/macro.py index 26f37719643..cda0fd2ac32 100644 --- a/core/dbt/artifacts/resources/v1/macro.py +++ b/core/dbt/artifacts/resources/v1/macro.py @@ -4,7 +4,7 @@ from dbt_common.contracts.util import Replaceable from dbt_common.dataclass_schema import dbtClassMixin -from dbt.artifacts.resources.base import BaseArtifactNode +from dbt.artifacts.resources.base import BaseResource from dbt.artifacts.resources.types import NodeType, ModelLanguage from dbt.artifacts.resources.v1.docs import Docs @@ -27,7 +27,7 @@ def add_macro(self, value: str): @dataclass -class Macro(BaseArtifactNode): +class Macro(BaseResource): macro_sql: str resource_type: Literal[NodeType.Macro] depends_on: MacroDependsOn = field(default_factory=MacroDependsOn) diff --git a/core/dbt/artifacts/resources/v1/owner.py b/core/dbt/artifacts/resources/v1/owner.py new file mode 100644 index 00000000000..065ad51bac2 --- /dev/null +++ b/core/dbt/artifacts/resources/v1/owner.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass +from typing import Optional + +from dbt_common.contracts.config.properties import AdditionalPropertiesAllowed +from dbt_common.contracts.util import Replaceable + + +@dataclass +class Owner(AdditionalPropertiesAllowed, Replaceable): + email: Optional[str] = None + name: Optional[str] = None diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index a335bd59d9b..666affe38ed 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -95,12 +95,13 @@ ) from dbt.artifacts.resources import ( - BaseArtifactNode, + BaseResource, Docs, MacroDependsOn, MacroArgument, Documentation as DocumentationResource, Macro as MacroResource, + Group as GroupResource, ) @@ -128,7 +129,7 @@ @dataclass -class BaseNode(BaseArtifactNode): +class BaseNode(BaseResource): """All nodes or node-like objects in this file should have this as a base class""" @property @@ -1642,10 +1643,8 @@ def same_contents(self, old: Optional["Metric"]) -> bool: @dataclass -class Group(BaseNode): - name: str - owner: Owner - resource_type: Literal[NodeType.Group] +class Group(GroupResource, BaseNode): + pass # ==================================== diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index b17a2912432..8e751e71a4a 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -29,7 +29,7 @@ from dbt.exceptions import ParsingError from dbt_semantic_interfaces.type_enums import ConversionCalculationType -from dbt.artifacts.resources import Docs, MacroArgument +from dbt.artifacts.resources import Docs, MacroArgument, Owner from dataclasses import dataclass, field from datetime import timedelta @@ -522,12 +522,6 @@ class MaturityType(StrEnum): High = "high" -@dataclass -class Owner(AdditionalPropertiesAllowed, Replaceable): - email: Optional[str] = None - name: Optional[str] = None - - @dataclass class UnparsedExposure(dbtClassMixin, Replaceable): name: str From cdfdb7abff6b81bc7c9c4e352581005f26e94432 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 25 Jan 2024 19:03:25 -0500 Subject: [PATCH 4/5] remove contracts.graph.nodes references in dbt.artifacts --- core/dbt/artifacts/schemas/manifest/v12/manifest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/dbt/artifacts/schemas/manifest/v12/manifest.py b/core/dbt/artifacts/schemas/manifest/v12/manifest.py index 806f6b35e0b..d00740346e1 100644 --- a/core/dbt/artifacts/schemas/manifest/v12/manifest.py +++ b/core/dbt/artifacts/schemas/manifest/v12/manifest.py @@ -9,7 +9,7 @@ get_artifact_schema_version, ) from dbt.artifacts.schemas.upgrades import upgrade_manifest_json -from dbt.artifacts.resources import Documentation +from dbt.artifacts.resources import Documentation, Group, Macro # TODO: remove usage of dbt modules other than dbt.artifacts from dbt import tracking @@ -17,8 +17,6 @@ from dbt.contracts.graph.nodes import ( Exposure, GraphMemberNode, - Group, - Macro, ManifestNode, Metric, SavedQuery, From 90f2155d39326b48f32ac4e967a4d9ec742be04c Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Thu, 25 Jan 2024 22:14:07 -0500 Subject: [PATCH 5/5] code quality fix --- core/dbt/graph/selector_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index 6d478356cc0..e9a627b55b2 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -593,7 +593,7 @@ def _macros_modified(self) -> List[str]: else: modified.append(uid) - for uid, macro in old_macros.items(): + for uid, _ in old_macros.items(): if uid not in new_macros: modified.append(uid)