diff --git a/.changes/unreleased/Under the Hood-20240125-095453.yaml b/.changes/unreleased/Under the Hood-20240125-095453.yaml new file mode 100644 index 00000000000..2edb1eb487b --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240125-095453.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Define Macro and Group resources in dbt/artifacts +time: 2024-01-25T09:54:53.974332-05:00 +custom: + Author: michelleark + Issue: 9381 9382 diff --git a/core/dbt/artifacts/resources/__init__.py b/core/dbt/artifacts/resources/__init__.py index e7c97147b06..2b94afe3681 100644 --- a/core/dbt/artifacts/resources/__init__.py +++ b/core/dbt/artifacts/resources/__init__.py @@ -1,4 +1,8 @@ -from dbt.artifacts.resources.base import BaseArtifactNode +from dbt.artifacts.resources.base import BaseResource # alias to latest resource definitions from dbt.artifacts.resources.v1.documentation import Documentation +from dbt.artifacts.resources.v1.macro import Macro, MacroDependsOn, MacroArgument +from dbt.artifacts.resources.v1.docs import Docs +from dbt.artifacts.resources.v1.group import Group +from dbt.artifacts.resources.v1.owner import Owner diff --git a/core/dbt/artifacts/resources/base.py b/core/dbt/artifacts/resources/base.py index 72ce677e1ce..f82f072bbcf 100644 --- a/core/dbt/artifacts/resources/base.py +++ b/core/dbt/artifacts/resources/base.py @@ -6,7 +6,7 @@ @dataclass -class BaseArtifactNode(dbtClassMixin, Replaceable): +class BaseResource(dbtClassMixin, Replaceable): name: str resource_type: NodeType package_name: str diff --git a/core/dbt/artifacts/resources/v1/docs.py b/core/dbt/artifacts/resources/v1/docs.py new file mode 100644 index 00000000000..5dca7a88421 --- /dev/null +++ b/core/dbt/artifacts/resources/v1/docs.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass +from dbt_common.dataclass_schema import dbtClassMixin +from dbt_common.contracts.util import Replaceable +from typing import Optional + + +@dataclass +class Docs(dbtClassMixin, Replaceable): + show: bool = True + node_color: Optional[str] = None diff --git a/core/dbt/artifacts/resources/v1/documentation.py b/core/dbt/artifacts/resources/v1/documentation.py index b62846512d0..59c19e4ee6d 100644 --- a/core/dbt/artifacts/resources/v1/documentation.py +++ b/core/dbt/artifacts/resources/v1/documentation.py @@ -1,11 +1,11 @@ from dataclasses import dataclass from typing import Literal -from dbt.artifacts.resources.base import BaseArtifactNode +from dbt.artifacts.resources.base import BaseResource from dbt.artifacts.resources.types import NodeType @dataclass -class Documentation(BaseArtifactNode): +class Documentation(BaseResource): resource_type: Literal[NodeType.Documentation] block_contents: str diff --git a/core/dbt/artifacts/resources/v1/group.py b/core/dbt/artifacts/resources/v1/group.py new file mode 100644 index 00000000000..c5351268eaa --- /dev/null +++ b/core/dbt/artifacts/resources/v1/group.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass +from typing import Literal + +from dbt.artifacts.resources.base import BaseResource +from dbt.artifacts.resources.types import NodeType +from dbt.artifacts.resources.v1.owner import Owner + + +@dataclass +class Group(BaseResource): + name: str + owner: Owner + resource_type: Literal[NodeType.Group] diff --git a/core/dbt/artifacts/resources/v1/macro.py b/core/dbt/artifacts/resources/v1/macro.py new file mode 100644 index 00000000000..cda0fd2ac32 --- /dev/null +++ b/core/dbt/artifacts/resources/v1/macro.py @@ -0,0 +1,40 @@ +from dataclasses import dataclass, field +import time +from typing import Literal, List, Dict, Optional, Any + +from dbt_common.contracts.util import Replaceable +from dbt_common.dataclass_schema import dbtClassMixin +from dbt.artifacts.resources.base import BaseResource +from dbt.artifacts.resources.types import NodeType, ModelLanguage +from dbt.artifacts.resources.v1.docs import Docs + + +@dataclass +class MacroArgument(dbtClassMixin): + name: str + type: Optional[str] = None + description: str = "" + + +@dataclass +class MacroDependsOn(dbtClassMixin, Replaceable): + macros: List[str] = field(default_factory=list) + + # 'in' on lists is O(n) so this is O(n^2) for # of macros + def add_macro(self, value: str): + if value not in self.macros: + self.macros.append(value) + + +@dataclass +class Macro(BaseResource): + macro_sql: str + resource_type: Literal[NodeType.Macro] + depends_on: MacroDependsOn = field(default_factory=MacroDependsOn) + description: str = "" + meta: Dict[str, Any] = field(default_factory=dict) + docs: Docs = field(default_factory=Docs) + patch_path: Optional[str] = None + arguments: List[MacroArgument] = field(default_factory=list) + created_at: float = field(default_factory=lambda: time.time()) + supported_languages: Optional[List[ModelLanguage]] = None diff --git a/core/dbt/artifacts/resources/v1/owner.py b/core/dbt/artifacts/resources/v1/owner.py new file mode 100644 index 00000000000..065ad51bac2 --- /dev/null +++ b/core/dbt/artifacts/resources/v1/owner.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass +from typing import Optional + +from dbt_common.contracts.config.properties import AdditionalPropertiesAllowed +from dbt_common.contracts.util import Replaceable + + +@dataclass +class Owner(AdditionalPropertiesAllowed, Replaceable): + email: Optional[str] = None + name: Optional[str] = None diff --git a/core/dbt/artifacts/schemas/manifest/v12/manifest.py b/core/dbt/artifacts/schemas/manifest/v12/manifest.py index 806f6b35e0b..d00740346e1 100644 --- a/core/dbt/artifacts/schemas/manifest/v12/manifest.py +++ b/core/dbt/artifacts/schemas/manifest/v12/manifest.py @@ -9,7 +9,7 @@ get_artifact_schema_version, ) from dbt.artifacts.schemas.upgrades import upgrade_manifest_json -from dbt.artifacts.resources import Documentation +from dbt.artifacts.resources import Documentation, Group, Macro # TODO: remove usage of dbt modules other than dbt.artifacts from dbt import tracking @@ -17,8 +17,6 @@ from dbt.contracts.graph.nodes import ( Exposure, GraphMemberNode, - Group, - Macro, ManifestNode, Metric, SavedQuery, diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 9b93e995cff..666affe38ed 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -27,12 +27,10 @@ ) from dbt.contracts.graph.unparsed import ( ConstantPropertyInput, - Docs, ExposureType, ExternalTable, FreshnessThreshold, HasYamlMetadata, - MacroArgument, MaturityType, Owner, Quoting, @@ -61,7 +59,6 @@ from dbt_common.events.contextvars import set_log_contextvars from dbt.flags import get_flags from dbt.node_types import ( - ModelLanguage, NodeType, AccessType, REFABLE_NODE_TYPES, @@ -97,7 +94,15 @@ SavedQueryConfig, ) -from dbt.artifacts.resources import BaseArtifactNode, Documentation as DocumentationContract +from dbt.artifacts.resources import ( + BaseResource, + Docs, + MacroDependsOn, + MacroArgument, + Documentation as DocumentationResource, + Macro as MacroResource, + Group as GroupResource, +) # ===================================================================== @@ -124,7 +129,7 @@ @dataclass -class BaseNode(BaseArtifactNode): +class BaseNode(BaseResource): """All nodes or node-like objects in this file should have this as a base class""" @property @@ -240,18 +245,6 @@ def quoting_dict(self) -> Dict[str, bool]: return {} -@dataclass -class MacroDependsOn(dbtClassMixin, Replaceable): - """Used only in the Macro class""" - - macros: List[str] = field(default_factory=list) - - # 'in' on lists is O(n) so this is O(n^2) for # of macros - def add_macro(self, value: str): - if value not in self.macros: - self.macros.append(value) - - @dataclass class DeferRelation(HasRelationMetadata): alias: str @@ -1149,18 +1142,7 @@ class SnapshotNode(CompiledNode): @dataclass -class Macro(BaseNode): - macro_sql: str - resource_type: Literal[NodeType.Macro] - depends_on: MacroDependsOn = field(default_factory=MacroDependsOn) - description: str = "" - meta: Dict[str, Any] = field(default_factory=dict) - docs: Docs = field(default_factory=Docs) - patch_path: Optional[str] = None - arguments: List[MacroArgument] = field(default_factory=list) - created_at: float = field(default_factory=lambda: time.time()) - supported_languages: Optional[List[ModelLanguage]] = None - +class Macro(MacroResource, BaseNode): def same_contents(self, other: Optional["Macro"]) -> bool: if other is None: return False @@ -1179,7 +1161,7 @@ def depends_on_macros(self): @dataclass -class Documentation(DocumentationContract, BaseNode): +class Documentation(DocumentationResource, BaseNode): @property def search_name(self): return self.name @@ -1661,10 +1643,8 @@ def same_contents(self, old: Optional["Metric"]) -> bool: @dataclass -class Group(BaseNode): - name: str - owner: Owner - resource_type: Literal[NodeType.Group] +class Group(GroupResource, BaseNode): + pass # ==================================== diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 5f355635872..8e751e71a4a 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -29,6 +29,7 @@ from dbt.exceptions import ParsingError from dbt_semantic_interfaces.type_enums import ConversionCalculationType +from dbt.artifacts.resources import Docs, MacroArgument, Owner from dataclasses import dataclass, field from datetime import timedelta @@ -83,12 +84,6 @@ class UnparsedRunHook(UnparsedNode): index: Optional[int] = None -@dataclass -class Docs(dbtClassMixin, Replaceable): - show: bool = True - node_color: Optional[str] = None - - @dataclass class HasColumnProps(AdditionalPropertiesMixin, ExtensibleDbtClassMixin, Replaceable): name: str @@ -266,13 +261,6 @@ def get_tests_for_version(self, version: NodeVersion) -> List[TestDef]: ) -@dataclass -class MacroArgument(dbtClassMixin): - name: str - type: Optional[str] = None - description: str = "" - - @dataclass class UnparsedMacroUpdate(HasConfig, HasColumnProps, HasYamlMetadata): arguments: List[MacroArgument] = field(default_factory=list) @@ -534,12 +522,6 @@ class MaturityType(StrEnum): High = "high" -@dataclass -class Owner(AdditionalPropertiesAllowed, Replaceable): - email: Optional[str] = None - name: Optional[str] = None - - @dataclass class UnparsedExposure(dbtClassMixin, Replaceable): name: str diff --git a/core/dbt/graph/selector_methods.py b/core/dbt/graph/selector_methods.py index 6d478356cc0..e9a627b55b2 100644 --- a/core/dbt/graph/selector_methods.py +++ b/core/dbt/graph/selector_methods.py @@ -593,7 +593,7 @@ def _macros_modified(self) -> List[str]: else: modified.append(uid) - for uid, macro in old_macros.items(): + for uid, _ in old_macros.items(): if uid not in new_macros: modified.append(uid)