From 52c61b3906f9c4c54b06e97bd5ff282f764e855a Mon Sep 17 00:00:00 2001 From: morgan-dgk Date: Fri, 31 Jan 2025 15:54:59 +1100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20ability=20to=20handle=20long?= =?UTF-8?q?=20file=20extensions=20for=20yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add new `get_yaml_path` helper to allow handling of both yml and yaml file extension for dbt project config files (currently `profiles` and `dbt_project` files). Small number of failing tests are due to hardcoded references to short file extension in tests which expected an error to be raised. --- core/dbt/clients/yaml_helper.py | 10 ++++++++++ core/dbt/config/profile.py | 4 ++-- core/dbt/config/project.py | 14 ++++++++------ core/dbt/constants.py | 4 ++-- core/dbt/task/base.py | 2 ++ 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/core/dbt/clients/yaml_helper.py b/core/dbt/clients/yaml_helper.py index a0a51099331..8a014d44cf7 100644 --- a/core/dbt/clients/yaml_helper.py +++ b/core/dbt/clients/yaml_helper.py @@ -1,3 +1,4 @@ +import os from typing import Any, Dict, Optional import yaml @@ -66,3 +67,12 @@ def load_yaml_text(contents, path=None): error = str(e) raise dbt_common.exceptions.base.DbtValidationError(error) + + +def get_yaml_path(path: str) -> str: + if os.path.exists("{}.yml".format(path)): + return path + ".yml" + elif os.path.exists("{}.yaml".format(path)): + return path + ".yaml" + else: + return "" diff --git a/core/dbt/config/profile.py b/core/dbt/config/profile.py index ada7f30711c..d53417f3d3c 100644 --- a/core/dbt/config/profile.py +++ b/core/dbt/config/profile.py @@ -3,7 +3,7 @@ from typing import Any, Dict, Optional, Tuple from dbt.adapters.contracts.connection import Credentials, HasCredentials -from dbt.clients.yaml_helper import load_yaml_text +from dbt.clients.yaml_helper import get_yaml_path, load_yaml_text from dbt.contracts.project import ProfileConfig from dbt.events.types import MissingProfileTarget from dbt.exceptions import ( @@ -31,7 +31,7 @@ def read_profile(profiles_dir: str) -> Dict[str, Any]: - path = os.path.join(profiles_dir, "profiles.yml") + path = get_yaml_path(os.path.join(profiles_dir, "profiles")) contents = None if os.path.isfile(path): diff --git a/core/dbt/config/project.py b/core/dbt/config/project.py index 1054c42041d..7156d33e2f0 100644 --- a/core/dbt/config/project.py +++ b/core/dbt/config/project.py @@ -8,7 +8,7 @@ from dbt import deprecations from dbt.adapters.contracts.connection import QueryComment -from dbt.clients.yaml_helper import load_yaml_text +from dbt.clients.yaml_helper import get_yaml_path, load_yaml_text from dbt.config.selectors import SelectorDict from dbt.config.utils import normalize_warn_error_options from dbt.constants import ( @@ -99,8 +99,10 @@ def load_yml_dict(file_path): def package_and_project_data_from_root(project_root): - packages_yml_dict = load_yml_dict(f"{project_root}/{PACKAGES_FILE_NAME}") - dependencies_yml_dict = load_yml_dict(f"{project_root}/{DEPENDENCIES_FILE_NAME}") + packages_yml_dict = load_yml_dict(get_yaml_path(f"{project_root}/{PACKAGES_FILE_NAME}")) + dependencies_yml_dict = load_yml_dict( + get_yaml_path(f"{project_root}/{DEPENDENCIES_FILE_NAME}") + ) if "packages" in packages_yml_dict and "packages" in dependencies_yml_dict: msg = "The 'packages' key cannot be specified in both packages.yml and dependencies.yml" @@ -184,7 +186,7 @@ def value_or(value: Optional[T], default: T) -> T: def load_raw_project(project_root: str) -> Dict[str, Any]: project_root = os.path.normpath(project_root) - project_yaml_filepath = os.path.join(project_root, DBT_PROJECT_FILE_NAME) + project_yaml_filepath = get_yaml_path(os.path.join(project_root, DBT_PROJECT_FILE_NAME)) # get the project.yml contents if not path_exists(project_yaml_filepath): @@ -197,7 +199,7 @@ def load_raw_project(project_root: str) -> Dict[str, Any]: project_dict = _load_yaml(project_yaml_filepath) if not isinstance(project_dict, dict): - raise DbtProjectError(f"{DBT_PROJECT_FILE_NAME} does not parse to a dictionary") + raise DbtProjectError(f"{project_yaml_filepath} does not parse to a dictionary") if "tests" in project_dict and "data_tests" not in project_dict: project_dict["data_tests"] = project_dict.pop("tests") @@ -792,7 +794,7 @@ def read_project_flags(project_dir: str, profiles_dir: str) -> ProjectFlags: # want to throw an error for non-existence of dbt_project.yml here # because it breaks things. project_root = os.path.normpath(project_dir) - project_yaml_filepath = os.path.join(project_root, DBT_PROJECT_FILE_NAME) + project_yaml_filepath = get_yaml_path(os.path.join(project_root, DBT_PROJECT_FILE_NAME)) if path_exists(project_yaml_filepath): try: project_dict = load_raw_project(project_root) diff --git a/core/dbt/constants.py b/core/dbt/constants.py index 0ff538910d5..b35c4d442aa 100644 --- a/core/dbt/constants.py +++ b/core/dbt/constants.py @@ -11,8 +11,8 @@ "https://docs.getdbt.com/docs/package-management#section-specifying-package-versions" ) -DBT_PROJECT_FILE_NAME = "dbt_project.yml" -PACKAGES_FILE_NAME = "packages.yml" +DBT_PROJECT_FILE_NAME = "dbt_project" +PACKAGES_FILE_NAME = "packages" DEPENDENCIES_FILE_NAME = "dependencies.yml" PACKAGE_LOCK_FILE_NAME = "package-lock.yml" MANIFEST_FILE_NAME = "manifest.json" diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 6295f794e99..5c2a93b99a2 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -21,6 +21,7 @@ ) from dbt.artifacts.schemas.run import RunResult from dbt.cli.flags import Flags +from dbt.clients.yaml_helper import get_yaml_path from dbt.compilation import Compiler from dbt.config import RuntimeConfig from dbt.config.profile import read_profile @@ -88,6 +89,7 @@ def get_nearest_project_dir(project_dir: Optional[str]) -> Path: if project_dir: cur_dir = Path(project_dir) project_file = Path(project_dir) / DBT_PROJECT_FILE_NAME + project_file = Path(get_yaml_path(str(project_file))) if project_file.is_file(): return cur_dir else: