From 9673c4a9d697f6aa43c0220aac2664afc11e8520 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Fri, 23 Feb 2024 22:09:50 +0000 Subject: [PATCH 1/7] wip --- .../Under the Hood-20240221-145058.yaml | 6 ++++++ .github/workflows/main.yml | 4 ++-- core/dbt/task/test.py | 4 +--- core/dbt/utils.py | 18 ++++++++++++++++++ 4 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240221-145058.yaml diff --git a/.changes/unreleased/Under the Hood-20240221-145058.yaml b/.changes/unreleased/Under the Hood-20240221-145058.yaml new file mode 100644 index 00000000000..a847bb68c53 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240221-145058.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Make dbt-core compatible with Python 3.12 +time: 2024-02-21T14:50:58.983559Z +custom: + Author: l1xnan aranke + Issue: "9007" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1819146369a..db9fade87f6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] env: TOXENV: "unit" @@ -157,7 +157,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] os: [ubuntu-20.04] split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }} include: ${{ fromJson(needs.integration-metadata.outputs.include) }} diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py index 9548f633eb7..0754ca2277f 100644 --- a/core/dbt/task/test.py +++ b/core/dbt/task/test.py @@ -1,12 +1,10 @@ -from distutils.util import strtobool - import agate import daff import io import json import re from dataclasses import dataclass -from dbt.utils import _coerce_decimal +from dbt.utils import _coerce_decimal, strtobool from dbt_common.events.format import pluralize from dbt_common.dataclass_schema import dbtClassMixin import threading diff --git a/core/dbt/utils.py b/core/dbt/utils.py index b4d510da8bb..8f7509a5dec 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -369,3 +369,21 @@ def args_to_dict(args): dict_args[key] = var_args[key] return dict_args + + +# Taken from https://github.com/python/cpython/blob/3.11/Lib/distutils/util.py +# This is a copy of the function from distutils.util, which was removed in Python 3.12. +def strtobool(val: str) -> bool: + """Convert a string representation of truth to True or False. + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + """ + val = val.lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return True + elif val in ("n", "no", "f", "false", "off", "0"): + return False + else: + raise ValueError("invalid truth value %r" % (val,)) From 1a2ac996e6bb9491f2cff8415f93645dec75fe5e Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Fri, 23 Feb 2024 23:27:45 +0000 Subject: [PATCH 2/7] add pythonpath --- core/dbt/task/init.py | 10 +++------- pytest.ini | 1 + 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index aa7e942c206..83f5f75d506 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -34,6 +34,9 @@ from dbt.task.base import BaseTask, move_to_nearest_project_dir +from dbt.include.starter_project import PACKAGE_PATH as starter_project_directory +from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME + DOCS_URL = "https://docs.getdbt.com/docs/configure-your-profile" SLACK_URL = "https://community.getdbt.com/" @@ -54,9 +57,6 @@ class InitTask(BaseTask): def copy_starter_repo(self, project_name: str) -> None: - # Lazy import to avoid ModuleNotFoundError - from dbt.include.starter_project import PACKAGE_PATH as starter_project_directory - fire_event(StarterProjectPath(dir=starter_project_directory)) shutil.copytree( starter_project_directory, project_name, ignore=shutil.ignore_patterns(*IGNORE_FILES) @@ -264,10 +264,6 @@ def setup_profile(self, profile_name: str) -> None: def get_valid_project_name(self) -> str: """Returns a valid project name, either from CLI arg or user prompt.""" - - # Lazy import to avoid ModuleNotFoundError - from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME - name = self.args.project_name internal_package_names = {GLOBAL_PROJECT_NAME} available_adapters = list(_get_adapter_plugin_names()) diff --git a/pytest.ini b/pytest.ini index 0760d49a55a..800dd6b9ece 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,3 +7,4 @@ env_files = testpaths = tests/functional tests/unit +pythonpath = core From 04c15eacd9ebe8bd4c67aeb17757f206aaf7211f Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Fri, 23 Feb 2024 23:31:33 +0000 Subject: [PATCH 3/7] Discard changes to core/dbt/task/init.py --- core/dbt/task/init.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index 83f5f75d506..aa7e942c206 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -34,9 +34,6 @@ from dbt.task.base import BaseTask, move_to_nearest_project_dir -from dbt.include.starter_project import PACKAGE_PATH as starter_project_directory -from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME - DOCS_URL = "https://docs.getdbt.com/docs/configure-your-profile" SLACK_URL = "https://community.getdbt.com/" @@ -57,6 +54,9 @@ class InitTask(BaseTask): def copy_starter_repo(self, project_name: str) -> None: + # Lazy import to avoid ModuleNotFoundError + from dbt.include.starter_project import PACKAGE_PATH as starter_project_directory + fire_event(StarterProjectPath(dir=starter_project_directory)) shutil.copytree( starter_project_directory, project_name, ignore=shutil.ignore_patterns(*IGNORE_FILES) @@ -264,6 +264,10 @@ def setup_profile(self, profile_name: str) -> None: def get_valid_project_name(self) -> str: """Returns a valid project name, either from CLI arg or user prompt.""" + + # Lazy import to avoid ModuleNotFoundError + from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME + name = self.args.project_name internal_package_names = {GLOBAL_PROJECT_NAME} available_adapters = list(_get_adapter_plugin_names()) From 86f08411fb9052b7665ad292c7c1ad923d7d57dd Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Fri, 23 Feb 2024 23:39:45 +0000 Subject: [PATCH 4/7] called_once_with -> assert_called_once_with --- core/dbt/task/init.py | 10 +++------- tests/unit/test_plugin_manager.py | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index aa7e942c206..83f5f75d506 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -34,6 +34,9 @@ from dbt.task.base import BaseTask, move_to_nearest_project_dir +from dbt.include.starter_project import PACKAGE_PATH as starter_project_directory +from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME + DOCS_URL = "https://docs.getdbt.com/docs/configure-your-profile" SLACK_URL = "https://community.getdbt.com/" @@ -54,9 +57,6 @@ class InitTask(BaseTask): def copy_starter_repo(self, project_name: str) -> None: - # Lazy import to avoid ModuleNotFoundError - from dbt.include.starter_project import PACKAGE_PATH as starter_project_directory - fire_event(StarterProjectPath(dir=starter_project_directory)) shutil.copytree( starter_project_directory, project_name, ignore=shutil.ignore_patterns(*IGNORE_FILES) @@ -264,10 +264,6 @@ def setup_profile(self, profile_name: str) -> None: def get_valid_project_name(self) -> str: """Returns a valid project name, either from CLI arg or user prompt.""" - - # Lazy import to avoid ModuleNotFoundError - from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME - name = self.args.project_name internal_package_names = {GLOBAL_PROJECT_NAME} available_adapters = list(_get_adapter_plugin_names()) diff --git a/tests/unit/test_plugin_manager.py b/tests/unit/test_plugin_manager.py index 4cfe01d0dfc..38cdea002f9 100644 --- a/tests/unit/test_plugin_manager.py +++ b/tests/unit/test_plugin_manager.py @@ -101,7 +101,7 @@ def test_get_nodes(self, tracking, get_nodes_plugins): nodes = pm.get_nodes() assert len(nodes.models) == 2 - assert tracking.track_plugin_get_nodes.called_once_with( + tracking.track_plugin_get_nodes.assert_called_once_with( { "plugin_name": get_nodes_plugins[0].name, "num_model_nodes": 2, From a67d62a1ef3891ac631dc359d9e8b627f05eb5e3 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Sat, 24 Feb 2024 00:09:08 +0000 Subject: [PATCH 5/7] Discard changes to core/dbt/task/init.py --- core/dbt/task/init.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/dbt/task/init.py b/core/dbt/task/init.py index 83f5f75d506..aa7e942c206 100644 --- a/core/dbt/task/init.py +++ b/core/dbt/task/init.py @@ -34,9 +34,6 @@ from dbt.task.base import BaseTask, move_to_nearest_project_dir -from dbt.include.starter_project import PACKAGE_PATH as starter_project_directory -from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME - DOCS_URL = "https://docs.getdbt.com/docs/configure-your-profile" SLACK_URL = "https://community.getdbt.com/" @@ -57,6 +54,9 @@ class InitTask(BaseTask): def copy_starter_repo(self, project_name: str) -> None: + # Lazy import to avoid ModuleNotFoundError + from dbt.include.starter_project import PACKAGE_PATH as starter_project_directory + fire_event(StarterProjectPath(dir=starter_project_directory)) shutil.copytree( starter_project_directory, project_name, ignore=shutil.ignore_patterns(*IGNORE_FILES) @@ -264,6 +264,10 @@ def setup_profile(self, profile_name: str) -> None: def get_valid_project_name(self) -> str: """Returns a valid project name, either from CLI arg or user prompt.""" + + # Lazy import to avoid ModuleNotFoundError + from dbt.include.global_project import PROJECT_NAME as GLOBAL_PROJECT_NAME + name = self.args.project_name internal_package_names = {GLOBAL_PROJECT_NAME} available_adapters = list(_get_adapter_plugin_names()) From df8b9f26ab97831df10194c12124eaaab589e460 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Mon, 26 Feb 2024 09:51:58 -0600 Subject: [PATCH 6/7] add expected_calls to test_plugin_manager --- tests/unit/test_plugin_manager.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_plugin_manager.py b/tests/unit/test_plugin_manager.py index 38cdea002f9..bf25d810729 100644 --- a/tests/unit/test_plugin_manager.py +++ b/tests/unit/test_plugin_manager.py @@ -101,13 +101,25 @@ def test_get_nodes(self, tracking, get_nodes_plugins): nodes = pm.get_nodes() assert len(nodes.models) == 2 - tracking.track_plugin_get_nodes.assert_called_once_with( - { - "plugin_name": get_nodes_plugins[0].name, - "num_model_nodes": 2, - "num_model_packages": 1, - } - ) + + expected_calls = [ + mock.call( + { + "plugin_name": get_nodes_plugins[0].name, + "num_model_nodes": 1, + "num_model_packages": 1, + } + ), + mock.call( + { + "plugin_name": get_nodes_plugins[1].name, + "num_model_nodes": 1, + "num_model_packages": 1, + } + ), + ] + + tracking.track_plugin_get_nodes.assert_has_calls(expected_calls) def test_get_manifest_artifact(self, get_artifacts_plugins): pm = PluginManager(plugins=get_artifacts_plugins) From 36c4671fc160e8a376a70298746998041e798ee7 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Mon, 26 Feb 2024 10:24:57 -0600 Subject: [PATCH 7/7] Add Python 3.12 tag --- core/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/setup.py b/core/setup.py index 3bd43cd31df..190b4501e41 100644 --- a/core/setup.py +++ b/core/setup.py @@ -95,6 +95,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], python_requires=">=3.8", )