Merge branch 'd2x' into feature/flow-skip-start

additional.yml

@@ -0,0 +1,14 @@
+flows:
+  my_custom_flow:
+    description: A custom flow loaded via --load-yml
+    group: Loaded YAML
+    steps:
+      1:
+        task: my_custom_task
+tasks:
+  my_custom_task:
+    description: A custom task loaded via --load-yml
+    group: Loaded YAML
+    class_path: cumulusci.tasks.util.Sleep
+    options:
+      seconds: 1

cumulusci/cli/cci.py

@@ -1,5 +1,6 @@
 import code
 import contextlib
+import os
 import pdb
 import runpy
 import sys
@@ -52,6 +53,7 @@ def main(args=None):
 
     This wraps the `click` library in order to do some initialization and centralized error handling.
     """
+
     with contextlib.ExitStack() as stack:
         args = args or sys.argv
 
@@ -71,13 +73,27 @@ def main(args=None):
             logger, tempfile_path = get_tempfile_logger()
             stack.enter_context(tee_stdout_stderr(args, logger, tempfile_path))
 
+        context_kwargs = {}
+
+        # Allow commands to load additional yaml configuration from a file
+        if "--load-yml" in args:
+            yml_path_index = args.index("--load-yml") + 1
+            try:
+                load_yml_path = args[yml_path_index]
+            except IndexError:
+                raise CumulusCIUsageError("No path specified for --load-yml")
+            if not os.path.isfile(load_yml_path):
+                raise CumulusCIUsageError(f"File not found: {load_yml_path}")
+            with open(load_yml_path, "r") as f:
+                context_kwargs["additional_yaml"] = f.read()
+
         debug = "--debug" in args
         if debug:
             args.remove("--debug")
 
         with set_debug_mode(debug):
             try:
-                runtime = CliRuntime(load_keychain=False)
+                runtime = CliRuntime(load_keychain=False, **context_kwargs)
             except Exception as e:
                 handle_exception(e, is_error_command, tempfile_path, debug)
                 sys.exit(1)

cumulusci/cli/flow.py

@@ -24,8 +24,12 @@ def flow():
 @click.option(
     "--project", "project", is_flag=True, help="Include project-specific flows only"
 )
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_project=False, require_keychain=True)
-def flow_doc(runtime, project=False):
+def flow_doc(runtime, project=False, load_yml=None):
     flow_info_path = Path(__file__, "..", "..", "..", "docs", "flows.yml").resolve()
     with open(flow_info_path, "r", encoding="utf-8") as f:
         flow_info = load_yaml_data(f)
@@ -79,8 +83,12 @@ def flow_doc(runtime, project=False):
 @flow.command(name="list", help="List available flows for the current context")
 @click.option("--plain", is_flag=True, help="Print the table using plain ascii.")
 @click.option("--json", "print_json", is_flag=True, help="Print a json string")
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_project=False)
-def flow_list(runtime, plain, print_json):
+def flow_list(runtime, plain, print_json, load_yml=None):
     plain = plain or runtime.universal_config.cli__plain_output
     flows = runtime.get_available_flows()
     if print_json:
@@ -132,6 +140,7 @@ def flow_info(
 ):
     if skip:
         skip = skip.split(",")
+
     try:
         coordinator = runtime.get_flow(
             flow_name,
@@ -182,6 +191,7 @@ def flow_info(
     help="Specify a task or flow name to start from. All prior steps will be skippped.",
 )
 @click.option(
+
     "--load-yml",
     help="If set, loads the specified yml file into the the project config as additional config",
 )
@@ -201,6 +211,7 @@ def flow_run(
 ):
     if skip:
         skip = skip.split(",")
+
     # Get necessary configs
     org, org_config = runtime.get_org(org)
     if delete_org and not org_config.scratch:

cumulusci/cli/task.py

@@ -25,8 +25,12 @@ def task():
 @task.command(name="list", help="List available tasks for the current context")
 @click.option("--plain", is_flag=True, help="Print the table using plain ascii.")
 @click.option("--json", "print_json", is_flag=True, help="Print a json string")
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_project=False)
-def task_list(runtime, plain, print_json):
+def task_list(runtime, plain, print_json, load_yml=None):
     tasks = runtime.get_available_tasks()
     plain = plain or runtime.universal_config.cli__plain_output
 
@@ -60,8 +64,12 @@ def task_list(runtime, plain, print_json):
     is_flag=True,
     help="If true, write output to a file (./docs/project_tasks.rst or ./docs/cumulusci_tasks.rst)",
 )
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_project=False)
-def task_doc(runtime, project=False, write=False):
+def task_doc(runtime, project=False, write=False, load_yml=None):
     if project and runtime.project_config is None:
         raise click.UsageError(
             "The --project option can only be used inside a project."
@@ -95,8 +103,12 @@ def task_doc(runtime, project=False, write=False):
 
 @task.command(name="info", help="Displays information for a task")
 @click.argument("task_name")
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_project=False, require_keychain=True)
-def task_info(runtime, task_name):
+def task_info(runtime, task_name, load_yml=None):
     task_config = (
         runtime.project_config.get_task(task_name)
         if runtime.project_config is not None
@@ -126,6 +138,10 @@ class RunTaskCommand(click.MultiCommand):
             "help": "Drops into the Python debugger at task completion.",
             "is_flag": True,
         },
+        "load-yml": {
+            "help": "If set, loads the specified yml file into the the project config as additional config",
+            "is_flag": False,
+        },
     }
 
     def list_commands(self, ctx):

cumulusci/cli/tests/test_cci.py

@@ -1,3 +1,4 @@
+import builtins
 import contextlib
 import io
 import os
@@ -8,17 +9,20 @@
 from unittest import mock
 
 import click
+from click import Command
+from click.testing import CliRunner
 import pkg_resources
 import pytest
 from requests.exceptions import ConnectionError
 from rich.console import Console
 
 import cumulusci
 from cumulusci.cli import cci
+from cumulusci.cli.task import task_list
 from cumulusci.cli.tests.utils import run_click_command
 from cumulusci.cli.utils import get_installed_version
 from cumulusci.core.config import BaseProjectConfig
-from cumulusci.core.exceptions import CumulusCIException
+from cumulusci.core.exceptions import CumulusCIException, CumulusCIUsageError
 from cumulusci.utils import temporary_dir
 
 MagicMock = mock.MagicMock()
@@ -209,6 +213,83 @@ def test_main__CliRuntime_error(CliRuntime, get_tempfile_logger, tee):
     tempfile.unlink()
 
 
+@mock.patch("cumulusci.cli.cci.init_logger")  # side effects break other tests
+@mock.patch("cumulusci.cli.cci.get_tempfile_logger")
+@mock.patch("cumulusci.cli.cci.tee_stdout_stderr")
+@mock.patch("sys.exit")
+def test_cci_load_yml__missing(
+    exit, tee_stdout_stderr, get_tempfile_logger, init_logger
+):
+    # get_tempfile_logger doesn't clean up after itself which breaks other tests
+    get_tempfile_logger.return_value = mock.Mock(), ""
+    runner = CliRunner()
+    # Mock the contents of the yaml file
+    with pytest.raises(CumulusCIUsageError):
+        cci.main(
+            [
+                "cci",
+                "task",
+                "list",
+                "--load-yml",
+            ],
+        )
+
+
+@mock.patch("cumulusci.cli.cci.init_logger")  # side effects break other tests
+@mock.patch("cumulusci.cli.cci.get_tempfile_logger")
+@mock.patch("cumulusci.cli.cci.tee_stdout_stderr")
+@mock.patch("sys.exit")
+# @mock.patch("cumulusci.cli.cci.CliRuntime")
+def test_cci_load_yml__notfound(
+    exit, tee_stdout_stderr, get_tempfile_logger, init_logger
+):
+    # get_tempfile_logger doesn't clean up after itself which breaks other tests
+    get_tempfile_logger.return_value = mock.Mock(), ""
+    runner = CliRunner()
+    with pytest.raises(CumulusCIUsageError):
+        cci.main(
+            [
+                "cci",
+                "task",
+                "list",
+                "--load-yml",
+                "/path/that/does/not/exist/anywhere",
+            ],
+        )
+
+
+@mock.patch("cumulusci.cli.cci.init_logger")  # side effects break other tests
+@mock.patch("cumulusci.cli.cci.get_tempfile_logger")
+@mock.patch("cumulusci.cli.cci.tee_stdout_stderr")
+@mock.patch("sys.exit")
+@mock.patch("cumulusci.cli.cci.CliRuntime")
+def test_cci_load_yml(
+    CliRuntime, exit, tee_stdout_stderr, get_tempfile_logger, init_logger
+):
+    # get_tempfile_logger doesn't clean up after itself which breaks other tests
+    get_tempfile_logger.return_value = mock.Mock(), ""
+
+    load_yml_path = [cumulusci.__path__[0][: -len("/cumulusci")]]
+    load_yml_path.append("additional.yml")
+    load_yml = os.path.join(*load_yml_path)
+
+    cci.main(
+        [
+            "cci",
+            "org",
+            "default",
+            "--load-yml",
+            load_yml,
+        ]
+    )
+
+    # Check that CliRuntime was called with the correct arguments
+    with open(load_yml, "r") as f:
+        CliRuntime.assert_called_once_with(
+            load_keychain=False, additional_yaml=f.read()
+        )
+
+
 @mock.patch("cumulusci.cli.cci.init_logger")  # side effects break other tests
 @mock.patch("cumulusci.cli.cci.get_tempfile_logger")
 @mock.patch("cumulusci.cli.cci.tee_stdout_stderr")
@@ -217,7 +298,6 @@ def test_main__CliRuntime_error(CliRuntime, get_tempfile_logger, tee):
 def test_handle_org_name(
     CliRuntime, tee_stdout_stderr, get_tempfile_logger, init_logger
 ):
-
     # get_tempfile_logger doesn't clean up after itself which breaks other tests
     get_tempfile_logger.return_value = mock.Mock(), ""
 

cumulusci/cli/tests/test_task.py

@@ -126,10 +126,10 @@ def test_format_help(runtime):
 
 def test_get_default_command_options():
     opts = task.RunTaskCommand()._get_default_command_options(is_salesforce_task=False)
-    assert len(opts) == 4
+    assert len(opts) == 5
 
     opts = task.RunTaskCommand()._get_default_command_options(is_salesforce_task=True)
-    assert len(opts) == 5
+    assert len(opts) == 6
     assert any([o.name == "org" for o in opts])
 
 

cumulusci/core/config/project_config.py

@@ -16,6 +16,7 @@
 
 from cumulusci.core.config.base_config import BaseConfig
 from cumulusci.core.debug import get_debug_mode
+from cumulusci.core.utils import import_global
 from cumulusci.core.versions import PackageVersionNumber
 from cumulusci.utils.version_strings import LooseVersion
 
@@ -165,6 +166,29 @@ def _load_config(self):
         if project_config:
             self.config_project.update(project_config)
 
+        self.config_plugins = {}
+        for plugin in project_config.get("plugins", []):
+            # Look for a cumulusci.yml file in the plugin's python package root directory
+            # and load it if found
+            try:
+                __import__(plugin)
+                plugin_config_path = (
+                    Path(sys.modules[plugin].__file__).parent / "cumulusci.yml"
+                )
+            except Exception as exc:
+                raise ConfigError(f"Could not load plugin {plugin}: {exc}. Please make sure the plugin is installed.")
+            if plugin_config_path.is_file():
+                plugin_config = cci_safe_load(
+                    str(plugin_config_path), logger=self.logger
+                )
+                plugin_config = plugin_config
+                if plugin_config:
+                    self.config_plugins[plugin] = plugin_config
+                self.logger.info(f"Loaded plugin {plugin}")
+            else:
+                self.logger.info(f"Loaded plugin {plugin} but no cumulusci.yml found")
+
+
         # Load the local project yaml config file if it exists
         if self.config_project_local_path:
             local_config = cci_safe_load(
@@ -183,15 +207,21 @@ def _load_config(self):
             if additional_yaml_config:
                 self.config_additional_yaml.update(additional_yaml_config)
 
-        self.config = merge_config(
-            {
+        merge_stack = {
                 "universal_config": self.config_universal,
                 "global_config": self.config_global,
+        }
+        if self.config_plugins:
+            for plugin, plugin_config in self.config_plugins.items():
+                merge_stack[plugin] = plugin_config
+
+        merge_stack.update({
                 "project_config": self.config_project,
                 "project_local_config": self.config_project_local,
                 "additional_yaml": self.config_additional_yaml,
-            }
-        )
+        })
+
+        self.config = merge_config(merge_stack)
 
         self._validate_config()