Added support for --load-yml option on cci task and cci flow commands

including full test coverage and docs.

additional.yml

@@ -0,0 +1,14 @@
+flows:
+  my_custom_flow:
+    description: A custom flow loaded via --load-yml
+    group: Loaded YAML
+    steps:
+      1:
+        task: my_custom_task
+tasks:
+  my_custom_task:
+    description: A custom task loaded via --load-yml
+    group: Loaded YAML
+    class_path: cumulusci.tasks.util.Sleep
+    options:
+      seconds: 1

cumulusci/cli/cci.py

@@ -1,5 +1,6 @@
 import code
 import contextlib
+import os
 import pdb
 import runpy
 import sys
@@ -52,6 +53,7 @@ def main(args=None):
 
     This wraps the `click` library in order to do some initialization and centralized error handling.
     """
+
     with contextlib.ExitStack() as stack:
         args = args or sys.argv
 
@@ -71,13 +73,27 @@ def main(args=None):
             logger, tempfile_path = get_tempfile_logger()
             stack.enter_context(tee_stdout_stderr(args, logger, tempfile_path))
 
+        context_kwargs = {}
+
+        # Allow commands to load additional yaml configuration from a file
+        if "--load-yml" in args:
+            yml_path_index = args.index("--load-yml") + 1
+            try:
+                load_yml_path = args[yml_path_index]
+            except IndexError:
+                raise CumulusCIUsageError("No path specified for --load-yml")
+            if not os.path.isfile(load_yml_path):
+                raise CumulusCIUsageError(f"File not found: {load_yml_path}")
+            with open(load_yml_path, "r") as f:
+                context_kwargs["additional_yaml"] = f.read()
+
         debug = "--debug" in args
         if debug:
             args.remove("--debug")
 
         with set_debug_mode(debug):
             try:
-                runtime = CliRuntime(load_keychain=False)
+                runtime = CliRuntime(load_keychain=False, **context_kwargs)
             except Exception as e:
                 handle_exception(e, is_error_command, tempfile_path, debug)
                 sys.exit(1)

cumulusci/cli/flow.py

@@ -24,8 +24,12 @@ def flow():
 @click.option(
     "--project", "project", is_flag=True, help="Include project-specific flows only"
 )
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_project=False, require_keychain=True)
-def flow_doc(runtime, project=False):
+def flow_doc(runtime, project=False, load_yml=None):
     flow_info_path = Path(__file__, "..", "..", "..", "docs", "flows.yml").resolve()
     with open(flow_info_path, "r", encoding="utf-8") as f:
         flow_info = load_yaml_data(f)
@@ -79,8 +83,12 @@ def flow_doc(runtime, project=False):
 @flow.command(name="list", help="List available flows for the current context")
 @click.option("--plain", is_flag=True, help="Print the table using plain ascii.")
 @click.option("--json", "print_json", is_flag=True, help="Print a json string")
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_project=False)
-def flow_list(runtime, plain, print_json):
+def flow_list(runtime, plain, print_json, load_yml=None):
     plain = plain or runtime.universal_config.cli__plain_output
     flows = runtime.get_available_flows()
     if print_json:
@@ -106,8 +114,12 @@ def flow_list(runtime, plain, print_json):
 
 @flow.command(name="info", help="Displays information for a flow")
 @click.argument("flow_name")
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_keychain=True)
-def flow_info(runtime, flow_name):
+def flow_info(runtime, flow_name, load_yml=None):
     try:
         coordinator = runtime.get_flow(flow_name)
         output = coordinator.get_summary(verbose=True)
@@ -141,9 +153,12 @@ def flow_info(runtime, flow_name):
     is_flag=True,
     help="Disables all prompts.  Set for non-interactive mode use such as calling from scripts or CI systems",
 )
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_keychain=True)
-def flow_run(runtime, flow_name, org, delete_org, debug, o, no_prompt):
-
+def flow_run(runtime, flow_name, org, delete_org, debug, o, no_prompt, load_yml=None):
     # Get necessary configs
     org, org_config = runtime.get_org(org)
     if delete_org and not org_config.scratch:

cumulusci/cli/task.py

@@ -25,8 +25,12 @@ def task():
 @task.command(name="list", help="List available tasks for the current context")
 @click.option("--plain", is_flag=True, help="Print the table using plain ascii.")
 @click.option("--json", "print_json", is_flag=True, help="Print a json string")
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_project=False)
-def task_list(runtime, plain, print_json):
+def task_list(runtime, plain, print_json, load_yml=None):
     tasks = runtime.get_available_tasks()
     plain = plain or runtime.universal_config.cli__plain_output
 
@@ -60,8 +64,12 @@ def task_list(runtime, plain, print_json):
     is_flag=True,
     help="If true, write output to a file (./docs/project_tasks.rst or ./docs/cumulusci_tasks.rst)",
 )
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_project=False)
-def task_doc(runtime, project=False, write=False):
+def task_doc(runtime, project=False, write=False, load_yml=None):
     if project and runtime.project_config is None:
         raise click.UsageError(
             "The --project option can only be used inside a project."
@@ -95,8 +103,12 @@ def task_doc(runtime, project=False, write=False):
 
 @task.command(name="info", help="Displays information for a task")
 @click.argument("task_name")
+@click.option(
+    "--load-yml",
+    help="If set, loads the specified yml file into the the project config as additional config",
+)
 @pass_runtime(require_project=False, require_keychain=True)
-def task_info(runtime, task_name):
+def task_info(runtime, task_name, load_yml=None):
     task_config = (
         runtime.project_config.get_task(task_name)
         if runtime.project_config is not None
@@ -126,6 +138,10 @@ class RunTaskCommand(click.MultiCommand):
             "help": "Drops into the Python debugger at task completion.",
             "is_flag": True,
         },
+        "load-yml": {
+            "help": "If set, loads the specified yml file into the the project config as additional config",
+            "is_flag": False,
+        },
     }
 
     def list_commands(self, ctx):

cumulusci/cli/tests/test_cci.py

@@ -1,3 +1,4 @@
+import builtins
 import contextlib
 import io
 import os
@@ -8,17 +9,20 @@
 from unittest import mock
 
 import click
+from click import Command
+from click.testing import CliRunner
 import pkg_resources
 import pytest
 from requests.exceptions import ConnectionError
 from rich.console import Console
 
 import cumulusci
 from cumulusci.cli import cci
+from cumulusci.cli.task import task_list
 from cumulusci.cli.tests.utils import run_click_command
 from cumulusci.cli.utils import get_installed_version
 from cumulusci.core.config import BaseProjectConfig
-from cumulusci.core.exceptions import CumulusCIException
+from cumulusci.core.exceptions import CumulusCIException, CumulusCIUsageError
 from cumulusci.utils import temporary_dir
 
 MagicMock = mock.MagicMock()
@@ -209,6 +213,83 @@ def test_main__CliRuntime_error(CliRuntime, get_tempfile_logger, tee):
     tempfile.unlink()
 
 
+@mock.patch("cumulusci.cli.cci.init_logger")  # side effects break other tests
+@mock.patch("cumulusci.cli.cci.get_tempfile_logger")
+@mock.patch("cumulusci.cli.cci.tee_stdout_stderr")
+@mock.patch("sys.exit")
+def test_cci_load_yml__missing(
+    exit, tee_stdout_stderr, get_tempfile_logger, init_logger
+):
+    # get_tempfile_logger doesn't clean up after itself which breaks other tests
+    get_tempfile_logger.return_value = mock.Mock(), ""
+    runner = CliRunner()
+    # Mock the contents of the yaml file
+    with pytest.raises(CumulusCIUsageError):
+        cci.main(
+            [
+                "cci",
+                "task",
+                "list",
+                "--load-yml",
+            ],
+        )
+
+
+@mock.patch("cumulusci.cli.cci.init_logger")  # side effects break other tests
+@mock.patch("cumulusci.cli.cci.get_tempfile_logger")
+@mock.patch("cumulusci.cli.cci.tee_stdout_stderr")
+@mock.patch("sys.exit")
+# @mock.patch("cumulusci.cli.cci.CliRuntime")
+def test_cci_load_yml__notfound(
+    exit, tee_stdout_stderr, get_tempfile_logger, init_logger
+):
+    # get_tempfile_logger doesn't clean up after itself which breaks other tests
+    get_tempfile_logger.return_value = mock.Mock(), ""
+    runner = CliRunner()
+    with pytest.raises(CumulusCIUsageError):
+        cci.main(
+            [
+                "cci",
+                "task",
+                "list",
+                "--load-yml",
+                "/path/that/does/not/exist/anywhere",
+            ],
+        )
+
+
+@mock.patch("cumulusci.cli.cci.init_logger")  # side effects break other tests
+@mock.patch("cumulusci.cli.cci.get_tempfile_logger")
+@mock.patch("cumulusci.cli.cci.tee_stdout_stderr")
+@mock.patch("sys.exit")
+@mock.patch("cumulusci.cli.cci.CliRuntime")
+def test_cci_load_yml(
+    CliRuntime, exit, tee_stdout_stderr, get_tempfile_logger, init_logger
+):
+    # get_tempfile_logger doesn't clean up after itself which breaks other tests
+    get_tempfile_logger.return_value = mock.Mock(), ""
+
+    load_yml_path = [cumulusci.__path__[0][: -len("/cumulusci")]]
+    load_yml_path.append("additional.yml")
+    load_yml = os.path.join(*load_yml_path)
+
+    cci.main(
+        [
+            "cci",
+            "org",
+            "default",
+            "--load-yml",
+            load_yml,
+        ]
+    )
+
+    # Check that CliRuntime was called with the correct arguments
+    with open(load_yml, "r") as f:
+        CliRuntime.assert_called_once_with(
+            load_keychain=False, additional_yaml=f.read()
+        )
+
+
 @mock.patch("cumulusci.cli.cci.init_logger")  # side effects break other tests
 @mock.patch("cumulusci.cli.cci.get_tempfile_logger")
 @mock.patch("cumulusci.cli.cci.tee_stdout_stderr")
@@ -217,7 +298,6 @@ def test_main__CliRuntime_error(CliRuntime, get_tempfile_logger, tee):
 def test_handle_org_name(
     CliRuntime, tee_stdout_stderr, get_tempfile_logger, init_logger
 ):
-
     # get_tempfile_logger doesn't clean up after itself which breaks other tests
     get_tempfile_logger.return_value = mock.Mock(), ""
 

cumulusci/cli/tests/test_task.py

@@ -126,10 +126,10 @@ def test_format_help(runtime):
 
 def test_get_default_command_options():
     opts = task.RunTaskCommand()._get_default_command_options(is_salesforce_task=False)
-    assert len(opts) == 4
+    assert len(opts) == 5
 
     opts = task.RunTaskCommand()._get_default_command_options(is_salesforce_task=True)
-    assert len(opts) == 5
+    assert len(opts) == 6
     assert any([o.name == "org" for o in opts])
 
 

docs/config.md

@@ -908,3 +908,25 @@ how a task or flow is _currently_ configured. The information output by
 these commands change as you make further customizations to your
 project's `cumulusci.yml` file.
 ```
+
+## Loading additional yaml configuration from a file
+
+CumulusCI supports loading in an additional yaml file from the command line with the `--load-yml <path/to/file>` option on `cci task` and `cci flow` commands. This can be useful if you have one-off automation configurations that you want to keep out of the main project's configuration and load only in special cases.
+
+A good example of this is upgrade or migration scripts to do things like enabling a new feature from a release. If you create a lot of these upgrade scripts as their own flows. All those flows and all their one-off custom tasks would show up for everyone in the `cci task list` and `cci flow list`. Instead, you could create a directory of yaml files such as `migrations/1.2.yml` and `migrations/1.3.yml` where each file contains a set of custom tasks and flows only needed for their migration logic.
+
+You could inspect the new commands added on top of the existing project config defined in cumulusci.yml with the following commands:
+
+```console
+# Tasks
+$ cci task list --load-yml migrations/1.2.yml
+$ cci task info custom_task_for_1.2 --load-yml migrations/1.2.yml
+$ cci task run custom_task_for_1.2 --load-yml migrations/1.2.yml
+
+# Flows
+$ cci flow list --load-yml migrations/1.2.yml
+$ cci flow info custom_flow_for_1.2 --load-yml migrations/1.2.yml
+$ cci flow run custom_flow_for_1.2 --load-yml migrations/1.2.yml
+```
+
+Behind the scenes, CumulusCI is merging the yaml file specified by `--load-yml` on top of the project config. This means any customizations you could make in cumulusci.yml can also be made in a file loaded via `--load-yml`