From 73c931bf56f893a877d62de4657c4dd57655b934 Mon Sep 17 00:00:00 2001 From: Jason Lantz Date: Mon, 1 Jan 2024 19:34:45 -0600 Subject: [PATCH] Added support for `--load-yml` option on cci task and cci flow commands including full test coverage and docs. --- additional.yml | 14 ++++++ cumulusci/cli/cci.py | 18 ++++++- cumulusci/cli/flow.py | 25 ++++++++-- cumulusci/cli/task.py | 22 +++++++-- cumulusci/cli/tests/test_cci.py | 84 +++++++++++++++++++++++++++++++- cumulusci/cli/tests/test_task.py | 4 +- docs/config.md | 22 +++++++++ 7 files changed, 176 insertions(+), 13 deletions(-) create mode 100644 additional.yml diff --git a/additional.yml b/additional.yml new file mode 100644 index 0000000000..7a78b7a871 --- /dev/null +++ b/additional.yml @@ -0,0 +1,14 @@ +flows: + my_custom_flow: + description: A custom flow loaded via --load-yml + group: Loaded YAML + steps: + 1: + task: my_custom_task +tasks: + my_custom_task: + description: A custom task loaded via --load-yml + group: Loaded YAML + class_path: cumulusci.tasks.util.Sleep + options: + seconds: 1 diff --git a/cumulusci/cli/cci.py b/cumulusci/cli/cci.py index adc50c49df..d950fc6591 100644 --- a/cumulusci/cli/cci.py +++ b/cumulusci/cli/cci.py @@ -1,5 +1,6 @@ import code import contextlib +import os import pdb import runpy import sys @@ -52,6 +53,7 @@ def main(args=None): This wraps the `click` library in order to do some initialization and centralized error handling. """ + with contextlib.ExitStack() as stack: args = args or sys.argv @@ -71,13 +73,27 @@ def main(args=None): logger, tempfile_path = get_tempfile_logger() stack.enter_context(tee_stdout_stderr(args, logger, tempfile_path)) + context_kwargs = {} + + # Allow commands to load additional yaml configuration from a file + if "--load-yml" in args: + yml_path_index = args.index("--load-yml") + 1 + try: + load_yml_path = args[yml_path_index] + except IndexError: + raise CumulusCIUsageError("No path specified for --load-yml") + if not os.path.isfile(load_yml_path): + raise CumulusCIUsageError(f"File not found: {load_yml_path}") + with open(load_yml_path, "r") as f: + context_kwargs["additional_yaml"] = f.read() + debug = "--debug" in args if debug: args.remove("--debug") with set_debug_mode(debug): try: - runtime = CliRuntime(load_keychain=False) + runtime = CliRuntime(load_keychain=False, **context_kwargs) except Exception as e: handle_exception(e, is_error_command, tempfile_path, debug) sys.exit(1) diff --git a/cumulusci/cli/flow.py b/cumulusci/cli/flow.py index 96bd8db9cf..8733331129 100644 --- a/cumulusci/cli/flow.py +++ b/cumulusci/cli/flow.py @@ -24,8 +24,12 @@ def flow(): @click.option( "--project", "project", is_flag=True, help="Include project-specific flows only" ) +@click.option( + "--load-yml", + help="If set, loads the specified yml file into the the project config as additional config", +) @pass_runtime(require_project=False, require_keychain=True) -def flow_doc(runtime, project=False): +def flow_doc(runtime, project=False, load_yml=None): flow_info_path = Path(__file__, "..", "..", "..", "docs", "flows.yml").resolve() with open(flow_info_path, "r", encoding="utf-8") as f: flow_info = load_yaml_data(f) @@ -79,8 +83,12 @@ def flow_doc(runtime, project=False): @flow.command(name="list", help="List available flows for the current context") @click.option("--plain", is_flag=True, help="Print the table using plain ascii.") @click.option("--json", "print_json", is_flag=True, help="Print a json string") +@click.option( + "--load-yml", + help="If set, loads the specified yml file into the the project config as additional config", +) @pass_runtime(require_project=False) -def flow_list(runtime, plain, print_json): +def flow_list(runtime, plain, print_json, load_yml=None): plain = plain or runtime.universal_config.cli__plain_output flows = runtime.get_available_flows() if print_json: @@ -106,8 +114,12 @@ def flow_list(runtime, plain, print_json): @flow.command(name="info", help="Displays information for a flow") @click.argument("flow_name") +@click.option( + "--load-yml", + help="If set, loads the specified yml file into the the project config as additional config", +) @pass_runtime(require_keychain=True) -def flow_info(runtime, flow_name): +def flow_info(runtime, flow_name, load_yml=None): try: coordinator = runtime.get_flow(flow_name) output = coordinator.get_summary(verbose=True) @@ -141,9 +153,12 @@ def flow_info(runtime, flow_name): is_flag=True, help="Disables all prompts. Set for non-interactive mode use such as calling from scripts or CI systems", ) +@click.option( + "--load-yml", + help="If set, loads the specified yml file into the the project config as additional config", +) @pass_runtime(require_keychain=True) -def flow_run(runtime, flow_name, org, delete_org, debug, o, no_prompt): - +def flow_run(runtime, flow_name, org, delete_org, debug, o, no_prompt, load_yml=None): # Get necessary configs org, org_config = runtime.get_org(org) if delete_org and not org_config.scratch: diff --git a/cumulusci/cli/task.py b/cumulusci/cli/task.py index cfbe749b91..1ee0ee7df3 100644 --- a/cumulusci/cli/task.py +++ b/cumulusci/cli/task.py @@ -25,8 +25,12 @@ def task(): @task.command(name="list", help="List available tasks for the current context") @click.option("--plain", is_flag=True, help="Print the table using plain ascii.") @click.option("--json", "print_json", is_flag=True, help="Print a json string") +@click.option( + "--load-yml", + help="If set, loads the specified yml file into the the project config as additional config", +) @pass_runtime(require_project=False) -def task_list(runtime, plain, print_json): +def task_list(runtime, plain, print_json, load_yml=None): tasks = runtime.get_available_tasks() plain = plain or runtime.universal_config.cli__plain_output @@ -60,8 +64,12 @@ def task_list(runtime, plain, print_json): is_flag=True, help="If true, write output to a file (./docs/project_tasks.rst or ./docs/cumulusci_tasks.rst)", ) +@click.option( + "--load-yml", + help="If set, loads the specified yml file into the the project config as additional config", +) @pass_runtime(require_project=False) -def task_doc(runtime, project=False, write=False): +def task_doc(runtime, project=False, write=False, load_yml=None): if project and runtime.project_config is None: raise click.UsageError( "The --project option can only be used inside a project." @@ -95,8 +103,12 @@ def task_doc(runtime, project=False, write=False): @task.command(name="info", help="Displays information for a task") @click.argument("task_name") +@click.option( + "--load-yml", + help="If set, loads the specified yml file into the the project config as additional config", +) @pass_runtime(require_project=False, require_keychain=True) -def task_info(runtime, task_name): +def task_info(runtime, task_name, load_yml=None): task_config = ( runtime.project_config.get_task(task_name) if runtime.project_config is not None @@ -126,6 +138,10 @@ class RunTaskCommand(click.MultiCommand): "help": "Drops into the Python debugger at task completion.", "is_flag": True, }, + "load-yml": { + "help": "If set, loads the specified yml file into the the project config as additional config", + "is_flag": False, + }, } def list_commands(self, ctx): diff --git a/cumulusci/cli/tests/test_cci.py b/cumulusci/cli/tests/test_cci.py index ccb06adaea..e5a01712f3 100644 --- a/cumulusci/cli/tests/test_cci.py +++ b/cumulusci/cli/tests/test_cci.py @@ -1,3 +1,4 @@ +import builtins import contextlib import io import os @@ -8,6 +9,8 @@ from unittest import mock import click +from click import Command +from click.testing import CliRunner import pkg_resources import pytest from requests.exceptions import ConnectionError @@ -15,10 +18,11 @@ import cumulusci from cumulusci.cli import cci +from cumulusci.cli.task import task_list from cumulusci.cli.tests.utils import run_click_command from cumulusci.cli.utils import get_installed_version from cumulusci.core.config import BaseProjectConfig -from cumulusci.core.exceptions import CumulusCIException +from cumulusci.core.exceptions import CumulusCIException, CumulusCIUsageError from cumulusci.utils import temporary_dir MagicMock = mock.MagicMock() @@ -209,6 +213,83 @@ def test_main__CliRuntime_error(CliRuntime, get_tempfile_logger, tee): tempfile.unlink() +@mock.patch("cumulusci.cli.cci.init_logger") # side effects break other tests +@mock.patch("cumulusci.cli.cci.get_tempfile_logger") +@mock.patch("cumulusci.cli.cci.tee_stdout_stderr") +@mock.patch("sys.exit") +def test_cci_load_yml__missing( + exit, tee_stdout_stderr, get_tempfile_logger, init_logger +): + # get_tempfile_logger doesn't clean up after itself which breaks other tests + get_tempfile_logger.return_value = mock.Mock(), "" + runner = CliRunner() + # Mock the contents of the yaml file + with pytest.raises(CumulusCIUsageError): + cci.main( + [ + "cci", + "task", + "list", + "--load-yml", + ], + ) + + +@mock.patch("cumulusci.cli.cci.init_logger") # side effects break other tests +@mock.patch("cumulusci.cli.cci.get_tempfile_logger") +@mock.patch("cumulusci.cli.cci.tee_stdout_stderr") +@mock.patch("sys.exit") +# @mock.patch("cumulusci.cli.cci.CliRuntime") +def test_cci_load_yml__notfound( + exit, tee_stdout_stderr, get_tempfile_logger, init_logger +): + # get_tempfile_logger doesn't clean up after itself which breaks other tests + get_tempfile_logger.return_value = mock.Mock(), "" + runner = CliRunner() + with pytest.raises(CumulusCIUsageError): + cci.main( + [ + "cci", + "task", + "list", + "--load-yml", + "/path/that/does/not/exist/anywhere", + ], + ) + + +@mock.patch("cumulusci.cli.cci.init_logger") # side effects break other tests +@mock.patch("cumulusci.cli.cci.get_tempfile_logger") +@mock.patch("cumulusci.cli.cci.tee_stdout_stderr") +@mock.patch("sys.exit") +@mock.patch("cumulusci.cli.cci.CliRuntime") +def test_cci_load_yml( + CliRuntime, exit, tee_stdout_stderr, get_tempfile_logger, init_logger +): + # get_tempfile_logger doesn't clean up after itself which breaks other tests + get_tempfile_logger.return_value = mock.Mock(), "" + + load_yml_path = [cumulusci.__path__[0][: -len("/cumulusci")]] + load_yml_path.append("additional.yml") + load_yml = os.path.join(*load_yml_path) + + cci.main( + [ + "cci", + "org", + "default", + "--load-yml", + load_yml, + ] + ) + + # Check that CliRuntime was called with the correct arguments + with open(load_yml, "r") as f: + CliRuntime.assert_called_once_with( + load_keychain=False, additional_yaml=f.read() + ) + + @mock.patch("cumulusci.cli.cci.init_logger") # side effects break other tests @mock.patch("cumulusci.cli.cci.get_tempfile_logger") @mock.patch("cumulusci.cli.cci.tee_stdout_stderr") @@ -217,7 +298,6 @@ def test_main__CliRuntime_error(CliRuntime, get_tempfile_logger, tee): def test_handle_org_name( CliRuntime, tee_stdout_stderr, get_tempfile_logger, init_logger ): - # get_tempfile_logger doesn't clean up after itself which breaks other tests get_tempfile_logger.return_value = mock.Mock(), "" diff --git a/cumulusci/cli/tests/test_task.py b/cumulusci/cli/tests/test_task.py index 398fb1b8b1..6b88aeaef1 100644 --- a/cumulusci/cli/tests/test_task.py +++ b/cumulusci/cli/tests/test_task.py @@ -126,10 +126,10 @@ def test_format_help(runtime): def test_get_default_command_options(): opts = task.RunTaskCommand()._get_default_command_options(is_salesforce_task=False) - assert len(opts) == 4 + assert len(opts) == 5 opts = task.RunTaskCommand()._get_default_command_options(is_salesforce_task=True) - assert len(opts) == 5 + assert len(opts) == 6 assert any([o.name == "org" for o in opts]) diff --git a/docs/config.md b/docs/config.md index 99ca4fc416..865202cff9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -908,3 +908,25 @@ how a task or flow is _currently_ configured. The information output by these commands change as you make further customizations to your project's `cumulusci.yml` file. ``` + +## Loading additional yaml configuration from a file + +CumulusCI supports loading in an additional yaml file from the command line with the `--load-yml ` option on `cci task` and `cci flow` commands. This can be useful if you have one-off automation configurations that you want to keep out of the main project's configuration and load only in special cases. + +A good example of this is upgrade or migration scripts to do things like enabling a new feature from a release. If you create a lot of these upgrade scripts as their own flows. All those flows and all their one-off custom tasks would show up for everyone in the `cci task list` and `cci flow list`. Instead, you could create a directory of yaml files such as `migrations/1.2.yml` and `migrations/1.3.yml` where each file contains a set of custom tasks and flows only needed for their migration logic. + +You could inspect the new commands added on top of the existing project config defined in cumulusci.yml with the following commands: + +```console +# Tasks +$ cci task list --load-yml migrations/1.2.yml +$ cci task info custom_task_for_1.2 --load-yml migrations/1.2.yml +$ cci task run custom_task_for_1.2 --load-yml migrations/1.2.yml + +# Flows +$ cci flow list --load-yml migrations/1.2.yml +$ cci flow info custom_flow_for_1.2 --load-yml migrations/1.2.yml +$ cci flow run custom_flow_for_1.2 --load-yml migrations/1.2.yml +``` + +Behind the scenes, CumulusCI is merging the yaml file specified by `--load-yml` on top of the project config. This means any customizations you could make in cumulusci.yml can also be made in a file loaded via `--load-yml` \ No newline at end of file