From a8da2c33f6bf124f05e24ac5c938b830f1bd81b5 Mon Sep 17 00:00:00 2001 From: Naman Jain Date: Thu, 21 Sep 2023 21:54:23 +0530 Subject: [PATCH 1/2] Added Namespace in org info (#3662) Co-authored-by: David Reed --- cumulusci/cli/org.py | 1 + cumulusci/cli/tests/test_org.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cumulusci/cli/org.py b/cumulusci/cli/org.py index 4dc5ed3e6f..c8ee718e6f 100644 --- a/cumulusci/cli/org.py +++ b/cumulusci/cli/org.py @@ -330,6 +330,7 @@ def org_info(runtime, org_name, print_json): "instance_url", "instance_name", "is_sandbox", + "namespace", "namespaced", "org_id", "org_type", diff --git a/cumulusci/cli/tests/test_org.py b/cumulusci/cli/tests/test_org.py index 01692e22a8..007e82249d 100644 --- a/cumulusci/cli/tests/test_org.py +++ b/cumulusci/cli/tests/test_org.py @@ -540,6 +540,7 @@ def test_org_info(self): "default": True, "password": None, "connected_app": "built-in", + "namespace": "test", } org_config.expires = date.today() org_config.latest_api_version = "42.0" @@ -557,6 +558,7 @@ def test_org_info(self): ["\x1b[1mconnected_app\x1b[0m", "built-in"], ["\x1b[1mdays\x1b[0m", "1"], ["\x1b[1mdefault\x1b[0m", "True"], + ["\x1b[1mnamespace\x1b[0m", "test"], ["\x1b[1mpassword\x1b[0m", "None"], ], ) From ebf3596831dd47e76c9f08c513d8744b18d0d2e4 Mon Sep 17 00:00:00 2001 From: aditya-balachander <139134092+aditya-balachander@users.noreply.github.com> Date: Thu, 21 Sep 2023 23:29:13 +0530 Subject: [PATCH 2/2] Extended 'Deploy' task to support REST API deployment (#3650) --- cumulusci/salesforce_api/rest_deploy.py | 148 ++++++++++ .../salesforce_api/tests/test_rest_deploy.py | 261 ++++++++++++++++++ cumulusci/tasks/salesforce/Deploy.py | 9 + .../tasks/salesforce/tests/test_Deploy.py | 52 +++- 4 files changed, 458 insertions(+), 12 deletions(-) create mode 100644 cumulusci/salesforce_api/rest_deploy.py create mode 100644 cumulusci/salesforce_api/tests/test_rest_deploy.py diff --git a/cumulusci/salesforce_api/rest_deploy.py b/cumulusci/salesforce_api/rest_deploy.py new file mode 100644 index 0000000000..70d532569a --- /dev/null +++ b/cumulusci/salesforce_api/rest_deploy.py @@ -0,0 +1,148 @@ +import base64 +import io +import json +import os +import time +import uuid +import zipfile +from typing import List, Union + +import requests + +PARENT_DIR_NAME = "metadata" + + +class RestDeploy: + def __init__( + self, + task, + package_zip: str, + purge_on_delete: Union[bool, str, None], + check_only: bool, + test_level: Union[str, None], + run_tests: List[str], + ): + # Initialize instance variables and configuration options + self.api_version = task.project_config.project__package__api_version + self.task = task + assert package_zip, "Package zip should not be None" + if purge_on_delete is None: + purge_on_delete = True + self._set_purge_on_delete(purge_on_delete) + self.check_only = "true" if check_only else "false" + self.test_level = test_level + self.package_zip = package_zip + self.run_tests = run_tests or [] + + def __call__(self): + self._boundary = str(uuid.uuid4()) + url = f"{self.task.org_config.instance_url}/services/data/v{self.api_version}/metadata/deployRequest" + headers = { + "Authorization": f"Bearer {self.task.org_config.access_token}", + "Content-Type": f"multipart/form-data; boundary={self._boundary}", + } + + # Prepare deployment options as JSON payload + deploy_options = { + "deployOptions": { + "allowMissingFiles": False, + "autoUpdatePackage": False, + "checkOnly": self.check_only, + "ignoreWarnings": False, + "performRetrieve": False, + "purgeOnDelete": self.purge_on_delete, + "rollbackOnError": False, + "runTests": self.run_tests, + "singlePackage": False, + "testLevel": self.test_level, + } + } + json_payload = json.dumps(deploy_options) + + # Construct the multipart/form-data request body + body = ( + f"--{self._boundary}\r\n" + f'Content-Disposition: form-data; name="json"\r\n' + f"Content-Type: application/json\r\n\r\n" + f"{json_payload}\r\n" + f"--{self._boundary}\r\n" + f'Content-Disposition: form-data; name="file"; filename="metadata.zip"\r\n' + f"Content-Type: application/zip\r\n\r\n" + ).encode("utf-8") + body += self._reformat_zip(self.package_zip) + body += f"\r\n--{self._boundary}--\r\n".encode("utf-8") + + response = requests.post(url, headers=headers, data=body) + response_json = response.json() + + if response.status_code == 201: + self.task.logger.info("Deployment request successful") + deploy_request_id = response_json["id"] + self._monitor_deploy_status(deploy_request_id) + else: + self.task.logger.error( + f"Deployment request failed with status code {response.status_code}" + ) + + # Set the purge_on_delete attribute based on org type + def _set_purge_on_delete(self, purge_on_delete): + if not purge_on_delete or purge_on_delete == "false": + self.purge_on_delete = "false" + else: + self.purge_on_delete = "true" + # Disable purge on delete entirely for non sandbox or DE orgs as it is + # not allowed + org_type = self.task.org_config.org_type + is_sandbox = self.task.org_config.is_sandbox + if org_type != "Developer Edition" and not is_sandbox: + self.purge_on_delete = "false" + + # Monitor the deployment status and log progress + def _monitor_deploy_status(self, deploy_request_id): + url = f"{self.task.org_config.instance_url}/services/data/v{self.api_version}/metadata/deployRequest/{deploy_request_id}?includeDetails=true" + headers = {"Authorization": f"Bearer {self.task.org_config.access_token}"} + + while True: + response = requests.get(url, headers=headers) + response_json = response.json() + self.task.logger.info( + f"Deployment {response_json['deployResult']['status']}" + ) + + if response_json["deployResult"]["status"] not in ["InProgress", "Pending"]: + # Handle the case when status has Failed + if response_json["deployResult"]["status"] == "Failed": + for failure in response_json["deployResult"]["details"][ + "componentFailures" + ]: + self.task.logger.error(self._construct_error_message(failure)) + return + time.sleep(5) + + # Reformat the package zip file to include parent directory + def _reformat_zip(self, package_zip): + zip_bytes = base64.b64decode(package_zip) + zip_stream = io.BytesIO(zip_bytes) + new_zip_stream = io.BytesIO() + + with zipfile.ZipFile(zip_stream, "r") as zip_ref: + with zipfile.ZipFile(new_zip_stream, "w") as new_zip_ref: + for item in zip_ref.infolist(): + # Choice of name for parent directory is irrelevant to functioning + new_item_name = os.path.join(PARENT_DIR_NAME, item.filename) + file_content = zip_ref.read(item.filename) + new_zip_ref.writestr(new_item_name, file_content) + + new_zip_bytes = new_zip_stream.getvalue() + return new_zip_bytes + + # Construct an error message from deployment failure details + def _construct_error_message(self, failure): + error_message = f"{str.upper(failure['problemType'])} in file {failure['fileName'][len(PARENT_DIR_NAME)+len('/'):]}: {failure['problem']}" + + if failure["lineNumber"] and failure["columnNumber"]: + error_message += ( + f" at line {failure['lineNumber']}:{failure['columnNumber']}" + ) + + return error_message diff --git a/cumulusci/salesforce_api/tests/test_rest_deploy.py b/cumulusci/salesforce_api/tests/test_rest_deploy.py new file mode 100644 index 0000000000..6a3794e35a --- /dev/null +++ b/cumulusci/salesforce_api/tests/test_rest_deploy.py @@ -0,0 +1,261 @@ +import base64 +import io +import unittest +import zipfile +from unittest.mock import MagicMock, Mock, call, patch + +from cumulusci.salesforce_api.rest_deploy import RestDeploy + + +def generate_sample_zip_data(parent=""): + # Create a sample ZIP with two files + zip_data = io.BytesIO() + with zipfile.ZipFile(zip_data, "w") as zip_file: + zip_file.writestr( + f"{parent}objects/mockfile1.obj", "Sample content for mockfile1" + ) + zip_file.writestr( + f"{parent}objects/mockfile2.obj", "Sample content for mockfile2" + ) + return base64.b64encode(zip_data.getvalue()).decode("utf-8") + + +class TestRestDeploy(unittest.TestCase): + # Setup method executed before each test method + def setUp(self): + self.mock_logger = Mock() + self.mock_task = MagicMock() + self.mock_task.logger = self.mock_logger + self.mock_task.org_config.instance_url = "https://example.com" + self.mock_task.org_config.access_token = "dummy_token" + self.mock_task.project_config.project__package__api_version = 58.0 + # Empty zip file for testing + self.mock_zip = generate_sample_zip_data() + + # Test case for a successful deployment and deploy status + @patch("requests.post") + @patch("requests.get") + def test_deployment_success(self, mock_get, mock_post): + + response_post = Mock(status_code=201) + response_post.json.return_value = {"id": "dummy_id"} + mock_post.return_value = response_post + + response_get = Mock(status_code=200) + response_get.json.side_effect = [ + {"deployResult": {"status": "InProgress"}}, + {"deployResult": {"status": "Succeeded"}}, + ] + mock_get.return_value = response_get + + deployer = RestDeploy( + self.mock_task, self.mock_zip, False, False, "NoTestRun", [] + ) + deployer() + + # Assertions to verify log messages + assert ( + call("Deployment request successful") + in self.mock_logger.info.call_args_list + ) + assert call("Deployment InProgress") in self.mock_logger.info.call_args_list + assert call("Deployment Succeeded") in self.mock_logger.info.call_args_list + + # Assertions to verify API Calls + expected_get_calls = [ + call( + "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true", + headers={"Authorization": "Bearer dummy_token"}, + ), + call( + "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true", + headers={"Authorization": "Bearer dummy_token"}, + ), + ] + + mock_post.assert_called_once() + mock_get.assert_has_calls(expected_get_calls, any_order=True) + + # Test case for a deployment failure + @patch("requests.post") + def test_deployment_failure(self, mock_post): + + response_post = Mock(status_code=500) + response_post.json.return_value = {"id": "dummy_id"} + mock_post.return_value = response_post + + deployer = RestDeploy( + self.mock_task, self.mock_zip, False, False, "NoTestRun", [] + ) + deployer() + + # Assertions to verify log messages + assert ( + call("Deployment request failed with status code 500") + in self.mock_logger.error.call_args_list + ) + + # Assertions to verify API Calls + mock_post.assert_called_once() + + # Test for deployment success but deploy status failure + @patch("requests.post") + @patch("requests.get") + def test_deployStatus_failure(self, mock_get, mock_post): + + response_post = Mock(status_code=201) + response_post.json.return_value = {"id": "dummy_id"} + mock_post.return_value = response_post + + response_get = Mock(status_code=200) + response_get.json.side_effect = [ + {"deployResult": {"status": "InProgress"}}, + { + "deployResult": { + "status": "Failed", + "details": { + "componentFailures": [ + { + "problemType": "Error", + "fileName": "metadata/classes/mockfile1.cls", + "problem": "someproblem1", + "lineNumber": 1, + "columnNumber": 1, + }, + { + "problemType": "Error", + "fileName": "metadata/objects/mockfile2.obj", + "problem": "someproblem2", + "lineNumber": 2, + "columnNumber": 2, + }, + ] + }, + } + }, + ] + mock_get.return_value = response_get + + deployer = RestDeploy( + self.mock_task, self.mock_zip, False, False, "NoTestRun", [] + ) + deployer() + + # Assertions to verify log messages + assert ( + call("Deployment request successful") + in self.mock_logger.info.call_args_list + ) + assert call("Deployment InProgress") in self.mock_logger.info.call_args_list + assert call("Deployment Failed") in self.mock_logger.info.call_args_list + assert ( + call("ERROR in file classes/mockfile1.cls: someproblem1 at line 1:1") + in self.mock_logger.error.call_args_list + ) + assert ( + call("ERROR in file objects/mockfile2.obj: someproblem2 at line 2:2") + in self.mock_logger.error.call_args_list + ) + + # Assertions to verify API Calls + expected_get_calls = [ + call( + "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true", + headers={"Authorization": "Bearer dummy_token"}, + ), + call( + "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true", + headers={"Authorization": "Bearer dummy_token"}, + ), + ] + + mock_post.assert_called_once() + mock_get.assert_has_calls(expected_get_calls, any_order=True) + + # Test case for a deployment with a pending status + @patch("requests.post") + @patch("requests.get") + def test_pending_call(self, mock_get, mock_post): + + response_post = Mock(status_code=201) + response_post.json.return_value = {"id": "dummy_id"} + mock_post.return_value = response_post + + response_get = Mock(status_code=200) + response_get.json.side_effect = [ + {"deployResult": {"status": "InProgress"}}, + {"deployResult": {"status": "Pending"}}, + {"deployResult": {"status": "Succeeded"}}, + ] + mock_get.return_value = response_get + + deployer = RestDeploy( + self.mock_task, self.mock_zip, False, False, "NoTestRun", [] + ) + deployer() + + # Assertions to verify log messages + assert ( + call("Deployment request successful") + in self.mock_logger.info.call_args_list + ) + assert call("Deployment InProgress") in self.mock_logger.info.call_args_list + assert call("Deployment Pending") in self.mock_logger.info.call_args_list + assert call("Deployment Succeeded") in self.mock_logger.info.call_args_list + + # Assertions to verify API Calls + expected_get_calls = [ + call( + "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true", + headers={"Authorization": "Bearer dummy_token"}, + ), + call( + "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true", + headers={"Authorization": "Bearer dummy_token"}, + ), + call( + "https://example.com/services/data/v58.0/metadata/deployRequest/dummy_id?includeDetails=true", + headers={"Authorization": "Bearer dummy_token"}, + ), + ] + + mock_post.assert_called_once() + mock_get.assert_has_calls(expected_get_calls, any_order=True) + + def test_reformat_zip(self): + input_zip = generate_sample_zip_data() + expected_zip = generate_sample_zip_data("metadata/") + + deployer = RestDeploy( + self.mock_task, self.mock_zip, False, False, "NoTestRun", [] + ) + actual_output_zip = deployer._reformat_zip(input_zip) + + self.assertEqual( + base64.b64encode(actual_output_zip).decode("utf-8"), expected_zip + ) + + def test_purge_on_delete(self): + test_data = [ + ("not_sandbox_developer", "Not Developer Edition", False, False, "false"), + ("purgeOnDelete_true", "Developer Edition", True, True, "true"), + ("purgeOnDelete_none", "Developer Edition", True, None, "true"), + ] + + for name, org_type, is_sandbox, purge_on_delete, expected_result in test_data: + with self.subTest(name=name): + self.mock_task.org_config.org_type = org_type + self.mock_task.org_config.is_sandbox = is_sandbox + deployer = RestDeploy( + self.mock_task, + self.mock_zip, + purge_on_delete, + False, + "NoTestRun", + [], + ) + self.assertEqual(deployer.purge_on_delete, expected_result) + + +if __name__ == "__main__": + unittest.main() diff --git a/cumulusci/tasks/salesforce/Deploy.py b/cumulusci/tasks/salesforce/Deploy.py index f84ae3ae24..4f42a8a14d 100644 --- a/cumulusci/tasks/salesforce/Deploy.py +++ b/cumulusci/tasks/salesforce/Deploy.py @@ -13,6 +13,7 @@ from cumulusci.core.utils import process_bool_arg, process_list_arg from cumulusci.salesforce_api.metadata import ApiDeploy from cumulusci.salesforce_api.package_zip import MetadataPackageZipBuilder +from cumulusci.salesforce_api.rest_deploy import RestDeploy from cumulusci.tasks.salesforce.BaseSalesforceMetadataApiTask import ( BaseSalesforceMetadataApiTask, ) @@ -55,6 +56,7 @@ class Deploy(BaseSalesforceMetadataApiTask): "transforms": { "description": "Apply source transforms before deploying. See the CumulusCI documentation for details on how to specify transforms." }, + "rest_deploy": {"description": "If True, deploy metadata using REST API"}, } namespaces = {"sf": "http://soap.sforce.com/2006/04/metadata"} @@ -99,6 +101,9 @@ def _init_options(self, kwargs): f"The validation error was {str(e)}" ) + # Set class variable to true if rest_deploy is set to True + self.rest_deploy = process_bool_arg(self.options.get("rest_deploy", False)) + def _get_api(self, path=None): if not path: path = self.options.get("path") @@ -110,6 +115,10 @@ def _get_api(self, path=None): self.logger.warning("Deployment package is empty; skipping deployment.") return + # If rest_deploy param is set, update api_class to be RestDeploy + if self.rest_deploy: + self.api_class = RestDeploy + return self.api_class( self, package_zip, diff --git a/cumulusci/tasks/salesforce/tests/test_Deploy.py b/cumulusci/tasks/salesforce/tests/test_Deploy.py index a08211f117..d971f3a230 100644 --- a/cumulusci/tasks/salesforce/tests/test_Deploy.py +++ b/cumulusci/tasks/salesforce/tests/test_Deploy.py @@ -15,7 +15,8 @@ class TestDeploy: - def test_get_api(self): + @pytest.mark.parametrize("rest_deploy", [True, False]) + def test_get_api(self, rest_deploy): with temporary_dir() as path: touch("package.xml") task = create_task( @@ -26,6 +27,7 @@ def test_get_api(self): "namespace_inject": "ns", "namespace_strip": "ns", "unmanaged": True, + "rest_deply": rest_deploy, }, ) @@ -34,11 +36,18 @@ def test_get_api(self): assert "package.xml" in zf.namelist() zf.close() - def test_get_api__managed(self): + @pytest.mark.parametrize("rest_deploy", [True, False]) + def test_get_api__managed(self, rest_deploy): with temporary_dir() as path: touch("package.xml") task = create_task( - Deploy, {"path": path, "namespace_inject": "ns", "unmanaged": False} + Deploy, + { + "path": path, + "namespace_inject": "ns", + "unmanaged": False, + "rest_deploy": rest_deploy, + }, ) api = task._get_api() @@ -46,7 +55,8 @@ def test_get_api__managed(self): assert "package.xml" in zf.namelist() zf.close() - def test_get_api__additional_options(self): + @pytest.mark.parametrize("rest_deploy", [True, False]) + def test_get_api__additional_options(self, rest_deploy): with temporary_dir() as path: touch("package.xml") task = create_task( @@ -56,6 +66,7 @@ def test_get_api__additional_options(self): "test_level": "RunSpecifiedTests", "specified_tests": "TestA,TestB", "unmanaged": False, + "rest_deploy": rest_deploy, }, ) @@ -63,7 +74,8 @@ def test_get_api__additional_options(self): assert api.run_tests == ["TestA", "TestB"] assert api.test_level == "RunSpecifiedTests" - def test_get_api__skip_clean_meta_xml(self): + @pytest.mark.parametrize("rest_deploy", [True, False]) + def test_get_api__skip_clean_meta_xml(self, rest_deploy): with temporary_dir() as path: touch("package.xml") task = create_task( @@ -72,6 +84,7 @@ def test_get_api__skip_clean_meta_xml(self): "path": path, "clean_meta_xml": False, "unmanaged": True, + "rest_deploy": rest_deploy, }, ) @@ -80,7 +93,8 @@ def test_get_api__skip_clean_meta_xml(self): assert "package.xml" in zf.namelist() zf.close() - def test_get_api__static_resources(self): + @pytest.mark.parametrize("rest_deploy", [True, False]) + def test_get_api__static_resources(self, rest_deploy): with temporary_dir() as path: with open("package.xml", "w") as f: f.write( @@ -107,6 +121,7 @@ def test_get_api__static_resources(self): "namespace_inject": "ns", "namespace_strip": "ns", "unmanaged": True, + "rest_deploy": rest_deploy, }, ) @@ -120,32 +135,37 @@ def test_get_api__static_resources(self): assert "TestBundle" in package_xml zf.close() - def test_get_api__missing_path(self): + @pytest.mark.parametrize("rest_deploy", [True, False]) + def test_get_api__missing_path(self, rest_deploy): task = create_task( Deploy, { "path": "BOGUS", "unmanaged": True, + "rest_deploy": rest_deploy, }, ) api = task._get_api() assert api is None - def test_get_api__empty_package_zip(self): + @pytest.mark.parametrize("rest_deploy", [True, False]) + def test_get_api__empty_package_zip(self, rest_deploy): with temporary_dir() as path: task = create_task( Deploy, { "path": path, "unmanaged": True, + "rest_deploy": rest_deploy, }, ) api = task._get_api() assert api is None - def test_init_options(self): + @pytest.mark.parametrize("rest_deploy", [True, False]) + def test_init_options(self, rest_deploy): with pytest.raises(TaskOptionsError): create_task( Deploy, @@ -153,6 +173,7 @@ def test_init_options(self): "path": "empty", "test_level": "RunSpecifiedTests", "unmanaged": False, + "rest_deploy": rest_deploy, }, ) @@ -169,34 +190,40 @@ def test_init_options(self): "test_level": "RunLocalTests", "specified_tests": ["TestA"], "unmanaged": False, + "rest_deploy": rest_deploy, }, ) - def test_init_options__transforms(self): + @pytest.mark.parametrize("rest_deploy", [True, False]) + def test_init_options__transforms(self, rest_deploy): d = create_task( Deploy, { "path": "src", "transforms": ["clean_meta_xml"], + "rest_deploy": rest_deploy, }, ) assert len(d.transforms) == 1 assert isinstance(d.transforms[0], CleanMetaXMLTransform) - def test_init_options__bad_transforms(self): + @pytest.mark.parametrize("rest_deploy", [True, False]) + def test_init_options__bad_transforms(self, rest_deploy): with pytest.raises(TaskOptionsError) as e: create_task( Deploy, { "path": "src", "transforms": [{}], + "rest_deploy": rest_deploy, }, ) assert "transform spec is not valid" in str(e) - def test_freeze_sets_kind(self): + @pytest.mark.parametrize("rest_deploy", [True, False]) + def test_freeze_sets_kind(self, rest_deploy): task = create_task( Deploy, { @@ -204,6 +231,7 @@ def test_freeze_sets_kind(self): "namespace_tokenize": "ns", "namespace_inject": "ns", "namespace_strip": "ns", + "rest_deploy": rest_deploy, }, ) step = StepSpec(