From 4a5647a2285b52014be441192dd99c920ea88e4a Mon Sep 17 00:00:00 2001 From: Vanessa Busch Date: Mon, 26 Jun 2017 06:39:12 -0400 Subject: [PATCH 1/2] Created plugin for getting artifacts from local dir Enable plugin with --artifact-local-enable Set the director with --artifact-local-dir (default is artifacts) Fixes: 114 --- dogen/generator.py | 3 ++ dogen/plugin.py | 3 ++ dogen/plugins/artifact.py | 56 +++++++++++++++++++++++++++ dogen/plugins/dist_git.py | 2 +- tests/test_plugin_artifact.py | 72 +++++++++++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 dogen/plugins/artifact.py create mode 100644 tests/test_plugin_artifact.py diff --git a/dogen/generator.py b/dogen/generator.py index 415dcb0..63e6d73 100644 --- a/dogen/generator.py +++ b/dogen/generator.py @@ -217,6 +217,9 @@ def run(self): else: self.log.warn("No scripts will be copied, mistake?") + for plugin in self.plugins: + plugin.before_sources() + self.handle_sources() self.render_from_template() diff --git a/dogen/plugin.py b/dogen/plugin.py index 66269bf..be4e08e 100644 --- a/dogen/plugin.py +++ b/dogen/plugin.py @@ -14,6 +14,9 @@ def inject_args(parser): def prepare(self, **kwargs): pass + def before_sources(self): + pass + def after_sources(self, **kwargs): pass diff --git a/dogen/plugins/artifact.py b/dogen/plugins/artifact.py new file mode 100644 index 0000000..5fc627a --- /dev/null +++ b/dogen/plugins/artifact.py @@ -0,0 +1,56 @@ +import glob +import os +import shutil + +from dogen.plugin import Plugin + +class Artifact(Plugin): + @staticmethod + def info(): + return "artifact", "Support overriding the sources from an artifact directory" + + @staticmethod + def inject_args(parser): + parser.add_argument('--artifact-local-enable', action='store_true', help='Uses local artifacts from the provided artifacts directory') + parser.add_argument('--artifact-local-dir', default='artifacts', help='Provides path to local artifacts directory') + + return parser + + def __init__(self, dogen, args): + super(Artifact, self).__init__(dogen, args) + self.artifact_files = [] + + def before_sources(self): + + if not self.artifact_files: + return + + if not os.path.exists(self.output): + self.log.warn("build directory does not exist") + return + self.log.info("Copying local artifact files from '%s'" % self.args.artifact_local_dir) + + for f in sorted(self.artifact_files): + self.log.debug("Copying local artifact %s" % os.path.basename(f)) + shutil.copy2(f, self.output) + + self.log.debug("Done.") + + def prepare(self, cfg): + + if not self.args.artifact_local_enable: + return + + if not self.args.artifact_local_dir: + self.log.debug("No directory with local artifact files specified, skipping artifact plugin") + return + + if not os.path.isdir(self.args.artifact_local_dir): + self.log.debug("Provided path to directory with local artifact files does not exists or is not a directory") + return + + self.artifact_files = glob.glob(os.path.join(self.args.artifact_local_dir, "*")) + + if not self.artifact_files: + self.log.debug("No local artifacts found") + return diff --git a/dogen/plugins/dist_git.py b/dogen/plugins/dist_git.py index d2fb807..5a49457 100644 --- a/dogen/plugins/dist_git.py +++ b/dogen/plugins/dist_git.py @@ -18,7 +18,7 @@ def inject_args(parser): def __init__(self, dogen, args): super(DistGitPlugin, self).__init__(dogen, args) - if not self.args.dist_git_enable: + if not self.args.dist_git_enable: return self.git = Git(self.log, os.path.dirname(self.descriptor), self.output, self.args.dist_git_assume_yes) diff --git a/tests/test_plugin_artifact.py b/tests/test_plugin_artifact.py new file mode 100644 index 0000000..10453a6 --- /dev/null +++ b/tests/test_plugin_artifact.py @@ -0,0 +1,72 @@ +import argparse +import mock +import os +import tempfile +import unittest +import shutil +import logging +import sys + +from dogen.plugins.artifact import Artifact +from dogen.generator import Generator + +class MockDogen(): + def __init__(self, log, cfg={}): + self.log = log + self.descriptor = 0 + self.output = "" + self.cfg = cfg + +class TestArtifactPlugin(unittest.TestCase): + def setUp(self): + self.workdir = tempfile.mkdtemp(prefix='test_artifact_plugin') + self.descriptor = tempfile.NamedTemporaryFile(delete=False) + self.target_dir = os.path.join(self.workdir, "target") + if not os.path.exists(self.target_dir): + os.makedirs(self.target_dir) + self.artifact_dir = os.path.join(self.workdir, "artifact") + if not os.path.exists(self.artifact_dir): + os.makedirs(self.artifact_dir) + + self.log = mock.Mock() + + def teardown(self): + shutil.rmtree(self.workdir) + + def write_config(self, config): + with self.descriptor as f: + f.write(config.encode()) + + def prepare_dogen(self, artifact_local_dir=None, output_dir=None): + self.write_config("version: '1'\ncmd:\nfrom: scratch\nname: someimage\n") + args = argparse.Namespace(path=self.descriptor.name, output=output_dir, without_sources=None, + template=None, scripts_path=None, additional_script=None, + skip_ssl_verification=None, repo_files_dir=None, artifact_local_enable=True, artifact_local_dir=artifact_local_dir) + self.dogen = Generator(self.log, args, [Artifact]) + + def test_should_skip_plugin_if_no_path_to_local_dir_is_provided(self): + self.prepare_dogen(None, self.target_dir) + self.dogen.run() + + self.log.debug.assert_any_call("No directory with local artifact files specified, skipping artifact plugin") + + def test_should_skip_plugin_if_local_dir_does_not_exist(self): + self.prepare_dogen("non_existing_local_artifact_dir", self.target_dir) + self.dogen.run() + + self.log.debug.assert_any_call("Provided path to directory with local artifact files does not exists or is not a directory") + + def test_should_skip_plugin_if_path_to_local_artifact_dir_is_provided_but_there_are_no_files(self): + self.prepare_dogen(self.artifact_dir, self.target_dir) + self.dogen.run() + self.log.debug.assert_any_call("No local artifacts found") + + def test_local_artifact_files_should_be_copied_to_target(self): + open(os.path.join(self.artifact_dir, "artifact1"), 'a').close() + open(os.path.join(self.artifact_dir, "artifact2"), 'a').close() + + self.prepare_dogen(self.artifact_dir, self.target_dir) + self.dogen.run() + + self.assertTrue(os.path.exists(os.path.join(self.target_dir, "artifact1"))) + self.assertTrue(os.path.exists(os.path.join(self.target_dir, "artifact2"))) From 83ab174aff7d451586c6a19763c551a1d10eb468 Mon Sep 17 00:00:00 2001 From: Vanessa Busch Date: Mon, 26 Jun 2017 08:34:42 -0400 Subject: [PATCH 2/2] Added support for subdirectories in target --- dogen/plugins/artifact.py | 17 +++++++++++++---- tests/test_plugin_artifact.py | 13 +++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/dogen/plugins/artifact.py b/dogen/plugins/artifact.py index 5fc627a..4ae24c1 100644 --- a/dogen/plugins/artifact.py +++ b/dogen/plugins/artifact.py @@ -30,9 +30,18 @@ def before_sources(self): return self.log.info("Copying local artifact files from '%s'" % self.args.artifact_local_dir) - for f in sorted(self.artifact_files): - self.log.debug("Copying local artifact %s" % os.path.basename(f)) - shutil.copy2(f, self.output) + for item in sorted(self.artifact_files): + src = os.path.join(self.args.artifact_local_dir, item) + dest = os.path.join(self.output, item) + self.log.debug("Copying local artifact %s" % os.path.basename(src)) + if os.path.isdir(src): + if os.path.exists(dest): + shutil.rmtree(dest) + shutil.copytree(src, dest) + else: + if os.path.exists(dest): + os.remove(dest) + shutil.copy2(src, dest) self.log.debug("Done.") @@ -49,7 +58,7 @@ def prepare(self, cfg): self.log.debug("Provided path to directory with local artifact files does not exists or is not a directory") return - self.artifact_files = glob.glob(os.path.join(self.args.artifact_local_dir, "*")) + self.artifact_files = os.listdir(self.args.artifact_local_dir) if not self.artifact_files: self.log.debug("No local artifacts found") diff --git a/tests/test_plugin_artifact.py b/tests/test_plugin_artifact.py index 10453a6..2edbe7e 100644 --- a/tests/test_plugin_artifact.py +++ b/tests/test_plugin_artifact.py @@ -70,3 +70,16 @@ def test_local_artifact_files_should_be_copied_to_target(self): self.assertTrue(os.path.exists(os.path.join(self.target_dir, "artifact1"))) self.assertTrue(os.path.exists(os.path.join(self.target_dir, "artifact2"))) + + def test_local_artifact_files_and_subdirectories_should_be_copied_to_target(self): + open(os.path.join(self.artifact_dir, "artifact1"), 'a').close() + subdir = os.path.join(self.artifact_dir, "subdir") + if not os.path.exists(subdir): + os.makedirs(subdir) + open(os.path.join(subdir, "artifact2"), 'a').close() + + self.prepare_dogen(self.artifact_dir, self.target_dir) + self.dogen.run() + + self.assertTrue(os.path.exists(os.path.join(self.target_dir, "artifact1"))) + self.assertTrue(os.path.exists(os.path.join(subdir, "artifact2")))