From 351a1b6e3f0d65108576ea6531fd8fb010592a00 Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Thu, 28 Jul 2022 16:58:41 -0600 Subject: [PATCH 1/7] Add support for specifying a minimum version of devlab for a project --- devlab | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/devlab b/devlab index f2357a0..dc46908 100755 --- a/devlab +++ b/devlab @@ -214,6 +214,23 @@ if __name__ == '__main__': LOGGER.error("No configured components found!... aborting") sys.exit(1) + #Check min devlab version if set + if devlab_bench.CONFIG.get('min_devlab_version', None): + MIN_DEVLAB_VERSION = devlab_bench.CONFIG['min_devlab_version'] + #Assume that "master" version is newer that min version and only if the version doesn't match + if __VERSION__ not in ["master", MIN_DEVLAB_VERSION]: + VERS_SORT = sorted( + [ + __VERSION__, + MIN_DEVLAB_VERSION + ], + key=devlab_bench.helpers.common.human_keys + ) + if VERS_SORT[-1] != __VERSION__: + LOGGER.error("This devlab project requuires a minimum version of: '%s' Found: '%s' installed. Please upgrade", MIN_DEVLAB_VERSION, __VERSION__) + sys.exit(1) + LOGGER.debug("Current version of devlab: '%s' matches or excedes required minimum version: '%s'", __VERSION__, MIN_DEVLAB_VERSION) + #Create our DockerHelper Object devlab_bench.helpers.docker.DOCKER = DockerHelper( filter_label=devlab_bench.CONFIG['project_filter'], From fe7f1844372b706d825e6c8654bb1ce282796dc6 Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Thu, 28 Jul 2022 16:59:29 -0600 Subject: [PATCH 2/7] Allow for forcing a image to NOT pull even if requested --- devlab_bench/actions/build.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/devlab_bench/actions/build.py b/devlab_bench/actions/build.py index 10a9af6..bf4128c 100644 --- a/devlab_bench/actions/build.py +++ b/devlab_bench/actions/build.py @@ -135,16 +135,23 @@ def action(images='*', clean=False, no_cache=False, pull=False, skip_pull_images log.debug(line) log.debug("Successfully removed image: %s", image) if pull: - with open(images_dict[image]['docker_file_full_path']) as dfile: - local_image = False - for line in dfile.readlines(): - if line.startswith('FROM '): - if line.split()[1].split(':')[0] in images_to_build + skip_pull_images: - local_image = True - log.debug("Skipping pull, as devlab manages this image's base image") - break - if not local_image: - images_dict[image]['build_opts'].append('--pull') + local_image = False + if image in skip_pull_images: + log.info("Image: %s was explicitly excluded from being pulled from a calling function, Skipping", image) + local_image = True + elif image not in base_images_to_build and config['runtime_images'][image].get('skip_pull', False): + log.info("Runtime image: %s is explicitely set to not be pulled in Devlabconfig, skipping pull argument when building", image) + local_image = True + else: + with open(images_dict[image]['docker_file_full_path']) as dfile: + for line in dfile.readlines(): + if line.startswith('FROM '): + if line.split()[1].split(':')[0] in images_to_build + skip_pull_images: + local_image = True + log.debug("Skipping pull, as devlab manages this image's base image") + break + if not local_image: + images_dict[image]['build_opts'].append('--pull') if no_cache: images_dict[image]['build_opts'].append('--no-cache') log.info("Building image: %s", image_n_tag) From 751b97f373ea40d9b2fa1c515ebc69ca491a204b Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Thu, 28 Jul 2022 17:00:28 -0600 Subject: [PATCH 3/7] Add support for specifying environment vars for components directly without run_opts --- devlab_bench/helpers/docker.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/devlab_bench/helpers/docker.py b/devlab_bench/helpers/docker.py index ccb825e..0574b92 100644 --- a/devlab_bench/helpers/docker.py +++ b/devlab_bench/helpers/docker.py @@ -452,7 +452,7 @@ def rm_image(self, name): logger=self.log ).run() return cmd_ret - def run_container(self, image, name, network=None, ports=None, background=True, interactive=False, ignore_nonzero_rc=False, cmd=None, logger=None, mounts=None, systemd_support=False, systemd_tmpfs_args=None, run_opts=None, **kwargs): #pylint: disable=too-many-arguments + def run_container(self, image, name, network=None, ports=None, background=True, env=None, env_file=None, interactive=False, ignore_nonzero_rc=False, cmd=None, logger=None, mounts=None, systemd_support=False, systemd_tmpfs_args=None, run_opts=None, **kwargs): #pylint: disable=too-many-arguments """ Run a docker_container @@ -461,6 +461,8 @@ def run_container(self, image, name, network=None, ports=None, background=True, name: str, The name of the container (this also sets the hostname) network: str, docker network to attach cmd: str, Command to run inside the container. (OPTIONAL) + env: dict, key/values of environment vars to set (OPTIONAL) + env_file: str, path to a file to set environment vars (OPTIONAL) ports: list/tuple, of ports to publish to the host. (OPTIONAL) background: Run the container in the background. (OPTIONAL) interactive: bool, whether or not the docker command could require @@ -499,9 +501,15 @@ def run_container(self, image, name, network=None, ports=None, background=True, opts.append("--detach") if network: opts.append("--network={}".format(network)) + if env: + for e_var, e_val in env.items(): + opts.append("--env") + opts.append("{}={}".format(e_var, e_val)) + if env_file: + opts.append("--env-file={}".format(env_file)) if systemd_support: if systemd_tmpfs_args: - systemd_tmpfs_args=':{}'.format(systemd_tmpfs_args) + systemd_tmpfs_args = ':{}'.format(systemd_tmpfs_args) opts += [ '--tmpfs=/run{}'.format(systemd_tmpfs_args), '--tmpfs=/run/lock{}'.format(systemd_tmpfs_args), From fe3c597873124f4bd9dca3dd933b2be6aebfac06 Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Thu, 28 Jul 2022 17:01:31 -0600 Subject: [PATCH 4/7] Move human_keys sort function to be reusable, switch script runners to use new env parameter --- devlab_bench/helpers/common.py | 37 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/devlab_bench/helpers/common.py b/devlab_bench/helpers/common.py index 2a8edf8..a6412d9 100644 --- a/devlab_bench/helpers/common.py +++ b/devlab_bench/helpers/common.py @@ -262,21 +262,6 @@ def get_ordinal_sorting(components, config_components): num = 100 ordinals['{}:{}|{}'.format(grp, num, comp)] = comp log.debug("Ordinals found for components: %s", ordinals) - def human_keys(astr): - """ - Sorts keys based on human order.. IE 1 is less than 10 etc.. - - alist.sort(key=human_keys) sorts in human order - """ - keys = [] - for elt in re.split(r'(\d+)', astr): - elt = elt.swapcase() - try: - elt = int(elt) - except ValueError: - pass - keys.append(elt) - return keys #Get the list of ordinals, and human sort them ordinal_list = sorted(tuple(ordinals.keys()), key=human_keys) log.debug("Sorted list of ordinals: '%s'", ', '.join(ordinal_list)) @@ -328,7 +313,7 @@ def get_proj_root(start_dir=None): start_dir = os.path.abspath(start_dir) cur_dir = start_dir found = False - while cur_dir != None: + while cur_dir is not None: if os.path.basename(cur_dir) != 'defaults': for cfile_name in devlab_bench.CONFIG_FILE_NAMES: if os.path.isfile('{}/{}'.format(cur_dir, cfile_name)): @@ -368,6 +353,22 @@ def get_shell_components(filter_list): """ return get_components(filter_list=filter_list, virtual_components=('adhoc',)) +def human_keys(astr): + """ + Sorts keys based on human order.. IE 1 is less than 10 etc.. + + alist.sort(key=human_keys) sorts in human order + """ + keys = [] + for elt in re.split(r'(\d+)', astr): + elt = elt.swapcase() + try: + elt = int(elt) + except ValueError: + pass + keys.append(elt) + return keys + def is_valid_hostname(hostname): """ Takes a hostname and tries to determine if it is valid or not @@ -540,8 +541,6 @@ def script_runner(script, name, ignore_nonzero_rc=False, interactive=True, log_o if '=' in script_arg: if not script_end_env: log.debug("Found environment variable for script: '%s'", script_arg) - script_run_opts.append('-e') - script_run_opts.append(script_arg) e_var, e_val = script_arg.split('=') env_map[e_var] = e_val continue @@ -570,6 +569,7 @@ def script_runner(script, name, ignore_nonzero_rc=False, interactive=True, log_o mounts=[ '{}:/devlab'.format(devlab_bench.PROJ_ROOT) ], + env=env_map, background=False, interactive=interactive, cmd=script_stripped, @@ -595,6 +595,7 @@ def script_runner(script, name, ignore_nonzero_rc=False, interactive=True, log_o name=name, background=False, interactive=interactive, + env=env_map, cmd=script_stripped, ignore_nonzero_rc=ignore_nonzero_rc, logger=log, From 7e0a3f11b1830892b54a432851772f739464dbc6 Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Thu, 28 Jul 2022 17:04:57 -0600 Subject: [PATCH 5/7] Update README.md with latest features --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 118437a..7718c4f 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ pip3 install pyyaml The configuration file has the following base structure: ``` { + "min_devlab_version": "", "components": {}, "domain": "", "foreground_component": {}, @@ -115,6 +116,7 @@ All Keys that are in **bold** are required to be in the config | domain | String | The domain name to assign to all component's hostname inside the container | | **components** | Hash of Hashes | Defines the components to start up. The First level key is a string of the name of the container. Structure conforms to the [Component Config Structure](#component-config-structure) | | foreground_component | Hash | Defines a component that will be startup up after ***all*** other components and will run in the foreground. After the process exits, all ther components will be stopped. Same structure as [Component Config Structure](#component-config-structure) with one additional key `name` to indicate the name of the foreground component | +| min_devlab_version | String | The minimum version of devlab that the project requires. Useful when taking advantage of new features and ensuring users of your project are updated to a version that supports the features you're using | | network | Hash | Defines a docker network to create and/or attach components to. Structure conforms to [Network Config Structure](#network-config-structure) | | **paths** | Hash | Defines the persistence directory for components, as well as files that should be deleted during the [reset](#reset-action) action. Structure conforms to [Paths Config Structure](#paths-config-structure) | | **project_filter** | String | A unique docker label that is used to identify containers and images that belong to the project. | @@ -130,6 +132,8 @@ The structure looks like this: "systemd_support": false, "systemd_tmpfs_args": "", "enabled": false, + "env": {}, + "env_file": "", "cmd": "", "ports": [], "mounts": [], @@ -157,7 +161,9 @@ All Keys that are in **bold** are required | systemd_tmpfs_args | String | If `systemd_support` is set to `true`, and this argument is set, then the value is appended to the tmpfs mounts as arguments for systemd support. This way you can specify things like: `rw`, `exec`, etc... | | **enabled** | Boolean | Whether or not the component should be brought [up](#up-action) and images [built](#build-action) | | **_name_** | String | This is only supported for `foreground_components` but required. It indicates the name of the component | -| type | String | This only only supported for `foreground_components`, but can be either `host` or `container`. If set to host then `cmd` is executed on the local system instead of a container | +| type | String | This is only supported for `foreground_components`, but can be either `host` or `container`. If set to host then `cmd` is executed on the local system instead of a container | +| env | Hash | Key value pairs of environment variables to set for the component | +| env_file | String | Path to a file containing environment variables for the component. This fills in the `--env-file` option for the `docker run` command | cmd | String | This is the command passed to the container as part of the `docker run` command. If `type` is set to `host` then the command is executed on the local system | | ports | List of Strings | The ports that should be "published" using the same notation as the `--publish` option to `docker run` | | mounts | List of Strings | List of mounts in the format `SOURCE_ON_HOST:DESTINATION_IN_CONTAINER`. If using a relative path then the paths are relative to the project's root | @@ -220,6 +226,7 @@ The structure looks like this: "tag": ""|[], "docker_file": "", "build_opts": [], + "skip_pull": BOOL, "ordinal": { "group": INT, "number": INT @@ -233,6 +240,7 @@ All Keys that are in **bold** are required | **tag** | String or List of Strings | This is a tag that should be applied to the image. If a list is passed, the first tag becomes a primary identifier. | | **docker_file** | String | Path to the docker file, relative to the project's root to use when building the image. ***[NOTE]*** The build context will be the parent directory of the dockerfile's path | | build_opts | List of Strings | Additional options to pass to the `docker build` command. Each CLI arg must be it's own element. For example: `[ '--build-arg', 'foo=bar' ]` would become `docker build --build-arg foo=bar PATH...` etc... | +| skip_pull | Boolean | Whether or not a forced pull does anything. There are cases where an image is built locally used elsewhere, so a pull will fail since it isn't on docker hub. If this is `true`, then even when a build is requesting a `pull` it will skip it for this image | | ordinal | Hash | This is used indicate the order of the images to build. When parallel execution is supported, the `group` key indicates the image that can be built at the same time, `number` indicates the order inside the group to start up | _**[NOTE]**_ Devlab supports a special label (`last_modified`). From 42b7a8e17fe0783e39ee00bd8d7b136cdcf70c7a Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Fri, 29 Jul 2022 18:06:26 -0600 Subject: [PATCH 6/7] Get rid of linting error, using globals() --- devlab_bench/helpers/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devlab_bench/helpers/common.py b/devlab_bench/helpers/common.py index a6412d9..dd610e6 100644 --- a/devlab_bench/helpers/common.py +++ b/devlab_bench/helpers/common.py @@ -19,7 +19,7 @@ #Python2/3 compatibility try: #Python2 - text_input = raw_input #pylint: disable=invalid-name + text_input = globals()['__builtins__'].raw_input #pylint: disable=invalid-name from pipes import quote #pylint: disable=unused-import try: from pathlib2 import Path #pylint: disable=unused-import From c98d102969c77b5278498b229b14a4b05a6f7541 Mon Sep 17 00:00:00 2001 From: Dan Farnsworth Date: Fri, 29 Jul 2022 18:24:20 -0600 Subject: [PATCH 7/7] Remove some trailing space from the readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7718c4f..3859b4e 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ The structure looks like this: "tag": ""|[], "docker_file": "", "build_opts": [], - "skip_pull": BOOL, + "skip_pull": BOOL, "ordinal": { "group": INT, "number": INT