From 85fb7a4a473f415902ff7820bb936fb1edfb0200 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 9 Nov 2024 13:53:06 +0100 Subject: [PATCH 1/6] Handle docker API schema v2 --- exegol/config/ConstantConfig.py | 2 +- exegol/model/ExegolImage.py | 3 ++- exegol/utils/DockerUtils.py | 9 +++---- exegol/utils/WebUtils.py | 46 ++++++++++++++++++++++++++++++--- 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/exegol/config/ConstantConfig.py b/exegol/config/ConstantConfig.py index 0c698243..b545a569 100644 --- a/exegol/config/ConstantConfig.py +++ b/exegol/config/ConstantConfig.py @@ -5,7 +5,7 @@ class ConstantConfig: """Constant parameters information""" # Exegol Version - version: str = "4.3.7" + version: str = "4.3.8b1" # Exegol documentation link documentation: str = "https://exegol.rtfd.io/" diff --git a/exegol/model/ExegolImage.py b/exegol/model/ExegolImage.py index 5bb3c1d3..49479e4b 100644 --- a/exegol/model/ExegolImage.py +++ b/exegol/model/ExegolImage.py @@ -79,7 +79,8 @@ def __init__(self, if meta_img and meta_img.meta_id is not None: self.__setDigest(meta_img.meta_id) self.__setLatestRemoteId(meta_img.meta_id) # Meta id is always the latest one - logger.debug(f"└── {self.__name}\t→ ({self.getType()}) {self.__digest}") + # Debug every Exegol image init + # logger.debug(f"└── {self.__name}\t→ ({self.getType()}) {self.__digest}") def __initFromDockerImage(self): """Parse Docker object to set up self configuration on creation.""" diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 5cd512c2..b0ed4e60 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -412,18 +412,17 @@ def __listRemoteImages(self) -> List[MetaImages]: remote_results = [] # Define max number of tags to download from dockerhub (in order to limit download time and discard historical versions) page_size = 20 - page_max = 2 - current_page = 0 - url: Optional[str] = f"https://{ConstantConfig.DOCKER_HUB}/v2/repositories/{ConstantConfig.IMAGE_NAME}/tags?page_size={page_size}" + page_max = 3 + current_page = 1 + url: Optional[str] = f"https://{ConstantConfig.DOCKER_HUB}/v2/repositories/{ConstantConfig.IMAGE_NAME}/tags?page=1&page_size={page_size}" # Handle multi-page tags from registry with console.status(f"Loading registry information from [green]{url}[/green]", spinner_style="blue") as s: while url is not None: - if current_page == page_max: + if current_page > page_max: logger.debug("Max page limit reached. In non-verbose mode, downloads will stop there.") if not logger.isEnabledFor(ExeLog.VERBOSE): break current_page += 1 - logger.debug(f"Fetching information from: {url}") s.update(status=f"Fetching registry information from [green]{url}[/green]") docker_repo_response = WebUtils.runJsonRequest(url, "Dockerhub") if docker_repo_response is None: diff --git a/exegol/utils/WebUtils.py b/exegol/utils/WebUtils.py index d6da2580..bd01973c 100644 --- a/exegol/utils/WebUtils.py +++ b/exegol/utils/WebUtils.py @@ -94,10 +94,47 @@ def getRemoteVersion(cls, tag: str) -> Optional[str]: version: Optional[str] = None if response is not None and response.status_code == 200: data = json.loads(response.content.decode("utf-8")) - # Parse metadata of the current image from v1 schema - metadata = json.loads(data.get("history", [])[0]['v1Compatibility']) - # Find version label and extract data - version = metadata.get("config", {}).get("Labels", {}).get("org.exegol.version", "") + received_media_type = data.get("mediaType") + if received_media_type == "application/vnd.docker.distribution.manifest.v1+json": + # Get image version from legacy v1 manifest (faster) + # Parse metadata of the current image from v1 schema + metadata = json.loads(data.get("history", [])[0]['v1Compatibility']) + # Find version label and extract data + version = metadata.get("config", {}).get("Labels", {}).get("org.exegol.version", "") + + # Convert image list to a specific image + elif received_media_type == "application/vnd.docker.distribution.manifest.list.v2+json": + # Get image version from v2 manifest list (slower) + # Retrieve image digest id from manifest image list + manifest = data.get("manifests") + # Get first image manifest + # Handle application/vnd.docker.distribution.manifest.list.v2+json spec + if type(manifest) is list and len(manifest) > 0: + # Get Image digest + first_digest = manifest[0].get("digest") + # Retrieve specific image detail from first image digest (architecture not sensitive) + manifest_headers["Accept"] = "application/vnd.docker.distribution.manifest.v2+json" + url = f"https://{ConstantConfig.DOCKER_REGISTRY}/v2/{ConstantConfig.IMAGE_NAME}/manifests/{first_digest}" + response = cls.__runRequest(url, service_name="Docker Registry", headers=manifest_headers, method="GET") + if response is not None and response.status_code == 200: + data = json.loads(response.content.decode("utf-8")) + # Update received media type to ba handle later + received_media_type = data.get("mediaType") + # Try to extract version tag from a specific image + if received_media_type == "application/vnd.docker.distribution.manifest.v2+json": + # Get image version from v2 manifest (slower) + # Retrieve config detail from config digest + config_digest: Optional[str] = data.get("config", {}).get('digest') + if config_digest is not None: + manifest_headers["Accept"] = "application/json" + url = f"https://{ConstantConfig.DOCKER_REGISTRY}/v2/{ConstantConfig.IMAGE_NAME}/blobs/{config_digest}" + response = cls.__runRequest(url, service_name="Docker Registry", headers=manifest_headers, method="GET") + if response is not None and response.status_code == 200: + data = json.loads(response.content.decode("utf-8")) + # Find version label and extract data + version = data.get("config", {}).get("Labels", {}).get("org.exegol.version") + else: + logger.debug(f"WARNING: Docker API not supported: {received_media_type}") return version @classmethod @@ -135,6 +172,7 @@ def __runRequest(cls, url: str, service_name: str, headers: Optional[Dict] = Non no_proxy = os.environ.get('NO_PROXY') or os.environ.get('no_proxy') if no_proxy: proxies['no_proxy'] = no_proxy + logger.debug(f"Fetching information from {url}") response = requests.request(method=method, url=url, timeout=(10, 20), verify=ParametersManager().verify, headers=headers, data=data, proxies=proxies if len(proxies) > 0 else None) return response except requests.exceptions.HTTPError as e: From 8c100dcc7d0b7f623d301d3f8c48cb7384bb8395 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 9 Nov 2024 14:13:18 +0100 Subject: [PATCH 2/6] Improve image loader wait time --- exegol/model/ExegolImage.py | 13 ++++++--- exegol/utils/DockerUtils.py | 53 +++++++++++++++++++++---------------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/exegol/model/ExegolImage.py b/exegol/model/ExegolImage.py index 49479e4b..596b88d3 100644 --- a/exegol/model/ExegolImage.py +++ b/exegol/model/ExegolImage.py @@ -3,6 +3,7 @@ from docker.models.containers import Container from docker.models.images import Image +from rich.status import Status from exegol.config.DataCache import DataCache from exegol.console import ConsoleFormat @@ -179,7 +180,7 @@ def setDockerObject(self, docker_image: Image): # backup plan: Use label to retrieve image version self.__labelVersionParsing() - def setMetaImage(self, meta: MetaImages): + def setMetaImage(self, meta: MetaImages, status: Status): dockerhub_data = meta.getDockerhubImageForArch(self.getArch()) self.__is_remote = True if self.__version_specific: @@ -189,6 +190,8 @@ def setMetaImage(self, meta: MetaImages): self.__outdated = self.__version_specific elif not meta.version: # nightly image don't have a version in their tag. The latest version must be fetch from label on the registry directly + logger.verbose(f"Fetch latest [green]{self.getName()}[/green] image version") + status.update(status=f"Retrieving latest [green]{self.getName()}[/green] image version") fetch_version = WebUtils.getRemoteVersion(self.__name) if fetch_version: meta.version = fetch_version @@ -269,7 +272,7 @@ def autoLoad(self, from_cache: bool = True) -> 'ExegolImage': logger.debug(f"Auto-load remote version for the specific image '{self.__name}'") # Find remote metadata for the specific current image with console.status(f"Synchronization of the [green]{self.__name}[/green] image status...", - spinner_style="blue"): + spinner_style="blue") as s: remote_digest = None version = None if from_cache: @@ -283,6 +286,8 @@ def autoLoad(self, from_cache: bool = True) -> 'ExegolImage': break if not from_cache or version is None: remote_digest = WebUtils.getMetaDigestId(self.__name) + logger.verbose(f"Fetch latest [green]{self.getName()}[/green] image version") + s.update(status=f"Retrieving latest [green]{self.getName()}[/green] image version") version = WebUtils.getRemoteVersion(self.__name) if remote_digest is not None: self.__setLatestRemoteId(remote_digest) @@ -347,7 +352,7 @@ def __mergeMetaImages(cls, images: List[MetaImages]): pass @classmethod - def mergeImages(cls, remote_images: List[MetaImages], local_images: List[Image]) -> List['ExegolImage']: + def mergeImages(cls, remote_images: List[MetaImages], local_images: List[Image], status: Status) -> List['ExegolImage']: """Compare and merge local images and remote images. Use case to process : - up-to-date : "Version specific" image can use exact digest_id matching. Latest image must match corresponding tag @@ -400,7 +405,7 @@ def mergeImages(cls, remote_images: List[MetaImages], local_images: List[Image]) selected = default if selected: # Remote match found - current_local_img.setMetaImage(selected) + current_local_img.setMetaImage(selected, status) else: if len(remote_images) > 0: # If there is some result from internet but no match and this is not a local image, this image is probably discontinued or the remote image is too old (relative to other images) diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index b0ed4e60..11000c56 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -10,6 +10,7 @@ from docker.models.images import Image from docker.models.volumes import Volume from requests import ReadTimeout +from rich.status import Status from exegol.config.ConstantConfig import ConstantConfig from exegol.config.DataCache import DataCache @@ -258,9 +259,14 @@ def listImages(self, include_version_tag: bool = False, include_locked: bool = F """List available docker images. Return a list of ExegolImage""" if self.__images is None: - remote_images = self.__listRemoteImages() - local_images = self.__listLocalImages() - self.__images = ExegolImage.mergeImages(remote_images, local_images) + logger.verbose("Loading every Exegol images") + with console.status(f"Loading Exegol images from registry", spinner_style="blue") as s: + remote_images = self.__listRemoteImages(s) + logger.verbose("Retrieve [green]local[/green] Exegol images") + s.update(status=f"Retrieving [green]local[/green] Exegol images") + local_images = self.__listLocalImages() + self.__images = ExegolImage.mergeImages(remote_images, local_images, s) + logger.verbose("Images fetched") result = self.__images assert result is not None # Caching latest images @@ -405,7 +411,8 @@ def __findLocalRecoveryImages(self, include_untag: bool = False) -> List[Image]: id_list.add(img.id) return result - def __listRemoteImages(self) -> List[MetaImages]: + @staticmethod + def __listRemoteImages(status: Status) -> List[MetaImages]: """List remote dockerhub images available. Return a list of ExegolImage""" logger.debug("Fetching remote image tags, digests and sizes") @@ -416,25 +423,25 @@ def __listRemoteImages(self) -> List[MetaImages]: current_page = 1 url: Optional[str] = f"https://{ConstantConfig.DOCKER_HUB}/v2/repositories/{ConstantConfig.IMAGE_NAME}/tags?page=1&page_size={page_size}" # Handle multi-page tags from registry - with console.status(f"Loading registry information from [green]{url}[/green]", spinner_style="blue") as s: - while url is not None: - if current_page > page_max: - logger.debug("Max page limit reached. In non-verbose mode, downloads will stop there.") - if not logger.isEnabledFor(ExeLog.VERBOSE): - break - current_page += 1 - s.update(status=f"Fetching registry information from [green]{url}[/green]") - docker_repo_response = WebUtils.runJsonRequest(url, "Dockerhub") - if docker_repo_response is None: - logger.warning("Skipping online queries.") - return [] - error_message = docker_repo_response.get("message") - if error_message: - logger.error(f"Dockerhub send an error message: {error_message}") - for docker_images in docker_repo_response.get("results", []): - meta_image = MetaImages(docker_images) - remote_results.append(meta_image) - url = docker_repo_response.get("next") # handle multiple page tags + while url is not None: + if current_page > page_max: + logger.debug("Max page limit reached. In non-verbose mode, downloads will stop there.") + if not logger.isEnabledFor(ExeLog.VERBOSE): + break + current_page += 1 + if logger.isEnabledFor(ExeLog.VERBOSE): + status.update(status=f"Fetching registry information from [green]{url}[/green]") + docker_repo_response = WebUtils.runJsonRequest(url, "Dockerhub") + if docker_repo_response is None: + logger.warning("Skipping online queries.") + return [] + error_message = docker_repo_response.get("message") + if error_message: + logger.error(f"Dockerhub send an error message: {error_message}") + for docker_images in docker_repo_response.get("results", []): + meta_image = MetaImages(docker_images) + remote_results.append(meta_image) + url = docker_repo_response.get("next") # handle multiple page tags # Remove duplication (version specific / latest release) return remote_results From 29b1c49130a941778ce5271eeac0523eead8c58b Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 9 Nov 2024 14:23:07 +0100 Subject: [PATCH 3/6] Keep updating latest wrapper version --- exegol/manager/ExegolManager.py | 4 ++-- exegol/manager/UpdateManager.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/exegol/manager/ExegolManager.py b/exegol/manager/ExegolManager.py index 627f8cc9..fec14e6a 100644 --- a/exegol/manager/ExegolManager.py +++ b/exegol/manager/ExegolManager.py @@ -217,9 +217,9 @@ def print_debug_banner(cls): logger.debug(f"Docker engine: {EnvInfo.getDockerEngine().value}") logger.debug(f"Docker desktop: {boolFormatter(EnvInfo.isDockerDesktop())}") logger.debug(f"Shell type: {EnvInfo.getShellType().value}") - if not UpdateManager.isUpdateTag() and UserConfig().auto_check_updates: + if UserConfig().auto_check_updates: UpdateManager.checkForWrapperUpdate() - if UpdateManager.isUpdateTag(): + if UpdateManager.isUpdateAvailable(): logger.empty_line() if Confirm(f"An [green]Exegol[/green] update is [orange3]available[/orange3] ({UpdateManager.display_current_version()} -> {UpdateManager.display_latest_version()}), do you want to update ?", default=True): UpdateManager.updateWrapper() diff --git a/exegol/manager/UpdateManager.py b/exegol/manager/UpdateManager.py index ef7fecde..438facf8 100644 --- a/exegol/manager/UpdateManager.py +++ b/exegol/manager/UpdateManager.py @@ -283,7 +283,7 @@ def __tagUpdateAvailable(cls, latest_version, current_version=None): DataCache().get_wrapper_data().current_version = cls.__get_current_version() if current_version is None else current_version @classmethod - def isUpdateTag(cls) -> bool: + def isUpdateAvailable(cls) -> bool: """Check if the cache file is present to announce an available update of the exegol wrapper.""" current_version = cls.__get_current_version() wrapper_data = DataCache().get_wrapper_data() From 4db334defb75f0f8de10f6521976c5af80b36dc5 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 9 Nov 2024 15:58:38 +0100 Subject: [PATCH 4/6] Fix startup sequence when applying X11 GUI with remote host --- exegol/model/ExegolContainer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index d834d527..e5d699d5 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -109,8 +109,8 @@ def start(self): """Start the docker container""" if not self.isRunning(): logger.info(f"Starting container {self.name}") - self.__preStartSetup() self.__start_container() + self.__postStartSetup() def __start_container(self): """ @@ -281,9 +281,9 @@ def __removeVolume(self): return logger.success("Private workspace volume removed successfully") - def __preStartSetup(self): + def __postStartSetup(self): """ - Operation to be performed before starting a container + Operation to be performed after starting a container :return: """ self.__applyX11ACLs() From 543e8a196181eafe18c72cb8b37729e6f134d98d Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 9 Nov 2024 17:09:25 +0100 Subject: [PATCH 5/6] Add pipx installation detection --- exegol/config/ConstantConfig.py | 1 + exegol/manager/ExegolManager.py | 3 ++- exegol/utils/GitUtils.py | 5 ++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/exegol/config/ConstantConfig.py b/exegol/config/ConstantConfig.py index b545a569..5b0b176b 100644 --- a/exegol/config/ConstantConfig.py +++ b/exegol/config/ConstantConfig.py @@ -28,6 +28,7 @@ class ConstantConfig: # Install mode, check if Exegol has been git cloned or installed using pip package git_source_installation: bool = (src_root_path_obj / '.git').is_dir() pip_installed: bool = src_root_path_obj.name == "site-packages" + pipx_installed: bool = "/pipx/venvs/" in src_root_path_obj.as_posix() # Dockerhub Exegol images repository DOCKER_HUB: str = "hub.docker.com" # Don't handle docker login operations DOCKER_REGISTRY: str = "registry-1.docker.io" # Don't handle docker login operations diff --git a/exegol/manager/ExegolManager.py b/exegol/manager/ExegolManager.py index fec14e6a..8c81f700 100644 --- a/exegol/manager/ExegolManager.py +++ b/exegol/manager/ExegolManager.py @@ -205,7 +205,8 @@ def print_version(cls): @classmethod def print_debug_banner(cls): """Print header debug info""" - logger.debug(f"Pip installation: {boolFormatter(ConstantConfig.pip_installed)}") + logger.debug(f"Pip installation: {boolFormatter(ConstantConfig.pip_installed)}" + f"{'[bright_black](pipx)[/bright_black]' if ConstantConfig.pipx_installed else ''}") logger.debug(f"Git source installation: {boolFormatter(ConstantConfig.git_source_installation)}") logger.debug(f"Host OS: {EnvInfo.getHostOs().value} [bright_black]({EnvInfo.getDockerEngine().value})[/bright_black]") logger.debug(f"Arch: {EnvInfo.arch}") diff --git a/exegol/utils/GitUtils.py b/exegol/utils/GitUtils.py index a924da5e..1fb6fbbf 100644 --- a/exegol/utils/GitUtils.py +++ b/exegol/utils/GitUtils.py @@ -49,7 +49,10 @@ def __init__(self, except ReferenceError: if self.__git_name == "wrapper": logger.warning("Exegol has [red]not[/red] been installed via git clone. Skipping wrapper auto-update operation.") - if ConstantConfig.pip_installed: + if ConstantConfig.pipx_installed: + logger.info("If you have installed Exegol with pipx, check for an update with the command " + "[green]pipx upgrade exegol[/green]") + elif ConstantConfig.pip_installed: logger.info("If you have installed Exegol with pip, check for an update with the command " "[green]pip3 install exegol --upgrade[/green]") abort_loading = True From 9d12b3b01b86c820b25b3b71393c2cbc75827425 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 9 Nov 2024 17:18:22 +0100 Subject: [PATCH 6/6] New minor version --- exegol/config/ConstantConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/config/ConstantConfig.py b/exegol/config/ConstantConfig.py index 5b0b176b..b8032413 100644 --- a/exegol/config/ConstantConfig.py +++ b/exegol/config/ConstantConfig.py @@ -5,7 +5,7 @@ class ConstantConfig: """Constant parameters information""" # Exegol Version - version: str = "4.3.8b1" + version: str = "4.3.8" # Exegol documentation link documentation: str = "https://exegol.rtfd.io/"