From caa2c668b08338e467e4d9540e2ef8f49cef946f Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Tue, 12 Apr 2022 17:44:29 +0200 Subject: [PATCH 1/9] implement version without conda-pack --- Dockerfile | 5 ----- Singularity | 21 +++++++++++++-------- conda_to_singularity.py | 33 +++++++++++++++++++++++++++++++++ conda_to_singularity.sh | 24 ------------------------ 4 files changed, 46 insertions(+), 37 deletions(-) delete mode 100644 Dockerfile create mode 100755 conda_to_singularity.py delete mode 100755 conda_to_singularity.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 82bf6eb..0000000 --- a/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM continuumio/miniconda3:latest - -ADD packed_environment.tar.gz /opt/conda/ - -RUN conda-unpack diff --git a/Singularity b/Singularity index a68417f..45c7bfc 100644 --- a/Singularity +++ b/Singularity @@ -1,14 +1,19 @@ -Bootstrap: docker - -From: continuumio/miniconda3 +Bootstrap: yum +OSVersion: 7 +MirrorURL: http://mirror.centos.org/centos-%{{OSVERSION}}/%{{OSVERSION}}/os/$basearch/ +Include: yum %files - packed_environment.tar.gz /packed_environment.tar.gz + {path} %environment - export NUMBA_CACHE_DIR=/tmp/numba_cache + export NUMBA_CACHE_DIR=/tmp/numba_cache %post - tar xvzf /packed_environment.tar.gz -C /opt/conda - conda-unpack - rm /packed_environment.tar.gz + yum install -y kernel-3.10.0-1160.11.1.el7 + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh > /install_conda.sh + chmod +x /install_conda.sh + /install_conda.sh -b -p /opt/conda + rm /install_conda.sh + echo "source /opt/conda/bin/activate {path}" >>$SINGULARITY_ENVIRONMENT + diff --git a/conda_to_singularity.py b/conda_to_singularity.py new file mode 100755 index 0000000..bf73d78 --- /dev/null +++ b/conda_to_singularity.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +""" +Usage: + ./conda_to_singularity.py CONDA_ENV OUTPUT.sif + +CONDA_ENV needs to be an absolute path to a conda environment. +""" + +import sys +import tempfile +from subprocess import call +from os.path import abspath, join as join_path + +conda_env, output_path = sys.argv[1:3] +output_path = abspath(output_path) + +TEMPLATE = "./Singularity" + +with open(TEMPLATE) as f: + template = "".join(f.readlines()) + +template = template.format(path=conda_env) + + +with tempfile.TemporaryDirectory() as tmpdir: + with open(join_path(tmpdir, "Singularity"), 'w') as f: + f.write(template) + print(tmpdir) + call(["singularity", "build", "--fakeroot", "--force", output_path, "Singularity"], cwd=tmpdir) + + + diff --git a/conda_to_singularity.sh b/conda_to_singularity.sh deleted file mode 100755 index b18cf84..0000000 --- a/conda_to_singularity.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -################################### -# -# Usage: -# conda_to_singularity.sh -# -################################### - -CONDAENV=$1 -CONTAINER=$(readlink -f $2) -DIR=${TMPDIR}/$(echo $CONDAENV | md5sum | awk '{print $1}') - -mkdir -p $DIR - -echo ENV=$CONDAENV -echo CONTAINER=$CONTAINER -echo DIR=$DIR - -conda-pack -n $CONDAENV -o $DIR/packed_environment.tar.gz --ignore-missing-files --force && \ - cp Singularity $DIR && \ - cd $DIR && \ - singularity build --fakeroot --force $CONTAINER Singularity - From 0cb0a4de02ee15d5c171dc742b073019d419e44b Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Tue, 12 Apr 2022 22:06:36 +0200 Subject: [PATCH 2/9] update readme --- README.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0355073..ea23459 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,26 @@ # Containerize an existing conda environment -I use conda environments for working on data analysis projects. -Sometimes I need to revert to install using `pip` or `R`'s -`install.packages` if a package is not on bioconda or conda-forge. +I use conda environments for working on data analysis projects. +Sometimes I need to revert to install using `pip` or `R`'s +`install.packages` if a package is not on bioconda or conda-forge. This makes it very hard to reproduce the environment, and therefore, the analysis, on another system. Even pure conda environments stored as an `environment.yml` file tend to [break after a -while](https://github.com/conda/conda/issues/9257). +while](https://github.com/conda/conda/issues/9257). Using the instructions below allows to package an existing environment into a Docker or Singularity container which should be more portable and can also easily be integrated into a [fully reproducible data analysis workflow](https://grst.github.io/bioinformatics/2019/12/23/reportsrender.html) -based on e.g. [Nextflow](https://www.nextflow.io/). +based on e.g. [Nextflow](https://www.nextflow.io/). ## Prerequisites * [conda-pack](https://conda.github.io/conda-pack/) * either Docker, Podman or Singularity - * source conda environment needs to be on a linux x64 machine. + * source conda environment needs to be on a linux x64 machine. ## Usage @@ -55,5 +55,9 @@ Conda environment can't be just "moved" to another location, as some paths are hardcoded into the environment. `conda-pack` takes care of replacing these paths back to placeholders and creates a `.tar.gz` archive that contains the environment. This environment can be unpacked to another machine (or, in our -case, a container). Running `conda-unpack` in the environment replaces the -placeholders back to the actual paths matching the new location. +case, a container). Running `conda-unpack` in the environment replaces the +placeholders back to the actual paths matching the new location. + +## Troubleshooting + + * `find . -xtype l` finds broken symbolic links which leads to a failed container creation... From ab2b574ddd67dbbace92a37b2d2ca86c2021962c Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Sun, 8 May 2022 14:57:40 +0200 Subject: [PATCH 3/9] Add wireframe how to get intact symbolic links into the singularity container --- README.md | 54 +++++++++++++-------------------------------- make_tar_archive.sh | 13 +++++++++++ 2 files changed, 28 insertions(+), 39 deletions(-) create mode 100644 make_tar_archive.sh diff --git a/README.md b/README.md index ea23459..e00ef82 100644 --- a/README.md +++ b/README.md @@ -16,48 +16,24 @@ data analysis workflow](https://grst.github.io/bioinformatics/2019/12/23/reportsrender.html) based on e.g. [Nextflow](https://www.nextflow.io/). -## Prerequisites +## Variant without conda-pack (singularity only) - * [conda-pack](https://conda.github.io/conda-pack/) - * either Docker, Podman or Singularity - * source conda environment needs to be on a linux x64 machine. +Conda envs cannot simply be "moved" as some paths are hardcoded into the environment. +I previously applied `conda-pack` to solve this issue, which works fine in most cases +but breaks in some (especially for old environments that have a long history +of manually installing stuff through R or pip) -## Usage +This is an other appraoch where the issue is solved by copying the conda environment +with its full absolute path to the container and append a line to the Singularity environment +file that activates the conda environment from that path once the container is started. -1. Clone this repository (retrieve `Dockerfile`/`Singularity`) +Naively, this could be solved with `%files /path/to/env`, however, this dereferences +all symbolic links, which breaks some environments. Instead, I involve some bash/tar +magic to keep all symbolic links intact *within* the conda environment, but at the +same time include all files that are outside the conda env, but referenced +by a symbolic link. -``` -git clone git@github.com:grst/containerize-conda.git -cd containerize-conda -``` +I don't have a lot of experience yet if it is really more stable than conda-pack +or just happens to fail in different cases. -2. Pack the environment -``` -conda-pack -n -o packed_environment.tar.gz -``` - -3. Build the container - -``` -# With singularity -singularity build --fakeroot Singularity - -# With Docker -docker build . -t - -# With Podman/Buildah -podman build . -t -``` - -## How it works -Conda environment can't be just "moved" to another location, as some paths are -hardcoded into the environment. `conda-pack` takes care of replacing these paths -back to placeholders and creates a `.tar.gz` archive that contains the -environment. This environment can be unpacked to another machine (or, in our -case, a container). Running `conda-unpack` in the environment replaces the -placeholders back to the actual paths matching the new location. - -## Troubleshooting - - * `find . -xtype l` finds broken symbolic links which leads to a failed container creation... diff --git a/make_tar_archive.sh b/make_tar_archive.sh new file mode 100644 index 0000000..5d00389 --- /dev/null +++ b/make_tar_archive.sh @@ -0,0 +1,13 @@ +cd PATH_TO_CONDA_ENV + +# find $(readlink -f .) gets all files from the conda env as their absolute path (NOTE: it should be an absolute path, but not necessarily the realpath, but the absolute path used by conda) +# find -L . -exect readlink -f) gets all files and dereferences them (this includes all filese outside the conda env) +# using sort/uniq duplicate filese are removed +cat <(find $(readlink -f .)) <(find -L . -exec readlink -f "{}" ";") | sort | uniq > ~/Downloads/pircher_sc_integrate2_all_files2.txt + +# create a tar archive from the file list above. +# DO NOT DEREFERNCE symbolic links +# dereferencing hard links is not necessary +# The tar file is a way to get symbolic links into the singularity container +tar cvf temp_conda_tar_archive.tar -T THE_FILE_WITH_ALL_FILENAMES_FROM_ABOVE + From 006ac48845a0d56382ea6837e209dad18d3618b1 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Sat, 24 Sep 2022 13:51:36 +0200 Subject: [PATCH 4/9] Initial version of conda2singularity script --- .editorconfig | 12 ++++ Singularity => Singularity.template | 11 +-- conda_to_singularity.py | 108 ++++++++++++++++++++++------ make_tar_archive.sh | 13 ---- 4 files changed, 106 insertions(+), 38 deletions(-) create mode 100644 .editorconfig rename Singularity => Singularity.template (50%) delete mode 100644 make_tar_archive.sh diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..198ee90 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = false \ No newline at end of file diff --git a/Singularity b/Singularity.template similarity index 50% rename from Singularity rename to Singularity.template index 45c7bfc..cf667c6 100644 --- a/Singularity +++ b/Singularity.template @@ -1,19 +1,20 @@ Bootstrap: yum OSVersion: 7 -MirrorURL: http://mirror.centos.org/centos-%{{OSVERSION}}/%{{OSVERSION}}/os/$basearch/ +MirrorURL: http://mirror.centos.org/centos-%{OSVERSION}/%{OSVERSION}/os/$basearch/ Include: yum %files - {path} + packed_env.tar /packed_env.tar %environment export NUMBA_CACHE_DIR=/tmp/numba_cache %post - yum install -y kernel-3.10.0-1160.11.1.el7 +# yum install -y kernel-3.10.0-1160.11.1.el7 curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh > /install_conda.sh chmod +x /install_conda.sh /install_conda.sh -b -p /opt/conda rm /install_conda.sh - echo "source /opt/conda/bin/activate {path}" >>$SINGULARITY_ENVIRONMENT - + tar xvf /packed_env.tar + rm /packed_env.tar + echo "source /opt/conda/bin/activate {conda_env}" >>$SINGULARITY_ENVIRONMENT \ No newline at end of file diff --git a/conda_to_singularity.py b/conda_to_singularity.py index bf73d78..1350518 100755 --- a/conda_to_singularity.py +++ b/conda_to_singularity.py @@ -1,33 +1,101 @@ #!/usr/bin/env python -""" -Usage: - ./conda_to_singularity.py CONDA_ENV OUTPUT.sif - -CONDA_ENV needs to be an absolute path to a conda environment. -""" - -import sys import tempfile from subprocess import call -from os.path import abspath, join as join_path +from os.path import abspath, join as join_path, dirname +from textwrap import dedent +import argparse +from pathlib import Path + + +def _generate_file_list(conda_env, filelist_path): + """ + Generate list of all files in the conda env. + + We need to include all files as absolute paths, and also the symbolic links they are pointing to (which + might be outside the environment). To this end, the first `find` lists all files in the conda env. + The second find finds the files the links point to. Using sort/uniq removes the duplicates files. + + TODO: While this covered all the cases I encountered so far, I believe this would still fail if there were nested + symbolic links outside the repository. + """ + command = f"""\ + #!/bin/bash + set -o pipefail + + cat <(find {conda_env}) <(find -L {conda_env} -exec readlink -f "{{}}" ";") | \\ + sort | \\ + uniq > {filelist_path} + """ + call(command, shell=True, executable="/bin/bash") + + +def _build_tar_archive(filelist_path, archive_path): + """Build a tar archive from the filelist""" + call(["tar", "cvf", archive_path, "-T", filelist_path]) + + +def _build_container(singularity_file, output_path): + """ + Actually builds the container. + """ + call( + [ + "singularity", + "build", + "--fakeroot", + "--force", + output_path, + singularity_file, + ], + ) + -conda_env, output_path = sys.argv[1:3] -output_path = abspath(output_path) +def conda2singularity(conda_env, output_path, template_path): + output_path = abspath(output_path) -TEMPLATE = "./Singularity" + with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = Path(tmpdir) + print("Using temporary directory: {tmpdir}") + singularity_file_path = tmpdir / "Singularity" + filelist_path = tmpdir / "filelist.txt" + tar_archive_path = tmpdir / "packed_env.tar" -with open(TEMPLATE) as f: - template = "".join(f.readlines()) + # Read Singularity template file + with open(template_path) as f: + template = "".join(f.readlines()) + template = template.format(conda_env=conda_env) -template = template.format(path=conda_env) + # Write formatted template file + with open(singularity_file_path, "w") as f: + f.write(template) + print("Building file list...") + _generate_file_list(conda_env, filelist_path) -with tempfile.TemporaryDirectory() as tmpdir: - with open(join_path(tmpdir, "Singularity"), 'w') as f: - f.write(template) - print(tmpdir) - call(["singularity", "build", "--fakeroot", "--force", output_path, "Singularity"], cwd=tmpdir) + print("Building tar archive...") + _build_tar_archive(filelist_path, tar_archive_path) + print("Building singularity container...") + _build_container(singularity_file_path, output_path) +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Convert a conda env to a singularity container." + ) + parser.add_argument( + "CONDA_ENV", + help="Absolute path to the conda enviornment. Must be exactely the path as it shows up in `conda env list`, not a symbolic link to it, nor a realpath. ", + ) + parser.add_argument( + "OUTPUT_CONTAINER", + help="Output path where the singularity container will be safed.", + ) + parser.add_argument( + "--template", + help="Path to a Singularity template file. Must contain a `{conda_env}` placeholder. If not specified, uses the default template shipped with this script.", + default=join_path(dirname(__file__), "Singularity.template"), + ) + args = parser.parse_args() + conda2singularity(args.CONDA_ENV, args.OUTPUT_CONTAINER, args.template) diff --git a/make_tar_archive.sh b/make_tar_archive.sh deleted file mode 100644 index 5d00389..0000000 --- a/make_tar_archive.sh +++ /dev/null @@ -1,13 +0,0 @@ -cd PATH_TO_CONDA_ENV - -# find $(readlink -f .) gets all files from the conda env as their absolute path (NOTE: it should be an absolute path, but not necessarily the realpath, but the absolute path used by conda) -# find -L . -exect readlink -f) gets all files and dereferences them (this includes all filese outside the conda env) -# using sort/uniq duplicate filese are removed -cat <(find $(readlink -f .)) <(find -L . -exec readlink -f "{}" ";") | sort | uniq > ~/Downloads/pircher_sc_integrate2_all_files2.txt - -# create a tar archive from the file list above. -# DO NOT DEREFERNCE symbolic links -# dereferencing hard links is not necessary -# The tar file is a way to get symbolic links into the singularity container -tar cvf temp_conda_tar_archive.tar -T THE_FILE_WITH_ALL_FILENAMES_FROM_ABOVE - From fe182fe70c080ea19d4e82b704675a27ee3664b8 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Sat, 24 Sep 2022 17:45:25 +0200 Subject: [PATCH 5/9] Fix template --- Singularity.template | 5 +++-- conda_to_singularity.py | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Singularity.template b/Singularity.template index cf667c6..5d2534a 100644 --- a/Singularity.template +++ b/Singularity.template @@ -1,6 +1,6 @@ Bootstrap: yum OSVersion: 7 -MirrorURL: http://mirror.centos.org/centos-%{OSVERSION}/%{OSVERSION}/os/$basearch/ +MirrorURL: http://mirror.centos.org/centos-%{{OSVERSION}}/%{{OSVERSION}}/os/$basearch/ Include: yum %files @@ -11,10 +11,11 @@ Include: yum %post # yum install -y kernel-3.10.0-1160.11.1.el7 + yum install -y tar curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh > /install_conda.sh chmod +x /install_conda.sh /install_conda.sh -b -p /opt/conda rm /install_conda.sh - tar xvf /packed_env.tar + tar xf /packed_env.tar rm /packed_env.tar echo "source /opt/conda/bin/activate {conda_env}" >>$SINGULARITY_ENVIRONMENT \ No newline at end of file diff --git a/conda_to_singularity.py b/conda_to_singularity.py index 1350518..a3ab68a 100755 --- a/conda_to_singularity.py +++ b/conda_to_singularity.py @@ -2,10 +2,10 @@ import tempfile from subprocess import call -from os.path import abspath, join as join_path, dirname -from textwrap import dedent +from os.path import abspath, join as join_path, dirname, realpath import argparse from pathlib import Path +from time import sleep def _generate_file_list(conda_env, filelist_path): @@ -32,12 +32,14 @@ def _generate_file_list(conda_env, filelist_path): def _build_tar_archive(filelist_path, archive_path): """Build a tar archive from the filelist""" - call(["tar", "cvf", archive_path, "-T", filelist_path]) + call(["tar", "cf", archive_path, "-T", filelist_path]) -def _build_container(singularity_file, output_path): +def _build_container(tmpdir, singularity_file, output_path): """ Actually builds the container. + + tmpdir is the temporary directory that already contains the tar archive. """ call( [ @@ -48,6 +50,7 @@ def _build_container(singularity_file, output_path): output_path, singularity_file, ], + cwd=tmpdir, ) @@ -56,7 +59,7 @@ def conda2singularity(conda_env, output_path, template_path): with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) - print("Using temporary directory: {tmpdir}") + print(f"Using temporary directory: {tmpdir}") singularity_file_path = tmpdir / "Singularity" filelist_path = tmpdir / "filelist.txt" tar_archive_path = tmpdir / "packed_env.tar" @@ -73,11 +76,13 @@ def conda2singularity(conda_env, output_path, template_path): print("Building file list...") _generate_file_list(conda_env, filelist_path) + # We are using a tar archive as tar is the only way of getting the symbolic links into the singularity + # container as symbolic links. print("Building tar archive...") _build_tar_archive(filelist_path, tar_archive_path) print("Building singularity container...") - _build_container(singularity_file_path, output_path) + _build_container(tmpdir, singularity_file_path, output_path) if __name__ == "__main__": @@ -95,7 +100,7 @@ def conda2singularity(conda_env, output_path, template_path): parser.add_argument( "--template", help="Path to a Singularity template file. Must contain a `{conda_env}` placeholder. If not specified, uses the default template shipped with this script.", - default=join_path(dirname(__file__), "Singularity.template"), + default=join_path(dirname(realpath(__file__)), "Singularity.template"), ) args = parser.parse_args() conda2singularity(args.CONDA_ENV, args.OUTPUT_CONTAINER, args.template) From ffcd32af9f139b9673fd551b557055904f2f215a Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Sat, 24 Sep 2022 17:50:17 +0200 Subject: [PATCH 6/9] re-add legacy conda-pack version --- conda-pack/Dockerfile | 5 +++++ conda-pack/Singularity | 14 ++++++++++++++ conda-pack/conda_to_singularity.sh | 24 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 conda-pack/Dockerfile create mode 100644 conda-pack/Singularity create mode 100755 conda-pack/conda_to_singularity.sh diff --git a/conda-pack/Dockerfile b/conda-pack/Dockerfile new file mode 100644 index 0000000..82bf6eb --- /dev/null +++ b/conda-pack/Dockerfile @@ -0,0 +1,5 @@ +FROM continuumio/miniconda3:latest + +ADD packed_environment.tar.gz /opt/conda/ + +RUN conda-unpack diff --git a/conda-pack/Singularity b/conda-pack/Singularity new file mode 100644 index 0000000..a68417f --- /dev/null +++ b/conda-pack/Singularity @@ -0,0 +1,14 @@ +Bootstrap: docker + +From: continuumio/miniconda3 + +%files + packed_environment.tar.gz /packed_environment.tar.gz + +%environment + export NUMBA_CACHE_DIR=/tmp/numba_cache + +%post + tar xvzf /packed_environment.tar.gz -C /opt/conda + conda-unpack + rm /packed_environment.tar.gz diff --git a/conda-pack/conda_to_singularity.sh b/conda-pack/conda_to_singularity.sh new file mode 100755 index 0000000..b18cf84 --- /dev/null +++ b/conda-pack/conda_to_singularity.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +################################### +# +# Usage: +# conda_to_singularity.sh +# +################################### + +CONDAENV=$1 +CONTAINER=$(readlink -f $2) +DIR=${TMPDIR}/$(echo $CONDAENV | md5sum | awk '{print $1}') + +mkdir -p $DIR + +echo ENV=$CONDAENV +echo CONTAINER=$CONTAINER +echo DIR=$DIR + +conda-pack -n $CONDAENV -o $DIR/packed_environment.tar.gz --ignore-missing-files --force && \ + cp Singularity $DIR && \ + cd $DIR && \ + singularity build --fakeroot --force $CONTAINER Singularity + From 4c6b3ffed54ab23a8ba4b7ae4b2960ab42cd108c Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Sat, 24 Sep 2022 18:03:26 +0200 Subject: [PATCH 7/9] Update README --- README.md | 46 +++++++++++++++++++++++++++++++++++++++----- conda-pack/README.md | 41 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 conda-pack/README.md diff --git a/README.md b/README.md index e00ef82..93bb300 100644 --- a/README.md +++ b/README.md @@ -16,20 +16,56 @@ data analysis workflow](https://grst.github.io/bioinformatics/2019/12/23/reportsrender.html) based on e.g. [Nextflow](https://www.nextflow.io/). -## Variant without conda-pack (singularity only) +## Usage + +**Note**: This is an updated version of my scripts that works without `conda-pack` and turned out +to work even in cases where the conda-pack variant failed. It works only with Singularity at the moment, though. +If you are looking for the previous scripts based on `conda-pack`, because you need a Docker variant, or they just +work for you, they are in the [conda-pack](conda-pack) folder with a dedicated [README](conda-pack/README.md). + + +``` +usage: conda_to_singularity.py [-h] [--template TEMPLATE] CONDA_ENV OUTPUT_CONTAINER + +Convert a conda env to a singularity container. + +positional arguments: + CONDA_ENV Absolute path to the conda enviornment. Must be exactely the path as it shows up in `conda env list`, not a symbolic link to it, nor a realpath. + OUTPUT_CONTAINER Output path where the singularity container will be safed. + +optional arguments: + -h, --help show this help message and exit + --template TEMPLATE Path to a Singularity template file. Must contain a `{conda_env}` placeholder. If not specified, uses the default template shipped with this script. +``` + +For example + +``` +conda_to_singularity.py /home/sturm/.conda/envs/whatever whatever.sif +``` + +By default, the image will be based on CentOS 7. If you want a different base image, +you can modify `Singularity.template`, and specify it with the `--template` argument. + + +## How it works Conda envs cannot simply be "moved" as some paths are hardcoded into the environment. I previously applied `conda-pack` to solve this issue, which works fine in most cases but breaks in some (especially for old environments that have a long history -of manually installing stuff through R or pip) +of manually installing stuff through R or pip). This is an other appraoch where the issue is solved by copying the conda environment with its full absolute path to the container and append a line to the Singularity environment -file that activates the conda environment from that path once the container is started. +file that activates the conda environment from that path once the container is started: + +``` +echo "source /opt/conda/bin/activate {conda_env}" >>$SINGULARITY_ENVIRONMENT +``` Naively, this could be solved with `%files /path/to/env`, however, this dereferences -all symbolic links, which breaks some environments. Instead, I involve some bash/tar -magic to keep all symbolic links intact *within* the conda environment, but at the +all symbolic links, which breaks some environments. Instead, I build a tar archive +that keeps all symbolic links intact *within* the conda environment, but at the same time include all files that are outside the conda env, but referenced by a symbolic link. diff --git a/conda-pack/README.md b/conda-pack/README.md new file mode 100644 index 0000000..8186112 --- /dev/null +++ b/conda-pack/README.md @@ -0,0 +1,41 @@ +## Prerequisites + + * [conda-pack](https://conda.github.io/conda-pack/) + * either Docker, Podman or Singularity + * source conda environment needs to be on a linux x64 machine. + +## Usage + +1. Clone this repository (retrieve `Dockerfile`/`Singularity`) + +``` +git clone git@github.com:grst/containerize-conda.git +cd containerize-conda/conda-pack +``` + +2. Pack the environment + +``` +conda-pack -n -o packed_environment.tar.gz +``` + +3. Build the container + +``` +# With singularity +singularity build --fakeroot Singularity + +# With Docker +docker build . -t + +# With Podman/Buildah +podman build . -t +``` + +## How it works +Conda environment can't be just "moved" to another location, as some paths are +hardcoded into the environment. `conda-pack` takes care of replacing these paths +back to placeholders and creates a `.tar.gz` archive that contains the +environment. This environment can be unpacked to another machine (or, in our +case, a container). Running `conda-unpack` in the environment replaces the +placeholders back to the actual paths matching the new location. From b195d7ee1544edcada2e989d3489dd7292145004 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Sat, 24 Sep 2022 18:05:12 +0200 Subject: [PATCH 8/9] update readme --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 93bb300..79c621d 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,6 @@ based on e.g. [Nextflow](https://www.nextflow.io/). ## Usage -**Note**: This is an updated version of my scripts that works without `conda-pack` and turned out -to work even in cases where the conda-pack variant failed. It works only with Singularity at the moment, though. -If you are looking for the previous scripts based on `conda-pack`, because you need a Docker variant, or they just -work for you, they are in the [conda-pack](conda-pack) folder with a dedicated [README](conda-pack/README.md). - - ``` usage: conda_to_singularity.py [-h] [--template TEMPLATE] CONDA_ENV OUTPUT_CONTAINER @@ -72,4 +66,11 @@ by a symbolic link. I don't have a lot of experience yet if it is really more stable than conda-pack or just happens to fail in different cases. +## Where's the conda-pack version? + +This is an updated version of my scripts that works without `conda-pack` and turned out +to work even in cases where the conda-pack variant failed. It works only with Singularity at the moment, though. +If you are looking for the previous scripts based on `conda-pack`, because you need a Docker variant, or they just +work for you, they are in the [conda-pack](conda-pack) folder with a dedicated [README](conda-pack/README.md). + From a22932b4a48716d688bb6883339cad15a4b3e342 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Sat, 24 Sep 2022 18:48:32 +0200 Subject: [PATCH 9/9] Fix merge confict --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 0b2208e..2140610 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ you can modify `Singularity.template`, and specify it with the `--template` argu ## How it works -<<<<<<< HEAD Conda envs cannot simply be "moved" as some paths are hardcoded into the environment. I previously applied `conda-pack` to solve this issue, which works fine in most cases