Merge branch 'develop' at v0.3

phnmnl · Jan 12, 2018 · 7a22f7d · 7a22f7d
2 parents e5a6b5e + aa82162
commit 7a22f7d
Show file tree

Hide file tree

Showing 23 changed files with 667 additions and 507 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -38,9 +38,8 @@ script:
 - tests/comparators/run_all
 - wft4galaxy -f examples/change_case/workflow-test.yml --server ${GALAXY_URL} --api-key ${GALAXY_API_KEY}
 - export GALAXY_URL=http://${GALAXY_ADDRESS} # use the address of the Galaxy container within the Docker network
-- branch=${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}
-- tests/docker/run-all.sh --branch ${branch} --server ${GALAXY_URL} --api-key ${GALAXY_API_KEY} --network ${GALAXY_NETWORK}
+- tests/docker/run-all.sh --server ${GALAXY_URL} --api-key ${GALAXY_API_KEY} --network ${GALAXY_NETWORK} --local-copy --image-owner travis-ci
 
 
 after_script:
-- docker rm -f ${GALAXY_CONTAINER_NAME}
+- docker rm -f ${GALAXY_CONTAINER_NAME}
diff --git a/examples/change_case/workflow-test.yml b/examples/change_case/workflow-test.yml
@@ -38,6 +38,15 @@
 # Enable to print yet more information
 #enable_debug: True
 
+# Maximum number of retries after a connection failure
+# max_retries: 10
+
+# Delay between consecutive retries (in seconds)
+# retry_delay: 1
+
+# Wait time between consecutive checks of the workflow state (in seconds)
+# polling_interval: 1
+
 ##########################################################################################
 #  Workflow tests
 ##########################################################################################

diff --git a/setup.py b/setup.py
@@ -145,7 +145,7 @@ def run(self):
     name='wft4galaxy',
     description='Utility module for testing Galaxy workflows',
     url='https://github.com/phnmnl/wft4galaxy',
-    version='0.1',
+    version='0.3',
     install_requires=requirements,
     package_data={'wft4galaxy': ['wft4galaxy.properties'], 'templates': ['*']},
     packages=["wft4galaxy", "wft4galaxy.comparators", "wft4galaxy.app", "templates"],

diff --git a/tests/docker/test-image.sh b/tests/docker/test-image.sh
@@ -11,150 +11,170 @@ image_root_path="${script_path}/../../utils/docker"
 
 # print help
 function print_usage(){
-    (   echo "USAGE: $0 [--server URL] [--api-key API-KEY] [--network ADDRESS] "
-        echo "          [-r|--image-registry REGISTRY] [-o|--image-owner OWNER] [-n|--image-name NAME] [-t|--image-tag TAG]"
-        echo "          {minimal,develop}"
-        echo ""
-        echo "If no options related to the Docker image are provided, this script will try to get the"
-        echo "required image information from the local repository itself") >&2
+    echo "USAGE: $0 [--server URL] [--api-key API-KEY] [--network ADDRESS] { any options build-image.sh accepts }" >&2
 }
 
-# init argument variables
-image_type=''
+function error() {
+  if [[ $# -gt 0 ]]; then
+    echo "$@" >&2
+  else
+    echo "Sorry.  There's been an error!" >&2
+  fi
+  exit 2
+}
+
+function usage_error() {
+  if [[ $# -gt 0 ]]; then
+    echo "$@" >&2
+  fi
+  print_usage
+  exit 2
+}
+
+function parse_args() {
+  # this function parses the arguments it receives in $@ and sets variables
+  while test $# -gt 0
+  do
+      case "$1" in
+          -h|--help)
+              print_usage
+              exit 0
+              ;;
+          --server )
+              [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; }
+              GALAXY_URL="$2"
+              shift
+              ;;
+          --api-key)
+              [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; }
+              GALAXY_API_KEY="$2"
+              shift
+              ;;
+          --network )
+              [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; }
+              GALAXY_NETWORK="$2"
+              shift
+              ;;
+          --debug )
+              debug="--debug"
+              shift
+              ;;
+          *)
+              OTHER_ARGS+=("${1}")
+              ;;
+      esac
+      shift
+  done
+
+  # check required options
+  local settings=(GALAXY_URL GALAXY_API_KEY)
+  for s in ${settings[@]}; do
+      if [[ -z ${!s} ]]; then
+          echo "No ${s} provided." >&2
+          exit 99
+      fi
+  done
+}
+
+function extract_build_image_info() { # args: (filename, tag)
+  if [[ $# -ne 2 ]]; then
+    error "BUG!  extract_build_image_tag called with $# arguments (expected 2)"
+  fi
+  local filename="${1}"
+  local tag="${2}"
+
+  local sed_expr="/${tag}/s/\s*${tag}\s*:\s*\([^,]*\),\?\s*/\1/p"
+  local value="$(sed -n -e "${sed_expr}" "${filename}")"
+  echo "Extracted ${tag}: '${value}'" &>2
+  echo "${value}"
+
+  return 0
+}
+
+########## main ###########
+
+# absolute path of the current script
+script_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# initialize variables to default values, if any
 GALAXY_URL=''
 GALAXY_API_KEY=''
 GALAXY_NETWORK=''
-repo_url=''
-GIT_BRANCH=''
-repo_branch=''
 
-# options
-opts="$@"
+# other options to be passed to build-image.sh
+OTHER_ARGS=()
 
 # disable debug
 debug=""
 
-# parse args
-while test $# -gt 0
-do
-    case "$1" in
-        -h|--help)
-            print_usage
-            exit 0
-            ;;
-        --server )
-            [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; }
-            GALAXY_URL="$2"
-            shift
-            ;;
-        --api-key)
-            [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; }
-            GALAXY_API_KEY="$2"
-            shift
-            ;;
-        --network )
-            [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; }
-            GALAXY_NETWORK="$2"
-            shift
-            ;;
-        --url|--repo-url)
-            [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; }
-            repo_url="--url $2"
-            shift
-            ;;
-        --branch|--repo-branch)
-            [ $# -ge 2 ] || { echo "Missing value for '$1'"; exit 1; }
-            GIT_BRANCH=$2
-            repo_branch="--branch $2"
-            shift
-            ;;
-        --debug )
-            debug="--debug"
-            shift
-            ;;
-        --*)
-            print_usage
-            exit -1
-            ;;
-        *)
-            # support only the first argument; skip all remaining
-            if [[ -z ${image_type} ]]; then
-                image_type=${1}
-            fi
-            ;;
-    esac
-    shift
-done
-
-# check required options
-settings=(image_type GALAXY_URL GALAXY_API_KEY)
-for s in ${settings[@]}; do
-    if [[ -z ${!s} ]]; then
-        echo "No ${s} provided." >&2
-        exit 99
-    fi
-done
-
-# check image type
-if [[ -z ${image_type} ]]; then
-    image_type="minimal"
-    echo "No image type provided. Using the default : ${image_type}">&2
-elif [[ ${image_type} != "minimal" && ${image_type} != "develop" ]]; then
-    echo -e "\nERROR: '${image_type}' not supported! Use 'minimal' or 'develop'."
-    exit 99
+# parse args. An empty $@ will result in the empty string "" being passed, which is ok
+# (we'll raise an error later for lack of mandatory arguments)
+parse_args "${@:-}"
+
+# copy the wft4galaxy script
+docker_runner="${script_path}/../../wft4galaxy/app/docker_runner.py"
+
+echo "Building docker image" >&2
+build_image="${script_path}/../../utils/docker/build-image.sh"
+
+# Use a temporary file to capture the name of the image being created
+# `mktemp -t` is deprecated on Linux, but it gives us compatibility with both Linux and MacOS
+tmpfile=$(mktemp -t wft4galaxy-test-image.XXXXX)
+trap "rm -f '${tmpfile}'" EXIT # remove that file on exit
+
+# fail if any step in the pipe files (we especially care about build-image.sh :-) )
+set -o pipefail
+# We want to capture the last section of the stdout from build-image.sh where it's
+# going to print the name of the docker image it created. At the same time, we
+# want the output to go to the console so that we can see what's going on.  >()
+# is the syntax for bash's process substitution: tee writes to an "anonymous"
+# pipe which is connected to sed's stdin. The sed command will delete everything
+# up to and including the line with the marker
+
+if [[ ${#OTHER_ARGS[@]} -eq 0 ]]; then
+  # This works around bash's quirky behaviour of raising an error if nounset is on and
+  # you expand an empty (but declared) array
+  "${build_image}" | tee >(sed -e '1,/^=== build-image.sh complete ===$/d' > "${tmpfile}")
 else
-    export IMAGE_TYPE="${image_type}"
+  "${build_image}" "${OTHER_ARGS[@]}" | tee >(sed -e '1,/^=== build-image.sh complete ===$/d' > "${tmpfile}")
 fi
 
-# extract git info & Docker image name
-source ${image_root_path}/set-git-repo-info.sh ${repo_url} ${repo_branch}
-source ${image_root_path}/set-docker-image-info.sh
+image_repo="$(extract_build_image_info "${tmpfile}" image_repository)"
+image_tag="$(extract_build_image_info "${tmpfile}" image_tag)"
 
-# download wft4galaxy script
-owner=${GIT_OWNER:-"phnmnl"}
-branch=${GIT_BRANCH:-"develop"}
-curl -s https://raw.githubusercontent.com/${owner}/wft4galaxy/${branch}/utils/docker/install.sh | bash /dev/stdin --repo "${owner}/wft4galaxy" --branch ${branch} .
-echo "Downloaded 'wft4galaxy-docker' Github repository: ${owner}/wft4galaxy (branch: ${branch})" >&2
+# array of arguments for the docker run
+cmd_args=(--server "${GALAXY_URL}" --api-key "${GALAXY_API_KEY}" --skip-update)
 
-# switch the Docker image context
-cd ${image_root_path} > /dev/null
-
-# build docker image
-echo "${image_root_path}/${image_type}/build.sh ${opts}" >&2
-"${image_root_path}/${image_type}/build.sh" ${repo_branch}
-cd - > /dev/null
-
-# set optional arguments
-cmd_other_opts="--repository ${IMAGE_REPOSITORY} --skip-update ${debug}"
-if [[ -n ${IMAGE_TAG} ]]; then
-    cmd_other_opts="${cmd_other_opts} --tag ${IMAGE_TAG}"
+if [[ -n "${image_repo}" ]]; then
+  cmd_args+=(--repository "${image_repo}")
+fi
+if [[ -n "${image_tag}" ]]; then
+  cmd_args+=(--tag "${image_tag}")
 fi
-if [[ -n ${GALAXY_NETWORK} ]]; then
-    cmd_other_opts="${cmd_other_opts} --network ${GALAXY_NETWORK}"
+if [[ -n "${debug}" ]]; then
+  cmd_args+=("${debug}")
 fi
+if [[ -n "${GALAXY_NETWORK}" ]]; then
+  cmd_args+=(--network "${GALAXY_NETWORK}")
+fi
+
+# finally, add the test definition argument
+cmd_args+=(-f "${script_path}/../../examples/change_case/workflow-test.yml")
 
 # uncomment for debug
 #echo "Trying to contact Galaxy sever..."
-#docker run --rm --network ${GALAXY_NETWORK} \
+#docker run --rm --network "${GALAXY_NETWORK}" \
 #            ubuntu bash -c "apt-get update && apt-get install -y iputils-ping && timeout 5 ping 172.18.0.22"
 
-# build cmd
-base_cmd="./wft4galaxy-docker ${cmd_other_opts} --server ${GALAXY_URL} --api-key ${GALAXY_API_KEY}"
-cmd="${base_cmd} -f examples/change_case/workflow-test.yml"
-echo -e "CMD: ${cmd}\n">&2
-
-# turn off command error checking
+# now run the tests
+echo -e "CMD: ${docker_runner} ${cmd_args[@]}\n" >&2
+# first turn off command error checking
 set +o errexit
-
-# run test
-${cmd} 
+"${docker_runner}" "${cmd_args[@]}"
 exit_code=$?
 
-# cleanup
-rm -f wft4galaxy-docker
-
-if [ ${exit_code} -ne 0 ]; then
-    echo "Test failed (exit code: ${exit_code}" >&2
+if [[ ${exit_code} -ne 0 ]]; then
+  echo "Test failed (exit code: ${exit_code}" >&2
 fi
 
 exit ${exit_code}
diff --git a/utils/docker/develop/bashrc → utils/docker/bashrc b/utils/docker/develop/bashrc → utils/docker/bashrc