From b277d26d8d76f28c0e05b0a60158b5d03009c96d Mon Sep 17 00:00:00 2001 From: Blazej Floch Date: Fri, 25 Oct 2019 14:58:47 -0400 Subject: [PATCH 1/6] Fixing PowerShell to conform to other shells. Initially adcac2176dedb06174abe229fed8c59ac03c4b06 added rez to the systempaths, but this should not be necessary. It creates failure in tests if rez is not in the systempath, which should not be a requirement for any shell. Also conformed the parameters in cmd's and powershell's spawn_shell that otherwise will end up in Popen_args. --- .../shell/_utils/powershell_base.py | 19 +++++++++---------- src/rezplugins/shell/cmd.py | 9 ++++++--- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/rezplugins/shell/_utils/powershell_base.py b/src/rezplugins/shell/_utils/powershell_base.py index d7e25e766..ad9b9d0db 100644 --- a/src/rezplugins/shell/_utils/powershell_base.py +++ b/src/rezplugins/shell/_utils/powershell_base.py @@ -6,6 +6,7 @@ from rez.config import config from rez.rex import RexExecutor, OutputStyle, EscapedString from rez.shells import Shell +from rez.system import system from rez.utils.platform_ import platform_ from rez.utils.execution import Popen @@ -121,12 +122,6 @@ def gen_expected_regex(parts): cls.syspaths = list(set([x for x in paths if x])) - # add Rez binaries - exe = which("rez-env") - assert exe, "Could not find rez binary, this is a bug" - rez_bin_dir = os.path.dirname(exe) - cls.syspaths.insert(0, rez_bin_dir) - return cls.syspaths def _bind_interactive_rez(self): @@ -154,6 +149,7 @@ def spawn_shell(self, env=None, quiet=False, pre_command=None, + add_rez=True, **Popen_args): startup_sequence = self.get_startup_sequence(rcfile, norc, bool(stdin), @@ -164,11 +160,14 @@ def _record_shell(ex, files, bind_rez=True, print_msg=False): ex.source(context_file) if startup_sequence["envvar"]: ex.unsetenv(startup_sequence["envvar"]) - if bind_rez: + if add_rez and bind_rez: ex.interpreter._bind_interactive_rez() - if print_msg and not quiet: - # Rez may not be available - ex.command("Try { rez context } Catch { }") + if print_msg and add_rez and not quiet: + ex.info('') + ex.info('You are now in a rez-configured environment.') + ex.info('') + if system.is_production_rez_install: + ex.command("rezolve context") executor = RexExecutor(interpreter=self.new_shell(), parent_environ={}, diff --git a/src/rezplugins/shell/cmd.py b/src/rezplugins/shell/cmd.py index f049b1f24..10fabb88f 100644 --- a/src/rezplugins/shell/cmd.py +++ b/src/rezplugins/shell/cmd.py @@ -150,7 +150,7 @@ def _bind_interactive_rez(self): def spawn_shell(self, context_file, tmpdir, rcfile=None, norc=False, stdin=False, command=None, env=None, quiet=False, - pre_command=None, **Popen_args): + pre_command=None, add_rez=True, **Popen_args): startup_sequence = self.get_startup_sequence(rcfile, norc, bool(stdin), command) shell_command = None @@ -159,9 +159,12 @@ def _record_shell(ex, files, bind_rez=True, print_msg=False): ex.source(context_file) if startup_sequence["envvar"]: ex.unsetenv(startup_sequence["envvar"]) - if bind_rez: + if add_rez and bind_rez: ex.interpreter._bind_interactive_rez() - if print_msg and not quiet: + if print_msg and add_rez and not quiet: + ex.info('') + ex.info('You are now in a rez-configured environment.') + ex.info('') if system.is_production_rez_install: # previously this was called with the /K flag, however # that would leave spawn_shell hung on a blocked call From 4bf62bfbd7776c221f0782c14fc91f705088bcfb Mon Sep 17 00:00:00 2001 From: Blazej Floch Date: Fri, 25 Oct 2019 15:23:12 -0400 Subject: [PATCH 2/6] Fix for test_rez_env_output failure in cmd-shell. Fresh Windows systems have unexpanded %systemroot% in their PATH variable, which causes `which()` to fail. --- src/rez/backport/shutilwhich.py | 2 ++ src/rez/tests/test_shells.py | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/rez/backport/shutilwhich.py b/src/rez/backport/shutilwhich.py index c873b44db..d20c546e2 100644 --- a/src/rez/backport/shutilwhich.py +++ b/src/rez/backport/shutilwhich.py @@ -57,6 +57,8 @@ def _access_check(fn, mode): seen.add(dir) for thefile in files: name = os.path.join(dir, thefile) + # On windows the system paths might have %systemroot% + name = os.path.expandvars(name) if _access_check(name, mode): return name return None diff --git a/src/rez/tests/test_shells.py b/src/rez/tests/test_shells.py index 2efe309bd..73a467090 100644 --- a/src/rez/tests/test_shells.py +++ b/src/rez/tests/test_shells.py @@ -188,7 +188,6 @@ def test_rcfile(self): def test_rez_env_output(self): # here we are making sure that running a command via rez-env prints # exactly what we expect. - sh = create_shell() # Assumes that the shell has an echo command, build-in or alias cmd = [os.path.join(system.rez_bin_path, "rez-env"), "--", "echo", "hey"] @@ -196,8 +195,8 @@ def test_rez_env_output(self): cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True ) - sh_out = process.communicate()[0] - self.assertEqual(sh_out.strip(), "hey") + sh_out = process.communicate() + self.assertEqual(sh_out[0].strip(), "hey") @per_available_shell() @install_dependent() From 21b28cc8b37543736749320c4f2c0781b9aa8eed Mon Sep 17 00:00:00 2001 From: Blazej Floch Date: Wed, 30 Oct 2019 16:58:11 -0400 Subject: [PATCH 3/6] Add docker based windows tests. Tests with regular github actions failed (potentially due to overflown PATH variable), so instead this tries to test via a clean windows docker image. --- .github/docker/rez-win-base/Dockerfile | 36 ++++++++ .github/docker/rez-win-py/Dockerfile | 44 +++++++++ .github/docker/rez-win-py/entrypoint.ps1 | 53 +++++++++++ .github/workflows/core.yaml | 11 ++- .github/workflows/mac.yaml | 11 ++- .github/workflows/ubuntu.yaml | 11 ++- .github/workflows/windows-docker-image.yaml | 98 +++++++++++++++++++++ .github/workflows/windows-docker.yaml | 56 ++++++++++++ .github/workflows/windows.yaml | 45 ---------- CONTRIBUTING.md | 12 +++ 10 files changed, 329 insertions(+), 48 deletions(-) create mode 100644 .github/docker/rez-win-base/Dockerfile create mode 100644 .github/docker/rez-win-py/Dockerfile create mode 100644 .github/docker/rez-win-py/entrypoint.ps1 create mode 100644 .github/workflows/windows-docker-image.yaml create mode 100644 .github/workflows/windows-docker.yaml delete mode 100644 .github/workflows/windows.yaml diff --git a/.github/docker/rez-win-base/Dockerfile b/.github/docker/rez-win-base/Dockerfile new file mode 100644 index 000000000..0e3890163 --- /dev/null +++ b/.github/docker/rez-win-base/Dockerfile @@ -0,0 +1,36 @@ +# escape=` + +# Please Note: Any " requires \" in the Dockerfile for windows. + +# The windows version has to match the host system. +# 1809 -> 10.0.17763.805 -> windows-2019 +# Compare: https://hub.docker.com/_/microsoft-windows-servercore +ARG WINDOWS_VERSION=1809-amd64 + +FROM mcr.microsoft.com/windows/servercore:$WINDOWS_VERSION + +SHELL ["powershell.exe", "-NoLogo", "-NoProfile", "-ExecutionPolicy", "ByPass"] + +# ------------------------------------------------------------------------------------------------------------ +# Chocolatey +# +ENV chocolateyUseWindowsCompression false +RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) +RUN choco feature disable --name showDownloadProgress + +# ------------------------------------------------------------------------------------------------------------ +# Git +# +RUN choco install git.install --yes --version=2.23.0 + +# ------------------------------------------------------------------------------------------------------------ +# Cmake +# +RUN choco install cmake --yes --version=3.15.4 --installargs 'ADD_CMAKE_TO_PATH=System' + +# ------------------------------------------------------------------------------------------------------------ +# pwsh (PowerShellCore) +# +RUN choco install pwsh --yes --version=6.2.2 + +ENTRYPOINT ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "ByPass"] diff --git a/.github/docker/rez-win-py/Dockerfile b/.github/docker/rez-win-py/Dockerfile new file mode 100644 index 000000000..1550ee84b --- /dev/null +++ b/.github/docker/rez-win-py/Dockerfile @@ -0,0 +1,44 @@ +# escape=` + +# Tag of base image to use +ARG BASE_TAG + +# Base image windows version +FROM rez-win-base:$BASE_TAG + +# NOTE: Any " requires \" in the Dockerfile for windows. +# NOTE: The order matters. ARG after the shell command will allow access via +# the PowerShell environment like ${ENV:PYTHON_VERSION}. +SHELL ["powershell.exe", "-NoLogo", "-NoProfile", "-ExecutionPolicy", "ByPass"] + +# Python version to install (full chocolatey compatible version required) +# For example: 3.7.5 +# +ARG PYTHON_VERSION + +# ------------------------------------------------------------------------------------------------------------ +# Python +# +# Installs given Python version to C:\Python +# Python 2.x uses msi while 3.x has an exe with separate arguments +# +RUN ${PYTHON_INSTALL_PATH} = 'C:\Python'; ` + ${PYTHON_MAJOR_VERSION} = ${ENV:PYTHON_VERSION}.Split('.')[0]; ` + if (${PYTHON_MAJOR_VERSION} -eq "2") { ` + ${INSTALLARGS} = \"'/qn /norestart ADDLOCAL=ALL ALLUSERS=1 TARGETDIR=`\"\" + ${PYTHON_INSTALL_PATH} + \"`\"'\"; ` + } else { ` + ${INSTALLARGS} = \"'/quiet InstallAllUsers=1 PrependPath=1 TargetDir=`\"\" + ${PYTHON_INSTALL_PATH} + \"`\"'\"; ` + } ` + choco install python${PYTHON_MAJOR_VERSION} --yes --version=\"${ENV:PYTHON_VERSION}\" --override --installargs=${INSTALLARGS}; ` + if (-not $?) {exit 1}; + +# ------------------------------------------------------------------------------------------------------------ +# Verify Python +# +RUN $python_relative_ver = (& python --version 2>&1).ToString().Trim().Split(" ")[1]; ` + $python_explicit_ver = (& C:\python\python.exe --version 2>&1).ToString().Trim().Split(" ")[1]; ` + if (-not ($python_explicit_ver -eq $python_relative_ver -and $python_explicit_ver -eq ${ENV:PYTHON_VERSION})) {exit 1}; + +COPY entrypoint.ps1 /entrypoint.ps1 + +ENTRYPOINT ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "ByPass", "-File", "/entrypoint.ps1"] diff --git a/.github/docker/rez-win-py/entrypoint.ps1 b/.github/docker/rez-win-py/entrypoint.ps1 new file mode 100644 index 000000000..080ece7df --- /dev/null +++ b/.github/docker/rez-win-py/entrypoint.ps1 @@ -0,0 +1,53 @@ +# +# Entrypoint that installs given python version and runs tests from commit. +# +# $repo is the repository like 'nerdvegas/rez' +# $commit is the revision to test +param( $repo, $commit ) + +# Stop on errors; .exe has to be checked manually +Set-StrictMode -Version latest +$ErrorActionPreference = "Stop" + +echo "Testing $repo at $commit" + +# Fixes encoding issue on Windows 10 local docker run. +# +${ENV:PYTHONIOENCODING} = "UTF-8" + +# Verify Python +# +python --version +if (-not $?) {exit 1} + +# Verify cmake +# +cmake.exe --version +if (-not $?) {exit 1} + +#Verify pwsh +pwsh --version +if (-not $?) {exit 1} + +#Verify git +git --version +if (-not $?) {exit 1} + +# Clone repo +git clone https://github.com/$repo rez +if (-not $?) {exit 1} +cd rez +git checkout $commit +if (-not $?) {exit 1} + +# Install rez +mkdir build +python install.py build +if (-not $?) {exit 1} + +# Run Rez Tests +# +.\build\Scripts\rez\rez-selftest.exe + +# Pass on exit code to runner +exit $LASTEXITCODE diff --git a/.github/workflows/core.yaml b/.github/workflows/core.yaml index 26c1868cc..8dad4181c 100644 --- a/.github/workflows/core.yaml +++ b/.github/workflows/core.yaml @@ -4,7 +4,16 @@ # * Platform dependent. # name: Core -on: [push, pull_request] +on: + # Do not run if Docker image has been updated + pull_request: + paths-ignore: + - '.github/docker/**' + - '.github/workflows/windows-docker-image.yaml' + push: + paths-ignore: + - '.github/docker/**' + - '.github/workflows/windows-docker-image.yaml' jobs: main: diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index 402657ba3..28b031358 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -1,5 +1,14 @@ name: MacOS -on: [push, pull_request] +on: + # Do not run if Docker image has been updated + pull_request: + paths-ignore: + - '.github/docker/**' + - '.github/workflows/windows-docker-image.yaml' + push: + paths-ignore: + - '.github/docker/**' + - '.github/workflows/windows-docker-image.yaml' jobs: main: diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index 4de672c0f..358e2b55d 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -1,5 +1,14 @@ name: Ubuntu -on: [push, pull_request] +on: + # Do not run if Docker image has been updated + pull_request: + paths-ignore: + - '.github/docker/**' + - '.github/workflows/windows-docker-image.yaml' + push: + paths-ignore: + - '.github/docker/**' + - '.github/workflows/windows-docker-image.yaml' jobs: main: diff --git a/.github/workflows/windows-docker-image.yaml b/.github/workflows/windows-docker-image.yaml new file mode 100644 index 000000000..2949aa2d4 --- /dev/null +++ b/.github/workflows/windows-docker-image.yaml @@ -0,0 +1,98 @@ +# +# Create base docker image and python version specific images for windows. +# Currently uses hub.docker.com as registry. +# +# Requires a DOCKERHUB_TOKEN secret and username/repository to match the github +# repository. For example: nerdvegas/rez -> User/Repository: nerdvegas +# +# Commit to windows-docker-image.yaml or .github/docker/** separately +# to trigger a image build. This Image will be used in consecutive commits. +# +# TODO: +# - Better way to pass the variables throughout steps +# - The reason for using docker is that the github supplied containers have +# a bloated PATH environment variable that can not be changed: +# https://github.community/t5/GitHub-Actions/Windows-image-bloat-in-PATH-disables-cmd-prompt-usage/m-p/36004#M2414 +# - Github offers its own package registry. Pushing to is broken in the BETA +# https://github.community/t5/GitHub-Actions/Docker-Push-to-Package-Registry-from-Windows/m-p/36393#M2560 + +name: Compose Windows Docker Image + +on: + push: + # Will only trigger building a docker image if any of these changed + paths: + - '.github/docker/rez-win-base/**' + - '.github/docker/rez-win-py/**' + - '.github/workflows/windows-docker-image.yaml' + +jobs: + + # ---------------------------------------------------------------------------- + # Create base image if necessary + # + base: + runs-on: windows-2019 + + steps: + - name: Login to docker repository + run: | + ${gh_user} = ("${{ github.repository }}" -Split '/')[0] + docker login -u ${gh_user} -p ${{ secrets.DOCKERHUB_TOKEN }} + + # TODO: Represent the conditional build as steps? + # https://github.community/t5/GitHub-Actions/Fallback-via-if-failure-without-failing-the-job/m-p/36596#M2636 + # + # By using dockers experimental CLI features we don't need to pull an + # image to check if it exists. + - uses: actions/checkout@v1 + - name: Build docker image if needed + run: | + ${Env:LAST_DOCKER_BASE_REVISION} = $( git log -n 1 --author-date-order --pretty=format:%H -- .\.github\docker\rez-win-base\ .\.github\workflows\windows-docker-image.yaml ).SubString(0, 8) + ${gh_user} = ("${{ github.repository }}" -Split '/')[0] + docker pull ${gh_user}/rez-win-base:${ENV:LAST_DOCKER_BASE_REVISION} + if (-not $?) { + cd .github\docker\rez-win-base + docker build --tag ${gh_user}/rez-win-base:${ENV:LAST_DOCKER_BASE_REVISION} . + docker push ${gh_user}/rez-win-base:${ENV:LAST_DOCKER_BASE_REVISION} + } + + # ---------------------------------------------------------------------------- + # Create python images off base image + # Each image will have a single python version installed as defined in the + # strategy-matrix. + # + python: + runs-on: windows-2019 + needs: base + + strategy: + matrix: + python-version: + - '2.7.17' + - '3.6.8' + - '3.7.5' + fail-fast: false + + steps: + - name: Login to docker repository + run: | + ${gh_user} = ("${{ github.repository }}" -Split '/')[0] + docker login -u ${gh_user} -p ${{ secrets.DOCKERHUB_TOKEN }} + + - uses: actions/checkout@v1 + - name: Pull base docker image + run: | + ${Env:LAST_DOCKER_BASE_REVISION} = $( git log -n 1 --author-date-order --pretty=format:%H -- .\.github\docker\rez-win-base\ .\.github\workflows\windows-docker-image.yaml ).SubString(0, 8) + ${gh_user} = ("${{ github.repository }}" -Split '/')[0] + docker pull ${gh_user}/rez-win-base:${ENV:LAST_DOCKER_BASE_REVISION} + docker tag ${gh_user}/rez-win-base:${ENV:LAST_DOCKER_BASE_REVISION} rez-win-base:${ENV:LAST_DOCKER_BASE_REVISION} + + - name: Build and push Docker image + run: | + ${gh_user} = ("${{ github.repository }}" -Split '/')[0] + ${ENV:DOCKER_TAG} = "${{ github.sha }}".Substring(0, 8) + ${Env:LAST_DOCKER_BASE_REVISION} = $( git log -n 1 --author-date-order --pretty=format:%H -- .\.github\docker\rez-win-base\ .\.github\workflows\windows-docker-image.yaml ).SubString(0, 8) + cd .github\docker\rez-win-py + docker build --tag ${gh_user}/rez-win-py:${{ matrix.python-version }}-${ENV:DOCKER_TAG} --build-arg PYTHON_VERSION="${{ matrix.python-version }}" --build-arg BASE_TAG=${ENV:LAST_DOCKER_BASE_REVISION} . + docker push ${gh_user}/rez-win-py:${{ matrix.python-version }}-${ENV:DOCKER_TAG} diff --git a/.github/workflows/windows-docker.yaml b/.github/workflows/windows-docker.yaml new file mode 100644 index 000000000..a4c27cd7d --- /dev/null +++ b/.github/workflows/windows-docker.yaml @@ -0,0 +1,56 @@ +# +# Picks up docker image from hub.docker.com as repository. +# +# Requires a DOCKERHUB_TOKEN secret and username/repository to match the github +# repository. For example: nerdvegas/rez -> User/Repository: nerdvegas +# +# Uses image tagged as **latest** commit revision of any file within: +# .github/docker/windows/** +# .github/workflows/windows-docker-image.yaml +# + +name: Windows Docker + +on: + # Do not run if Docker image has been updated + pull_request: + paths-ignore: + - '.github/docker/**' + - '.github/workflows/windows-docker-image.yaml' + push: + paths-ignore: + - '.github/docker/**' + - '.github/workflows/windows-docker-image.yaml' + +jobs: + main: + runs-on: windows-2019 + + strategy: + matrix: + # Needs to match python version of images (see windows-docker-image.yaml) + python-version: + - '2.7.17' + - '3.6.8' + - '3.7.5' + fail-fast: false + + steps: + - uses: actions/checkout@v1 + - name: Login to docker repository + run: | + ${gh_user} = ("${{ github.repository }}" -Split '/')[0] + docker login -u ${gh_user} -p ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull docker image + run: | + ${gh_user} = ("${{ github.repository }}" -Split '/')[0] + ${Env:LAST_DOCKER_REVISION} = $( git log -n 1 --author-date-order --pretty=format:%H -- .\.github\docker\rez-win-py\ .\.github\workflows\windows-docker-image.yaml ).SubString(0, 8) + echo "Pulling rez-win-py:${{ matrix.python-version }}-${Env:LAST_DOCKER_REVISION}" + docker pull ${gh_user}/rez-win-py:${{ matrix.python-version }}-${Env:LAST_DOCKER_REVISION} + + - name: Run the Docker image + run: | + ${gh_user} = ("${{ github.repository }}" -Split '/')[0] + ${Env:LAST_DOCKER_REVISION} = $( git log -n 1 --author-date-order --pretty=format:%H -- .\.github\docker\rez-win-py\ .\.github\workflows\windows-docker-image.yaml ).SubString(0, 8) + docker run ${gh_user}/rez-win-py:${{ matrix.python-version }}-${Env:LAST_DOCKER_REVISION} ${{ github.repository }} ${{ github.sha }} diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml deleted file mode 100644 index 8269c94b5..000000000 --- a/.github/workflows/windows.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Windows -on: [push, pull_request] - -jobs: - main: - name: main - runs-on: windows-${{ matrix.os-version }} - - strategy: - matrix: - os-version: - - '2019' - python-version: - - '2.7' - - '3.6' - - '3.7' - fail-fast: false - - steps: - - name: Checkout - uses: actions/checkout@v1 - with: - fetch-depth: 1 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Verify cmake - run: | - cmake.exe --version - - - name: Verify pwsh - run: | - pwsh --version - - - name: Install Rez - run: | - mkdir build - python install.py build - - - name: Run Rez Tests - run: | - .\build\Scripts\rez\rez-selftest.exe diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e52ca43ca..136e9d619 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,18 @@ sending a pull request. Please follow these guidelines: document the setting. The comments in this file are extracted and turned into Wiki content. Pay attention to the comment formatting and follow the existing style closely. +## Windows Docker Workflow + +The Windows tests currently build a Python image for each version to test. Each is based on a common +base image. Any changes to the following Docker images sources should be a separate commit: + +- `.github/docker/rez-win-base/**` +- `.github/docker/rez-win-py/**` +- `.github/workflows/windows-docker-image.yaml` + +The base and Python images will be automatically rebuild. +Any future commits will pickup the correct image via `windows-docker.yaml` + ## Reporting Bugs If you report a bug, please ensure to specify the following: From 2fd5848c8312da82f8c1549c7ff0271eea828db0 Mon Sep 17 00:00:00 2001 From: Blazej Floch Date: Fri, 1 Nov 2019 13:40:29 -0400 Subject: [PATCH 4/6] Do not pull to check (uses experimental docker cli) --- .github/workflows/windows-docker-image.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows-docker-image.yaml b/.github/workflows/windows-docker-image.yaml index 2949aa2d4..033e0b7bc 100644 --- a/.github/workflows/windows-docker-image.yaml +++ b/.github/workflows/windows-docker-image.yaml @@ -31,6 +31,7 @@ jobs: # ---------------------------------------------------------------------------- # Create base image if necessary # + # State depends on .github\docker\rez-win-base\** and this file base: runs-on: windows-2019 @@ -50,13 +51,16 @@ jobs: run: | ${Env:LAST_DOCKER_BASE_REVISION} = $( git log -n 1 --author-date-order --pretty=format:%H -- .\.github\docker\rez-win-base\ .\.github\workflows\windows-docker-image.yaml ).SubString(0, 8) ${gh_user} = ("${{ github.repository }}" -Split '/')[0] - docker pull ${gh_user}/rez-win-base:${ENV:LAST_DOCKER_BASE_REVISION} + docker manifest inspect ${gh_user}/rez-win-base:${ENV:LAST_DOCKER_BASE_REVISION} | Out-Null if (-not $?) { cd .github\docker\rez-win-base docker build --tag ${gh_user}/rez-win-base:${ENV:LAST_DOCKER_BASE_REVISION} . docker push ${gh_user}/rez-win-base:${ENV:LAST_DOCKER_BASE_REVISION} } + env: + DOCKER_CLI_EXPERIMENTAL: enabled + # ---------------------------------------------------------------------------- # Create python images off base image # Each image will have a single python version installed as defined in the From 0666152ae5e7239700ab47a32f5b443be8009034 Mon Sep 17 00:00:00 2001 From: Blazej Floch Date: Mon, 4 Nov 2019 10:06:18 -0500 Subject: [PATCH 5/6] Optimizations and security improvements --- .github/docker/rez-win-base/Dockerfile | 32 +++++++++------------ .github/docker/rez-win-py/Dockerfile | 4 +++ .github/workflows/windows-docker-image.yaml | 4 +-- .github/workflows/windows-docker.yaml | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/docker/rez-win-base/Dockerfile b/.github/docker/rez-win-base/Dockerfile index 0e3890163..862f185cd 100644 --- a/.github/docker/rez-win-base/Dockerfile +++ b/.github/docker/rez-win-base/Dockerfile @@ -11,26 +11,22 @@ FROM mcr.microsoft.com/windows/servercore:$WINDOWS_VERSION SHELL ["powershell.exe", "-NoLogo", "-NoProfile", "-ExecutionPolicy", "ByPass"] -# ------------------------------------------------------------------------------------------------------------ -# Chocolatey -# -ENV chocolateyUseWindowsCompression false -RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) -RUN choco feature disable --name showDownloadProgress - -# ------------------------------------------------------------------------------------------------------------ -# Git -# -RUN choco install git.install --yes --version=2.23.0 +ARG GIT_VERSION=2.23.0 +ARG CMAKE_VERSION=3.15.4 +ARG PWSH_VERSION=6.2.2 # ------------------------------------------------------------------------------------------------------------ -# Cmake +# Install: +# - Chocolatey +# - Git +# - Cmake +# - PowerShellCore # -RUN choco install cmake --yes --version=3.15.4 --installargs 'ADD_CMAKE_TO_PATH=System' - -# ------------------------------------------------------------------------------------------------------------ -# pwsh (PowerShellCore) -# -RUN choco install pwsh --yes --version=6.2.2 +ENV chocolateyUseWindowsCompression false +RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); ` + choco feature disable --name showDownloadProgress; ` + choco install git.install --yes --version=${ENV:GIT_VERSION}; ` + choco install cmake --yes --version=${ENV:CMAKE_VERSION} --installargs 'ADD_CMAKE_TO_PATH=System'; ` + choco install pwsh --yes --version=${PWSH_VERSION}; ENTRYPOINT ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "ByPass"] diff --git a/.github/docker/rez-win-py/Dockerfile b/.github/docker/rez-win-py/Dockerfile index 1550ee84b..a3b3c2559 100644 --- a/.github/docker/rez-win-py/Dockerfile +++ b/.github/docker/rez-win-py/Dockerfile @@ -21,6 +21,7 @@ ARG PYTHON_VERSION # # Installs given Python version to C:\Python # Python 2.x uses msi while 3.x has an exe with separate arguments +# Verifies the installation by running python explicitly and via `python` # RUN ${PYTHON_INSTALL_PATH} = 'C:\Python'; ` ${PYTHON_MAJOR_VERSION} = ${ENV:PYTHON_VERSION}.Split('.')[0]; ` @@ -36,6 +37,9 @@ RUN ${PYTHON_INSTALL_PATH} = 'C:\Python'; # Verify Python # RUN $python_relative_ver = (& python --version 2>&1).ToString().Trim().Split(" ")[1]; ` + $python_explicit_ver = (& C:\python\python.exe --version 2>&1).ToString().Trim().Split(" ")[1]; ` + if (-not $?) {exit 1}; ` + $python_relative_ver = (& python --version 2>&1).ToString().Trim().Split(" ")[1]; ` $python_explicit_ver = (& C:\python\python.exe --version 2>&1).ToString().Trim().Split(" ")[1]; ` if (-not ($python_explicit_ver -eq $python_relative_ver -and $python_explicit_ver -eq ${ENV:PYTHON_VERSION})) {exit 1}; diff --git a/.github/workflows/windows-docker-image.yaml b/.github/workflows/windows-docker-image.yaml index 033e0b7bc..87750e787 100644 --- a/.github/workflows/windows-docker-image.yaml +++ b/.github/workflows/windows-docker-image.yaml @@ -39,7 +39,7 @@ jobs: - name: Login to docker repository run: | ${gh_user} = ("${{ github.repository }}" -Split '/')[0] - docker login -u ${gh_user} -p ${{ secrets.DOCKERHUB_TOKEN }} + '${{ secrets.DOCKERHUB_TOKEN }}' | docker login -u ${gh_user} --password-stdin # TODO: Represent the conditional build as steps? # https://github.community/t5/GitHub-Actions/Fallback-via-if-failure-without-failing-the-job/m-p/36596#M2636 @@ -82,7 +82,7 @@ jobs: - name: Login to docker repository run: | ${gh_user} = ("${{ github.repository }}" -Split '/')[0] - docker login -u ${gh_user} -p ${{ secrets.DOCKERHUB_TOKEN }} + '${{ secrets.DOCKERHUB_TOKEN }}' | docker login -u ${gh_user} --password-stdin - uses: actions/checkout@v1 - name: Pull base docker image diff --git a/.github/workflows/windows-docker.yaml b/.github/workflows/windows-docker.yaml index a4c27cd7d..82bfeca4a 100644 --- a/.github/workflows/windows-docker.yaml +++ b/.github/workflows/windows-docker.yaml @@ -40,7 +40,7 @@ jobs: - name: Login to docker repository run: | ${gh_user} = ("${{ github.repository }}" -Split '/')[0] - docker login -u ${gh_user} -p ${{ secrets.DOCKERHUB_TOKEN }} + '${{ secrets.DOCKERHUB_TOKEN }}' | docker login -u ${gh_user} --password-stdin - name: Pull docker image run: | From 49f89bb654cfd251ac1b40a5f2d1fdbab219778e Mon Sep 17 00:00:00 2001 From: Blazej Floch Date: Mon, 4 Nov 2019 11:06:15 -0500 Subject: [PATCH 6/6] Reduce Image size by deleting temporary chocolatey files. --- .github/docker/rez-win-base/Dockerfile | 5 ++++- .github/docker/rez-win-py/Dockerfile | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/docker/rez-win-base/Dockerfile b/.github/docker/rez-win-base/Dockerfile index 862f185cd..9dbe021f1 100644 --- a/.github/docker/rez-win-base/Dockerfile +++ b/.github/docker/rez-win-base/Dockerfile @@ -27,6 +27,9 @@ RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/insta choco feature disable --name showDownloadProgress; ` choco install git.install --yes --version=${ENV:GIT_VERSION}; ` choco install cmake --yes --version=${ENV:CMAKE_VERSION} --installargs 'ADD_CMAKE_TO_PATH=System'; ` - choco install pwsh --yes --version=${PWSH_VERSION}; + choco install pwsh --yes --version=${PWSH_VERSION}; ` + choco install --yes choco-cleaner; ` + C:\ProgramData\chocolatey\bin\choco-cleaner.ps1; ` + choco uninstall --yes choco-cleaner ENTRYPOINT ["powershell.exe", "-NoLogo", "-ExecutionPolicy", "ByPass"] diff --git a/.github/docker/rez-win-py/Dockerfile b/.github/docker/rez-win-py/Dockerfile index a3b3c2559..db63cc99c 100644 --- a/.github/docker/rez-win-py/Dockerfile +++ b/.github/docker/rez-win-py/Dockerfile @@ -41,7 +41,10 @@ RUN $python_relative_ver = (& python --version 2>&1).ToString().Trim().Split(" " if (-not $?) {exit 1}; ` $python_relative_ver = (& python --version 2>&1).ToString().Trim().Split(" ")[1]; ` $python_explicit_ver = (& C:\python\python.exe --version 2>&1).ToString().Trim().Split(" ")[1]; ` - if (-not ($python_explicit_ver -eq $python_relative_ver -and $python_explicit_ver -eq ${ENV:PYTHON_VERSION})) {exit 1}; + if (-not ($python_explicit_ver -eq $python_relative_ver -and $python_explicit_ver -eq ${ENV:PYTHON_VERSION})) {exit 1}; ` + choco install --yes choco-cleaner; ` + C:\ProgramData\chocolatey\bin\choco-cleaner.ps1; ` + choco uninstall --yes choco-cleaner COPY entrypoint.ps1 /entrypoint.ps1