From 41e299e214c02c48e22f57d98a35e2ca1975e2ba Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 19 Dec 2024 10:01:09 +0100 Subject: [PATCH 01/10] Removed cmake update from docs ci --- util/ci_utils.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 1adb4b7d8c5..b0883e1f6ad 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -340,11 +340,7 @@ test_cpp_example() { install_docs_dependencies() { echo echo Install ubuntu dependencies - echo Update cmake needed in Ubuntu 20.04 - sudo apt-key adv --fetch-keys https://apt.kitware.com/keys/kitware-archive-latest.asc - sudo apt-add-repository --yes 'deb https://apt.kitware.com/ubuntu/ focal main' ./util/install_deps_ubuntu.sh assume-yes - sudo apt-get install --yes cmake sudo apt-get install --yes libxml2-dev libxslt-dev python3-dev sudo apt-get install --yes doxygen sudo apt-get install --yes texlive From 8d3f5c9081e5a849143eaa2daf65d549b9e9cfd0 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 19 Dec 2024 11:34:16 +0100 Subject: [PATCH 02/10] Centralize python dependency versions to requirements files --- .github/workflows/style.yml | 2 +- .github/workflows/windows.yml | 31 +++++++++++++++---------------- util/ci_utils.sh | 23 +++++++++-------------- 3 files changed, 25 insertions(+), 31 deletions(-) diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 043cb0c7b97..9871c13eed3 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -27,7 +27,7 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - pip install -U clang-format~=10.0.0 yapf==0.30.0 nbformat + pip install -U -r python/requirements_style.txt - name: Run style check run: | python util/check_style.py diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 04a408802d0..2587602f791 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -21,10 +21,6 @@ concurrency: env: PIP_VER: "23.2.1" - WHEEL_VER: "0.38.4" - STOOLS_VER: "67.3.2" - JEDI_VER: "0.17.2" # https://github.com/ipython/ipython/issues/12740 - IDNA_VER: "2.8" # https://github.com/psf/requests/issues/5710 CUDA_VERSION: "12.1.0" SRC_DIR: "D:\\a\\open3d\\open3d" BUILD_DIR: "C:\\Open3D\\build" @@ -213,15 +209,16 @@ jobs: cmake --build . --config ${{ matrix.CONFIG }} .\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test Remove-Item "C:\Program Files\Open3D" -Recurse + - name: Install Open3D python build requirements + working-directory: ${{ env.SOURCE_DIR }} - name: Install Python package + run: | + $ErrorActionPreference = 'Stop' + python -m pip install -U pip==${{ env.PIP_VER }} + python -m pip install -U -c python/requirements_build.txt wheel setuptools working-directory: ${{ env.BUILD_DIR }} run: | $ErrorActionPreference = 'Stop' - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} ` - jedi==${{ env.JEDI_VER }} ` - idna==${{ env.IDNA_VER }} cmake --build . --config ${{ matrix.CONFIG }} --target install-pip-package - name: Import python package # If BUILD_SHARED_LIBS == ON, Open3D.dll needs to be copied, which is not recommended for python. @@ -296,13 +293,17 @@ jobs: -DBUILD_JUPYTER_EXTENSION=ON ` ${{ env.SRC_DIR }} + - name: Install Open3D python build requirements + working-directory: ${{ env.SOURCE_DIR }} + run: | + $ErrorActionPreference = 'Stop' + python -m pip install -U pip==${{ env.PIP_VER }} + python -m pip install -U -c python/requirements_build.txt wheel setuptools + - name: Build Python package working-directory: ${{ env.BUILD_DIR }} run: | $ErrorActionPreference = 'Stop' - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} cmake --build . --parallel ${{ env.NPROC }} --config Release --target pip-package $PIP_PKG_NAME=(Get-ChildItem lib/python_package/pip_package/open3d*.whl).Name echo "PIP_PKG_NAME=$PIP_PKG_NAME" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append @@ -365,10 +366,8 @@ jobs: python -V python -m venv open3d_test_venv open3d_test_venv\Scripts\Activate.ps1 - - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} + python -m pip install -U pip==${{ env.PIP_VER }} + python -m pip install -U -c python/requirements_build.txt wheel setuptools python -m pip install -U -r python/requirements_test.txt $py_tag=(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") if (Test-Path -Path "pip_package") { diff --git a/util/ci_utils.sh b/util/ci_utils.sh index b0883e1f6ad..4a94810485c 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -30,9 +30,6 @@ TORCH_VER="2.2.2" TORCH_REPO_URL="https://download.pytorch.org/whl/torch/" # Python PIP_VER="23.2.1" -WHEEL_VER="0.38.4" -STOOLS_VER="67.3.2" -YAPF_VER="0.30.0" PROTOBUF_VER="4.24.0" OPEN3D_INSTALL_DIR=~/open3d_install @@ -42,10 +39,10 @@ install_python_dependencies() { echo "Installing Python dependencies" options="$(echo "$@" | tr ' ' '|')" - python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \ - setuptools=="$STOOLS_VER" + python -m pip install -U pip=="$PIP_VER" + python -m pip install -U -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" wheel setuptools if [[ "with-unit-test" =~ ^($options)$ ]]; then - python -m pip install -U -r python/requirements_test.txt + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}python/requirements_test.txt" fi if [[ "with-cuda" =~ ^($options)$ ]]; then TF_ARCH_NAME=tensorflow @@ -64,7 +61,6 @@ install_python_dependencies() { TORCH_GLNX="torch==${TORCH_VER}+cpu" fi - # TODO: modify other locations to use requirements.txt python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements.txt" if [[ "with-jupyter" =~ ^($options)$ ]]; then python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements_jupyter_build.txt" @@ -88,7 +84,7 @@ install_python_dependencies() { fi fi if [ "$BUILD_TENSORFLOW_OPS" == "ON" ] || [ "$BUILD_PYTORCH_OPS" == "ON" ]; then - python -m pip install -U yapf=="$YAPF_VER" + python -m pip install -U -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" yapf # Fix Protobuf compatibility issue # https://stackoverflow.com/a/72493690/1255535 # https://github.com/protocolbuffers/protobuf/issues/10051 @@ -241,8 +237,8 @@ test_wheel() { python -m venv open3d_test.venv # shellcheck disable=SC1091 source open3d_test.venv/bin/activate - python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \ - setuptools=="$STOOLS_VER" + python -m pip install -U pip=="$PIP_VER" + python -m pip install -U -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" wheel setuptools echo -n "Using python: $(command -v python)" python --version echo -n "Using pip: " @@ -285,7 +281,7 @@ test_wheel() { run_python_tests() { # shellcheck disable=SC1091 source open3d_test.venv/bin/activate - python -m pip install -U -r python/requirements_test.txt + python -m pip install -U -r "$OPEN3D_SOURCE_ROOT/python/requirements_test.txt" echo Add --randomly-seed=SEED to the test command to reproduce test order. pytest_args=("$OPEN3D_SOURCE_ROOT"/python/test/) if [ "$BUILD_PYTORCH_OPS" == "OFF" ] && [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then @@ -352,9 +348,8 @@ install_docs_dependencies() { echo Install Python dependencies for building docs command -v python python -V - python -m pip install -U -q "wheel==$WHEEL_VER" \ - "pip==$PIP_VER" - python -m pip install -U -q "yapf==$YAPF_VER" + python -m pip install -U -q "pip==$PIP_VER" + python -m pip install -U -q -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" wheel yapf if [[ -d "$1" ]]; then OPEN3D_ML_ROOT="$1" echo Installing Open3D-ML dependencies from "${OPEN3D_ML_ROOT}" From 3af6a63dad50de9982e83888e7392e9aad8790ae Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 19 Dec 2024 11:34:44 +0100 Subject: [PATCH 03/10] Updated pyproject.toml to be in sync with requirements files --- python/pyproject.toml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 3d229011876..a57218b3de4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,3 +1,10 @@ [build-system] -requires = ["ipywidgets>=8.0.3", "pygments>=2.7.4", "jupyter_packaging~=0.12", "jupyterlab>=3.0.0,==3.*", "setuptools>=50.3.2", "wheel==0.38.4"] +requires = [ + "ipywidgets>=8.0.4", + "pygments>=2.7.4", + "jupyter_packaging~=0.12", + "jupyterlab>=3.0.0,==3.*", + "setuptools>=67.3.2", + "wheel==0.38.4", +] build-backend = "setuptools.build_meta" From 870109e793ef1910abb64d03030c34374b0cb09e Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 19 Dec 2024 12:09:31 +0100 Subject: [PATCH 04/10] Fixed wrong order of lines in windows CI --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 2587602f791..fdf8168b3a6 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -211,11 +211,11 @@ jobs: Remove-Item "C:\Program Files\Open3D" -Recurse - name: Install Open3D python build requirements working-directory: ${{ env.SOURCE_DIR }} - - name: Install Python package run: | $ErrorActionPreference = 'Stop' python -m pip install -U pip==${{ env.PIP_VER }} python -m pip install -U -c python/requirements_build.txt wheel setuptools + - name: Install Python package working-directory: ${{ env.BUILD_DIR }} run: | $ErrorActionPreference = 'Stop' From d253dca1fa89ac2b32f2048e53cf48d6b04acc67 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 19 Dec 2024 12:14:07 +0100 Subject: [PATCH 05/10] Added missing slash --- util/ci_utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 4a94810485c..2929ceb61ab 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -42,7 +42,7 @@ install_python_dependencies() { python -m pip install -U pip=="$PIP_VER" python -m pip install -U -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" wheel setuptools if [[ "with-unit-test" =~ ^($options)$ ]]; then - python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}python/requirements_test.txt" + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_test.txt" fi if [[ "with-cuda" =~ ^($options)$ ]]; then TF_ARCH_NAME=tensorflow From 8c63c8adbb1d163b45d46cb7c5c8752efbc61e8f Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 19 Dec 2024 12:48:11 +0100 Subject: [PATCH 06/10] Added missing requirements_build.txt to dockerfile --- docker/Dockerfile.wheel | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index eb60286b620..ef0fc44b955 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -111,6 +111,7 @@ COPY ./util/ci_utils.sh /root/Open3D/util/ COPY ./python/requirements.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_build.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_install.txt /root/Open3D/python/ +COPY ./python/requirements_build.txt /root/Open3D/python/ RUN source /root/Open3D/util/ci_utils.sh \ && install_python_dependencies with-jupyter From 3ada9655e1bbc2f0f7677743de704113698a2ab7 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 19 Dec 2024 12:55:47 +0100 Subject: [PATCH 07/10] Updated pip (23.2.1 -> 24.3.1) --- .github/workflows/windows.yml | 2 +- util/ci_utils.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index fdf8168b3a6..2b05b2a07a9 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -20,7 +20,7 @@ concurrency: cancel-in-progress: true env: - PIP_VER: "23.2.1" + PIP_VER: "24.3.1" CUDA_VERSION: "12.1.0" SRC_DIR: "D:\\a\\open3d\\open3d" BUILD_DIR: "C:\\Open3D\\build" diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 2929ceb61ab..0090ffc8943 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -29,7 +29,7 @@ TENSORFLOW_VER="2.16.2" TORCH_VER="2.2.2" TORCH_REPO_URL="https://download.pytorch.org/whl/torch/" # Python -PIP_VER="23.2.1" +PIP_VER="24.3.1" PROTOBUF_VER="4.24.0" OPEN3D_INSTALL_DIR=~/open3d_install From 909b5f0a6f2dacf8d7f9ad5b5d4bb106a3b19804 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 19 Dec 2024 13:04:52 +0100 Subject: [PATCH 08/10] Added explicit cmake install and removed Ubuntu 20.04 from comment --- util/ci_utils.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 0090ffc8943..ce921111092 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -331,12 +331,13 @@ test_cpp_example() { # Now I am in Open3D/build/ } -# Install dependencies needed for building documentation (on Ubuntu 20.04) +# Install dependencies needed for building documentation # Usage: install_docs_dependencies "${OPEN3D_ML_ROOT}" install_docs_dependencies() { echo echo Install ubuntu dependencies ./util/install_deps_ubuntu.sh assume-yes + sudo apt-get install --yes cmake sudo apt-get install --yes libxml2-dev libxslt-dev python3-dev sudo apt-get install --yes doxygen sudo apt-get install --yes texlive From f3e3d93f0ba16fad0db412490d15f33cba9b1c16 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 19 Dec 2024 14:09:59 +0100 Subject: [PATCH 09/10] Fixed Documentation CI Ubuntu version to 22.04 --- .github/workflows/documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index f6f1dae9f5e..b0d99479c31 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -22,7 +22,7 @@ jobs: headless-docs: # Build headless and docs permissions: contents: write # Artifact upload and release upload - runs-on: ubuntu-latest # Warn about build issues in new versions + runs-on: ubuntu-22.04 env: OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} From dfc107567a0ddb7c0dea74b9e76c40c263446afa Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 19 Dec 2024 23:42:13 +0100 Subject: [PATCH 10/10] Added comment why ubuntu is pinned to 22.04 instead of latest --- .github/workflows/documentation.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index b0d99479c31..2086b9a7a73 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -22,6 +22,8 @@ jobs: headless-docs: # Build headless and docs permissions: contents: write # Artifact upload and release upload + # Pinning to 22.04 instead of ubuntu-latest until 24.04 is supported + # (see https://github.com/isl-org/Open3D/pull/7105) runs-on: ubuntu-22.04 env: OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML