Fix conda builds (#1776)

* test conda packages in a test environment as part of CI * do not test sleap import using conda build * use github environment variables to define build path for each OS in the matrix and add print statements for testing * figure out paths one OS at a time * github environment variables work in subsequent steps not current step * use local builds first * print env info * try simple environment creation * try conda instead of mamba * fix windows build path * fix windows build path * add comment to reference pull request * remove test stage from conda build for macs and test instead by creating the environment in a workflow * test workflow by pushing to current branch * test conda package on macos runner * Mac build does not need nvidia channel * qudida and albumentations are conda installed now * add comment with original issue * use python 3.9 * use conda match specifications syntax * make print statements more readable for troubleshooting python versioning * clean up build file * update version for pre-release * add TODO * add tests for conda packages before uploading * update ci comments and branches * remove macos test of pip wheel since python 3.9 is not supported by setup-python action
talmolab · May 20, 2024 · 6a0cf9c · 6a0cf9c
1 parent 43a4f13
commit 6a0cf9c
Show file tree

Hide file tree

Showing 9 changed files with 156 additions and 17 deletions.
diff --git a/.conda/meta.yaml b/.conda/meta.yaml
@@ -91,6 +91,8 @@ requirements:
     - conda-forge::ndx-pose
     - conda-forge::importlib-metadata ==4.11.4
 
-test:
-  imports:
-    - sleap
+# This no longer works so we have moved it to the build workflow
+# https://github.com/talmolab/sleap/pull/1744
+# test:
+#   imports:
+#     - sleap
diff --git a/.conda_mac/build.sh b/.conda_mac/build.sh
@@ -2,11 +2,10 @@
 
 # Install anything that didn't get conda installed via pip.
 # We need to turn pip index back on because Anaconda turns it off for some reason. 
-
 export PIP_NO_INDEX=False
 export PIP_NO_DEPENDENCIES=False
 export PIP_IGNORE_INSTALLED=False
 
-pip install --no-cache-dir -r requirements.txt --no-binary qudida,albumentations
+pip install --no-cache-dir -r requirements.txt
 
 python setup.py install --single-version-externally-managed --record=record.txt
diff --git a/.conda_mac/condarc.yaml b/.conda_mac/condarc.yaml
@@ -1,4 +1,3 @@
-# This file is not used at the moment, but when github actions can be used to build the package, it needs to be listed.
 # https://github.com/github/roadmap/issues/528
 
 channels:

diff --git a/.conda_mac/meta.yaml b/.conda_mac/meta.yaml
@@ -23,7 +23,7 @@ source:
 
 requirements:
   host:  
-    - conda-forge::python ~=3.9
+    - conda-forge::python >=3.9.0, <3.10.0
     - anaconda::numpy >=1.19.5,<1.23.0
     - conda-forge::setuptools
     - conda-forge::packaging
@@ -59,7 +59,7 @@ requirements:
     - conda-forge::ndx-pose
 
   run:
-    - conda-forge::python ~=3.9
+    - conda-forge::python >=3.9.0, <3.10.0
     - conda-forge::attrs >=21.2.0
     - conda-forge::cattrs ==1.1.1
     - conda-forge::h5py
@@ -89,6 +89,6 @@ requirements:
     - conda-forge::albumentations
     - conda-forge::ndx-pose
 
-test:
-  imports:
-    - sleap
+# test:
+#   imports:
+#     - sleap
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -84,6 +84,74 @@ jobs:
         run: |
           conda build .conda_mac --output-folder build
 
+      # Test built conda package (Ubuntu and Windows)
+      - name: Test built conda package (Ubuntu and Windows)
+        if: matrix.os != 'macos-14'
+        shell: bash -l {0}
+        run: |
+          echo "Current build path: $BUILD_PATH"
+          conda deactivate
+
+          echo "Python executable before activating environment:"
+          which python
+          echo "Python version before activating environment:"
+          python --version
+          echo "Conda info before activating environment:"
+          conda info
+
+          echo "Creating and testing conda environment with sleap package..."
+          conda create -y -n sleap_test -c file://$BUILD_PATH -c sleap/label/dev -c conda-forge -c nvidia -c anaconda sleap
+          conda activate sleap_test
+
+          echo "Python executable after activating sleap_test environment:"
+          which python
+          echo "Python version after activating sleap_test environment:"
+          python --version
+          echo "Conda info after activating sleap_test environment:"
+          conda info
+          echo "List of installed conda packages in the sleap_test environment:"
+          conda list
+          echo "List of installed pip packages in the sleap_test environment:"
+          pip list
+
+          echo "Testing sleap package installation..."
+          sleap_version=$(python -c "import sleap; print(sleap.__version__)")
+          echo "Test completed using sleap version: $sleap_version"
+
+      # Test built conda package (Mac)
+      - name: Test built conda package (Mac)
+        if: matrix.os == 'macos-14'
+        shell: bash -l {0}
+        run: |
+          echo "Current build path: $BUILD_PATH"
+          conda deactivate
+
+          echo "Python executable before activating environment:"
+          which python
+          echo "Python version before activating environment:"
+          python --version
+          echo "Conda info before activating environment:"
+          conda info
+
+          echo "Creating and testing conda environment with sleap package..."
+          conda create -y -n sleap_test -c file://$BUILD_PATH -c conda-forge -c anaconda sleap
+          conda activate sleap_test
+
+          echo "Python executable after activating sleap_test environment:"
+          which python
+          echo "Python version after activating sleap_test environment:"
+          python --version
+          echo "Conda info after activating sleap_test environment:"
+          conda info
+          echo "List of installed conda packages in the sleap_test environment:"
+          conda list
+          echo "List of installed pip packages in the sleap_test environment:"
+          pip list
+
+          echo "Testing sleap package installation..."
+          sleap_version=$(python -c "import sleap; print(sleap.__version__)")
+          echo "Test completed using sleap version: $sleap_version"
+      
       # Login to conda (Ubuntu)
       - name: Login to Anaconda (Ubuntu)
         if: matrix.os == 'ubuntu-22.04'

diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml
@@ -1,4 +1,4 @@
-# Run tests using built conda packages and wheels.
+# Run tests using built wheels.
 name: Build CI (no upload)
 
 # Run when changes to pip wheel
@@ -79,7 +79,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-22.04", "windows-2022", "macos-14"]
+        os: ["ubuntu-22.04", "windows-2022"]
+        # os: ["ubuntu-22.04", "windows-2022", "macos-14"]  # removing macos-14 for now since the setup-python action only support py>=3.10, which is breaking this CI.
         # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategymatrixinclude
         include:
           # Default values

diff --git a/.github/workflows/build_manual.yml b/.github/workflows/build_manual.yml
@@ -12,7 +12,6 @@ on:
     branches:
       # - develop
       - fakebranch
-      # - talmo/fix-mac-v140
 
 jobs:
   build:
@@ -83,20 +82,91 @@ jobs:
         shell: bash -l {0}
         run: |
           conda build .conda --output-folder build
+          echo "BUILD_PATH=$(pwd)/build" >> "$GITHUB_ENV"
 
-      # Build conda package (Windows)
+      # Build conda package (Windows) 
       - name: Build conda package (Windows)
         if: matrix.os == 'windows-2022'
         shell: powershell
         run: |
           conda build .conda --output-folder build
+          echo "BUILD_PATH=\$(pwd)\build" >> "$env:GITHUB_ENV"
 
       # Build conda package (Mac)
       - name: Build conda package (Mac)
         if: matrix.os == 'macos-14'
         shell: bash -l {0}
         run: |
           conda build .conda_mac --output-folder build
+          echo "BUILD_PATH=$(pwd)/build" >> "$GITHUB_ENV"
+
+      # Test built conda package (Ubuntu and Windows)
+      - name: Test built conda package (Ubuntu and Windows)
+        if: matrix.os != 'macos-14'
+        shell: bash -l {0}
+        run: |
+          echo "Current build path: $BUILD_PATH"
+          conda deactivate
+    
+          echo "Python executable before activating environment:"
+          which python
+          echo "Python version before activating environment:"
+          python --version
+          echo "Conda info before activating environment:"
+          conda info
+
+          echo "Creating and testing conda environment with sleap package..."
+          conda create -y -n sleap_test -c file://$BUILD_PATH -c sleap/label/dev -c conda-forge -c nvidia -c anaconda sleap
+          conda activate sleap_test
+
+          echo "Python executable after activating sleap_test environment:"
+          which python
+          echo "Python version after activating sleap_test environment:"
+          python --version
+          echo "Conda info after activating sleap_test environment:"
+          conda info
+          echo "List of installed conda packages in the sleap_test environment:"
+          conda list
+          echo "List of installed pip packages in the sleap_test environment:"
+          pip list
+
+          echo "Testing sleap package installation..."
+          sleap_version=$(python -c "import sleap; print(sleap.__version__)")
+          echo "Test completed using sleap version: $sleap_version"
+
+      # Test built conda package (Mac)
+      - name: Test built conda package (Mac)
+        if: matrix.os == 'macos-14'
+        shell: bash -l {0}
+        run: |
+          echo "Current build path: $BUILD_PATH"
+          conda deactivate
+    
+          echo "Python executable before activating environment:"
+          which python
+          echo "Python version before activating environment:"
+          python --version
+          echo "Conda info before activating environment:"
+          conda info
+
+          echo "Creating and testing conda environment with sleap package..."
+          conda create -y -n sleap_test -c file://$BUILD_PATH -c conda-forge -c anaconda sleap
+          conda activate sleap_test
+
+          echo "Python executable after activating sleap_test environment:"
+          which python
+          echo "Python version after activating sleap_test environment:"
+          python --version
+          echo "Conda info after activating sleap_test environment:"
+          conda info
+          echo "List of installed conda packages in the sleap_test environment:"
+          conda list
+          echo "List of installed pip packages in the sleap_test environment:"
+          pip list
+
+          echo "Testing sleap package installation..."
+          sleap_version=$(python -c "import sleap; print(sleap.__version__)")
+          echo "Test completed using sleap version: $sleap_version"
 
       # # Login to conda (Ubuntu)
       # - name: Login to Anaconda (Ubuntu)

diff --git a/requirements.txt b/requirements.txt
@@ -1,7 +1,7 @@
 # This file contains the minimal requirements to be installed via pip when using conda.
 
 # No conda packages for these
-imgstore<0.3.0  # 0.3.3 results in https://github.com/O365/python-o365/issues/591
+imgstore<0.3.0  # 0.3.3 results in https://github.com/O365/python-o365/issues/591 which is from https://github.com/regebro/tzlocal/issues/112 when tzlocal is v3.0
 nixio>=1.5.3  # Constrain put on by @jgrewe from G-Node
 qimage2ndarray  # ==1.9.0
 segmentation-models

diff --git a/sleap/version.py b/sleap/version.py
@@ -12,7 +12,7 @@
 """
 
 
-__version__ = "1.4.0"
+__version__ = "1.4.0a0"
 
 
 def versions():
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,7 +12,7 @@ @@
     """
-    __version__ = "1.4.0"
+    __version__ = "1.4.0a0"
     def versions():
@@ Expand Down @@