Build Python wheels (#159)

This PR rearranges this code to facilitate building and installing the `spt3g` python package with standard python tools like `pip` and `build`.

The shared libraries, headers, and cmake configuration files are installed relative to `CMAKE_INSTALL_PREFIX` as usual.  The `pip` installer handles versioning, and passes appropriate cmake definitions to the cmake builder that compiles all of the libraries and modules, then installs the python package into the appropriate location, and triggers the cmake installer.

The `pip` installer can also be run in an `--editable` mode that works much like the cmake build directory, enabling code development without the need to make changes to various path environment variables.  It also should install all of the components into appropriate locations when run in a virtual environment.

Versioning is done using `setuptools_scm` with the `pip` build system, or with a custom shell script with the `cmake` build system.  In both cases, the version information is determined by running a series of `git` commands or by parsing the `.git_archival.txt` file, then output to `spt3g/version.py` for access in the Python package and to the `SPT3G_VERSION` preprocessor directive for access in C++.

Finally, this PR includes a github workflow for producing pre-compiled wheels for Linux and OSX platforms.

.git_archival.txt

@@ -0,0 +1,3 @@
+node: $Format:%H$
+node-date: $Format:%cI$
+describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$

.gitattributes

@@ -1,3 +1,3 @@
-VERSION filter=exportversion
 .gitattributes export-ignore
 .gitignore export-ignore
+.git_archival.txt export-subst

.github/workflows/wheels.yml

@@ -0,0 +1,131 @@
+name: Wheels
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  release:
+    types: [ published ]
+
+jobs:
+  build_wheels:
+    name: Build wheels for ${{ matrix.build }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # macos-13 is an intel runner, macos-14 is apple silicon
+          - os: macos-13
+            build: cp38-macosx_x86_64
+            target: 13.0
+          - os: macos-13
+            build: cp39-macosx_x86_64
+            target: 13.0
+          - os: macos-13
+            build: cp310-macosx_x86_64
+            target: 13.0
+          - os: macos-13
+            build: cp311-macosx_x86_64
+            target: 13.0
+          - os: macos-13
+            build: cp312-macosx_x86_64
+            target: 13.0
+
+          - os: macos-14
+            build: cp39-macosx_arm64
+            target: 14.0
+          - os: macos-14
+            build: cp310-macosx_arm64
+            target: 14.0
+          - os: macos-14
+            build: cp311-macosx_arm64
+            target: 14.0
+          - os: macos-14
+            build: cp312-macosx_arm64
+            target: 14.0
+
+          - os: ubuntu-latest
+            build: cp38-manylinux_x86_64
+          - os: ubuntu-latest
+            build: cp39-manylinux_x86_64
+          - os: ubuntu-latest
+            build: cp310-manylinux_x86_64
+          - os: ubuntu-latest
+            build: cp311-manylinux_x86_64
+          - os: ubuntu-latest
+            build: cp312-manylinux_x86_64
+
+    steps:
+      - name: Set macOS deployment target
+        if: runner.os == 'macOS'
+        run: echo "MACOSX_DEPLOYMENT_TARGET=${{ matrix.target }}" >> $GITHUB_ENV
+
+      # missing build tools on some macos runners
+      - name: Install macOS build dependencies
+        if: runner.os == 'macOS'
+        run: brew install automake libtool
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD: ${{ matrix.build }}
+          CIBW_BEFORE_ALL_LINUX: yum install -y bzip2-devel flac-devel netcdf-devel
+          CIBW_BEFORE_ALL_MACOS: brew install bzip2 flac netcdf
+          CIBW_BEFORE_BUILD: ./deps/install_boost.sh
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
+            DYLD_LIBRARY_PATH=$PWD/deps/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
+          CIBW_BUILD_VERBOSITY: 1
+          CIBW_ENVIRONMENT: >
+            CMAKE_BUILD_PARALLEL_LEVEL=4 CMAKE_PREFIX_PATH=$PWD/deps
+          CIBW_TEST_COMMAND: ctest --test-dir {package}/build --output-on-failure
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheel-${{ matrix.build }}
+          path: ./wheelhouse/spt3g*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-sdist
+          path: dist/*.tar.gz
+
+  upload_pypi:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    if: github.event_name == 'release' && github.event.action == 'published'
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          # unpacks all CIBW artifacts into dist/
+          pattern: cibw-*
+          path: dist
+          merge-multiple: true
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

CMakeLists.txt

@@ -40,7 +40,13 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 # Find all the Boost and Python libraries
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
-find_package(Python COMPONENTS Interpreter Development REQUIRED)
+
+if(DEFINED ENV{CIBUILDWHEEL})
+	# Can't compile binaries against libpython in CI
+	find_package(Python COMPONENTS Interpreter Development.Module REQUIRED)
+else()
+	find_package(Python COMPONENTS Interpreter Development REQUIRED)
+endif()
 
 set(Boost_USE_STATIC_LIBS OFF)
 set(Boost_USE_MULTITHREADED ON)
@@ -91,7 +97,7 @@ configure_file(${CMAKE_SOURCE_DIR}/cmake/env-shell.sh.in ${CMAKE_BINARY_DIR}/env
 execute_process(COMMAND mkdir -p ${CMAKE_LIBRARY_OUTPUT_DIRECTORY})
 execute_process(COMMAND mkdir -p ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
 execute_process(COMMAND mkdir -p ${SPT3G_MODULE_DIR})
-execute_process(COMMAND ln -fsn ${CMAKE_SOURCE_DIR}/cmake/init.py ${SPT3G_MODULE_DIR}/__init__.py)
+execute_process(COMMAND ln -fsn ${CMAKE_SOURCE_DIR}/cmake/package/__init__.py ${SPT3G_MODULE_DIR}/__init__.py)
 
 set(BUILD_PROJECTS "${BUILD_PROJECTS}" CACHE STRING "The subset of available projects to actually build")
 if(NOT "${BUILD_PROJECTS}" STREQUAL "")
@@ -106,16 +112,6 @@ if(${USE_PROJECT_LIST} AND WILL_BUILD_CORE EQUAL -1)
 	list(APPEND BUILD_PROJECTS core)
 endif(${USE_PROJECT_LIST} AND WILL_BUILD_CORE EQUAL -1)
 
-# If not set by the user, ask python where it keeps modules
-if(NOT PYTHON_MODULE_DIR)
-	EXECUTE_PROCESS (COMMAND ${Python_EXECUTABLE} -c
-		"import sysconfig; print(sysconfig.get_path(\"platlib\"))"
-		OUTPUT_VARIABLE PYTHON_MODULE_DIR
-		OUTPUT_STRIP_TRAILING_WHITESPACE
-	)
-endif(NOT PYTHON_MODULE_DIR)
-message(STATUS "Python modules will be installed to ${PYTHON_MODULE_DIR}")
-
 include(GNUInstallDirs)
 if(APPLE)
        # See: https://gitlab.kitware.com/cmake/community/-/wikis/doc/cmake/RPATH-handling
@@ -161,11 +157,13 @@ if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.17)
 endif(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.17)
 
 # Target for version string
-add_custom_target(version ALL
-	COMMAND sh ${CMAKE_SOURCE_DIR}/cmake/getvers.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}
-	BYPRODUCTS ${CMAKE_BINARY_DIR}/spt3g/version.py ${CMAKE_BINARY_DIR}/pyproject.toml
-	COMMENT "Regenerating VCS version information"
-)
+if(NOT PIP_SPT3G_VERSION_FILE)
+	add_custom_target(version ALL
+		COMMAND sh ${CMAKE_SOURCE_DIR}/cmake/getvers.sh ${CMAKE_SOURCE_DIR} ${SPT3G_MODULE_DIR}/version.py
+		BYPRODUCTS ${SPT3G_MODULE_DIR}/version.py
+		COMMENT "Regenerating VCS version information"
+	)
+endif()
 
 # Add mechanism to make a tarball for the grid
 add_custom_target(tarball
@@ -188,37 +186,6 @@ add_custom_target(docs
 # Export the phony library target
 INSTALL(TARGETS spt3g EXPORT ${PROJECT_NAME}Config)
 
-# The python module
-# First, one-off files
-INSTALL(FILES ${CMAKE_SOURCE_DIR}/cmake/init.py DESTINATION ${PYTHON_MODULE_DIR}/spt3g RENAME __init__.py)
-INSTALL(FILES ${CMAKE_BINARY_DIR}/spt3g/version.py DESTINATION ${PYTHON_MODULE_DIR}/spt3g)
-
-# Python scripts
-foreach(dir ${SPT3G_PYTHON_DIRS})
-	get_filename_component(dir_name "${dir}" NAME)
-	if("${dir_name}" STREQUAL "python")
-		# If the directory name is "python", need to walk up the path to find out what it should be called.
-		get_filename_component(dir_name "${dir}" DIRECTORY)
-		get_filename_component(lib "${dir_name}" NAME)
-		# The INSTALL conmmand does not support the RENAME option in directory mode, which makes this a pain.
-		file(GLOB_RECURSE project_py_files LIST_DIRECTORIES false RELATIVE ${dir} ${dir}/*)
-		foreach(file ${project_py_files})
-			get_filename_component(subdir "${file}" DIRECTORY)
-			if(file MATCHES ".*pyc" OR file MATCHES ".*pyo")
-				continue() # skip existing compiled python files
-			endif(file MATCHES ".*pyc" OR file MATCHES ".*pyo")
-			install(FILES "${dir}/${file}" DESTINATION ${PYTHON_MODULE_DIR}/spt3g/${lib}/${subdir})
-		endforeach(file ${project_py_files})
-	else("${dir_name}" STREQUAL "python")
-		install(DIRECTORY ${dir} DESTINATION ${PYTHON_MODULE_DIR}/spt3g/
-			PATTERN "*.pyc" EXCLUDE # skip existing compiled python files
-			PATTERN "*.pyo" EXCLUDE)
-	endif("${dir_name}" STREQUAL "python")
-endforeach(dir SPT3G_PYTHON_DIRS)
-# Ensure that python scripts are precompiled, in case they will be executed from a read-only perspective
-install(CODE "MESSAGE(STATUS \"Pre-compiling python scripts in ${PYTHON_MODULE_DIR}/spt3g/\")
-EXECUTE_PROCESS(COMMAND ${Python_EXECUTABLE} -O -m compileall -fq ${PYTHON_MODULE_DIR}/spt3g/)")
-
 # The exectutables
 foreach(program ${SPT3G_PROGRAMS})
 	# programs are likely to by symlinks, so resolve those before installing

README.rst

@@ -1,3 +1,12 @@
+.. image:: https://badge.fury.io/py/spt3g.svg
+    :target: https://badge.fury.io/py/spt3g
+
+.. image:: https://github.com/CMB-S4/spt3g_software/actions/workflows/cmake.yml/badge.svg
+    :target: https://github.com/CMB-S4/spt3g_software/actions/workflows/cmake.yml
+
+.. image:: https://github.com/CMB-S4/spt3g_software/actions/workflows/wheels.yml/badge.svg
+    :target: https://github.com/CMB-S4/spt3g_software/actions/workflows/wheels.yml
+
 About
 -----
 
@@ -96,40 +105,65 @@ Installation
 
 For various reasons it may be useful to install the software after building, instead of continuing to use it out of the build directory. Several CMake variables control how the software is installed:
 
- * ``WITH_BZIP2``, which defaults to ``TRUE``, is used to control whether the core library is built with support for bzip2 compression of G3 files.  Use ``-DWITH_BZIP2=FALSE`` when calling ``cmake`` to disable.
- * ``CMAKE_INSTALL_PREFIX``, which defaults to ``/usr/local`` is used as the root directory for installing all non-python components (header files, cmake export scripts, etc.)
- * ``PYTHON_MODULE_DIR``, which if not explicitly set defaults to the result of running `distutils.sysconfig.get_python_lib <https://docs.python.org/3/distutils/apiref.html#distutils.sysconfig.get_python_lib>` with the selected python interpreter, is where the python module will be installed.
+* ``WITH_BZIP2``, which defaults to ``TRUE``, is used to control whether the core library is built with support for bzip2 compression of G3 files.  Use ``-DWITH_BZIP2=FALSE`` when calling ``cmake`` to disable.
+* ``CMAKE_INSTALL_PREFIX``, which defaults to ``/usr/local`` is used as the root directory for installing all non-python components (header files, cmake export scripts, etc.).  This variable is frequently useful when installing into a python virtual environment.
+* ``CMAKE_BUILD_PARALLEL_LEVEL`` is an environment variable (*not* a cmake option) used to control how many parallel processes are used to compile the shared libraries.  This option provides the same behavior as running ``make`` with the ``-j`` flag (e.g. ``make -j4``).
+
+Installation with Pip
+---------------------
+
+Use ``pip`` to install the python package.  Ensure that you use the appropriate options as necessary for your installation, e.g. ``--user`` or ``--prefix``.
+
+For pre-built wheels hosted on PyPI, available for most Linux x86_64, macOS x86_64 and macOS arm64 platforms, simply install the package without any additional options:
+
+.. code-block:: shell
+
+	pip install spt3g
+
+The hosted wheels will include the necessary libraries (Boost, etc) bundled with the package.  Otherwise, ensure that the dependency libraries are installed as explained above, and processed to one of the following steps.
 
-It is rarely necessary to set ``PYTHON_MODULE_DIR`` if ``python`` has been detected correctly, but setting ``CMAKE_INSTALL_PREFIX`` is frequently useful when installing into a python virtual environment. In such a case, one may want build as follows:
+To install the package from the github repo, run ``pip`` as usual (this may take a while, so consider setting the ``CMAKE_BUILD_PARALLEL_LEVEL`` environment variable):
 
 .. code-block:: shell
 
 	cd spt3g_software
-	mkdir build
-	cd build
-	cmake .. -DCMAKE_INSTALL_PREFIX="${VIRTUAL_ENV}"
-	make
-	make install
+	CMAKE_BUILD_PARALLEL_LEVEL=4 pip install -v .
 
-After this completes, it should be possible when using the virtual environment to ``import spt3g`` in python without needing to make use of ``env-shell.sh``.
+By default this will create a directory called ``build`` in the repo and run the ``cmake`` build from there.  The build directory location can be changed by setting the ``BUILD_DIR`` environment variable, but keep in mind that ``pip`` requires that the build directory must be a path inside the repo file tree.
+For development builds, use the ``--editable`` option to assemble the python package from the appropriate compiled extensions and python directories:
 
-Release Version Tracking
-------------------------
+.. code-block:: shell
 
-Use git tags to keep track of release versions.  Tags should be of the form "v0.1.2" for release with major version 0, minor version 1 and patch version 2.
-If such a tag is defined, cmake will populate the following outputs:
+	cd spt3g_software
+	CMAKE_BUILD_PARALLEL_LEVEL=4 BUILD_DIR=build pip install -v --editable .
+
+An editable build adds references to the python directories to your python path, so that edits to library python files are immediately reflected in a fresh python session.
 
- * A `cmake/Spt3gConfigVersion.cmake` file that contains the version number to be checked when including the Spt3g libraries in another cmake project
- * A `spt3g/version.py` file containing VCS parameters for access in python and stored in PipelineInfo frames
- * Add a `SPT3G_VERSION` compiler definition for accessing the version string in C++ code
+To pass arguments to the cmake build system, use the ``CMAKE_ARGS`` environment variable with arguments separated by spaces.  For example:
+
+.. code-block:: shell
+
+	cd spt3g_software
+	CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_MODULE_PATH=/usr/local/share/cmake" pip install -v --prefix=/usr/local .
 
-When exporting the source tree to a standalone archive, run the following command in the source directory to ensure that the source version is correctly exported:
+To run the test suite on the compiled package, you must have ``cmake``, and in particular the ``ctest`` utility, available on your path.  You must also know the location of the build directory where the cmake build was assembled (e.g. the value of ``$BUILD_DIR`` above).
 
 .. code-block:: shell
 
-	cmake/config_export.sh
+	ctest --test-dir path/to/spt3g_software/build --output-on-failure
+
+
+Release Version Tracking
+------------------------
+
+Use git tags to keep track of release versions.  Tags should be of the form "v0.1.2" for release with major version 0, minor version 1 and patch version 2.
+If such a tag is defined, cmake will populate the following outputs:
+
+* A ``cmake/Spt3gConfigVersion.cmake`` file that contains the version number to be checked when including the Spt3g libraries in another cmake project
+* A ``spt3g/version.py`` file containing VCS parameters for access in python and stored in PipelineInfo frames
+* Add a ``SPT3G_VERSION`` compiler definition for accessing the version string in C++ code
 
-Then archive the source tree using  `git archive` as usual.
+Use the ``git archive`` command or the Python ``build`` package to export the source tree to a standalone archive.
 
 Version Control Hygiene
 -----------------------

cmake/Spt3gConfig.cmake.in

@@ -32,7 +32,11 @@ include(CMakeFindDependencyMacro)
 set(THREADS_PREFER_PTHREAD_FLAG @THREADS_PREFER_PTHREAD_FLAG@)
 find_dependency(Threads REQUIRED)
 
-find_dependency(Python COMPONENTS Interpreter Development REQUIRED)
+if(DEFINED ENV{CIBUILDWHEEL})
+	find_dependency(Python COMPONENTS Interpreter Development.Module REQUIRED)
+else()
+	find_dependency(Python COMPONENTS Interpreter Development REQUIRED)
+endif()
 
 set(Boost_USE_STATIC_LIBS @Boost_USE_STATIC_LIBS@)
 set(Boost_USE_MULTITHREADED @Boost_USE_MULTITHREADED@)